Spaces:
Running
Running
Sync from GitHub 567c1a39900defc0bb5a8dce17616ee3bb817643
Browse files- .gitattributes +4 -0
- cert_study_app/demo_data/questions_seed.json +0 -0
- cert_study_app/services/demo_seed_service.py +63 -33
- scripts/export_hf_seed.py +89 -8
- static/question_assets/AZ-104_dump_56cadaca/page_12.jpg +3 -0
- static/question_assets/AZ-104_dump_56cadaca/page_21.jpg +3 -0
- static/question_assets/AZ-104_dump_56cadaca/page_30.jpg +3 -0
- static/question_assets/AZ-104_dump_56cadaca/page_32.jpg +3 -0
.gitattributes
CHANGED
|
@@ -245,3 +245,7 @@ static/question_assets/AZ-104_dump_56cadaca/page_95.jpg filter=lfs diff=lfs merg
|
|
| 245 |
static/question_assets/AZ-104_dump_56cadaca/page_97.jpg filter=lfs diff=lfs merge=lfs -text
|
| 246 |
static/question_assets/AZ-104_dump_56cadaca/page_98.jpg filter=lfs diff=lfs merge=lfs -text
|
| 247 |
static/question_assets/AZ-104_dump_56cadaca/page_99.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 245 |
static/question_assets/AZ-104_dump_56cadaca/page_97.jpg filter=lfs diff=lfs merge=lfs -text
|
| 246 |
static/question_assets/AZ-104_dump_56cadaca/page_98.jpg filter=lfs diff=lfs merge=lfs -text
|
| 247 |
static/question_assets/AZ-104_dump_56cadaca/page_99.jpg filter=lfs diff=lfs merge=lfs -text
|
| 248 |
+
static/question_assets/AZ-104_dump_56cadaca/page_12.jpg filter=lfs diff=lfs merge=lfs -text
|
| 249 |
+
static/question_assets/AZ-104_dump_56cadaca/page_21.jpg filter=lfs diff=lfs merge=lfs -text
|
| 250 |
+
static/question_assets/AZ-104_dump_56cadaca/page_30.jpg filter=lfs diff=lfs merge=lfs -text
|
| 251 |
+
static/question_assets/AZ-104_dump_56cadaca/page_32.jpg filter=lfs diff=lfs merge=lfs -text
|
cert_study_app/demo_data/questions_seed.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
cert_study_app/services/demo_seed_service.py
CHANGED
|
@@ -30,48 +30,78 @@ def _json_text(value, fallback):
|
|
| 30 |
return fallback
|
| 31 |
|
| 32 |
|
| 33 |
-
def
|
| 34 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
return 0
|
| 36 |
|
| 37 |
-
|
| 38 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
return 0
|
| 40 |
|
| 41 |
with _seed_data_path().open("r", encoding="utf-8") as file:
|
| 42 |
rows = json.load(file)
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
inserted = 0
|
| 45 |
for row in rows:
|
| 46 |
-
question = Question(
|
| 47 |
-
|
| 48 |
-
answer=_json_text(row.get("answer"), None),
|
| 49 |
-
explanation=row.get("explanation"),
|
| 50 |
-
question_type=row.get("question_type", "MCQ"),
|
| 51 |
-
question_number=row.get("question_number"),
|
| 52 |
-
group_id=row.get("group_id"),
|
| 53 |
-
parent_stem=row.get("parent_stem"),
|
| 54 |
-
parent_image_paths=_json_text(row.get("parent_image_paths"), "[]"),
|
| 55 |
-
page=row.get("page"),
|
| 56 |
-
category=row.get("category"),
|
| 57 |
-
subcategory=row.get("subcategory"),
|
| 58 |
-
source=row.get("source") or DEMO_SOURCE,
|
| 59 |
-
image_path=row.get("image_path"),
|
| 60 |
-
raw_text=row.get("raw_text") or row.get("stem"),
|
| 61 |
-
structured_data_json=json.dumps(row, ensure_ascii=False),
|
| 62 |
-
parse_status=row.get("parse_status") or "approved",
|
| 63 |
-
quality_score=row.get("quality_score") or 100,
|
| 64 |
-
quality_status=row.get("quality_status") or "seed",
|
| 65 |
-
quality_issues=_json_text(row.get("quality_issues"), "[]"),
|
| 66 |
-
chunk_key=row.get("chunk_key"),
|
| 67 |
-
chunk_index=row.get("chunk_index"),
|
| 68 |
-
parser_version=row.get("parser_version") or "seed-v1",
|
| 69 |
-
code=row.get("code"),
|
| 70 |
-
pairs=_json_text(row.get("pairs"), None),
|
| 71 |
-
sequence=_json_text(row.get("sequence"), None),
|
| 72 |
-
)
|
| 73 |
-
question.set_options(row.get("options") or [])
|
| 74 |
-
question.set_concept_tags(row.get("concept_tags") or [])
|
| 75 |
db.add(question)
|
| 76 |
inserted += 1
|
| 77 |
|
|
|
|
| 30 |
return fallback
|
| 31 |
|
| 32 |
|
| 33 |
+
def _apply_seed_row(question: Question, row: dict) -> None:
|
| 34 |
+
question.stem = row["stem"]
|
| 35 |
+
question.answer = _json_text(row.get("answer"), None)
|
| 36 |
+
question.explanation = row.get("explanation")
|
| 37 |
+
question.question_type = row.get("question_type", "MCQ")
|
| 38 |
+
question.question_number = row.get("question_number")
|
| 39 |
+
question.group_id = row.get("group_id")
|
| 40 |
+
question.parent_stem = row.get("parent_stem")
|
| 41 |
+
question.parent_image_paths = _json_text(row.get("parent_image_paths"), "[]")
|
| 42 |
+
question.page = row.get("page")
|
| 43 |
+
question.category = row.get("category")
|
| 44 |
+
question.subcategory = row.get("subcategory")
|
| 45 |
+
question.source = row.get("source") or DEMO_SOURCE
|
| 46 |
+
question.image_path = row.get("image_path")
|
| 47 |
+
question.raw_text = row.get("raw_text") or row.get("stem")
|
| 48 |
+
question.structured_data_json = json.dumps(row, ensure_ascii=False)
|
| 49 |
+
question.parse_status = row.get("parse_status") or "approved"
|
| 50 |
+
question.quality_score = row.get("quality_score") or 100
|
| 51 |
+
question.quality_status = row.get("quality_status") or "seed"
|
| 52 |
+
question.quality_issues = _json_text(row.get("quality_issues"), "[]")
|
| 53 |
+
question.chunk_key = row.get("chunk_key")
|
| 54 |
+
question.chunk_index = row.get("chunk_index")
|
| 55 |
+
question.parser_version = row.get("parser_version") or "seed-v1"
|
| 56 |
+
question.code = row.get("code")
|
| 57 |
+
question.pairs = _json_text(row.get("pairs"), None)
|
| 58 |
+
question.sequence = _json_text(row.get("sequence"), None)
|
| 59 |
+
question.set_options(row.get("options") or [])
|
| 60 |
+
question.set_concept_tags(row.get("concept_tags") or [])
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def _refresh_seed_questions(db, rows: list[dict]) -> int:
|
| 64 |
+
if os.getenv("CERT_STUDY_REFRESH_SEED", "1") == "0":
|
| 65 |
return 0
|
| 66 |
|
| 67 |
+
updated = 0
|
| 68 |
+
for row in rows:
|
| 69 |
+
question_number = row.get("question_number")
|
| 70 |
+
if not question_number:
|
| 71 |
+
continue
|
| 72 |
+
|
| 73 |
+
question = (
|
| 74 |
+
db.query(Question)
|
| 75 |
+
.filter(Question.question_number == question_number)
|
| 76 |
+
.order_by(Question.id.asc())
|
| 77 |
+
.first()
|
| 78 |
+
)
|
| 79 |
+
if not question:
|
| 80 |
+
continue
|
| 81 |
+
|
| 82 |
+
_apply_seed_row(question, row)
|
| 83 |
+
updated += 1
|
| 84 |
+
|
| 85 |
+
if updated:
|
| 86 |
+
db.commit()
|
| 87 |
+
return updated
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def seed_demo_questions_if_empty(db) -> int:
|
| 91 |
+
if os.getenv("CERT_STUDY_SEED_DEMO", "1") == "0":
|
| 92 |
return 0
|
| 93 |
|
| 94 |
with _seed_data_path().open("r", encoding="utf-8") as file:
|
| 95 |
rows = json.load(file)
|
| 96 |
|
| 97 |
+
existing = db.query(func.count(Question.id)).scalar() or 0
|
| 98 |
+
if existing:
|
| 99 |
+
return _refresh_seed_questions(db, rows)
|
| 100 |
+
|
| 101 |
inserted = 0
|
| 102 |
for row in rows:
|
| 103 |
+
question = Question()
|
| 104 |
+
_apply_seed_row(question, row)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
db.add(question)
|
| 106 |
inserted += 1
|
| 107 |
|
scripts/export_hf_seed.py
CHANGED
|
@@ -9,6 +9,7 @@ from typing import Any
|
|
| 9 |
|
| 10 |
ROOT = Path(__file__).resolve().parent.parent
|
| 11 |
DEFAULT_SOURCE_JSON = ROOT / "data" / "Json" / "questions.json"
|
|
|
|
| 12 |
DEFAULT_IMAGE_ROOT = ROOT / "data" / "images"
|
| 13 |
DEFAULT_SEED_PATH = ROOT / "cert_study_app" / "demo_data" / "questions_seed.json"
|
| 14 |
DEFAULT_ASSET_ROOT = ROOT / "static" / "question_assets"
|
|
@@ -63,19 +64,84 @@ def _copy_image(source_path: Path, asset_root: Path) -> str:
|
|
| 63 |
return target_path.relative_to(ROOT).as_posix()
|
| 64 |
|
| 65 |
|
| 66 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
source_pages = row.get("source_pages") or []
|
| 68 |
if not isinstance(source_pages, list):
|
| 69 |
source_pages = [source_pages]
|
| 70 |
|
| 71 |
-
image_path = row.get("image_path")
|
| 72 |
if not image_path:
|
| 73 |
source_image = _first_existing_page_image(source_pages, image_root)
|
| 74 |
if source_image:
|
| 75 |
image_path = _copy_image(source_image, asset_root)
|
| 76 |
|
| 77 |
-
question_number =
|
| 78 |
page = source_pages[0] if source_pages else row.get("page")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
|
| 80 |
return {
|
| 81 |
"question_number": question_number,
|
|
@@ -90,8 +156,9 @@ def _export_row(row: dict[str, Any], image_root: Path, asset_root: Path) -> dict
|
|
| 90 |
"page": page,
|
| 91 |
"image_path": image_path,
|
| 92 |
"raw_text": row.get("raw_text") or row.get("stem") or row.get("question") or "",
|
| 93 |
-
"
|
| 94 |
-
"
|
|
|
|
| 95 |
"concept_tags": row.get("concept_tags") or [],
|
| 96 |
"parse_status": row.get("parse_status") or "approved",
|
| 97 |
"quality_score": row.get("quality_score") or 100,
|
|
@@ -101,15 +168,27 @@ def _export_row(row: dict[str, Any], image_root: Path, asset_root: Path) -> dict
|
|
| 101 |
"chunk_index": row.get("chunk_index"),
|
| 102 |
"parser_version": row.get("parser_version") or "hf-seed-export-v1",
|
| 103 |
"source_pages": source_pages,
|
| 104 |
-
|
| 105 |
|
| 106 |
|
| 107 |
-
def export_seed(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 108 |
rows = _load_rows(source_json)
|
|
|
|
|
|
|
|
|
|
| 109 |
asset_root.mkdir(parents=True, exist_ok=True)
|
| 110 |
seed_path.parent.mkdir(parents=True, exist_ok=True)
|
| 111 |
|
| 112 |
-
exported = [
|
|
|
|
|
|
|
|
|
|
| 113 |
with seed_path.open("w", encoding="utf-8") as file:
|
| 114 |
json.dump(exported, file, ensure_ascii=False, indent=2)
|
| 115 |
file.write("\n")
|
|
@@ -119,6 +198,7 @@ def export_seed(source_json: Path, image_root: Path, seed_path: Path, asset_root
|
|
| 119 |
def main() -> None:
|
| 120 |
parser = argparse.ArgumentParser(description="Export deployable question seed data for Hugging Face Space.")
|
| 121 |
parser.add_argument("--source-json", type=Path, default=DEFAULT_SOURCE_JSON)
|
|
|
|
| 122 |
parser.add_argument("--image-root", type=Path, default=DEFAULT_IMAGE_ROOT)
|
| 123 |
parser.add_argument("--seed-path", type=Path, default=DEFAULT_SEED_PATH)
|
| 124 |
parser.add_argument("--asset-root", type=Path, default=DEFAULT_ASSET_ROOT)
|
|
@@ -126,6 +206,7 @@ def main() -> None:
|
|
| 126 |
|
| 127 |
count = export_seed(
|
| 128 |
source_json=args.source_json,
|
|
|
|
| 129 |
image_root=args.image_root,
|
| 130 |
seed_path=args.seed_path,
|
| 131 |
asset_root=args.asset_root,
|
|
|
|
| 9 |
|
| 10 |
ROOT = Path(__file__).resolve().parent.parent
|
| 11 |
DEFAULT_SOURCE_JSON = ROOT / "data" / "Json" / "questions.json"
|
| 12 |
+
DEFAULT_PARENT_SOURCE_JSON = ROOT / "data" / "parsed_json" / "reparsed_multipage_images_20260601_205501.json"
|
| 13 |
DEFAULT_IMAGE_ROOT = ROOT / "data" / "images"
|
| 14 |
DEFAULT_SEED_PATH = ROOT / "cert_study_app" / "demo_data" / "questions_seed.json"
|
| 15 |
DEFAULT_ASSET_ROOT = ROOT / "static" / "question_assets"
|
|
|
|
| 64 |
return target_path.relative_to(ROOT).as_posix()
|
| 65 |
|
| 66 |
|
| 67 |
+
def _copy_image_reference(image_path: Any, asset_root: Path) -> str | None:
|
| 68 |
+
if not image_path:
|
| 69 |
+
return None
|
| 70 |
+
source_path = Path(str(image_path))
|
| 71 |
+
if not source_path.is_absolute():
|
| 72 |
+
source_path = ROOT / source_path
|
| 73 |
+
if not source_path.exists():
|
| 74 |
+
return str(image_path)
|
| 75 |
+
return _copy_image(source_path, asset_root)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def _question_number(row: dict[str, Any]) -> Any:
|
| 79 |
+
return row.get("question_number") or row.get("question_id") or row.get("number")
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _parent_context_by_number(parent_rows: list[dict[str, Any]]) -> dict[int, dict[str, Any]]:
|
| 83 |
+
contexts = {}
|
| 84 |
+
for row in parent_rows:
|
| 85 |
+
number = _question_number(row)
|
| 86 |
+
try:
|
| 87 |
+
number = int(number)
|
| 88 |
+
except Exception:
|
| 89 |
+
continue
|
| 90 |
+
if not row.get("parent_stem"):
|
| 91 |
+
continue
|
| 92 |
+
contexts[number] = row
|
| 93 |
+
return contexts
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def _parent_context_by_scenario(
|
| 97 |
+
rows: list[dict[str, Any]], parent_contexts: dict[int, dict[str, Any]]
|
| 98 |
+
) -> dict[str, dict[str, Any]]:
|
| 99 |
+
contexts = {}
|
| 100 |
+
for row in rows:
|
| 101 |
+
scenario = row.get("scenario")
|
| 102 |
+
if not scenario or scenario == "단일문제" or scenario in contexts:
|
| 103 |
+
continue
|
| 104 |
+
try:
|
| 105 |
+
number = int(_question_number(row))
|
| 106 |
+
except Exception:
|
| 107 |
+
continue
|
| 108 |
+
parent_context = parent_contexts.get(number)
|
| 109 |
+
if parent_context:
|
| 110 |
+
contexts[str(scenario)] = parent_context
|
| 111 |
+
return contexts
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def _export_row(
|
| 115 |
+
row: dict[str, Any],
|
| 116 |
+
image_root: Path,
|
| 117 |
+
asset_root: Path,
|
| 118 |
+
parent_contexts: dict[int, dict[str, Any]],
|
| 119 |
+
scenario_contexts: dict[str, dict[str, Any]],
|
| 120 |
+
) -> dict[str, Any]:
|
| 121 |
source_pages = row.get("source_pages") or []
|
| 122 |
if not isinstance(source_pages, list):
|
| 123 |
source_pages = [source_pages]
|
| 124 |
|
| 125 |
+
image_path = _copy_image_reference(row.get("image_path"), asset_root)
|
| 126 |
if not image_path:
|
| 127 |
source_image = _first_existing_page_image(source_pages, image_root)
|
| 128 |
if source_image:
|
| 129 |
image_path = _copy_image(source_image, asset_root)
|
| 130 |
|
| 131 |
+
question_number = _question_number(row)
|
| 132 |
page = source_pages[0] if source_pages else row.get("page")
|
| 133 |
+
try:
|
| 134 |
+
parent_context = parent_contexts.get(int(question_number)) or {}
|
| 135 |
+
except Exception:
|
| 136 |
+
parent_context = {}
|
| 137 |
+
if not parent_context and row.get("scenario"):
|
| 138 |
+
parent_context = scenario_contexts.get(str(row.get("scenario"))) or {}
|
| 139 |
+
|
| 140 |
+
parent_image_paths = []
|
| 141 |
+
for parent_image_path in parent_context.get("parent_image_paths") or row.get("parent_image_paths") or []:
|
| 142 |
+
copied_path = _copy_image_reference(parent_image_path, asset_root)
|
| 143 |
+
if copied_path:
|
| 144 |
+
parent_image_paths.append(copied_path)
|
| 145 |
|
| 146 |
return {
|
| 147 |
"question_number": question_number,
|
|
|
|
| 156 |
"page": page,
|
| 157 |
"image_path": image_path,
|
| 158 |
"raw_text": row.get("raw_text") or row.get("stem") or row.get("question") or "",
|
| 159 |
+
"group_id": parent_context.get("group_id") or row.get("group_id"),
|
| 160 |
+
"parent_stem": parent_context.get("parent_stem") or row.get("parent_stem") or row.get("scenario"),
|
| 161 |
+
"parent_image_paths": parent_image_paths,
|
| 162 |
"concept_tags": row.get("concept_tags") or [],
|
| 163 |
"parse_status": row.get("parse_status") or "approved",
|
| 164 |
"quality_score": row.get("quality_score") or 100,
|
|
|
|
| 168 |
"chunk_index": row.get("chunk_index"),
|
| 169 |
"parser_version": row.get("parser_version") or "hf-seed-export-v1",
|
| 170 |
"source_pages": source_pages,
|
| 171 |
+
}
|
| 172 |
|
| 173 |
|
| 174 |
+
def export_seed(
|
| 175 |
+
source_json: Path,
|
| 176 |
+
parent_source_json: Path,
|
| 177 |
+
image_root: Path,
|
| 178 |
+
seed_path: Path,
|
| 179 |
+
asset_root: Path,
|
| 180 |
+
) -> int:
|
| 181 |
rows = _load_rows(source_json)
|
| 182 |
+
parent_rows = _load_rows(parent_source_json) if parent_source_json.exists() else []
|
| 183 |
+
parent_contexts = _parent_context_by_number(parent_rows)
|
| 184 |
+
scenario_contexts = _parent_context_by_scenario(rows, parent_contexts)
|
| 185 |
asset_root.mkdir(parents=True, exist_ok=True)
|
| 186 |
seed_path.parent.mkdir(parents=True, exist_ok=True)
|
| 187 |
|
| 188 |
+
exported = [
|
| 189 |
+
_export_row(row, image_root, asset_root, parent_contexts, scenario_contexts)
|
| 190 |
+
for row in rows
|
| 191 |
+
]
|
| 192 |
with seed_path.open("w", encoding="utf-8") as file:
|
| 193 |
json.dump(exported, file, ensure_ascii=False, indent=2)
|
| 194 |
file.write("\n")
|
|
|
|
| 198 |
def main() -> None:
|
| 199 |
parser = argparse.ArgumentParser(description="Export deployable question seed data for Hugging Face Space.")
|
| 200 |
parser.add_argument("--source-json", type=Path, default=DEFAULT_SOURCE_JSON)
|
| 201 |
+
parser.add_argument("--parent-source-json", type=Path, default=DEFAULT_PARENT_SOURCE_JSON)
|
| 202 |
parser.add_argument("--image-root", type=Path, default=DEFAULT_IMAGE_ROOT)
|
| 203 |
parser.add_argument("--seed-path", type=Path, default=DEFAULT_SEED_PATH)
|
| 204 |
parser.add_argument("--asset-root", type=Path, default=DEFAULT_ASSET_ROOT)
|
|
|
|
| 206 |
|
| 207 |
count = export_seed(
|
| 208 |
source_json=args.source_json,
|
| 209 |
+
parent_source_json=args.parent_source_json,
|
| 210 |
image_root=args.image_root,
|
| 211 |
seed_path=args.seed_path,
|
| 212 |
asset_root=args.asset_root,
|
static/question_assets/AZ-104_dump_56cadaca/page_12.jpg
ADDED
|
Git LFS Details
|
static/question_assets/AZ-104_dump_56cadaca/page_21.jpg
ADDED
|
Git LFS Details
|
static/question_assets/AZ-104_dump_56cadaca/page_30.jpg
ADDED
|
Git LFS Details
|
static/question_assets/AZ-104_dump_56cadaca/page_32.jpg
ADDED
|
Git LFS Details
|