Add difficulty distribution

app.py

@@ -8,7 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 
 # Import the user's RAGMCQ implementation
-from generator import RAGMCQ
+from generator import RAGMCQWithDifficulty, RAGMCQ
 from utils import log_pipeline
 
 app = FastAPI(title="RAG MCQ Generator API")
@@ -24,6 +24,7 @@ app.add_middleware(
 
 # global rag instance
 rag: Optional[RAGMCQ] = None
+rag_difficulty: Optional[RAGMCQWithDifficulty] = None
 
 class GenerateResponse(BaseModel):
     mcqs: dict
@@ -34,15 +35,17 @@ class ListResponse(BaseModel):
 
 @app.on_event("startup")
 def startup_event():
+    global rag_difficulty
     global rag
 
     # instantiate the heavy object once
     rag = RAGMCQ()
+    rag_difficulty = RAGMCQWithDifficulty()
     print("RAGMCQ instance created on startup.")
 
 @app.get("/health")
 def health():
-    return {"status": "ok", "ready": rag is not None}
+    return {"status": "ok", "ready": rag_difficulty is not None and rag is not None}
 
 def _save_upload_to_temp(upload: UploadFile) -> str:
     suffix = ".pdf"
@@ -57,11 +60,11 @@ def _save_upload_to_temp(upload: UploadFile) -> str:
 async def list_collection_files_endpoint(
     collection_name: str = "programming"
 ):
-    global rag
-    if rag is None:
+    global rag_difficulty
+    if rag_difficulty is None:
         raise HTTPException(status_code=503, detail="RAGMCQ not ready on server.")
 
-    files = rag.list_files_in_collection(collection_name)
+    files = rag_difficulty.list_files_in_collection(collection_name)
 
     return {"files": files}
 
@@ -81,8 +84,8 @@ async def upload_multiple_files(
     - overwrite: if true, existing points for each filename will be removed
     - qdrant_filename_prefix: optional prefix; if provided each file will be saved under "<prefix>_<original_filename>"
     """
-    global rag
-    if rag is None:
+    global rag_difficulty
+    if rag_difficulty is None:
         raise HTTPException(status_code=503, detail="RAGMCQ not ready on server.")
 
     saved_files = []
@@ -112,7 +115,7 @@ async def upload_multiple_files(
         )
 
         try:
-            rag.save_pdf_to_qdrant(tmp_path, filename=qdrant_filename, collection=collection_name, overwrite=overwrite)
+            rag_difficulty.save_pdf_to_qdrant(tmp_path, filename=qdrant_filename, collection=collection_name, overwrite=overwrite)
             saved_files.append(qdrant_filename)
         except Exception as e:
             # collect failure info rather than aborting all uploads
@@ -122,6 +125,185 @@ async def upload_multiple_files(
 
 
 
+@app.post("/generate_saved_with_difficulty", response_model=GenerateResponse)
+async def generate_saved_with_difficulty_endpoint(
+    n_easy_questions: int = Form(3),
+    n_medium_questions: int = Form(5), 
+    n_hard_questions: int = Form(2),
+    qdrant_filename: str = Form("default_filename"),
+    collection_name: str = Form("programming"),
+    mode: str = Form("rag"),
+    questions_per_chunk: int = Form(3),
+    top_k: int = Form(3),
+    temperature: float = Form(0.2),
+    validate_mcqs: bool = Form(False),
+    enable_fiddler: bool = Form(False),
+):
+    global rag_difficulty
+    if rag_difficulty is None:
+        raise HTTPException(status_code=503, detail="RAGMCQ not ready on server.")
+    
+    difficulty_counts = {
+        "easy": n_easy_questions,
+        "medium": n_medium_questions,
+        "hard": n_hard_questions
+    }
+    
+    all_mcqs = {}
+    counter = 1
+
+    for difficulty, n_questions in difficulty_counts.items():
+        try:
+            mcqs = rag_difficulty.generate_from_qdrant(
+                filename=qdrant_filename,
+                collection=collection_name,
+                n_questions=n_questions,
+                mode=mode,
+                questions_per_chunk=questions_per_chunk,
+                top_k=top_k,
+                temperature=temperature,
+                enable_fiddler=enable_fiddler,
+                target_difficulty=difficulty,
+            )
+            questions_list = []
+            if isinstance(mcqs, dict):
+                for v in mcqs.values():
+                    if isinstance(v, list):
+                        questions_list.extend(v)
+                    else:
+                        questions_list.append(v)
+            elif isinstance(mcqs, list):
+                questions_list = mcqs
+            else:
+                continue
+
+            for qobj in questions_list:
+                if isinstance(qobj, dict):
+                    qobj["_difficulty"] = difficulty
+                all_mcqs[str(counter)] = qobj
+                counter += 1
+
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Generation from saved file failed: {e}")
+
+    validation_report = None
+
+    if validate_mcqs:
+        try:
+            # validate_mcqs expects keys as strings and the normalized content
+            validation_report = rag_difficulty.validate_mcqs(all_mcqs, top_k=top_k)
+        except Exception as e:
+            # don't fail the whole request for a validation error — return generator output and note the error
+            validation_report = {"error": f"Validation failed: {e}"}
+
+    # log_pipeline('test/mcq_output.json', content={"mcqs": mcqs, "validation": validation_report})
+
+    return {"mcqs": all_mcqs, "validation": validation_report}
+
+
+
+
+@app.post("/generate_with_difficulty", response_model=GenerateResponse)
+async def generate_with_difficulty_endpoint(
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(...),
+    n_easy_questions: int = Form(3),
+    n_medium_questions: int = Form(5), 
+    n_hard_questions: int = Form(2),
+    qdrant_filename: str = Form("default_filename"),
+    collection_name: str = Form("programming"),
+    mode: str = Form("rag"),
+    questions_per_page: int = Form(3),
+    top_k: int = Form(3),
+    temperature: float = Form(0.2),
+    validate_mcqs: bool = Form(False),
+    enable_fiddler: bool = Form(False)
+):
+    global rag_difficulty
+    if rag_difficulty is None:
+        raise HTTPException(status_code=503, detail="RAGMCQ not ready on server.")
+
+    # basic file validation
+    if not file.filename.lower().endswith(".pdf"):
+        raise HTTPException(status_code=400, detail="Only PDF files are supported.")
+
+    # save uploaded file to a temp location
+    tmp_path = _save_upload_to_temp(file)
+
+    # ensure file removed afterward
+    def _cleanup(path: str):
+        try:
+            os.remove(path)
+        except Exception:
+            pass
+
+    background_tasks.add_task(_cleanup, tmp_path)
+
+    # save pdf
+    try:
+        rag_difficulty.save_pdf_to_qdrant(tmp_path, filename=qdrant_filename, collection=collection_name, overwrite=True)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Could not save file to Qdrant Cloud: {e}")
+
+    difficulty_counts = {
+        "easy": n_easy_questions,
+        "medium": n_medium_questions,
+        "hard": n_hard_questions
+    }
+    
+    all_mcqs = {}
+    counter = 1
+
+    for difficulty, n_questions in difficulty_counts.items():
+        try:
+            mcqs = rag_difficulty.generate_from_pdf(
+                pdf_path=tmp_path,
+                n_questions=n_questions,
+                mode=mode,
+                questions_per_page=questions_per_page,
+                top_k=top_k,
+                temperature=temperature,
+                enable_fiddler=enable_fiddler,
+                target_difficulty=difficulty,
+            )
+            questions_list = []
+            if isinstance(mcqs, dict):
+                for v in mcqs.values():
+                    if isinstance(v, list):
+                        questions_list.extend(v)
+                    else:
+                        questions_list.append(v)
+            elif isinstance(mcqs, list):
+                questions_list = mcqs
+            else:
+                continue
+
+            for qobj in questions_list:
+                if isinstance(qobj, dict):
+                    qobj["_difficulty"] = difficulty
+                all_mcqs[str(counter)] = qobj
+                counter += 1
+
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Generation from file failed: {e}")
+
+    validation_report = None
+
+    if validate_mcqs:
+        try:
+            # rag.build_index_from_pdf(tmp_path)
+            # validate_mcqs expects keys as strings and the normalized content
+            validation_report = rag_difficulty.validate_mcqs(all_mcqs, top_k=top_k)
+        except Exception as e:
+            # don't fail the whole request for a validation error — return generator output and note the error
+            validation_report = {"error": f"Validation failed: {e}"}
+
+
+    # log_pipeline('test/mcq_output.json', content={"mcqs": mcqs, "validation": validation_report})
+
+    return {"mcqs": all_mcqs, "validation": validation_report}
+
+
 @app.post("/generate_saved", response_model=GenerateResponse)
 async def generate_saved_endpoint(
     n_questions: int = Form(10),

generator.py

@@ -9,6 +9,7 @@ from sentence_transformers import SentenceTransformer, CrossEncoder
 from transformers import pipeline
 from uuid import uuid4
 import pymupdf4llm
+from typing_extensions import override
 
 try:
     from qdrant_client import QdrantClient
@@ -31,7 +32,7 @@ try:
 except Exception:
     _HAS_FAISS = False
 
-from utils import generate_mcqs_from_text, _post_chat, _safe_extract_json, save_to_local
+from utils import generate_mcqs_from_text, _post_chat, _safe_extract_json, save_to_local, structure_context_for_llm, new_generate_mcqs_from_text
 
 from huggingface_hub import login
 login(token=os.environ['HF_MODEL_TOKEN'])
@@ -1074,3 +1075,422 @@ class RAGMCQ:
             label = "khó"
 
         return score, label
+
+class RAGMCQWithDifficulty(RAGMCQ):
+    def __init__(
+        self,
+        embedder_model: str = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
+        generation_model: str = "openai/gpt-oss-120b",
+        qdrant_url: str = os.environ.get('QDRANT_URL') or "",
+        qdrant_api_key: str = os.environ.get('QDRANT_API_KEY') or "",
+        qdrant_prefer_grpc: bool = False,
+    ):
+        super().__init__(embedder_model, generation_model, qdrant_url, qdrant_api_key, qdrant_prefer_grpc)
+
+    @override
+    def generate_from_pdf(
+        self,
+        pdf_path: str,
+        n_questions: int = 10,
+        mode: str = "rag", # per_page or rag
+        questions_per_page: int = 3, # for per_page mode
+        top_k: int = 3, # chunks to retrieve for each question in rag mode
+        temperature: float = 0.2,
+        enable_fiddler: bool = False,
+        target_difficulty: str = 'easy'  # easy, mid, difficult
+    ) -> Dict[str, Any]:
+        # build index
+        self.build_index_from_pdf(pdf_path)
+
+        output: Dict[str, Any] = {}
+        qcount = 0
+
+        if mode == "per_page":
+            # iterate pages -> chunks
+            for idx, meta in enumerate(self.metadata):
+                chunk_text = self.texts[idx]
+
+                if not chunk_text.strip():
+                    continue
+                to_gen = questions_per_page
+
+                # ask generator
+                try:
+                    structured_context = structure_context_for_llm(context, model=self.generation_model, temperature=0.2, enable_fiddler=False)
+                    mcq_block = generate_mcqs_from_text(
+                        source_text=chunk_text, n=to_gen, model=self.generation_model, temperature=temperature, enable_fiddler=enable_fiddler
+                    )
+                except Exception as e:
+                    # skip this chunk if generator fails
+                    print(f"Generator failed on page {meta['page']} chunk {meta['chunk_id']}: {e}")
+                    continue
+
+                if "error" in list(mcq_block.keys()):
+                    return output
+
+                for item in sorted(mcq_block.keys(), key=lambda x: int(x)):
+                    qcount += 1
+                    output[str(qcount)] = mcq_block[item]
+                    if qcount >= n_questions:
+                        return output
+
+            return output
+
+        # pdf gene
+        elif mode == "rag":
+            # strategy: create a few natural short queries by sampling sentences or using chunk summaries.
+            # create queries by sampling chunk text sentences.
+            # stop when n_questions reached or max_attempts exceeded.
+            attempts = 0
+            max_attempts = n_questions * 4
+
+            while qcount < n_questions and attempts < max_attempts:
+                attempts += 1
+                # create a seed query: pick a random chunk, pick a sentence from it
+                seed_idx = random.randrange(len(self.texts))
+                chunk = self.texts[seed_idx]
+
+                #? investigate better Chunking Strategy
+                #with open("chunks.txt", "a", encoding="utf-8") as f:
+                    #f.write(chunk + "\n")
+
+                sents = re.split(r'(?<=[\.\?\!])\s+', chunk)
+                seed_sent = random.choice([s for s in sents if len(s.strip()) > 20]) if sents else chunk[:200]
+                query = f"Create questions about: {seed_sent}"
+
+                # retrieve top_k chunks
+                retrieved = self._retrieve(query, top_k=top_k)
+                context_parts = []
+                for ridx, score in retrieved:
+                    md = self.metadata[ridx]
+                    context_parts.append(f"[page {md['page']}] {self.texts[ridx]}")
+                context = "\n\n".join(context_parts)
+
+                # save_to_local('test/context.md', content=context)
+
+                # call generator for 1 question (or small batch) with the retrieved context
+                try:
+                    # request 1 question at a time to keep diversity
+                    structured_context = structure_context_for_llm(context, model=self.generation_model, temperature=0.2, enable_fiddler=False)
+                    mcq_block = new_generate_mcqs_from_text(structured_context, n=questions_per_page, model=self.generation_model, temperature=temperature, enable_fiddler=False, target_difficulty=target_difficulty)
+                except Exception as e:
+                    print(f"Generator failed during RAG attempt {attempts}: {e}")
+                    continue
+
+                if "error" in list(mcq_block.keys()):
+                    return output
+
+                # append result(s)
+                for item in sorted(mcq_block.keys(), key=lambda x: int(x)):
+                    payload = mcq_block[item]
+                    q_text = (payload.get("câu hỏi") or payload.get("question") or payload.get("stem") or "").strip()
+                    options = payload.get("lựa chọn") or payload.get("options") or payload.get("choices") or {}
+                    if isinstance(options, list):
+                        options = {str(i+1): o for i, o in enumerate(options)}
+                    correct_key = payload.get("đáp án") or payload.get("answer") or payload.get("correct") or None
+                    concepts = payload.get("khái niệm sử dụng") or payload.get("concepts") or payload.get("concepts used") or None
+                    correct_text = ""
+                    if isinstance(correct_key, str) and correct_key.strip() in options:
+                        correct_text = options[correct_key.strip()]
+                    else:
+                        correct_text = payload.get("correct_text") or correct_key or ""
+                        
+                    diff_score, diff_label, components = self._estimate_difficulty_for_generation( # type: ignore
+                        q_text=q_text, options={k: str(v) for k,v in options.items()}, correct_text=str(correct_text), context_text=structured_context, concepts_used=concepts 
+                    )
+
+                    payload["độ khó"] = {"điểm": diff_score, "mức độ": diff_label}
+
+                    qcount += 1
+                    output[str(qcount)] = mcq_block[item]
+                    if qcount >= n_questions:
+                        return output
+
+            return output
+        else:
+            raise ValueError("mode must be 'per_page' or 'rag'.")
+
+    @override
+    def generate_from_qdrant(
+        self,
+        filename: str,
+        collection: str,
+        n_questions: int = 10,
+        mode: str = "rag",               # 'per_chunk' or 'rag'
+        questions_per_chunk: int = 3,    # used for 'per_chunk'
+        top_k: int = 3,                  # retrieval size used in RAG
+        temperature: float = 0.2,
+        enable_fiddler: bool = False,
+        target_difficulty: str = 'easy',
+
+    ) -> Dict[str, Any]:
+        if self.qdrant is None:
+            raise RuntimeError("Qdrant client not connected. Call connect_qdrant(...) first.")
+
+        # get all chunks for this filename (payload should contain 'text', 'page', 'chunk_id', etc.)
+        file_points = self.list_chunks_for_filename(collection=collection, filename=filename)
+        if not file_points:
+            raise RuntimeError(f"No chunks found for filename={filename} in collection={collection}.")
+
+        # create a local list of texts & metadata for sampling
+        texts = []
+        metas = []
+        for p in file_points:
+            payload = p.get("payload", {})
+            text = payload.get("text", "")
+            texts.append(text)
+            metas.append(payload)
+
+        self.texts = texts
+        self.metadata = metas
+        embeddings = self.embedder.encode(texts, convert_to_numpy=True, show_progress_bar=True)
+        if embeddings is None or len(embeddings) == 0:
+            self.embeddings = None
+            self.index = None
+        else:
+            self.embeddings = embeddings.astype("float32")
+
+            # update dim in case embedder changed unexpectedly
+            self.dim = int(self.embeddings.shape[1])
+
+            # build index
+            self._build_faiss_index()
+
+        output = {}
+        qcount = 0
+
+        if mode == "per_chunk":
+            # iterate all chunks (in payload order) and request questions_per_chunk from each
+            for i, txt in enumerate(texts):
+                if not txt.strip():
+                    continue
+                to_gen = questions_per_chunk
+                try:
+                    mcq_block = new_generate_mcqs_from_text(txt, n=to_gen, model=self.generation_model, temperature=temperature, enable_fiddler=False)
+                except Exception as e:
+                    print(f"Generator failed on chunk (index {i}): {e}")
+                    continue
+
+                if "error" in list(mcq_block.keys()):
+                    return output
+
+                for item in sorted(mcq_block.keys(), key=lambda x: int(x)):
+                    qcount += 1
+                    output[str(qcount)] = mcq_block[item]
+                    if qcount >= n_questions:
+                        return output
+            return output
+
+        elif mode == "rag":
+            attempts = 0
+            max_attempts = n_questions * 4
+            while qcount < n_questions and attempts < max_attempts:
+                attempts += 1
+                # create a seed query: pick a random chunk, pick a sentence from it
+                seed_idx = random.randrange(len(self.texts))
+                chunk = self.texts[seed_idx]
+                sents = re.split(r'(?<=[\.\?\!])\s+', chunk)
+                candidate = [s for s in sents if len(s.strip()) > 20]
+                if candidate:
+                    seed_sent = random.choice(candidate)
+                else:
+                    stripped = chunk.strip()
+                    seed_sent = (stripped[:200] if stripped else "[no text available]")
+                query = f"Create questions about: {seed_sent}"
+
+
+                # retrieve top_k chunks from the same file (restricted by filename filter)
+                retrieved = self._retrieve_qdrant(query=query, collection=collection, filename=filename, top_k=top_k)
+                print('retrieved qdrant', retrieved)
+                context_parts = []
+                for payload, score in retrieved:
+                    # payload should contain page & chunk_id and text
+                    page = payload.get("page", "?")
+                    ctxt = payload.get("text", "")
+                    context_parts.append(f"[page {page}] {ctxt}")
+                context = "\n\n".join(context_parts)
+
+
+                # q generation
+                try:
+                    # Difficulty pipeline: easy, mid, difficult
+                    structured_context = structure_context_for_llm(context, model=self.generation_model, temperature=0.2, enable_fiddler=False)
+                    mcq_block = new_generate_mcqs_from_text(structured_context, n=questions_per_chunk, model=self.generation_model, temperature=temperature, enable_fiddler=False, target_difficulty=target_difficulty)
+                except Exception as e:
+                    print(f"Generator failed during RAG attempt {attempts}: {e}")
+                    continue
+
+                if "error" in list(mcq_block.keys()):
+                    return output
+
+                for item in sorted(mcq_block.keys(), key=lambda x: int(x)):
+                    payload = mcq_block[item]
+                    q_text = (payload.get("câu hỏi") or payload.get("question") or payload.get("stem") or "").strip()
+                    options = payload.get("lựa chọn") or payload.get("options") or payload.get("choices") or {}
+
+                    if isinstance(options, list):
+                        options = {str(i+1): o for i, o in enumerate(options)}
+
+                    correct_key = payload.get("đáp án") or payload.get("answer") or payload.get("correct") or None
+                    concepts = payload.get("khái niệm sử dụng") or payload.get("concepts") or payload.get("concepts used") or None
+
+                    correct_text = ""
+                    if isinstance(correct_key, str) and correct_key.strip() in options:
+                        correct_text = options[correct_key.strip()]
+                    else:
+                        correct_text = payload.get("correct_text") or correct_key or ""
+
+                    #? change estimate
+                    diff_score, diff_label, components = self._estimate_difficulty_for_generation( # type: ignore
+                        q_text=q_text, options={k: str(v) for k,v in options.items()}, correct_text=str(correct_text), context_text=structured_context, concepts_used=concepts 
+                    )
+
+                    payload["độ khó"] = {"điểm": diff_score, "mức độ": diff_label}
+
+                    # CHECK n generation: if number of request mcqs < default generation number e.g. 5 - 3 = 2 < 3 then only genearate 2 mcqs
+                    if n_questions - qcount < questions_per_chunk:
+                      questions_per_chunk = n_questions - qcount
+
+                    qcount += 1 # count number of question
+                    print('qcount:', qcount)
+                    print('questions_per_chunk:', questions_per_chunk)
+
+                    output[str(qcount)] = mcq_block[item]
+                    if qcount >= n_questions:
+                        return output
+
+            if output is not None:
+              print("output available")
+            return output
+        else:
+            raise ValueError("mode must be 'per_chunk' or 'rag'.")
+
+    @override
+    def _estimate_difficulty_for_generation(
+        self,
+        q_text: str,
+        options: Dict[str, str],
+        correct_text: str,
+        context_text: str = "",
+        concepts_used: Dict = {}
+    ) -> Tuple[float, str]:
+        def safe_map_sim(s):
+            # map potentially [-1,1] cosine-like to [0,1], clamp
+            try:
+                s = float(s)
+            except Exception:
+                return 0.0
+            mapped = (s + 1.0) / 2.0
+            return max(0.0, min(1.0, mapped))
+        
+        # embedding support
+        emb_support = 0.0
+        try:
+            stmt = (q_text or "").strip()
+            if correct_text:
+                stmt = f"{stmt} Answer: {correct_text}"
+
+            # use internal retrieve but map returned score
+            res = []
+            try:
+                res = self._retrieve(stmt, top_k=1)
+            except Exception:
+                res = []
+
+            if res:
+                raw_score = float(res[0][1])
+                emb_support = safe_map_sim(raw_score)
+            else:
+                emb_support = 0.0
+        except Exception:
+            emb_support = 0.0
+
+        # distractor sims
+        mean_sim = 0.0
+        distractor_penalty = 0.0
+        amb_flag = 0.0
+        try:
+            keys = list(options.keys())
+            texts = [options[k] for k in keys]
+            if correct_text is None:
+                correct_text = ""
+
+            all_texts = [correct_text] + texts
+            embs = self.embedder.encode(all_texts, convert_to_numpy=True)
+            embs = np.asarray(embs, dtype=float)
+            norms = np.linalg.norm(embs, axis=1, keepdims=True) + 1e-12
+            embs = embs / norms
+            corr = embs[0]
+            opts = embs[1:]
+
+            if opts.size == 0:
+                mean_sim = 0.0
+                distractor_penalty = 0.0
+                gap = 0.0
+            else:
+                sims = (opts @ corr).tolist() # [-1,1]
+                sims_mapped = [safe_map_sim(s) for s in sims] # [0,1]
+                mean_sim = float(sum(sims_mapped) / len(sims_mapped))
+                # gap between best distractor and second best (higher gap -> easier)
+                sorted_s = sorted(sims_mapped, reverse=True)
+                top = sorted_s[0]
+                second = sorted_s[1] if len(sorted_s) > 1 else 0.0
+                gap = top - second
+                # penalties: if distractors are extremely close to correct -> higher penalty
+                too_close_count = sum(1 for s in sims_mapped if s >= 0.85)
+                too_far_count = sum(1 for s in sims_mapped if s <= 0.15)
+                distractor_penalty = min(1.0, 0.5 * mean_sim + 0.2 * (too_close_count / max(1, len(sims_mapped))) - 0.2 * (too_far_count / max(1, len(sims_mapped))))
+                amb_flag = 1.0 if top >= 0.8 else 0.0
+        except Exception:
+            mean_sim = 0.0
+            distractor_penalty = 0.0
+            amb_flag = 0.0
+            gap = 0.0
+
+        # question length normalized
+        question_len = len((q_text or "").strip())
+        question_len_norm = min(1.0, question_len / 300.0)
+
+        # count number of concept from string
+        concepts_num = len(concepts_used.keys())
+        if concepts_num < 2:
+          concepts_penalty = 0
+        else:
+          concepts_penalty = concepts_num
+
+        # combine signals using safer semantics:
+        #    higher emb_support -> easier (so we subtract a term)
+        #    higher distractor_penalty -> harder (add)
+        #    better gap -> easier (subtract)
+        # compute score (higher -> harder)
+
+        score = 0
+        score += 0.35 * float(distractor_penalty)
+        score += 0.20 * float(mean_sim)
+        score += 0.22 * float(amb_flag)
+        score += 0.05 * float(question_len_norm)
+        score -= 0.20 * float(gap)
+
+        # clamp
+        score = max(0.0, min(1.0, float(score)))
+        components = {
+            "base": 0.3,
+            "distractor_penalty": 0.35 * float(distractor_penalty),
+            "mean_sim": 0.15 * float(mean_sim),
+            "amb_flag": 0.05 * float(amb_flag),
+            "concepts_num": 0.1 * float(concepts_num),
+            "gap": -0.12 * float(gap),
+            "question_len_norm": 0.05 * float(question_len_norm),
+            "emb_support": -0.45 * float(emb_support),
+            "total_score": score,
+        }
+
+        # label
+        if score <= 0.35:
+            label = "dễ"
+        elif score <= 0.65 and score > 0.35:
+            label = "trung bình"
+        else:
+            label = "khó"
+
+        return score, label, components

requirements.txt

@@ -8,3 +8,4 @@ qdrant-client
 pymupdf4llm
 uuid
 huggingface_hub
+typing_extensions

utils.py

@@ -53,12 +53,6 @@ def _post_chat(messages: list, model: str, temperature: float = 0.2, timeout: in
     resp.raise_for_status()
     data = resp.json()
 
-    #save_to_local('test/raw_resp.json', content=data)
-
-    #? Must update within _post_chat because it the original function for LLM generation
-    update_token_count(token_usage=data['usage']) # get data['usages']['prompt_tokens'] & data['usages']['completion_tokens']
-    # update_time_info(time_info=data['time_info'])
-
     # handle various shapes
     if "choices" in data and len(data["choices"]) > 0:
         # prefer message.content
@@ -70,7 +64,6 @@ def _post_chat(messages: list, model: str, temperature: float = 0.2, timeout: in
         if "text" in ch:
             return ch["text"]
 
-    print(f'Generation Time: {data["time_info"]}')
     # final fallback
     raise RuntimeError("Unexpected HF response shape: " + json.dumps(data)[:200])
 
@@ -92,10 +85,164 @@ def _safe_extract_json(text: str) -> dict:
         return json.loads(fixed)
 
 
+def structure_context_for_llm(
+    source_text: str,
+    model: str = "openai/gpt-oss-120b",
+    temperature: float = 0.2,
+    enable_fiddler = False,
+) -> Dict[str, Any]:
+    """
+    Take a long source_text, split into N chunks, and restructure them
+    so each chunk is self-contained, structured, and semantically meaningful.
+    """
+
+    system_message = {
+        "role": "system",
+        "content": (
+            "Bạn là một trợ lý hữu ích chuyên xử lý và cấu trúc văn bản để phục vụ mô hình ngôn ngữ (LLM). Trả lời bằng Tiếng Việt\n"
+            "Nhiệm vụ của bạn là:\n"
+            "- Nếu văn bản dài trên 500 từ chia văn bản thành 2 đoạn (chunk) có ý nghĩa rõ ràng.\n"
+            "- Mỗi chunk phải **tự chứa đủ thông tin** (self-contained) để LLM có thể hiểu độc lập.\n"
+            "- Xác định **chủ đề chính (topic)** của mỗi chunk và dùng nó làm KEY trong JSON.\n"
+            "- Trong mỗi topic, tổ chức thông tin thành cấu trúc rõ ràng gồm các trường:\n"
+            "   - 'đoạn văn': nội dung gốc đã cấu trúc đầy đủ\n"
+            "   - 'khái niệm chính': từ điểm chứa các khái niệm chính với khái niệm phụ hỗ trợ khái niệm chính đi kèm nếu có\n"
+            "   - 'công thức': danh sách công thức (nếu có)\n"
+            "   - 'ví dụ': ví dụ minh họa (nếu có)\n"
+            "   - 'tóm tắt': tóm tắt nội dung, dễ hiểu\n"
+            "- Giữ ngữ nghĩa liền mạch.\n"
+            "- Chỉ TRẢ VỀ MỘT JSON hợp lệ theo schema, không kèm văn bản khác.\n\n"
+
+            "Chỉ TRẢ VỀ duy nhất MỘT đối tượng JSON theo schema sau và không có bất kỳ văn bản nào khác:\n\n"
+            "{\n"
+            '  "Tên topic": {"đoạn văn": "nội dung đã cấu trúc của topic 1", "khái niệm chính": {"khái niệm chính 1":["khái niệm phụ", "..."],"khái niệm chính 2":["khái niệm phụ", "..."]}, "công thức": ["..."], "ví dụ": ["..."], "tóm tắt": "tóm tắt ngắn gọn"},\n'
+            "}\n"
+        )
+    }
+
+    user_message = {
+        "role": "user",
+        "content": (
+            "Hãy chia văn bản sau thành nhiều chunk theo hướng dẫn trên và xuất JSON hợp lệ.\n"
+            f"### Văn bản nguồn:\n{source_text}"
+        )
+    }
+
+    if enable_fiddler:
+        max_conf, max_cat = text_safety_check(user_message['content'])
+        if max_conf > 0.5:
+            print(f"Harmful content detected: ({max_cat} : {max_conf})")
+            return {}
+
+    raw = _post_chat([system_message, user_message], model=model, temperature=temperature)
+    parsed = _safe_extract_json(raw)
+    if not isinstance(parsed, dict):
+        raise ValueError(f"Generator returned invalid structure. Raw:\n{raw}")
+    return parsed
+
+
+def new_generate_mcqs_from_text(
+    source_text: str,
+    n: int = 3,
+    model: str = "openai/openai/gpt-oss-120b",
+    temperature: float = 0.2,
+    enable_fiddler = False,
+    target_difficulty: str = "easy",
+
+) -> Dict[str, Any]:
+
+
+    expected_concepts = {
+      "easy": 1,
+      "medium": 2,
+      "hard": (3, 4)
+    }
+    if isinstance(expected_concepts[target_difficulty], tuple):
+      min_concepts, max_concepts = expected_concepts[target_difficulty]
+      concept_range = f"{min_concepts}-{max_concepts}"
+    else:
+      concept_range = expected_concepts[target_difficulty]
+
+
+    difficulty_prompts = {
+      "easy": (
+          "- Câu hỏi DỄ: kiểm tra duy nhất 1 khái niệm chính cơ bản dễ hiểu, định nghĩa, hoặc công thức đơn giản."
+          "- Đáp án có thể tìm thấy trực tiếp trong văn bản."
+          "- Ngữ cảnh đủ để hiểu khái niệm chính."
+          "- Distractors khác biệt rõ ràng, dễ loại bỏ."
+          "- Độ dài câu hỏi ngắn gọn không quá 10-20 từ hoặc ít hơn 120 ký tự, tập trung vào một ý duy nhất.\n"
+      ),
+      "medium": (
+          "- Câu hỏi TRUNG BÌNH kiểm tra khái niệm chính trong văn bản"
+          "- Nếu câu hỏi thuộc dạng áp dụng và suy luận thiếu dữ liệu để trả lời câu hỏi, thêm nội dung hoặc ví dụ từ văn bản nguồn."
+          "- Các Distractors không quá giống nhau."
+          "- Độ dài câu hỏi vừa phải khoảng 23–30 từ hoặc khoảng 150 - 180 ký tự, có thêm chi tiết phụ để suy luận.\n"
+      ),
+      "hard": (
+          "- Câu hỏi KHÓ kiểm tra thông tin được phân tích/tổng hợp"
+          "- Nếu câu hỏi thuộc dạng áp dụng và suy luận thiếu dữ liệu để trả lời câu hỏi, thêm nội dung hoặc ví dụ từ văn bản nguồn."
+          "- Ít nhất 2 distractors gần giống đáp án đúng, độ tương đồng cao. "
+          f"- Đáp án yêu cầu học sinh suy luận hoặc áp dụng công thức vào ví dụ nếu có."
+          "- Độ dài câu hỏi dài hơn 35 từ hoặc hơn 200 ký tự.\n \n"
+      )
+    }
+
+    difficult_criteria = difficulty_prompts[target_difficulty] # "easy", "medium", "hard"
+    print(concept_range)
+    system_message = {
+      "role": "system",
+      "content": (
+          "Bạn là một trợ lý hữu ích chuyên tạo câu hỏi trắc nghiệm (MCQ). Luôn trả lời bằng tiếng việt"
+          f"Đảm bảo chỉ tạo sinh câu trắc nghiệm có độ khó sau {difficult_criteria}"
+          f"Quan trọng: Mỗi câu hỏi chỉ sử dụng chính xác {concept_range} khái niệm chính (mỗi khái niệm chính có 1 danh sách khái niệm phụ) từ văn bản nguồn. "
+          "Mỗi câu hỏi và đáp án phải dựa trên thông tin từ văn bản nguồn. Không được đưa kiến thức ngoài vào."
+          "Chỉ TRẢ VỀ duy nhất một đối tượng JSON theo đúng schema sau và không kèm giải thích hay trường thêm:\n\n"
+          "{\n"
+          '  "1": { "câu hỏi": "...", "lựa chọn": {"a":"...","b":"...","c":"...","d":"..."}, "đáp án":"...", "khái niệm sử dụng": {"khái niệm chính":["khái niệm phụ", "..."], "..."]}},\n'
+          '  "2": { ... }\n'
+          "}\n\n"
+          "Lưu ý:\n"
+          f"- Tạo đúng {n} mục, đánh số từ 1 tới {n}.\n"
+          "- Khóa 'lựa chọn' phải có các phím a, b, c, d.\n"
+          "- 'đáp án' phải là toàn văn đáp án đúng (không phải ký tự chữ cái), và giá trị này phải khớp chính xác với một trong các giá trị trong 'options'.\n"
+          "- Toàn bộ thông tin cần thiết để trả lời phải nằm trong chính câu hỏi, không tham chiếu lại văn bản nguồn."
+          f"- Sử dụng chính xác {concept_range} khái niệm chính"
+        )
+    }
+
+    user_message = {
+        "role": "user",
+        "content": (
+            f"Hãy tạo {n} câu hỏi trắc nghiệm từ nội dung dưới đây. Chỉ sử dụng nội dung này làm nguồn duy nhất để xây dựng câu hỏi.\n\n"
+
+            "### Yêu cầu:\n"
+            "- Bám sát vào thông tin trong văn bản; không thêm kiến thức ngoài.\n"
+            "- Nếu văn bản thiếu chi tiết, hãy tạo phương án nhiễu (distractors) hợp lý, nhưng phải có thể biện minh từ nội dung hoặc ngữ cảnh.\n"
+            f"### Văn bản nguồn:\n{source_text}"
+        )
+    }
+
+
+    if enable_fiddler:
+        max_conf, max_cat = text_safety_check(user_message['content'])
+        if max_conf > 0.5:
+            print(f"Harmful content detected: ({max_cat} : {max_conf})")
+            return {}
+
+    raw = _post_chat([system_message, user_message], model=model, temperature=temperature)
+    # print('\n\n',raw)
+    parsed = _safe_extract_json(raw)
+    # basic validation
+    if not isinstance(parsed, dict) or len(parsed) != n:
+        raise ValueError(f"Generator returned invalid structure. Raw:\n{raw}")
+    return parsed
+
+
+
 def generate_mcqs_from_text(
     source_text: str,
     n: int = 3,
-    model: str = "gpt-oss-120b",
+    model: str = "openai/gpt-oss-120b",
     temperature: float = 0.2,
     enable_fiddler: bool = False,
 ) -> Dict[str, Any]: