Upload 19 files

.dockerignore

@@ -1,6 +1,15 @@
 .git
 .gitignore
+
+.env
+.env.*
+!.env.example
+
 __pycache__/
 *.py[cod]
 *.log
+.pytest_cache/
+.mypy_cache/
+
 faiss_storage_custom-topic-quiz-generator/
+sources/custom-topic-quiz-generator/quiz_questions.csv

.gitignore

@@ -0,0 +1,12 @@
+.env
+.env.*
+!.env.example
+
+__pycache__/
+*.py[cod]
+*.log
+.pytest_cache/
+.mypy_cache/
+
+faiss_storage_custom-topic-quiz-generator/
+sources/custom-topic-quiz-generator/quiz_questions.csv

CUSTOM_QUIZ_DEPLOYMENT.md

@@ -1,96 +1,80 @@
-# Custom Topic Quiz Generator Deployment
+# Custom Topic Quiz Generator API
 
-This directory is the dedicated Hugging Face Docker Space for
-`custom-topic-quiz-generator`. The authoritative runtime entrypoint is the root
-`app.py`, and the authoritative container definition is the root `Dockerfile`.
+This is the standalone Hugging Face/Docker retrieval API. The Supabase Edge
+Function is deployed separately as `custom-topic-quiz-generator`.
 
-## 1. Export the two-column source CSV
+## Source CSV
 
-Run the export job with service-role credentials only in the trusted export
-environment:
+The CSV columns are:
 
-```bash
-export QUIZ_EXPORT_SUPABASE_URL="https://supabase.example.com"
-export QUIZ_EXPORT_SERVICE_ROLE_KEY="..."
-export QUIZ_EXPORT_OUTPUT="sources/custom-topic-quiz-generator/quiz_questions.csv"
-python scripts/export_quiz_questions.py
+```text
+id
+question
+source_id
+category_id
+difficulty
+flash
+is_deleted
 ```
 
-The CSV contains only `id` and `question`. Upload the resulting file to the
-dedicated Google Drive file configured through `QUIZ_CSV_URL`. The retrieval
-API never receives options, answers, hints, explanations, source IDs, category
-IDs, difficulty, Flash flags, or deletion flags. The Supabase Edge Function
-hydrates and validates every returned ID against the live database.
+Only `question` is embedded. All other columns are stored as retrieval metadata
+for exhaustive source/category/difficulty/mode filtering before reranking. The
+Edge Function still hydrates and validates every returned ID against the live
+`quiz_questions` table.
 
-## 2. Build and run the API
+Export with:
 
 ```bash
-docker build -t custom-topic-quiz-generator .
-docker run --rm -p 7860:7860 --env-file .env custom-topic-quiz-generator
+export QUIZ_EXPORT_SUPABASE_URL="https://supabase.sealednectar.org"
+export QUIZ_EXPORT_SERVICE_ROLE_KEY="the service-role key"
+python scripts/export_quiz_questions.py
+```
+
+Upload the generated CSV to the Google Drive file configured by `QUIZ_CSV_URL`.
+The downloader accepts any of these forms:
+
+```text
+Google Drive file ID
+https://drive.google.com/file/d/FILE_ID
+https://drive.google.com/file/d/FILE_ID/view
+https://drive.google.com/open?id=FILE_ID
 ```
 
-The container starts Gunicorn with `app:app` on port `7860`. Initial model and
-index loading happens without changing the HTTP entrypoint. A failed refresh
-leaves the previous valid index active.
+The file must be shared so the Hugging Face Space can download it.
 
-## 3. Retrieval API environment
+## Hugging Face configuration
 
-Required credentials:
+Store credentials as Hugging Face Space secrets. Do not commit or upload `.env`.
+Required values:
 
 ```text
+QUIZ_CSV_URL
 CUSTOM_QUIZ_API_USERNAME
 CUSTOM_QUIZ_API_PASSWORD
 CUSTOM_QUIZ_ADMIN_USERNAME
 CUSTOM_QUIZ_ADMIN_PASSWORD
 ```
 
-Source/index settings commonly used by this deployment:
+Optional:
 
 ```text
-QUIZ_CSV_URL
-QUIZ_CSV_PATH
-QUIZ_SOURCE_DIR
-QUIZ_INDEX_DIR
-QUIZ_ALIASES_PATH
+HF_TOKEN
 QUIZ_INDEX_REFRESH_MINUTES
+QUIZ_RERANKER_ENABLED
 ```
 
-The public search endpoint is:
-
-```text
-POST /v1/custom-topic-quiz-generator/search
-```
-
-## 4. Supabase Edge Function
-
-The only authoritative Edge Function folder is:
-
-```text
-supabase/functions/custom-topic-quiz-generator
-```
-
-Configure these values in `supabase/functions/.env`:
-
-```text
-CUSTOM_QUIZ_API_BASE_URL
-CUSTOM_QUIZ_API_USERNAME
-CUSTOM_QUIZ_API_PASSWORD
-CUSTOM_QUIZ_API_TIMEOUT_MS
-CUSTOM_QUIZ_MAX_EXCLUDE_IDS
-CUSTOM_QUIZ_STALE_ID_ALERT_THRESHOLD
-```
-
-`CUSTOM_QUIZ_API_USERNAME` and `CUSTOM_QUIZ_API_PASSWORD` must match the
-retrieval API values. The Edge Function authenticates the Flutter user itself,
-so its `config.toml` entry uses `verify_jwt = false`.
-
-## 5. Flutter feature flag
+`HF_TOKEN` removes unauthenticated Hugging Face Hub download warnings and gives
+normal authenticated download limits.
 
-The feature is enabled by default. It can be disabled at build time with:
+The Docker runtime listens on port `7860` and exposes:
 
 ```text
---dart-define=CUSTOM_TOPIC_QUIZ_ENABLED=false
+GET  /
+GET  /health
+POST /v1/custom-topic-quiz-generator/search
+GET  /internal/index/status
+POST /internal/index/rebuild
+POST /internal/index/refresh-from-google
 ```
 
-Flutter calls only the `custom-topic-quiz-generator` Supabase Edge Function. It
-never receives the retrieval API URL or credentials.
+The search and admin endpoints use HTTP Basic Auth with separate credentials.

app.py

@@ -286,7 +286,7 @@ def search():
             query=query,
             limit=limit,
             exclude_ids=_exclude_ids(data.get("exclude_ids")),
-            filters=SearchFilters.from_payload(data.get("filters")),
+            filters=SearchFilters.from_payload(data.get("filters"), mode=mode),
         )
     except ValueError as exc:
         return jsonify({"error": str(exc)}), 400

note.txt

@@ -0,0 +1 @@
+https://homemademirpur-ed-cad-ref.hf.space

requirements.txt

@@ -1,9 +1,11 @@
-Flask
-python-dotenv
-gdown
-numpy
-faiss-cpu
-torch
-sentence-transformers
-gunicorn
-einops
+Flask==3.0.3
+python-dotenv==1.0.1
+gdown==5.2.0
+requests==2.32.3
+numpy==1.26.4
+faiss-cpu==1.8.0.post1
+torch==2.3.1
+transformers==4.40.0
+sentence-transformers==3.0.1
+einops==0.8.0
+gunicorn==22.0.0

retrieval/gdrive.py

@@ -31,27 +31,24 @@ def _file_id(value: str) -> Optional[str]:
 def download_gdrive_file(file_id_or_url: str, target_path: str) -> bool:
     file_id = _file_id(file_id_or_url)
     if not file_id:
-        logger.error("[CUSTOM_TOPIC_QUIZ_GENERATOR] Invalid Google Drive file URL or ID.")
+        logger.error(
+            "[CUSTOM_TOPIC_QUIZ_GENERATOR] Invalid Google Drive file URL or ID."
+        )
         return False
 
     destination = Path(target_path)
     destination.parent.mkdir(parents=True, exist_ok=True)
 
     try:
-        try:
-            result = gdown.download(
-                id=file_id,
-                output=str(destination),
-                quiet=False,
-            )
-        except TypeError:
-            result = gdown.download(
-                url=f"https://drive.google.com/uc?id={file_id}",
-                output=str(destination),
-                quiet=False,
-            )
+        result = gdown.download(
+            id=file_id,
+            output=str(destination),
+            quiet=False,
+        )
     except Exception:
-        logger.exception("[CUSTOM_TOPIC_QUIZ_GENERATOR] Google Drive download failed.")
+        logger.exception(
+            "[CUSTOM_TOPIC_QUIZ_GENERATOR] Google Drive download failed."
+        )
         return False
 
     return bool(

retrieval/parser.py

@@ -5,18 +5,21 @@ from pathlib import Path
 from typing import Dict, List, Optional
 
 
-# The retrieval index intentionally contains only the authoritative database ID
-# and exact question text. All mutable eligibility/filter metadata is checked by
-# the Supabase Edge Function against the live quiz_questions row.
-REQUIRED_COLUMNS = {"id", "question"}
+REQUIRED_COLUMNS = {
+    "id",
+    "question",
+    "source_id",
+    "category_id",
+    "difficulty",
+    "flash",
+    "is_deleted",
+}
 
 
 @dataclass(frozen=True)
 class QuestionRecord:
     question_id: int
     question: str
-    # Retained for backwards-compatible index deserialization. New two-column
-    # CSV builds leave these unset because Supabase owns live filtering.
     source_id: Optional[int] = None
     category_id: Optional[int] = None
     chapter_id: Optional[int] = None
@@ -24,9 +27,17 @@ class QuestionRecord:
     flash: bool = False
     event_seerah: bool = False
     updated_at: str = ""
+    is_deleted: bool = False
 
     def metadata(self) -> Dict[str, object]:
-        return {"question_id": self.question_id}
+        return {
+            "question_id": self.question_id,
+            "source_id": self.source_id,
+            "category_id": self.category_id,
+            "difficulty": self.difficulty,
+            "flash": self.flash,
+            "is_deleted": self.is_deleted,
+        }
 
 
 @dataclass(frozen=True)
@@ -57,13 +68,39 @@ def _required_positive_int(value: object, field: str, row_number: int) -> int:
             f"Row {row_number}: {field} is not an integer."
         ) from exc
     if not numeric.is_integer():
-        raise QuizCsvValidationError(f"Row {row_number}: {field} is not an integer.")
+        raise QuizCsvValidationError(
+            f"Row {row_number}: {field} is not an integer."
+        )
     parsed = int(numeric)
     if parsed <= 0:
-        raise QuizCsvValidationError(f"Row {row_number}: {field} must be positive.")
+        raise QuizCsvValidationError(
+            f"Row {row_number}: {field} must be positive."
+        )
     return parsed
 
 
+def _optional_positive_int(
+    value: object,
+    field: str,
+    row_number: int,
+) -> Optional[int]:
+    text = _text(value)
+    if not text:
+        return None
+    return _required_positive_int(text, field, row_number)
+
+
+def _boolean(value: object, field: str, row_number: int) -> bool:
+    text = _text(value).lower()
+    if text in {"1", "true", "t", "yes", "y"}:
+        return True
+    if text in {"0", "false", "f", "no", "n", ""}:
+        return False
+    raise QuizCsvValidationError(
+        f"Row {row_number}: {field} is not a boolean."
+    )
+
+
 def file_sha256(path: Path) -> str:
     digest = hashlib.sha256()
     with path.open("rb") as handle:
@@ -79,10 +116,14 @@ def parse_questions_csv(path: Path) -> ParsedQuestions:
     records: List[QuestionRecord] = []
     seen_ids = set()
     invalid_count = 0
+    deleted_count = 0
 
     with path.open("r", encoding="utf-8-sig", newline="") as handle:
         reader = csv.DictReader(handle)
-        fieldnames = {str(name or "").strip() for name in (reader.fieldnames or [])}
+        fieldnames = {
+            str(name or "").strip()
+            for name in (reader.fieldnames or [])
+        }
         missing = sorted(REQUIRED_COLUMNS - fieldnames)
         if missing:
             raise QuizCsvValidationError(
@@ -91,30 +132,74 @@ def parse_questions_csv(path: Path) -> ParsedQuestions:
 
         for row_number, raw in enumerate(reader, start=2):
             try:
-                question_id = _required_positive_int(raw.get("id"), "id", row_number)
+                question_id = _required_positive_int(
+                    raw.get("id"),
+                    "id",
+                    row_number,
+                )
                 question = _text(raw.get("question"))
                 if not question:
-                    raise QuizCsvValidationError(f"Row {row_number}: question is blank.")
+                    raise QuizCsvValidationError(
+                        f"Row {row_number}: question is blank."
+                    )
                 if question_id in seen_ids:
                     raise QuizCsvValidationError(
                         f"Row {row_number}: duplicate id {question_id}."
                     )
+
+                is_deleted = _boolean(
+                    raw.get("is_deleted"),
+                    "is_deleted",
+                    row_number,
+                )
                 seen_ids.add(question_id)
+                if is_deleted:
+                    deleted_count += 1
+                    continue
+
+                difficulty = _optional_positive_int(
+                    raw.get("difficulty"),
+                    "difficulty",
+                    row_number,
+                )
+                if difficulty is not None and difficulty not in {1, 2, 3}:
+                    raise QuizCsvValidationError(
+                        f"Row {row_number}: difficulty must be 1, 2, or 3."
+                    )
+
                 records.append(
-                    QuestionRecord(question_id=question_id, question=question)
+                    QuestionRecord(
+                        question_id=question_id,
+                        question=question,
+                        source_id=_optional_positive_int(
+                            raw.get("source_id"),
+                            "source_id",
+                            row_number,
+                        ),
+                        category_id=_optional_positive_int(
+                            raw.get("category_id"),
+                            "category_id",
+                            row_number,
+                        ),
+                        difficulty=difficulty,
+                        flash=_boolean(
+                            raw.get("flash"),
+                            "flash",
+                            row_number,
+                        ),
+                        is_deleted=False,
+                    )
                 )
             except QuizCsvValidationError:
                 invalid_count += 1
                 raise
 
     if not records:
-        raise QuizCsvValidationError("Quiz CSV contains no questions.")
+        raise QuizCsvValidationError("Quiz CSV contains no active questions.")
 
     return ParsedQuestions(
         records=records,
         source_sha256=file_sha256(path),
         invalid_count=invalid_count,
-        # Deleted rows cannot be represented in a two-column CSV. The export
-        # excludes them, and the Edge Function rechecks is_deleted live.
-        deleted_count=0,
+        deleted_count=deleted_count,
     )

retrieval/search.py

@@ -15,12 +15,13 @@ logger = logging.getLogger(__name__)
 
 @dataclass(frozen=True)
 class SearchFilters:
+    mode: str = "quiz"
     source_ids: Tuple[int, ...] = ()
     category_ids: Tuple[int, ...] = ()
     difficulty: Tuple[int, ...] = ()
 
     @classmethod
-    def from_payload(cls, raw: object) -> "SearchFilters":
+    def from_payload(cls, raw: object, *, mode: str) -> "SearchFilters":
         data = raw if isinstance(raw, dict) else {}
 
         def positive_ints(
@@ -41,21 +42,42 @@ class SearchFilters:
                     clean.add(parsed)
             return tuple(sorted(clean))
 
+        clean_mode = str(mode or "quiz").strip().lower()
+        if clean_mode not in {"quiz", "exam", "flash"}:
+            raise ValueError("mode must be quiz, exam, or flash.")
+
         return cls(
+            mode=clean_mode,
             source_ids=positive_ints(data.get("source_ids")),
             category_ids=positive_ints(data.get("category_ids")),
             difficulty=positive_ints(data.get("difficulty"), {1, 2, 3}),
         )
 
     def cache_key(self) -> Tuple[object, ...]:
-        # The two-column RAG index has no live database metadata. Candidate
-        # ranking is therefore query/index-version specific only. Supabase
-        # applies source/category/difficulty eligibility after hydration.
-        return ()
+        return (
+            self.mode,
+            self.source_ids,
+            self.category_ids,
+            self.difficulty,
+        )
 
     def matches(self, metadata: Dict[str, object]) -> bool:
-        # Intentionally do not filter in the retrieval service. The Edge
-        # Function owns all authoritative live database filtering.
+        if metadata.get("is_deleted") is True:
+            return False
+
+        if self.mode in {"exam", "flash"} and metadata.get("flash") is not True:
+            return False
+
+        source_id = metadata.get("source_id")
+        category_id = metadata.get("category_id")
+        difficulty = metadata.get("difficulty")
+
+        if self.source_ids and source_id not in self.source_ids:
+            return False
+        if self.category_ids and category_id not in self.category_ids:
+            return False
+        if self.difficulty and difficulty not in self.difficulty:
+            return False
         return True
 
 

scripts/export_quiz_questions.py

@@ -1,9 +1,9 @@
 #!/usr/bin/env python3
-"""Export the two-column custom-topic-quiz-generator CSV from Supabase.
+"""Export custom-topic retrieval rows from Supabase.
 
-The retrieval index receives only the authoritative question ID and exact
-question text. The Supabase Edge Function hydrates and validates every returned
-ID against the live quiz_questions row.
+Only the ``question`` field is embedded. The remaining fields are stored as
+FAISS-side metadata so source/category/difficulty/mode filtering can happen
+before reranking. The Edge Function still performs final live validation.
 """
 
 from __future__ import annotations
@@ -17,7 +17,15 @@ from typing import Dict, Iterable, List
 
 import requests
 
-FIELDS = ("id", "question")
+FIELDS = (
+    "id",
+    "question",
+    "source_id",
+    "category_id",
+    "difficulty",
+    "flash",
+    "is_deleted",
+)
 PAGE_SIZE = 1000
 
 
@@ -41,7 +49,10 @@ def fetch_rows(base_url: str, service_key: str) -> Iterable[Dict[str, object]]:
     while True:
         response = requests.get(
             endpoint,
-            headers={**headers, "Range": f"{offset}-{offset + PAGE_SIZE - 1}"},
+            headers={
+                **headers,
+                "Range": f"{offset}-{offset + PAGE_SIZE - 1}",
+            },
             params={
                 "select": ",".join(FIELDS),
                 "is_deleted": "eq.false",
@@ -64,6 +75,7 @@ def fetch_rows(base_url: str, service_key: str) -> Iterable[Dict[str, object]]:
 def validate_rows(rows: List[Dict[str, object]]) -> None:
     if not rows:
         raise RuntimeError("The export contains no active quiz questions.")
+
     seen = set()
     for index, row in enumerate(rows, start=2):
         question_id = row.get("id")
@@ -74,6 +86,10 @@ def validate_rows(rows: List[Dict[str, object]]) -> None:
             raise RuntimeError(f"Row {index}: duplicate id {question_id}.")
         if not question:
             raise RuntimeError(f"Row {index}: blank question.")
+        if row.get("is_deleted") is True:
+            raise RuntimeError(
+                f"Row {index}: deleted question {question_id} was exported."
+            )
         seen.add(question_id)
 
 
@@ -89,7 +105,11 @@ def write_atomic(rows: List[Dict[str, object]], output: Path) -> None:
         delete=False,
     ) as handle:
         temp_path = Path(handle.name)
-        writer = csv.DictWriter(handle, fieldnames=FIELDS, extrasaction="ignore")
+        writer = csv.DictWriter(
+            handle,
+            fieldnames=FIELDS,
+            extrasaction="ignore",
+        )
         writer.writeheader()
         writer.writerows(rows)
     os.replace(temp_path, output)

shared_models.py

@@ -63,7 +63,11 @@ class SharedReranker:
             "[CUSTOM_TOPIC_QUIZ_GENERATOR] Loading reranker model %s",
             model_name,
         )
-        self._model = CrossEncoder(model_name, trust_remote_code=True)
+        self._model = CrossEncoder(
+            model_name,
+            trust_remote_code=True,
+            automodel_args={"use_flash_attn": False},
+        )
         logger.info(
             "[CUSTOM_TOPIC_QUIZ_GENERATOR] Reranker loaded in %.2fs",
             time.time() - started,