Add clean HF snapshot deployment path

Dockerfile

@@ -24,4 +24,4 @@ COPY . ${APP_HOME}
 
 EXPOSE 7860
 
-CMD ["bash", "scripts/hf_space_boot.sh"]
+CMD ["bash", "scripts/hf_space_boot_dispatch.sh"]

deployment/hf_eval_backend/README.md

@@ -16,19 +16,35 @@ It no longer acts as the active challenge Text2SPARQL generation API.
 
 - DBpedia proxy forwards to the local DBpedia Virtuoso endpoint restored from the snapshot bucket.
 - Corporate proxy forwards to the same local Virtuoso instance, but through the corporate graph-aware endpoint URL.
-- The startup flow is unchanged:
-  - restore DBpedia snapshot
-  - start Virtuoso
-  - load the corporate graph
-  - start FastAPI
+- The Space now supports two boot modes through `HF_BOOT_MODE`:
+  - `legacy_snapshot`: old restore path from the existing snapshot scripts
+  - `clean_snapshot`: restore a separately prepared clean DBpedia snapshot from `/data/dbpedia_snapshot_paper_clean`, then load corporate and start FastAPI
+- The clean path exists so DBpedia can be served from a snapshot known to preserve the explicit graph URI `http://dbpedia.org`.
 
 ## Environment Variables
 
 - `DBPEDIA_ENDPOINT_URL` optional override for the internal DBpedia upstream
 - `CORPORATE_ENDPOINT_URL` optional override for the internal corporate upstream
 - `CORPORATE_GRAPH_URI` optional corporate graph URI override
+- `HF_BOOT_MODE` chooses `legacy_snapshot` or `clean_snapshot`
 - `PORT` optional FastAPI port, default `7860`
 
+## Clean Snapshot Preparation
+
+These scripts are kept separate from the legacy path:
+
+- `scripts/prepare_dbpedia_snapshot_from_container_clean.sh`
+  - checkpoints the working local Docker Virtuoso and copies its DB files into `hf_upload/dbpedia_snapshot_paper_clean`
+- `scripts/upload_dbpedia_snapshot_clean_sync.sh`
+  - syncs that directory into the HF bucket path `hf://buckets/InsanAlex/iris-at-text2sparql-storage/dbpedia_snapshot_paper_clean`
+- `scripts/hf_restore_db_snapshot_clean.sh`
+- `scripts/hf_space_boot_clean.sh`
+
+Optional offline RDF export helper:
+
+- `scripts/export_dbpedia_graph_partitions.py`
+  - exports `http://dbpedia.org` into deterministic RDF partitions by MD5(subject) prefix
+
 ## Expected Public URLs
 
 If the Space URL is:

scripts/export_dbpedia_graph_partitions.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import itertools
+from pathlib import Path
+
+import requests
+
+
+DEFAULT_ENDPOINT = "http://127.0.0.1:8890/sparql"
+DEFAULT_GRAPH = "http://dbpedia.org"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Export a Virtuoso graph into deterministic RDF partitions by MD5(subject) prefix."
+    )
+    parser.add_argument("--endpoint", default=DEFAULT_ENDPOINT)
+    parser.add_argument("--graph-uri", default=DEFAULT_GRAPH)
+    parser.add_argument("--output-dir", required=True)
+    parser.add_argument(
+        "--prefixes",
+        nargs="*",
+        help="Hex prefixes to export, e.g. 00 01 ff. Default is all 256 two-hex-digit prefixes.",
+    )
+    parser.add_argument("--overwrite", action="store_true")
+    parser.add_argument("--timeout-sec", type=int, default=1800)
+    return parser.parse_args()
+
+
+def all_prefixes() -> list[str]:
+    return [a + b for a, b in itertools.product("0123456789abcdef", repeat=2)]
+
+
+def build_query(graph_uri: str, prefix: str) -> str:
+    return f"""
+CONSTRUCT {{ ?s ?p ?o }}
+WHERE {{
+  GRAPH <{graph_uri}> {{
+    ?s ?p ?o .
+    FILTER (SUBSTR(MD5(STR(?s)), 1, 2) = "{prefix}")
+  }}
+}}
+""".strip()
+
+
+def export_partition(
+    endpoint: str,
+    graph_uri: str,
+    prefix: str,
+    output_path: Path,
+    timeout_sec: int,
+) -> None:
+    params = {
+        "query": build_query(graph_uri, prefix),
+        "format": "text/plain",
+    }
+    with requests.get(endpoint, params=params, stream=True, timeout=timeout_sec) as response:
+        response.raise_for_status()
+        with output_path.open("wb") as handle:
+            for chunk in response.iter_content(chunk_size=1 << 20):
+                if chunk:
+                    handle.write(chunk)
+
+
+def main() -> int:
+    args = parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    prefixes = args.prefixes or all_prefixes()
+    for prefix in prefixes:
+        if len(prefix) != 2 or any(ch not in "0123456789abcdefABCDEF" for ch in prefix):
+            raise ValueError(f"Invalid prefix: {prefix}")
+        prefix = prefix.lower()
+        output_path = output_dir / f"dbpedia_graph_{prefix}.nt"
+        if output_path.exists() and output_path.stat().st_size > 0 and not args.overwrite:
+            print(f"[skip] {output_path}")
+            continue
+        print(f"[export] {prefix} -> {output_path}")
+        export_partition(args.endpoint, args.graph_uri, prefix, output_path, args.timeout_sec)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/hf_restore_db_snapshot_clean.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SNAPSHOT_DIR="${VIRTUOSO_SNAPSHOT_DIR:-/data/dbpedia_snapshot_paper_clean}"
+RUNTIME_DIR="${VIRTUOSO_RUNTIME_DIR:-/tmp/virtuoso_runtime}"
+RUNTIME_DB_DIR="$RUNTIME_DIR/database"
+
+if [[ ! -d "$SNAPSHOT_DIR" ]]; then
+  echo "Missing clean Virtuoso snapshot directory: $SNAPSHOT_DIR"
+  echo "Upload the prepared clean DB snapshot there before starting the Space."
+  exit 1
+fi
+
+mkdir -p "$RUNTIME_DB_DIR"
+
+echo "Restoring clean Virtuoso snapshot from $SNAPSHOT_DIR to $RUNTIME_DB_DIR ..."
+find "$RUNTIME_DB_DIR" -maxdepth 1 -type f -name 'virtuoso*' -delete
+
+for name in \
+  virtuoso.db \
+  virtuoso-temp.db \
+  virtuoso.pxa \
+  virtuoso.key \
+  virtuoso.crt; do
+  if [[ -f "$SNAPSHOT_DIR/$name" ]]; then
+    cp -f "$SNAPSHOT_DIR/$name" "$RUNTIME_DB_DIR/$name"
+  fi
+done
+
+rm -f \
+  "$RUNTIME_DB_DIR/virtuoso.trx" \
+  "$RUNTIME_DB_DIR/virtuoso-temp.trx"
+
+if [[ ! -f "$RUNTIME_DB_DIR/virtuoso.db" ]]; then
+  echo "Clean snapshot restore failed: virtuoso.db not found in $RUNTIME_DB_DIR"
+  exit 1
+fi
+
+echo "Clean Virtuoso snapshot restored."

scripts/hf_space_boot_clean.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd /app
+export PYTHONPATH=/app:${PYTHONPATH:-}
+
+export CORPORATE_GRAPH_URI="${CORPORATE_GRAPH_URI:-http://ld.company.org/prod}"
+export DBPEDIA_GRAPH_URI="${DBPEDIA_GRAPH_URI:-http://dbpedia.org}"
+export DBPEDIA_ENDPOINT_URL="${DBPEDIA_ENDPOINT_URL:-http://127.0.0.1:8890/sparql?default-graph-uri=${DBPEDIA_GRAPH_URI}}"
+export CORPORATE_ENDPOINT_URL="${CORPORATE_ENDPOINT_URL:-http://127.0.0.1:8890/sparql?default-graph-uri=${DBPEDIA_GRAPH_URI}&default-graph-uri=${CORPORATE_GRAPH_URI}}"
+export PORT="${PORT:-7860}"
+
+bash scripts/hf_restore_db_snapshot_clean.sh
+bash scripts/hf_prepare_virtuoso_ini.sh
+bash scripts/hf_start_virtuoso.sh
+
+echo "Verifying DBpedia graph availability ..."
+DBPEDIA_VERIFY_RESPONSE="$(
+  curl -fsG \
+    --data-urlencode "query=ASK WHERE { <http://dbpedia.org/resource/Angela_Merkel> ?p ?o }" \
+    --data-urlencode "format=application/sparql-results+json" \
+    "${DBPEDIA_ENDPOINT_URL}" || true
+)"
+if [[ "${DBPEDIA_VERIFY_RESPONSE}" != *'"boolean": true'* && "${DBPEDIA_VERIFY_RESPONSE}" != *'"boolean":true'* ]]; then
+  echo "DBpedia verification failed."
+  echo "Response: ${DBPEDIA_VERIFY_RESPONSE}"
+  exit 1
+fi
+
+bash scripts/hf_load_corporate_graph.sh
+
+exec uvicorn service.app:app --host 0.0.0.0 --port "$PORT"

scripts/hf_space_boot_dispatch.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BOOT_MODE="${HF_BOOT_MODE:-legacy_snapshot}"
+
+case "$BOOT_MODE" in
+  legacy_snapshot)
+    exec bash scripts/hf_space_boot.sh
+    ;;
+  clean_snapshot)
+    exec bash scripts/hf_space_boot_clean.sh
+    ;;
+  *)
+    echo "Unknown HF_BOOT_MODE: $BOOT_MODE"
+    echo "Supported values: legacy_snapshot, clean_snapshot"
+    exit 1
+    ;;
+esac

scripts/prepare_dbpedia_snapshot_from_container_clean.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONTAINER_NAME="${1:-dbpedia-virtuoso}"
+SNAPSHOT_DIR="${2:-/home/alatipov/iris-at-text2sparql/hf_upload/dbpedia_snapshot_paper_clean}"
+CONTAINER_DB_DIR="${CONTAINER_DB_DIR:-/database}"
+
+export DOCKER_HOST="${DOCKER_HOST:-unix:///run/user/$(id -u)/docker.sock}"
+
+FILES=(
+  "virtuoso.db"
+  "virtuoso-temp.db"
+  "virtuoso.trx"
+  "virtuoso.pxa"
+  "virtuoso.key"
+  "virtuoso.crt"
+)
+
+docker inspect "$CONTAINER_NAME" >/dev/null
+
+echo "Checkpointing Virtuoso in container ${CONTAINER_NAME} ..."
+docker exec "$CONTAINER_NAME" sh -lc "cat >/tmp/checkpoint.sql <<'SQL'
+checkpoint;
+SQL
+/opt/virtuoso-opensource/bin/isql 1111 dba dba < /tmp/checkpoint.sql >/tmp/checkpoint.log 2>&1"
+
+mkdir -p "$SNAPSHOT_DIR"
+rm -f "$SNAPSHOT_DIR"/virtuoso*
+
+echo "Copying snapshot files from ${CONTAINER_NAME}:${CONTAINER_DB_DIR} to ${SNAPSHOT_DIR} ..."
+for name in "${FILES[@]}"; do
+  docker cp "${CONTAINER_NAME}:${CONTAINER_DB_DIR}/${name}" "$SNAPSHOT_DIR/$name"
+done
+
+echo "Clean snapshot directory ready:"
+ls -lh "$SNAPSHOT_DIR"

scripts/upload_dbpedia_snapshot_clean_sync.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+LOCAL_DIR="${1:-/home/alatipov/iris-at-text2sparql/hf_upload/dbpedia_snapshot_paper_clean}"
+BUCKET_URI="${2:-hf://buckets/InsanAlex/iris-at-text2sparql-storage/dbpedia_snapshot_paper_clean}"
+
+if [[ ! -d "$LOCAL_DIR" ]]; then
+  echo "Local clean snapshot directory not found: $LOCAL_DIR"
+  exit 1
+fi
+
+echo "Syncing $LOCAL_DIR -> $BUCKET_URI"
+hf buckets sync "$LOCAL_DIR" "$BUCKET_URI"