Upload validate.sh with huggingface_hub

validate.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# marvy-14B quick validation harness.
+#
+# Runs the task-coverage probes (Test 2 in VALIDATION.md) against any
+# OpenAI-compatible endpoint — LM Studio, mlx_lm server, vLLM, etc. — and prints
+# each artifact plus a lightweight heuristic PASS/FAIL on domain keywords.
+#
+# Usage:
+#   bash validate.sh                                   # defaults to LM Studio
+#   BASE_URL=http://localhost:8080/v1 MODEL=marvy-14B bash validate.sh
+#   API_KEY=xxx BASE_URL=https://... MODEL=MainStack/marvy-14B bash validate.sh
+set -uo pipefail
+
+BASE_URL="${BASE_URL:-http://localhost:1234/v1}"   # LM Studio default
+MODEL="${MODEL:-marvy-14B}"
+API_KEY="${API_KEY:-lm-studio}"
+TEMP="${TEMP:-0.4}"
+MAXTOK="${MAXTOK:-700}"
+
+SYSTEM="You are a senior ServiceNow delivery consultant. You produce precise, implementation-grade artifacts: business analyses, requirements, solution design documents, user stories with acceptance criteria, test cases, and validation reviews. You favor out-of-the-box capabilities, cite concrete tables/plugins/sys_ids when relevant, and write in clear professional English."
+
+# probe | expected-keyword-regex (case-insensitive) for a heuristic pass
+PROMPTS=(
+  "Write a user story with acceptance criteria for auto-escalating P1 incidents that breach a 15-minute response SLA.|as a.*i want.*so that|acceptance|sla"
+  "Draft the Incident Management section of an SDD for a greenfield ITSM implementation. Include assignment rules and SLA design.|assignment|sla|incident"
+  "Extract structured requirements (id, category, priority, target phase, success metric) from: replace email-based access requests with a catalog item routed for manager approval.|priority|requirement|catalog"
+  "Write a test case for the story: Restrict the Assignment Group field on incidents to groups with the itil role.|pre-condition|step|expected|itil"
+  "Validate this requirement against best practice and list follow-up questions: All incidents must auto-close after 3 days.|follow-up|risk|question"
+)
+
+command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required (brew install jq)"; exit 1; }
+
+echo "Endpoint: $BASE_URL   Model: $MODEL   Temp: $TEMP"
+echo "============================================================"
+pass=0; total=0
+for entry in "${PROMPTS[@]}"; do
+  total=$((total+1))
+  prompt="${entry%%|*}"
+  rest="${entry#*|}"; regex="$rest"
+  payload=$(jq -n --arg m "$MODEL" --arg s "$SYSTEM" --arg p "$prompt" \
+    --argjson t "$TEMP" --argjson mx "$MAXTOK" \
+    '{model:$m,temperature:$t,max_tokens:$mx,messages:[{role:"system",content:$s},{role:"user",content:$p}]}')
+  resp=$(curl -s "$BASE_URL/chat/completions" -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $API_KEY" -d "$payload")
+  content=$(echo "$resp" | jq -r '.choices[0].message.content // .error.message // "<<no response>>"')
+  echo ""
+  echo "### Probe $total: $prompt"
+  echo "------------------------------------------------------------"
+  echo "$content" | head -40
+  if echo "$content" | grep -iqE "$regex"; then
+    echo ">>> heuristic: PASS"
+    pass=$((pass+1))
+  else
+    echo ">>> heuristic: REVIEW (expected pattern not matched: $regex)"
+  fi
+  echo "============================================================"
+done
+echo ""
+echo "Heuristic result: $pass/$total probes matched domain patterns."
+echo "Pass threshold: >= 4/5 with implementation-grade, ServiceNow-specific content."
+echo "Note: heuristics are a sanity check — read the outputs to judge true quality."