validate.sh

#!/usr/bin/env bash
# marvy-14B quick validation harness.
#
# Runs the task-coverage probes (Test 2 in VALIDATION.md) against any
# OpenAI-compatible endpoint — LM Studio, mlx_lm server, vLLM, etc. — and prints
# each artifact plus a lightweight heuristic PASS/FAIL on domain keywords.
#
# Usage:
#   bash validate.sh                                   # defaults to LM Studio
#   BASE_URL=http://localhost:8080/v1 MODEL=marvy-14B bash validate.sh
#   API_KEY=xxx BASE_URL=https://... MODEL=MainStack/marvy-14B bash validate.sh
set -uo pipefail

BASE_URL="${BASE_URL:-http://localhost:1234/v1}"   # LM Studio default
MODEL="${MODEL:-marvy-14B}"
API_KEY="${API_KEY:-lm-studio}"
TEMP="${TEMP:-0.4}"
MAXTOK="${MAXTOK:-700}"

SYSTEM="You are a senior ServiceNow delivery consultant. You produce precise, implementation-grade artifacts: business analyses, requirements, solution design documents, user stories with acceptance criteria, test cases, and validation reviews. You favor out-of-the-box capabilities, cite concrete tables/plugins/sys_ids when relevant, and write in clear professional English."

# probe | expected-keyword-regex (case-insensitive) for a heuristic pass
PROMPTS=(
  "Write a user story with acceptance criteria for auto-escalating P1 incidents that breach a 15-minute response SLA.|as a.*i want.*so that|acceptance|sla"
  "Draft the Incident Management section of an SDD for a greenfield ITSM implementation. Include assignment rules and SLA design.|assignment|sla|incident"
  "Extract structured requirements (id, category, priority, target phase, success metric) from: replace email-based access requests with a catalog item routed for manager approval.|priority|requirement|catalog"
  "Write a test case for the story: Restrict the Assignment Group field on incidents to groups with the itil role.|pre-condition|step|expected|itil"
  "Validate this requirement against best practice and list follow-up questions: All incidents must auto-close after 3 days.|follow-up|risk|question"
)

command -v jq >/dev/null 2>&1 || { echo "ERROR: jq is required (brew install jq)"; exit 1; }

echo "Endpoint: $BASE_URL   Model: $MODEL   Temp: $TEMP"
echo "============================================================"
pass=0; total=0
for entry in "${PROMPTS[@]}"; do
  total=$((total+1))
  prompt="${entry%%|*}"
  rest="${entry#*|}"; regex="$rest"
  payload=$(jq -n --arg m "$MODEL" --arg s "$SYSTEM" --arg p "$prompt" \
    --argjson t "$TEMP" --argjson mx "$MAXTOK" \
    '{model:$m,temperature:$t,max_tokens:$mx,messages:[{role:"system",content:$s},{role:"user",content:$p}]}')
  resp=$(curl -s "$BASE_URL/chat/completions" -H "Content-Type: application/json" \
    -H "Authorization: Bearer $API_KEY" -d "$payload")
  content=$(echo "$resp" | jq -r '.choices[0].message.content // .error.message // "<<no response>>"')
  echo ""
  echo "### Probe $total: $prompt"
  echo "------------------------------------------------------------"
  echo "$content" | head -40
  if echo "$content" | grep -iqE "$regex"; then
    echo ">>> heuristic: PASS"
    pass=$((pass+1))
  else
    echo ">>> heuristic: REVIEW (expected pattern not matched: $regex)"
  fi
  echo "============================================================"
done
echo ""
echo "Heuristic result: $pass/$total probes matched domain patterns."
echo "Pass threshold: >= 4/5 with implementation-grade, ServiceNow-specific content."
echo "Note: heuristics are a sanity check — read the outputs to judge true quality."