scripts/backfill-imported-sessions.sh

#!/usr/bin/env bash
# Backfill memory artifacts for sessions imported via `agentmemory import-jsonl`.
#
# The import path only persists Session + Observation rows (via synthetic,
# zero-LLM compression) and the deterministic crystal/lesson derivation.
# It does NOT call mem::summarize, so the semantic/procedural/reflect tiers
# of the consolidation pipeline have nothing to roll up.
#
# This script walks every session tagged `jsonl-import` and:
#   1. POSTs /agentmemory/summarize per session  (LLM call)
#   2. POSTs /agentmemory/consolidate-pipeline once at the end
#
# Graph extraction (/agentmemory/graph/extract) is intentionally skipped —
# its API takes a per-observation payload, which is cost-prohibitive for
# bulk imports. `reflect` falls back to a no-graph clustering mode.
#
# Usage:
#   scripts/backfill-imported-sessions.sh --dry-run
#   scripts/backfill-imported-sessions.sh --limit 5
#   scripts/backfill-imported-sessions.sh                 # process all

set -euo pipefail

URL="${AGENTMEMORY_URL:-http://localhost:3111}"
DRY_RUN=0
LIMIT=0           # 0 = no limit
ONLY_TAG="jsonl-import"
SKIP_CONSOLIDATE=0
SKIP_AGENTS=0     # drop sessions whose project starts with "agent-"
MAX_OBS=0         # 0 = no cap; skip sessions with more observations than this
DEBUG_ON_ERROR=0  # on failure, dump session metadata + obs to DEBUG_DIR
DEBUG_DIR="${AGENTMEMORY_DEBUG_DIR:-./agentmemory-debug}"
PROJECT_PATTERN=""  # jq test() regex against .project; "" means no filter

# Cost-estimate knobs (defaults tuned for DeepSeek V4 Flash on DeepInfra:
# $0.14 / 1M input, $0.28 / 1M output). Override via env if needed.
COST_IN_PER_1M="${AGENTMEMORY_COST_IN_PER_1M:-0.14}"
COST_OUT_PER_1M="${AGENTMEMORY_COST_OUT_PER_1M:-0.28}"
# Rough token weight per compressed observation, derived from inspecting
# real synthetic-compression payloads in the kv store (mostly 100-300 tok,
# heavy-tailed). Override if your sessions are unusually verbose.
TOKENS_PER_OBS="${AGENTMEMORY_TOKENS_PER_OBS:-200}"
# Reserved per-call output budget (XML summary is small).
TOKENS_OUT_PER_SESSION="${AGENTMEMORY_TOKENS_OUT_PER_SESSION:-500}"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run)         DRY_RUN=1; shift ;;
    --limit)           LIMIT="${2:?--limit needs a number}"; shift 2 ;;
    --tag)             ONLY_TAG="${2:?--tag needs a value (use empty string for all)}"; shift 2 ;;
    --skip-consolidate) SKIP_CONSOLIDATE=1; shift ;;
    --skip-agents)     SKIP_AGENTS=1; shift ;;
    --max-obs)         MAX_OBS="${2:?--max-obs needs a number}"; shift 2 ;;
    --debug-on-error)  DEBUG_ON_ERROR=1; shift ;;
    --project-pattern) PROJECT_PATTERN="${2:?--project-pattern needs a regex}"; shift 2 ;;
    -h|--help)
      sed -n '2,28p' "$0"
      exit 0 ;;
    *) echo "unknown flag: $1" >&2; exit 2 ;;
  esac
done

for bin in curl jq; do
  command -v "$bin" >/dev/null || { echo "missing dependency: $bin" >&2; exit 1; }
done

# Curl timeout profiles. Metadata reads (livez, sessions list, observations
# pull for debug dumps) should fail fast and retry transient blips. The LLM
# work calls (summarize, consolidate) intentionally have no --retry and a
# wide --max-time: each call can legitimately take minutes for chunked
# summarize on large sessions, and retrying a half-finished LLM job is
# expensive both in dollars and in duplicated server-side work.
META_CURL_OPTS=(--connect-timeout 10 --max-time 30 --retry 2 --retry-delay 1)
WORK_CURL_OPTS=(--connect-timeout 10 --max-time 1800)

echo "agentmemory backfill — server: $URL"
[[ "$DRY_RUN" == 1 ]] && echo "DRY RUN: no POSTs will be made."

# --- liveness ---
if ! curl -fsS "${META_CURL_OPTS[@]}" "$URL/agentmemory/livez" >/dev/null; then
  echo "server not reachable at $URL (try: npx @agentmemory/agentmemory)" >&2
  exit 1
fi

# --- collect session ids ---
sessions_json="$(curl -fsS "${META_CURL_OPTS[@]}" "$URL/agentmemory/sessions")"
filter='.sessions[] | select(.status=="completed")'
if [[ -n "$ONLY_TAG" ]]; then
  filter+=" | select((.tags // []) | index(\"$ONLY_TAG\"))"
fi
if [[ "$SKIP_AGENTS" == 1 ]]; then
  filter+=' | select((.project // "") | startswith("agent-") | not)'
fi
if [[ -n "$PROJECT_PATTERN" ]]; then
  # jq's test() applies a regex against the project string.
  filter+=" | select((.project // \"\") | test(\"$PROJECT_PATTERN\"))"
fi
if [[ "$MAX_OBS" -gt 0 ]]; then
  filter+=" | select((.observationCount // 0) <= $MAX_OBS)"
fi
filter+=' | "\(.id)\t\(.observationCount // 0)\t\(.project // "")"'

rows=()
while IFS= read -r line; do
  rows+=("$line")
done < <(echo "$sessions_json" | jq -r "$filter")
total="${#rows[@]}"

if [[ "$total" -eq 0 ]]; then
  echo "no sessions matched (tag='$ONLY_TAG'); nothing to do."
  exit 0
fi

if [[ "$LIMIT" -gt 0 && "$LIMIT" -lt "$total" ]]; then
  rows=("${rows[@]:0:$LIMIT}")
fi

echo "matched $total session(s); will process ${#rows[@]}."
total_obs=0
for row in "${rows[@]}"; do
  obs="$(cut -f2 <<<"$row")"
  total_obs=$(( total_obs + obs ))
done
est_in=$(( total_obs * TOKENS_PER_OBS + ${#rows[@]} * 500 ))
est_out=$(( ${#rows[@]} * TOKENS_OUT_PER_SESSION ))
est_cost="$(awk -v i="$est_in" -v o="$est_out" -v ci="$COST_IN_PER_1M" -v co="$COST_OUT_PER_1M" \
  'BEGIN { printf "%.2f", (i*ci + o*co) / 1000000 }')"

echo "≈ ${#rows[@]} summarize LLM calls (one per session, covering $total_obs observations)"
printf '≈ %d input tok + %d output tok → $%s  (rates: in=$%s/1M out=$%s/1M, %s tok/obs)\n' \
  "$est_in" "$est_out" "$est_cost" "$COST_IN_PER_1M" "$COST_OUT_PER_1M" "$TOKENS_PER_OBS"
echo

if [[ "$DRY_RUN" == 1 ]]; then
  printf '%-40s %10s  %s\n' "session" "obs" "project"
  for row in "${rows[@]}"; do
    id="$(cut -f1 <<<"$row")"
    obs="$(cut -f2 <<<"$row")"
    proj="$(cut -f3 <<<"$row")"
    printf '%-40s %10s  %s\n' "$id" "$obs" "$proj"
  done
  echo
  echo "(dry run) next steps if you re-run without --dry-run:"
  echo "  for each session above: POST $URL/agentmemory/summarize {sessionId}"
  if [[ "$SKIP_CONSOLIDATE" == 0 ]]; then
    echo "  then: POST $URL/agentmemory/consolidate-pipeline {}"
  fi
  exit 0
fi

# --- summarize loop ---
if [[ "$DEBUG_ON_ERROR" == 1 ]]; then
  mkdir -p "$DEBUG_DIR"
  echo "debug mode: failed calls will dump to $DEBUG_DIR/"
  echo
fi

dump_failure() {
  local id="$1" obs="$2" resp="$3"
  # Replace anything outside [A-Za-z0-9._-] with `_` before joining with
  # DEBUG_DIR. Session IDs from the API are UUIDs in practice, but the
  # server doesn't enforce that — a hostile or buggy id containing `/` or
  # `..` would otherwise escape the debug directory.
  local safe_id
  safe_id="$(printf '%s' "$id" | tr -c 'A-Za-z0-9._-' '_')"
  local file="$DEBUG_DIR/${safe_id}.json"
  # Pull the raw observations (what would have gone into the prompt) so the
  # operator can reconstruct the upstream payload locally. We also compute
  # narrative size stats so size-related rejections are immediately visible.
  # Stream observations through stdin (avoids exec-arg overflow on
  # multi-thousand-obs sessions — macOS argv ceiling is ~256k).
  # `--get --data-urlencode` percent-encodes the session id so special
  # characters can't corrupt the query string.
  curl -fsS "${META_CURL_OPTS[@]}" --get \
       --data-urlencode "sessionId=$id" \
       "$URL/agentmemory/observations" \
    | jq \
        --arg id "$id" \
        --argjson obsCount "$obs" \
        --arg url "$URL/agentmemory/summarize" \
        --argjson response "$resp" \
        '. as $root
         | .observations as $obs
         | {
             sessionId: $id,
             observationCount: $obsCount,
             request: { url: $url, method: "POST", body: { sessionId: $id } },
             response: $response,
             observations: $obs,
             stats: {
               totalNarrativeBytes: ($obs | map(.narrative // "" | length) | add // 0),
               maxNarrativeBytes:   ($obs | map(.narrative // "" | length) | max // 0),
               titleHistogram:      ($obs | group_by(.title) | map({title: .[0].title, count: length}) | sort_by(-.count))
             }
           }' >"$file"
  echo "      → $file"
}

ok=0; skipped=0; failed=0
i=0
for row in "${rows[@]}"; do
  i=$(( i + 1 ))
  id="$(cut -f1 <<<"$row")"
  obs="$(cut -f2 <<<"$row")"

  body="$(jq -nc --arg id "$id" '{sessionId:$id}')"
  resp="$(curl -sS "${WORK_CURL_OPTS[@]}" -X POST "$URL/agentmemory/summarize" \
    -H 'content-type: application/json' --data "$body" || echo '{"success":false,"error":"curl_failed"}')"
  # iii's HTTP layer occasionally returns non-JSON (HTML 5xx, empty body
  # on timeout, etc.). Validate before parsing so `set -e` doesn't abort
  # the whole backfill loop on a single bad response.
  if jq -e . >/dev/null 2>&1 <<<"$resp"; then
    status="$(jq -r '.success // false' <<<"$resp")"
    err="$(jq -r '.error // ""' <<<"$resp")"
    title="$(jq -r '.summary.title // ""' <<<"$resp")"
  else
    status="false"
    err="invalid_json_response"
    title=""
  fi

  if [[ "$status" == "true" ]]; then
    ok=$(( ok + 1 ))
    printf '[%3d/%3d] OK    %s  obs=%-5s  %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$title"
  elif [[ "$err" == "no_observations" || "$err" == "no_provider" ]]; then
    skipped=$(( skipped + 1 ))
    printf '[%3d/%3d] SKIP  %s  obs=%-5s  %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$err"
  else
    failed=$(( failed + 1 ))
    printf '[%3d/%3d] FAIL  %s  obs=%-5s  %s\n' "$i" "${#rows[@]}" "$id" "$obs" "$err"
    [[ "$DEBUG_ON_ERROR" == 1 ]] && dump_failure "$id" "$obs" "$resp"
  fi
done

echo
echo "summarize: ok=$ok skipped=$skipped failed=$failed"

# --- consolidate ---
if [[ "$SKIP_CONSOLIDATE" == 1 ]]; then
  echo "skipping consolidate-pipeline (--skip-consolidate)"
  exit 0
fi

if [[ "$ok" -eq 0 ]]; then
  echo "no summaries produced; skipping consolidate-pipeline."
  exit 0
fi

echo
echo "running consolidate-pipeline …"
resp="$(curl -sS "${WORK_CURL_OPTS[@]}" -X POST "$URL/agentmemory/consolidate-pipeline" \
  -H 'content-type: application/json' --data '{}' || echo '{"success":false,"error":"curl_failed"}')"
if jq -e . >/dev/null 2>&1 <<<"$resp"; then
  echo "$resp" | jq .
else
  echo "consolidate-pipeline returned non-JSON (likely a timeout or upstream error):"
  printf '%s\n' "$resp" | head -c 500
  echo
fi