HyperFlow / backend /fde /trace.py
Gaurav711's picture
Deploy 891f179
2b86c60 verified
Raw History Blame Contribute Delete
7.86 kB
"""Forensic timeline trace tool for troubleshooting customer incidents.
Inspects the persisted event timeline for any order, cart, proposal,
action, delivery, review or audit event ID with exact timestamps,
state transitions and dispatch telemetry.
"""
from __future__ import annotations
import json
from typing import Any, Dict, List, Optional
from sqlalchemy import select
def _clean_str(val: Any) -> str:
if val is None:
return ""
return str(val).strip()
def redact_trace_data(value: Any) -> Any:
"""Remove secret-bearing fields recursively before either JSON or text output."""
if isinstance(value, dict):
result = {}
for key, item in value.items():
normalized = ''.join(c for c in str(key).lower() if c.isalnum())
sensitive = any(part in normalized for part in (
'token', 'secret', 'password', 'ciphertext', 'credential', 'codeverifier',
'authorization', 'cookie', 'privatekey', 'apikey', 'nonce',
'phone', 'email', 'latitude', 'longitude'))
# Raw upstream text/headers may echo credentials; inspect it separately
# through an explicitly redacted integration record, not default trace.
sensitive = sensitive or normalized in {'headers', 'raw', 'rawresponse', 'providerresponse', 'code',
'address', 'addresses', 'position', 'coordinates'}
result[key] = '[REDACTED]' if sensitive else redact_trace_data(item)
return result
if isinstance(value, list):
return [redact_trace_data(item) for item in value]
return value
def run_trace(store: Any, record_id: str) -> Dict[str, Any]:
"""Inspects persisted records and associated events for a local administrator."""
clean_id = _clean_str(record_id)
timeline: List[Dict[str, Any]] = []
record_data: Optional[Dict[str, Any]] = None
from backend.flagship.store import records, events
with store.engine.connect() as conn:
# 1. Lookup in primary hf_records
row = conn.execute(select(records).where(records.c.id == clean_id)).first()
if row:
record_data = store.unpack(row)
# 2. Query immutable event store (hf_events)
ev_rows = conn.execute(select(events).order_by(events.c.id)).all()
for r in ev_rows:
ev = store.unpack(r)
p = ev.get("payload", {})
matched = False
# Check primary and related ID fields
if (
str(ev.get("id")) == clean_id
or p.get("record_id") == clean_id
or p.get("order_id") == clean_id
or p.get("cart_id") == clean_id
or p.get("delivery_id") == clean_id
or p.get("proposal_id") == clean_id
or p.get("action_id") == clean_id
or p.get("review_id") == clean_id
or p.get("audit_id") == clean_id
):
matched = True
if matched:
timeline.append({
"event_id": ev["id"],
"source": "hf_events",
"type": ev["type"],
"account_id": ev["account_id"],
"created_at": ev["created_at"],
"payload": p,
})
# Sort timeline chronologically
timeline.sort(key=lambda x: str(x.get("created_at", "")))
# Analyze state transitions, dispatch telemetry, and anomalies
state_transitions: List[Dict[str, Any]] = []
dispatch_events: List[Dict[str, Any]] = []
refund_audit_logs: List[Dict[str, Any]] = []
anomalies: List[str] = []
last_state: Optional[str] = None
for ev in timeline:
p = ev.get("payload", {})
ev_type = ev.get("type", "")
# State transition detection
new_state = p.get("state") or p.get("status")
if not new_state and ev_type.endswith(".created"):
rec_d = record_data.get("data", {}) if record_data else {}
new_state = rec_d.get("initial_state") or (rec_d.get("state") if len(timeline) == 1 else None) or "CREATED"
if new_state:
if last_state is not None and new_state != last_state:
state_transitions.append({
"from_state": last_state,
"to_state": new_state,
"at": ev["created_at"],
"actor_id": ev.get("account_id"),
"event_type": ev_type,
})
last_state = new_state
# Dispatch telemetry
if "delivery" in ev_type or "dispatch" in ev_type or "eta" in ev_type:
dispatch_events.append(ev)
# Refund / Fraud logs
if "refund" in ev_type or "review" in ev_type:
refund_audit_logs.append(ev)
if p.get("fraud_probability", 0) > 0.70:
anomalies.append(f"High fraud risk flagged: {p.get('fraud_probability'):.2f} on {ev_type}")
# Jitter detection
if p.get("is_jitter") is True:
anomalies.append(f"Transient GPS jitter detected at {ev['created_at']}")
# Inventory reservation failure
if ev_type == "inventory.reservation" and p.get("outcome") != "success":
anomalies.append(f"Inventory reservation failed: {p.get('outcome')} for SKU {p.get('sku_id')}")
return {
"record_id": clean_id,
"record": redact_trace_data({**record_data, "data": "[REDACTED]"})
if record_data and any(part in record_data.get("kind", "").lower()
for part in ("credential", "oauth", "pkce", "session"))
else redact_trace_data(record_data),
"events_count": len(timeline),
"timeline": redact_trace_data(timeline),
"state_transitions": state_transitions,
"dispatch_events_count": len(dispatch_events),
"refund_audit_logs_count": len(refund_audit_logs),
"anomalies": anomalies,
}
def format_trace_report(res: Dict[str, Any]) -> str:
"""Formats forensic trace results into human-readable terminal output."""
rec = res.get("record")
lines = [
"=" * 75,
f"HYPERFLOW FORENSIC CAUSAL TRACE: {res['record_id']}",
"=" * 75,
]
if rec:
lines.append(
f"Kind: {rec.get('kind', 'N/A'):<14} | Account: {rec.get('account_id', 'N/A'):<16} | Version: {rec.get('version', 1)}"
)
lines.append(f"Created: {rec.get('created_at', 'N/A')} | Updated: {rec.get('updated_at', 'N/A')}")
else:
lines.append("Record not currently present in active records table (inferred from event logs).")
lines.append(f"Associated Lifecycle Events: {res['events_count']}")
lines.append("-" * 75)
if not res["timeline"]:
lines.append(" [NO EVENTS RECORDED FOR THIS IDENTIFIER]")
else:
for idx, ev in enumerate(res["timeline"], 1):
p = ev.get("payload", {})
p_summary = ", ".join(f"{k}={v}" for k, v in p.items() if k not in ("record_id", "order_id"))
if not p_summary:
p_summary = "-"
lines.append(
f" [{idx:02d}] {ev['created_at']} | {ev['type']:<22} | {ev.get('account_id', ''):<12} | {p_summary}"
)
if res.get("state_transitions"):
lines.append("-" * 75)
lines.append("State Transitions:")
for st in res["state_transitions"]:
lines.append(f" * {st['from_state']} -> {st['to_state']} at {st['at']} (actor: {st['actor_id']})")
if res.get("anomalies"):
lines.append("-" * 75)
lines.append("Detected Forensic Anomalies:")
for an in res["anomalies"]:
lines.append(f" ! {an}")
lines.append("=" * 75)
return "\n".join(lines)