// The output check: the model answers freely, then its output is checked against the tool schema and the scene: // "ok" plain text, or every call names a known tool with known arguments, allowed values and known ids // "invalid_tool" a tool the schema does not have -> the agent refuses (Refusal) // "invalid_value" an argument, allowed value or room the schema/scene does not have -> the agent refuses // "invalid_id" an id outside the scene, the hands, the gaze and earlier tool results -> generate again under the grammar // "malformed" a call that does not parse -> generate again under the grammar // The first out-of-schema call decides at once; an unknown id is remembered while the rest is checked. using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; namespace SceneAgent { public static partial class SceneGrammar { public const string Refusal = "I can't do that; none of my tools can."; static readonly HashSet IdKeys = new HashSet { "object_id", "target_id" }; static readonly HashSet IdListKeys = new HashSet { "object_ids", "candidate_ids" }; static readonly HashSet RoomKeys = new HashSet { "location_id", "room" }; static readonly Regex CallRe = new Regex(@"\s*(.*?)\s*", RegexOptions.Singleline); /// The ids the model may name (scene objects, hands, gaze, ids in earlier tool results) and the rooms. public static void Vocab(string sceneJson, IEnumerable toolResultJsons, out List ids, out List rooms) { var scene = (Dictionary)MiniJson.Parse(sceneJson); ids = new List(); if (scene.TryGetValue("visible_objects", out var vo) && vo is List objs) foreach (var o in objs) if (o is Dictionary d && d.TryGetValue("id", out var id) && id is string s) ids.Add(s); if (scene.TryGetValue("player", out var pl) && pl is Dictionary player) foreach (var k in new[] { "left_hand", "right_hand", "gaze" }) if (player.TryGetValue(k, out var v) && v is string s && s.Length > 0) ids.Add(s); foreach (var r in toolResultJsons ?? Enumerable.Empty()) { try { CollectIds(MiniJson.Parse(r), ids); } catch (System.FormatException) { } } ids = ids.Distinct().ToList(); rooms = scene.TryGetValue("locations", out var l) && l is List ls ? ls.OfType().Distinct().ToList() : new List(); } public static string Check(string text, string sceneJson, IEnumerable toolResultJsons) { text = text ?? ""; var blocks = CallRe.Matches(text).Cast().Select(m => m.Groups[1].Value).ToList(); if (blocks.Count == 0) return text.Contains("") ? "malformed" : "ok"; Vocab(sceneJson, toolResultJsons, out var idList, out var rooms); var ids = new HashSet(idList); string verdict = "ok"; foreach (var b in blocks) { object parsed; try { parsed = MiniJson.ParseStrict(b); } catch (System.Exception) { return "malformed"; } if (!(parsed is Dictionary obj) || !obj.TryGetValue("name", out var nameObj)) return "malformed"; if (!(nameObj is string name) || !Tools.TryGetValue(name, out var spec)) return "invalid_tool"; obj.TryGetValue("arguments", out var argsObj); // an empty or false "arguments" value counts as no arguments bool empty = argsObj == null || argsObj is List el && el.Count == 0 || argsObj is string es && es.Length == 0 || argsObj is double dz && dz == 0 || argsObj is bool bf && !bf; if (!empty && !(argsObj is Dictionary)) return "malformed"; var args = empty ? new Dictionary() : (Dictionary)argsObj; foreach (var kv in args) { string k = kv.Key; object v = kv.Value; if (!spec.TryGetValue(k, out var allowed)) return "invalid_value"; if (allowed != null && !(v is string sv && allowed.Contains(sv))) return "invalid_value"; if (RoomKeys.Contains(k) && rooms.Count > 0 && !(v is string rv && rooms.Contains(rv))) return "invalid_value"; if (IdKeys.Contains(k) && !(v is string iv && ids.Contains(iv))) verdict = "invalid_id"; if (IdListKeys.Contains(k) && NamesUnknownId(v, ids)) verdict = "invalid_id"; } } return verdict; } // a string where a list is expected is checked character by character, as in training static bool NamesUnknownId(object v, HashSet ids) { switch (v) { case null: return false; case List l: return l.Any(x => !(x is string s && ids.Contains(s))); case string s: return s.Length > 0 && s.Any(ch => !ids.Contains(ch.ToString())); default: return true; } } } }