Spaces:
Sleeping
Sleeping
| /** | |
| * CommandInterpreter.js | |
| * The missing layer: LLM output β validated commands β 3D engine actions. | |
| * | |
| * Handles: | |
| * - Structured command schema validation | |
| * - Target resolution ("selected", "car", model name β model ID) | |
| * - Relative transforms (translate DELTA vs absolute SET) | |
| * - Physics-aware movement (velocity/impulse if physics on, else transform) | |
| * - Multi-step animation generation | |
| * - Context memory ("it", "the model", "that object" β last selected) | |
| * - Error recovery (graceful fallback on bad LLM output) | |
| */ | |
| import * as THREE from 'three' | |
| import useStore from '../store/useStore' | |
| import { applyImpulse, setBodyVelocity } from './PhysicsEngine' | |
| // ββ Direction mapping βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| // Maps natural language directions to axis deltas | |
| const DIR_MAP = { | |
| forward: { axis:'z', sign:-1 }, | |
| backward: { axis:'z', sign: 1 }, | |
| back: { axis:'z', sign: 1 }, | |
| left: { axis:'x', sign:-1 }, | |
| right: { axis:'x', sign: 1 }, | |
| up: { axis:'y', sign: 1 }, | |
| down: { axis:'y', sign:-1 }, | |
| north: { axis:'z', sign:-1 }, | |
| south: { axis:'z', sign: 1 }, | |
| east: { axis:'x', sign: 1 }, | |
| west: { axis:'x', sign:-1 }, | |
| } | |
| // ββ Target resolver ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export function resolveTarget(target) { | |
| const s = useStore.getState() | |
| const models = s.models | |
| if (!target || target === 'selected' || target === 'it' || target === 'this' | |
| || target === 'the model' || target === 'that' || target === 'the object') { | |
| // Use selected, or fall back to last selected, or first model | |
| const id = s.selectedModelId || s.lastSelectedModelId || models[0]?.id | |
| return models.find(m => m.id === id) || null | |
| } | |
| if (target === 'all' || target === 'everything') { | |
| return models // returns array | |
| } | |
| // Try exact ID | |
| const byId = models.find(m => m.id === target) | |
| if (byId) return byId | |
| // Try name match (case-insensitive, partial) | |
| const lower = target.toLowerCase() | |
| const byName = models.find(m => m.name.toLowerCase().includes(lower)) | |
| if (byName) return byName | |
| // Try common aliases | |
| const aliases = { | |
| car: ['car','vehicle','automobile','auto','race'], | |
| character:['soldier','fox','robot','character','person','human'], | |
| tree: ['tree','plant','bush'], | |
| city: ['city','building','block','urban'], | |
| } | |
| for (const [, terms] of Object.entries(aliases)) { | |
| if (terms.some(t => lower.includes(t))) { | |
| const found = models.find(m => terms.some(t => m.name.toLowerCase().includes(t))) | |
| if (found) return found | |
| } | |
| } | |
| return null | |
| } | |
| // ββ Command schema validation βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| function validateCommand(cmd) { | |
| const errors = [] | |
| if (!cmd || typeof cmd !== 'object') { errors.push('Not an object'); return errors } | |
| if (!cmd.action && !cmd.type) { errors.push('Missing action/type field') } | |
| return errors | |
| } | |
| // ββ Apply a single validated command βββββββββββββββββββββββββββββββββββββββββ | |
| export async function applyCommand(cmd) { | |
| const s = useStore.getState() | |
| const action = cmd.action || cmd.type || '' | |
| // Resolve target | |
| let target = resolveTarget(cmd.target || cmd.modelId || 'selected') | |
| if (!target && action !== 'add_model' && action !== 'set_lighting' | |
| && action !== 'set_physics' && action !== 'set_playing' | |
| && action !== 'set_frame' && action !== 'add_model') { | |
| // No target β try first model | |
| target = s.models[0] || null | |
| } | |
| const targets = Array.isArray(target) ? target : (target ? [target] : []) | |
| switch (action.toLowerCase().replace(/-/g,'_')) { | |
| // ββ TRANSLATE (relative delta) βββββββββββββββββββββββββββββββββββββββββββ | |
| case 'translate': | |
| case 'move': | |
| case 'move_by': { | |
| for (const m of targets) { | |
| const pos = [...m.position] | |
| const t = cmd.translate || cmd.by || cmd.delta || {} | |
| // Handle direction shorthand: { direction:"forward", amount:3 } | |
| if (cmd.direction && cmd.amount != null) { | |
| const d = DIR_MAP[cmd.direction.toLowerCase()] | |
| if (d) pos[d.axis==='x'?0:d.axis==='y'?1:2] += cmd.amount * d.sign | |
| } else { | |
| pos[0] += t.x || 0 | |
| pos[1] += t.y || 0 | |
| pos[2] += t.z || 0 | |
| } | |
| // Physics-aware: if physics on + dynamic body β set velocity | |
| if (s.physicsEnabled && (s.modelPhysics[m.id]?.type || 'dynamic') === 'dynamic') { | |
| const vel = { x: (t.x||0)*2, y: (t.y||0)*2, z: (t.z||0)*2 } | |
| setBodyVelocity(m.id, vel) | |
| } else { | |
| s.updateModelTransform(m.id, 'position', pos) | |
| } | |
| } | |
| break | |
| } | |
| // ββ SET POSITION (absolute) ββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'set_position': | |
| case 'set_transform': | |
| case 'teleport': { | |
| for (const m of targets) { | |
| const p = cmd.position || cmd.translate || cmd.pos || {} | |
| const r = cmd.rotation || cmd.rotate || {} | |
| const sc = cmd.scale || {} | |
| if (cmd.position || cmd.pos) s.updateModelTransform(m.id, 'position', | |
| [p.x??m.position[0], p.y??m.position[1], p.z??m.position[2]]) | |
| if (cmd.rotation || cmd.rotate) s.updateModelTransform(m.id, 'rotation', | |
| [r.x??m.rotation[0], r.y??m.rotation[1], r.z??m.rotation[2]]) | |
| if (cmd.scale) s.updateModelTransform(m.id, 'scale', | |
| [sc.x??m.scale[0], sc.y??m.scale[1], sc.z??m.scale[2]]) | |
| } | |
| break | |
| } | |
| // ββ ROTATE (relative delta in degrees or radians) ββββββββββββββββββββββββ | |
| case 'rotate': | |
| case 'rotate_by': { | |
| for (const m of targets) { | |
| const rot = [...m.rotation] | |
| const r = cmd.rotate || cmd.by || cmd.delta || {} | |
| const useDeg = cmd.unit !== 'rad' // default degrees | |
| const toRad = useDeg ? (Math.PI/180) : 1 | |
| rot[0] += (r.x || 0) * toRad | |
| rot[1] += (r.y || 0) * toRad | |
| rot[2] += (r.z || 0) * toRad | |
| s.updateModelTransform(m.id, 'rotation', rot) | |
| } | |
| break | |
| } | |
| // ββ SCALE ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'scale': | |
| case 'resize': | |
| case 'scale_by': { | |
| for (const m of targets) { | |
| const sc = [...m.scale] | |
| const factor = cmd.scale || cmd.factor || cmd.by || {} | |
| if (typeof factor === 'number') { | |
| s.updateModelTransform(m.id, 'scale', [sc[0]*factor, sc[1]*factor, sc[2]*factor]) | |
| } else { | |
| s.updateModelTransform(m.id, 'scale', [ | |
| sc[0] * (factor.x || factor.uniform || 1), | |
| sc[1] * (factor.y || factor.uniform || 1), | |
| sc[2] * (factor.z || factor.uniform || 1), | |
| ]) | |
| } | |
| } | |
| break | |
| } | |
| // ββ ANIMATE SEQUENCE (keyframes over time) βββββββββββββββββββββββββββββββ | |
| case 'animate': | |
| case 'animate_sequence': | |
| case 'create_animation': { | |
| for (const m of targets) { | |
| const kfs = cmd.keyframes || [] | |
| for (const kf of kfs) { | |
| s.setCurrentFrame(kf.frame) | |
| await new Promise(r => setTimeout(r, 40)) | |
| if (kf.position) s.updateModelTransform(m.id, 'position', | |
| Array.isArray(kf.position) ? kf.position : [kf.position.x||0, kf.position.y||0, kf.position.z||0]) | |
| if (kf.rotation) s.updateModelTransform(m.id, 'rotation', | |
| Array.isArray(kf.rotation) ? kf.rotation : [kf.rotation.x||0, kf.rotation.y||0, kf.rotation.z||0]) | |
| if (kf.scale) s.updateModelTransform(m.id, 'scale', | |
| Array.isArray(kf.scale) ? kf.scale : [kf.scale.x||1, kf.scale.y||1, kf.scale.z||1]) | |
| s.addKeyframe(kf.frame, m.id) | |
| } | |
| s.setCurrentFrame(0) | |
| } | |
| break | |
| } | |
| // ββ PHYSICS IMPULSE ββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'impulse': | |
| case 'apply_impulse': | |
| case 'push': | |
| case 'launch': { | |
| for (const m of targets) { | |
| const imp = cmd.impulse || cmd.force || { x:0, y:8, z:0 } | |
| applyImpulse(m.id, imp) | |
| } | |
| break | |
| } | |
| case 'set_velocity': | |
| case 'velocity': { | |
| for (const m of targets) { | |
| const vel = cmd.velocity || cmd.vel || { x:0, y:0, z:0 } | |
| setBodyVelocity(m.id, vel) | |
| } | |
| break | |
| } | |
| case 'stop': { | |
| for (const m of targets) setBodyVelocity(m.id, { x:0, y:0, z:0 }) | |
| break | |
| } | |
| // ββ GLB ANIMATION ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'set_animation': | |
| case 'play_animation': { | |
| for (const m of targets) { | |
| if (cmd.animation) s.setModelActiveAnimation(m.id, cmd.animation) | |
| if (cmd.speed != null) s.setModelAnimSpeed(m.id, cmd.speed) | |
| s.setModelAnimPlaying?.(m.id, true) | |
| } | |
| break | |
| } | |
| case 'pause_animation': { | |
| for (const m of targets) s.setModelAnimPlaying?.(m.id, false) | |
| break | |
| } | |
| // ββ TIMELINE βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'play': s.setIsPlaying(true); break | |
| case 'pause': s.setIsPlaying(false); break | |
| case 'stop_playing': s.setIsPlaying(false); s.setCurrentFrame(0); break | |
| case 'set_playing': s.setIsPlaying(cmd.value ?? cmd.playing ?? true); break | |
| case 'set_frame': s.setCurrentFrame(cmd.frame ?? 0); break | |
| case 'goto_frame': s.setCurrentFrame(cmd.frame ?? 0); break | |
| // ββ PHYSICS ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'set_physics': | |
| case 'physics': { | |
| if (cmd.enabled != null) s.setPhysicsEnabled(cmd.enabled) | |
| if (cmd.gravity != null) s.setGravity(cmd.gravity) | |
| break | |
| } | |
| // ββ LIGHTING βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'set_lighting': | |
| case 'lighting': { | |
| if (cmd.preset) s.setLightingPreset(cmd.preset) | |
| break | |
| } | |
| // ββ MODEL MANAGEMENT βββββββββββββββββββββββββββββββββββββββββββββββββββββ | |
| case 'add_model': { | |
| if (cmd.url) s.addModel(cmd.url, cmd.name) | |
| break | |
| } | |
| case 'remove_model': | |
| case 'delete': { | |
| for (const m of targets) s.removeModel(m.id) | |
| break | |
| } | |
| case 'select': { | |
| if (targets.length > 0) s.selectModel(targets[0].id) | |
| break | |
| } | |
| case 'hide': { | |
| for (const m of targets) if (m.visible) s.toggleModelVisibility(m.id) | |
| break | |
| } | |
| case 'show': { | |
| for (const m of targets) if (!m.visible) s.toggleModelVisibility(m.id) | |
| break | |
| } | |
| case 'reset_transform': { | |
| for (const m of targets) { | |
| s.updateModelTransform(m.id, 'position', [0,0,0]) | |
| s.updateModelTransform(m.id, 'rotation', [0,0,0]) | |
| s.updateModelTransform(m.id, 'scale', [1,1,1]) | |
| } | |
| break | |
| } | |
| default: | |
| console.warn('[CommandInterpreter] Unknown action:', action) | |
| } | |
| } | |
| // ββ Execute array of commands βββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export async function executeCommands(commands) { | |
| const validated = [] | |
| const errors = [] | |
| for (const cmd of (commands || [])) { | |
| const errs = validateCommand(cmd) | |
| if (errs.length) { errors.push({ cmd, errs }); continue } | |
| validated.push(cmd) | |
| } | |
| if (errors.length) console.warn('[CommandInterpreter] Validation errors:', errors) | |
| for (const cmd of validated) { | |
| await applyCommand(cmd) | |
| await new Promise(r => setTimeout(r, 60)) | |
| } | |
| return { executed: validated.length, errors: errors.length } | |
| } | |
| // ββ Parse LLM response robustly βββββββββββββββββββββββββββββββββββββββββββββββ | |
| export function parseLLMResponse(rawText) { | |
| if (!rawText) return null | |
| // Strip markdown | |
| let text = rawText.replace(/```json\s*/gi,'').replace(/```\s*/gi,'').trim() | |
| // Find JSON object | |
| const start = text.indexOf('{') | |
| const end = text.lastIndexOf('}') | |
| if (start === -1 || end === -1) return null | |
| try { | |
| const obj = JSON.parse(text.slice(start, end+1)) | |
| // Normalize: support both "action" and "type" keys | |
| // Support both "commands" array and single "action" | |
| if (obj.commands && Array.isArray(obj.commands)) { | |
| return obj // already correct format | |
| } | |
| if (obj.actions && Array.isArray(obj.actions)) { | |
| return { message: obj.message, commands: obj.actions } | |
| } | |
| if (obj.action || obj.type) { | |
| // Single command β wrap it | |
| return { message: obj.message || '', commands: [obj] } | |
| } | |
| return obj | |
| } catch (e) { | |
| // Try to extract partial JSON arrays | |
| const arrMatch = text.match(/\[[\s\S]+?\]/) | |
| if (arrMatch) { | |
| try { | |
| const arr = JSON.parse(arrMatch[0]) | |
| return { message: '', commands: arr } | |
| } catch {} | |
| } | |
| return null | |
| } | |
| } | |
| // ββ Generate animation presets ββββββββββββββββββββββββββββββββββββββββββββββββ | |
| export function generatePresetAnimation(type, modelId, totalFrames) { | |
| const s = useStore.getState() | |
| const model = s.models.find(m => m.id === modelId) | |
| if (!model) return [] | |
| const [px, py, pz] = model.position | |
| const tf = totalFrames || s.totalFrames | |
| const presets = { | |
| drive_straight: [ | |
| { frame:0, position:[px-10, py, pz], rotation:[0,0,0], scale:model.scale }, | |
| { frame:tf, position:[px+10, py, pz], rotation:[0,0,0], scale:model.scale }, | |
| ], | |
| spin_360: [ | |
| { frame:0, position:[px,py,pz], rotation:[0,0,0], scale:model.scale }, | |
| { frame:tf/2, position:[px,py,pz], rotation:[0,Math.PI,0], scale:model.scale }, | |
| { frame:tf, position:[px,py,pz], rotation:[0,Math.PI*2,0], scale:model.scale }, | |
| ], | |
| bounce: [ | |
| { frame:0, position:[px,py,pz], rotation:model.rotation, scale:model.scale }, | |
| { frame:tf*0.25,position:[px,py+3,pz], rotation:model.rotation, scale:model.scale }, | |
| { frame:tf*0.5, position:[px,py,pz], rotation:model.rotation, scale:model.scale }, | |
| { frame:tf*0.75,position:[px,py+3,pz], rotation:model.rotation, scale:model.scale }, | |
| { frame:tf, position:[px,py,pz], rotation:model.rotation, scale:model.scale }, | |
| ], | |
| figure_eight: (() => { | |
| const kfs = [] | |
| const steps = 12 | |
| for (let i = 0; i <= steps; i++) { | |
| const t = (i / steps) * Math.PI * 2 | |
| const x = px + Math.sin(t) * 5 | |
| const z = pz + Math.sin(t) * Math.cos(t) * 5 | |
| const rot = [0, -t, 0] | |
| kfs.push({ frame: Math.round((i/steps)*tf), position:[x,py,z], rotation:rot, scale:model.scale }) | |
| } | |
| return kfs | |
| })(), | |
| circle: (() => { | |
| const kfs = [] | |
| const steps = 8 | |
| for (let i = 0; i <= steps; i++) { | |
| const t = (i / steps) * Math.PI * 2 | |
| kfs.push({ | |
| frame: Math.round((i/steps)*tf), | |
| position: [px + Math.cos(t)*5, py, pz + Math.sin(t)*5], | |
| rotation: [0, t, 0], | |
| scale: model.scale, | |
| }) | |
| } | |
| return kfs | |
| })(), | |
| fly_in: [ | |
| { frame:0, position:[px, py+20, pz-20], rotation:[0.5,0,0], scale:[0.1,0.1,0.1] }, | |
| { frame:tf, position:[px, py, pz], rotation:[0,0,0], scale:model.scale }, | |
| ], | |
| } | |
| return presets[type] || [] | |
| } | |