glb-studio / src /components /CommandInterpreter.js
GLB Studio Deploy
fix: animation detection + AI 404 errors
d9dc1a9
/**
* CommandInterpreter.js
* The missing layer: LLM output β†’ validated commands β†’ 3D engine actions.
*
* Handles:
* - Structured command schema validation
* - Target resolution ("selected", "car", model name β†’ model ID)
* - Relative transforms (translate DELTA vs absolute SET)
* - Physics-aware movement (velocity/impulse if physics on, else transform)
* - Multi-step animation generation
* - Context memory ("it", "the model", "that object" β†’ last selected)
* - Error recovery (graceful fallback on bad LLM output)
*/
import * as THREE from 'three'
import useStore from '../store/useStore'
import { applyImpulse, setBodyVelocity } from './PhysicsEngine'
// ── Direction mapping ─────────────────────────────────────────────────────────
// Maps natural language directions to axis deltas
const DIR_MAP = {
forward: { axis:'z', sign:-1 },
backward: { axis:'z', sign: 1 },
back: { axis:'z', sign: 1 },
left: { axis:'x', sign:-1 },
right: { axis:'x', sign: 1 },
up: { axis:'y', sign: 1 },
down: { axis:'y', sign:-1 },
north: { axis:'z', sign:-1 },
south: { axis:'z', sign: 1 },
east: { axis:'x', sign: 1 },
west: { axis:'x', sign:-1 },
}
// ── Target resolver ────────────────────────────────────────────────────────────
export function resolveTarget(target) {
const s = useStore.getState()
const models = s.models
if (!target || target === 'selected' || target === 'it' || target === 'this'
|| target === 'the model' || target === 'that' || target === 'the object') {
// Use selected, or fall back to last selected, or first model
const id = s.selectedModelId || s.lastSelectedModelId || models[0]?.id
return models.find(m => m.id === id) || null
}
if (target === 'all' || target === 'everything') {
return models // returns array
}
// Try exact ID
const byId = models.find(m => m.id === target)
if (byId) return byId
// Try name match (case-insensitive, partial)
const lower = target.toLowerCase()
const byName = models.find(m => m.name.toLowerCase().includes(lower))
if (byName) return byName
// Try common aliases
const aliases = {
car: ['car','vehicle','automobile','auto','race'],
character:['soldier','fox','robot','character','person','human'],
tree: ['tree','plant','bush'],
city: ['city','building','block','urban'],
}
for (const [, terms] of Object.entries(aliases)) {
if (terms.some(t => lower.includes(t))) {
const found = models.find(m => terms.some(t => m.name.toLowerCase().includes(t)))
if (found) return found
}
}
return null
}
// ── Command schema validation ─────────────────────────────────────────────────
function validateCommand(cmd) {
const errors = []
if (!cmd || typeof cmd !== 'object') { errors.push('Not an object'); return errors }
if (!cmd.action && !cmd.type) { errors.push('Missing action/type field') }
return errors
}
// ── Apply a single validated command ─────────────────────────────────────────
export async function applyCommand(cmd) {
const s = useStore.getState()
const action = cmd.action || cmd.type || ''
// Resolve target
let target = resolveTarget(cmd.target || cmd.modelId || 'selected')
if (!target && action !== 'add_model' && action !== 'set_lighting'
&& action !== 'set_physics' && action !== 'set_playing'
&& action !== 'set_frame' && action !== 'add_model') {
// No target β€” try first model
target = s.models[0] || null
}
const targets = Array.isArray(target) ? target : (target ? [target] : [])
switch (action.toLowerCase().replace(/-/g,'_')) {
// ── TRANSLATE (relative delta) ───────────────────────────────────────────
case 'translate':
case 'move':
case 'move_by': {
for (const m of targets) {
const pos = [...m.position]
const t = cmd.translate || cmd.by || cmd.delta || {}
// Handle direction shorthand: { direction:"forward", amount:3 }
if (cmd.direction && cmd.amount != null) {
const d = DIR_MAP[cmd.direction.toLowerCase()]
if (d) pos[d.axis==='x'?0:d.axis==='y'?1:2] += cmd.amount * d.sign
} else {
pos[0] += t.x || 0
pos[1] += t.y || 0
pos[2] += t.z || 0
}
// Physics-aware: if physics on + dynamic body β†’ set velocity
if (s.physicsEnabled && (s.modelPhysics[m.id]?.type || 'dynamic') === 'dynamic') {
const vel = { x: (t.x||0)*2, y: (t.y||0)*2, z: (t.z||0)*2 }
setBodyVelocity(m.id, vel)
} else {
s.updateModelTransform(m.id, 'position', pos)
}
}
break
}
// ── SET POSITION (absolute) ──────────────────────────────────────────────
case 'set_position':
case 'set_transform':
case 'teleport': {
for (const m of targets) {
const p = cmd.position || cmd.translate || cmd.pos || {}
const r = cmd.rotation || cmd.rotate || {}
const sc = cmd.scale || {}
if (cmd.position || cmd.pos) s.updateModelTransform(m.id, 'position',
[p.x??m.position[0], p.y??m.position[1], p.z??m.position[2]])
if (cmd.rotation || cmd.rotate) s.updateModelTransform(m.id, 'rotation',
[r.x??m.rotation[0], r.y??m.rotation[1], r.z??m.rotation[2]])
if (cmd.scale) s.updateModelTransform(m.id, 'scale',
[sc.x??m.scale[0], sc.y??m.scale[1], sc.z??m.scale[2]])
}
break
}
// ── ROTATE (relative delta in degrees or radians) ────────────────────────
case 'rotate':
case 'rotate_by': {
for (const m of targets) {
const rot = [...m.rotation]
const r = cmd.rotate || cmd.by || cmd.delta || {}
const useDeg = cmd.unit !== 'rad' // default degrees
const toRad = useDeg ? (Math.PI/180) : 1
rot[0] += (r.x || 0) * toRad
rot[1] += (r.y || 0) * toRad
rot[2] += (r.z || 0) * toRad
s.updateModelTransform(m.id, 'rotation', rot)
}
break
}
// ── SCALE ────────────────────────────────────────────────────────────────
case 'scale':
case 'resize':
case 'scale_by': {
for (const m of targets) {
const sc = [...m.scale]
const factor = cmd.scale || cmd.factor || cmd.by || {}
if (typeof factor === 'number') {
s.updateModelTransform(m.id, 'scale', [sc[0]*factor, sc[1]*factor, sc[2]*factor])
} else {
s.updateModelTransform(m.id, 'scale', [
sc[0] * (factor.x || factor.uniform || 1),
sc[1] * (factor.y || factor.uniform || 1),
sc[2] * (factor.z || factor.uniform || 1),
])
}
}
break
}
// ── ANIMATE SEQUENCE (keyframes over time) ───────────────────────────────
case 'animate':
case 'animate_sequence':
case 'create_animation': {
for (const m of targets) {
const kfs = cmd.keyframes || []
for (const kf of kfs) {
s.setCurrentFrame(kf.frame)
await new Promise(r => setTimeout(r, 40))
if (kf.position) s.updateModelTransform(m.id, 'position',
Array.isArray(kf.position) ? kf.position : [kf.position.x||0, kf.position.y||0, kf.position.z||0])
if (kf.rotation) s.updateModelTransform(m.id, 'rotation',
Array.isArray(kf.rotation) ? kf.rotation : [kf.rotation.x||0, kf.rotation.y||0, kf.rotation.z||0])
if (kf.scale) s.updateModelTransform(m.id, 'scale',
Array.isArray(kf.scale) ? kf.scale : [kf.scale.x||1, kf.scale.y||1, kf.scale.z||1])
s.addKeyframe(kf.frame, m.id)
}
s.setCurrentFrame(0)
}
break
}
// ── PHYSICS IMPULSE ──────────────────────────────────────────────────────
case 'impulse':
case 'apply_impulse':
case 'push':
case 'launch': {
for (const m of targets) {
const imp = cmd.impulse || cmd.force || { x:0, y:8, z:0 }
applyImpulse(m.id, imp)
}
break
}
case 'set_velocity':
case 'velocity': {
for (const m of targets) {
const vel = cmd.velocity || cmd.vel || { x:0, y:0, z:0 }
setBodyVelocity(m.id, vel)
}
break
}
case 'stop': {
for (const m of targets) setBodyVelocity(m.id, { x:0, y:0, z:0 })
break
}
// ── GLB ANIMATION ────────────────────────────────────────────────────────
case 'set_animation':
case 'play_animation': {
for (const m of targets) {
if (cmd.animation) s.setModelActiveAnimation(m.id, cmd.animation)
if (cmd.speed != null) s.setModelAnimSpeed(m.id, cmd.speed)
s.setModelAnimPlaying?.(m.id, true)
}
break
}
case 'pause_animation': {
for (const m of targets) s.setModelAnimPlaying?.(m.id, false)
break
}
// ── TIMELINE ─────────────────────────────────────────────────────────────
case 'play': s.setIsPlaying(true); break
case 'pause': s.setIsPlaying(false); break
case 'stop_playing': s.setIsPlaying(false); s.setCurrentFrame(0); break
case 'set_playing': s.setIsPlaying(cmd.value ?? cmd.playing ?? true); break
case 'set_frame': s.setCurrentFrame(cmd.frame ?? 0); break
case 'goto_frame': s.setCurrentFrame(cmd.frame ?? 0); break
// ── PHYSICS ──────────────────────────────────────────────────────────────
case 'set_physics':
case 'physics': {
if (cmd.enabled != null) s.setPhysicsEnabled(cmd.enabled)
if (cmd.gravity != null) s.setGravity(cmd.gravity)
break
}
// ── LIGHTING ─────────────────────────────────────────────────────────────
case 'set_lighting':
case 'lighting': {
if (cmd.preset) s.setLightingPreset(cmd.preset)
break
}
// ── MODEL MANAGEMENT ─────────────────────────────────────────────────────
case 'add_model': {
if (cmd.url) s.addModel(cmd.url, cmd.name)
break
}
case 'remove_model':
case 'delete': {
for (const m of targets) s.removeModel(m.id)
break
}
case 'select': {
if (targets.length > 0) s.selectModel(targets[0].id)
break
}
case 'hide': {
for (const m of targets) if (m.visible) s.toggleModelVisibility(m.id)
break
}
case 'show': {
for (const m of targets) if (!m.visible) s.toggleModelVisibility(m.id)
break
}
case 'reset_transform': {
for (const m of targets) {
s.updateModelTransform(m.id, 'position', [0,0,0])
s.updateModelTransform(m.id, 'rotation', [0,0,0])
s.updateModelTransform(m.id, 'scale', [1,1,1])
}
break
}
default:
console.warn('[CommandInterpreter] Unknown action:', action)
}
}
// ── Execute array of commands ─────────────────────────────────────────────────
export async function executeCommands(commands) {
const validated = []
const errors = []
for (const cmd of (commands || [])) {
const errs = validateCommand(cmd)
if (errs.length) { errors.push({ cmd, errs }); continue }
validated.push(cmd)
}
if (errors.length) console.warn('[CommandInterpreter] Validation errors:', errors)
for (const cmd of validated) {
await applyCommand(cmd)
await new Promise(r => setTimeout(r, 60))
}
return { executed: validated.length, errors: errors.length }
}
// ── Parse LLM response robustly ───────────────────────────────────────────────
export function parseLLMResponse(rawText) {
if (!rawText) return null
// Strip markdown
let text = rawText.replace(/```json\s*/gi,'').replace(/```\s*/gi,'').trim()
// Find JSON object
const start = text.indexOf('{')
const end = text.lastIndexOf('}')
if (start === -1 || end === -1) return null
try {
const obj = JSON.parse(text.slice(start, end+1))
// Normalize: support both "action" and "type" keys
// Support both "commands" array and single "action"
if (obj.commands && Array.isArray(obj.commands)) {
return obj // already correct format
}
if (obj.actions && Array.isArray(obj.actions)) {
return { message: obj.message, commands: obj.actions }
}
if (obj.action || obj.type) {
// Single command β€” wrap it
return { message: obj.message || '', commands: [obj] }
}
return obj
} catch (e) {
// Try to extract partial JSON arrays
const arrMatch = text.match(/\[[\s\S]+?\]/)
if (arrMatch) {
try {
const arr = JSON.parse(arrMatch[0])
return { message: '', commands: arr }
} catch {}
}
return null
}
}
// ── Generate animation presets ────────────────────────────────────────────────
export function generatePresetAnimation(type, modelId, totalFrames) {
const s = useStore.getState()
const model = s.models.find(m => m.id === modelId)
if (!model) return []
const [px, py, pz] = model.position
const tf = totalFrames || s.totalFrames
const presets = {
drive_straight: [
{ frame:0, position:[px-10, py, pz], rotation:[0,0,0], scale:model.scale },
{ frame:tf, position:[px+10, py, pz], rotation:[0,0,0], scale:model.scale },
],
spin_360: [
{ frame:0, position:[px,py,pz], rotation:[0,0,0], scale:model.scale },
{ frame:tf/2, position:[px,py,pz], rotation:[0,Math.PI,0], scale:model.scale },
{ frame:tf, position:[px,py,pz], rotation:[0,Math.PI*2,0], scale:model.scale },
],
bounce: [
{ frame:0, position:[px,py,pz], rotation:model.rotation, scale:model.scale },
{ frame:tf*0.25,position:[px,py+3,pz], rotation:model.rotation, scale:model.scale },
{ frame:tf*0.5, position:[px,py,pz], rotation:model.rotation, scale:model.scale },
{ frame:tf*0.75,position:[px,py+3,pz], rotation:model.rotation, scale:model.scale },
{ frame:tf, position:[px,py,pz], rotation:model.rotation, scale:model.scale },
],
figure_eight: (() => {
const kfs = []
const steps = 12
for (let i = 0; i <= steps; i++) {
const t = (i / steps) * Math.PI * 2
const x = px + Math.sin(t) * 5
const z = pz + Math.sin(t) * Math.cos(t) * 5
const rot = [0, -t, 0]
kfs.push({ frame: Math.round((i/steps)*tf), position:[x,py,z], rotation:rot, scale:model.scale })
}
return kfs
})(),
circle: (() => {
const kfs = []
const steps = 8
for (let i = 0; i <= steps; i++) {
const t = (i / steps) * Math.PI * 2
kfs.push({
frame: Math.round((i/steps)*tf),
position: [px + Math.cos(t)*5, py, pz + Math.sin(t)*5],
rotation: [0, t, 0],
scale: model.scale,
})
}
return kfs
})(),
fly_in: [
{ frame:0, position:[px, py+20, pz-20], rotation:[0.5,0,0], scale:[0.1,0.1,0.1] },
{ frame:tf, position:[px, py, pz], rotation:[0,0,0], scale:model.scale },
],
}
return presets[type] || []
}