services/omniParserAdapter.ts

export interface OmniParserElement {
  id: string;
  type: 'button' | 'input' | 'link' | 'panel' | 'graph-node' | 'text';
  label: string;
  bbox: [number, number, number, number];
  interactable: boolean;
  description: string;
}

export interface OmniParserTransition {
  source: 'microsoft/OmniParser-adapter';
  screenshotId: string;
  elements: OmniParserElement[];
  llmLanguage: string;
}

export function buildOmniParserTransition(screenName: string, elements: OmniParserElement[]): OmniParserTransition {
  const interactable = elements.filter((element) => element.interactable);
  const llmLanguage = [
    `Screen: ${screenName}.`,
    `Detected ${elements.length} UI elements; ${interactable.length} are interactable.`,
    ...interactable.map((element) => `Use ${element.type} "${element.label}" at bbox [${element.bbox.join(', ')}] to ${element.description}.`),
  ].join(' ');

  return {
    source: 'microsoft/OmniParser-adapter',
    screenshotId: `${screenName.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-synthetic`,
    elements,
    llmLanguage,
  };
}

export function buildSuiteOmniParserElements(activeTab: string, activeView: string): OmniParserElement[] {
  return [
    {
      id: 'main-tab',
      type: 'button',
      label: activeTab,
      bbox: [8, 6, 18, 11],
      interactable: true,
      description: `open the ${activeTab} workspace`,
    },
    {
      id: 'subview-slider',
      type: 'panel',
      label: `${activeView} subview slider`,
      bbox: [3, 13, 96, 25],
      interactable: true,
      description: `switch between sorted subviews inside ${activeTab}`,
    },
    {
      id: 'mindwalk-graph',
      type: 'graph-node',
      label: 'Mindwalk navigation graph',
      bbox: [5, 28, 62, 92],
      interactable: true,
      description: 'select UI touch-state nodes to generate Nova Act prompt context',
    },
    {
      id: 'limitations-button',
      type: 'button',
      label: 'Inject selected limitations',
      bbox: [72, 19, 94, 25],
      interactable: true,
      description: 'inject persona-specific limitation functions into the Nova Act context',
    },
  ];
}