File size: 2,132 Bytes
4fdaf19 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | export interface OmniParserElement {
id: string;
type: 'button' | 'input' | 'link' | 'panel' | 'graph-node' | 'text';
label: string;
bbox: [number, number, number, number];
interactable: boolean;
description: string;
}
export interface OmniParserTransition {
source: 'microsoft/OmniParser-adapter';
screenshotId: string;
elements: OmniParserElement[];
llmLanguage: string;
}
export function buildOmniParserTransition(screenName: string, elements: OmniParserElement[]): OmniParserTransition {
const interactable = elements.filter((element) => element.interactable);
const llmLanguage = [
`Screen: ${screenName}.`,
`Detected ${elements.length} UI elements; ${interactable.length} are interactable.`,
...interactable.map((element) => `Use ${element.type} "${element.label}" at bbox [${element.bbox.join(', ')}] to ${element.description}.`),
].join(' ');
return {
source: 'microsoft/OmniParser-adapter',
screenshotId: `${screenName.toLowerCase().replace(/[^a-z0-9]+/g, '-')}-synthetic`,
elements,
llmLanguage,
};
}
export function buildSuiteOmniParserElements(activeTab: string, activeView: string): OmniParserElement[] {
return [
{
id: 'main-tab',
type: 'button',
label: activeTab,
bbox: [8, 6, 18, 11],
interactable: true,
description: `open the ${activeTab} workspace`,
},
{
id: 'subview-slider',
type: 'panel',
label: `${activeView} subview slider`,
bbox: [3, 13, 96, 25],
interactable: true,
description: `switch between sorted subviews inside ${activeTab}`,
},
{
id: 'mindwalk-graph',
type: 'graph-node',
label: 'Mindwalk navigation graph',
bbox: [5, 28, 62, 92],
interactable: true,
description: 'select UI touch-state nodes to generate Nova Act prompt context',
},
{
id: 'limitations-button',
type: 'button',
label: 'Inject selected limitations',
bbox: [72, 19, 94, 25],
interactable: true,
description: 'inject persona-specific limitation functions into the Nova Act context',
},
];
}
|