"""The "Try a screenshot" view: ground one instruction on your own screenshot. Plain HTML + script inside one gr.HTML. The image is sent with the component's `upload()` helper, and grounding runs through a server function, so no stock Gradio widgets are involved. Example screenshots are served as static files. """ TEMPLATE = '
' SCRIPT = r""" const EX = props.examples || [], MODELS = props.models || []; const st = { path: null, preview: null, text: '', model: MODELS[0], busy: false, out: null, err: '' }; const root = element.querySelector('.dsh') || element; const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({'&': '&', '<': '<', '>': '>', '"': '"'}[c])); function render() { root.innerHTML = `
Your screenshot
Instruction
${MODELS.map(m => ``).join('')}
${st.err ? `
${esc(st.err)}
` : ''}
Where the model would click
${st.out ? `Predicted click
${esc(st.out.code)}
${esc(st.out.coords)}${esc(st.out.time)}
` : `
${st.busy ? ' Looking at the screen…' : 'The predicted click point shows up here.'}
`}
Or try an example
${EX.map((e, i) => ``).join('')}
`; } async function useFile(file) { if (!file || !file.type.startsWith('image/')) return; st.preview = URL.createObjectURL(file); st.out = null; st.err = ''; st.path = null; render(); try { const r = await upload(file); st.path = r.path; } catch (e) { st.err = 'Upload failed, please try again.'; render(); } } async function go() { const text = (element.querySelector('.dsh-text') || {}).value || st.text; st.text = text; if (!st.path) { st.err = st.preview ? 'Still uploading, one moment…' : 'Choose a screenshot first.'; render(); return; } if (!text.trim()) { st.err = 'Write an instruction first.'; render(); return; } st.busy = true; st.err = ''; st.out = null; render(); try { st.out = await gpuCall('ground_screenshot', { src: st.path, text, model: st.model }); } catch (e) { const msg = e && e.message ? e.message : String(e), f = friendlyGpuError(msg); st.err = f ? f.title + '. ' + f.text : 'Grounding failed: ' + msg; } st.busy = false; render(); } element.addEventListener('change', e => { if (e.target.classList.contains('dsh-file')) useFile(e.target.files[0]); }); element.addEventListener('input', e => { if (e.target.classList.contains('dsh-text')) st.text = e.target.value; }); element.addEventListener('keydown', e => { if (e.target.classList.contains('dsh-text') && e.key === 'Enter') go(); }); element.addEventListener('dragover', e => e.preventDefault()); element.addEventListener('drop', e => { e.preventDefault(); useFile(e.dataTransfer.files[0]); }); element.addEventListener('paste', e => { const f = [...(e.clipboardData || {}).files || []][0]; if (f) useFile(f); }); element.addEventListener('click', e => { const t = e.target.closest('[data-model],[data-act],[data-ex]'); if (!t) return; if (t.dataset.model) { st.model = t.dataset.model; render(); } else if (t.dataset.act === 'go') go(); else if (t.dataset.ex) { const ex = EX[+t.dataset.ex]; st.path = 'example:' + ex.name; st.preview = ex.url; st.text = ex.text; st.out = null; st.err = ''; render(); go(); } }); render(); """ CSS = """ #df-view-shot { display: none; background: #F3EDE2; padding: 22px 26px !important; } #df-view-shot .dsh { font-family: 'Hanken Grotesk', system-ui, sans-serif; color: #2B2722; } #df-view-shot .dsh-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; } @media (max-width: 1000px) { .dsh-grid { grid-template-columns: 1fr; } } #df-view-shot .dsh-card { background: #fff; border: 1px solid #E6E3DF; border-radius: 12px; padding: 14px; display: flex; flex-direction: column; gap: 9px; box-shadow: 0 1px 2px rgba(40,30,20,.04); } #df-view-shot .dsh-cap { font: 500 10px 'JetBrains Mono', monospace; letter-spacing: .06em; text-transform: uppercase; color: #9a9389; } #df-view-shot .dsh-drop { min-height: 230px; border: 1.5px dashed #DCD8D2; border-radius: 10px; display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 4px; cursor: pointer; background: #FAF9F7; text-align: center; padding: 14px; overflow: hidden; } #df-view-shot .dsh-drop:hover { border-color: #D2603A; } #df-view-shot .dsh-drop.has { padding: 0; border-style: solid; background: #fff; } #df-view-shot .dsh-drop img { width: 100%; display: block; } #df-view-shot .dsh-drop-main { font-size: 14px; font-weight: 600; color: #3a332b; } #df-view-shot .dsh-drop-sub { font-size: 12.5px; color: #8a8378; } #df-view-shot .dsh-text { font: 14px 'Hanken Grotesk', system-ui, sans-serif; color: #2B2722; background: #fff; border: 1px solid #E6E3DF; border-radius: 9px; padding: 9px 11px; outline: none; } #df-view-shot .dsh-text:focus { border-color: #D2603A; box-shadow: 0 0 0 3px rgba(210,96,58,.10); } #df-view-shot .dsh-row { display: flex; align-items: center; justify-content: space-between; gap: 10px; flex-wrap: wrap; } #df-view-shot .dsh-seg { display: inline-flex; border: 1px solid #E6E3DF; border-radius: 9px; overflow: hidden; background: #fff; } #df-view-shot .dsh-seg button { font: 500 13px 'Hanken Grotesk', system-ui, sans-serif; color: #6f6a62; background: transparent; border: none; padding: 7px 12px; cursor: pointer; } #df-view-shot .dsh-seg button + button { border-left: 1px solid #E6E3DF; } #df-view-shot .dsh-seg button.on { background: #FBF3EC; color: #C0552F; font-weight: 600; } #df-view-shot .dsh-go { font: 600 14px 'Hanken Grotesk', system-ui, sans-serif; color: #fff; background: #D2603A; border: 1px solid #C0552F; border-radius: 9px; padding: 8px 16px; cursor: pointer; } #df-view-shot .dsh-go:disabled { opacity: .6; cursor: default; } #df-view-shot .dsh-err { font-size: 13px; color: #A8452A; } #df-view-shot .dsh-result { width: 100%; border-radius: 9px; border: 1px solid #E6E3DF; display: block; } #df-view-shot .dsh-code { font: 500 12.5px/1.6 'JetBrains Mono', monospace; color: #3a332b; background: #FAF9F7; border: 1px solid #EEECE8; border-radius: 8px; padding: 8px 10px; overflow-wrap: anywhere; } #df-view-shot .dsh-meta { display: flex; justify-content: space-between; font: 500 11.5px 'JetBrains Mono', monospace; color: #9a9389; } #df-view-shot .dsh-empty { min-height: 230px; display: flex; align-items: center; justify-content: center; gap: 8px; border-radius: 10px; background: #FAF9F7; border: 1px solid #EEECE8; font-size: 13.5px; color: #8a8378; } #df-view-shot .dsh-spin { width: 13px; height: 13px; border-radius: 50%; border: 2px solid #E6E3DF; border-top-color: #D2603A; animation: df-spin .7s linear infinite; } #df-view-shot .dsh-excap { margin: 18px 0 8px; } #df-view-shot .dsh-ex { display: grid; grid-template-columns: repeat(auto-fill, minmax(170px, 1fr)); gap: 10px; } #df-view-shot .dsh-exi { display: flex; flex-direction: column; gap: 6px; background: #fff; border: 1px solid #E6E3DF; border-radius: 10px; padding: 6px; cursor: pointer; text-align: left; font: 12.5px/1.35 'Hanken Grotesk', system-ui, sans-serif; color: #3a332b; } #df-view-shot .dsh-exi:hover { border-color: #D2603A; } #df-view-shot .dsh-exi img { width: 100%; height: 92px; object-fit: cover; object-position: top left; border-radius: 6px; border: 1px solid #EEECE8; } #df-view-shot .dsh-exi span { display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; } """