shot_tool.py

"""The "Try a screenshot" view: ground one instruction on your own screenshot.

Plain HTML + script inside one gr.HTML. The image is sent with the component's
`upload()` helper, and grounding runs through a server function, so no stock
Gradio widgets are involved. Example screenshots are served as static files.
"""

TEMPLATE = '<div class="dsh"></div>'

SCRIPT = r"""
const EX = props.examples || [], MODELS = props.models || [];
const st = { path: null, preview: null, text: '', model: MODELS[0], busy: false, out: null, err: '' };
const root = element.querySelector('.dsh') || element;
const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;'}[c]));

function render() {
  root.innerHTML = `
  <div class="dsh-grid">
    <div class="dsh-card">
      <div class="dsh-cap">Your screenshot</div>
      <label class="dsh-drop ${st.preview ? 'has' : ''}">
        ${st.preview ? `<img src="${esc(st.preview)}" alt="Screenshot">` :
          `<span class="dsh-drop-main">Drop, paste or click to choose a screenshot</span>
           <span class="dsh-drop-sub">PNG or JPG of a desktop, ideally a Linux desktop</span>`}
        <input type="file" accept="image/*" class="dsh-file" hidden>
      </label>
      <div class="dsh-cap">Instruction</div>
      <input class="dsh-text" type="text" placeholder="e.g. Open the File menu" value="${esc(st.text)}">
      <div class="dsh-row">
        <span class="dsh-seg">${MODELS.map(m => `<button type="button" data-model="${esc(m)}" class="${st.model === m ? 'on' : ''}">${esc(m)}</button>`).join('')}</span>
        <button type="button" class="dsh-go" data-act="go" ${st.busy ? 'disabled' : ''}>${st.busy ? 'Grounding…' : 'Ground it ↑'}</button>
      </div>
      ${st.err ? `<div class="dsh-err">${esc(st.err)}</div>` : ''}
    </div>
    <div class="dsh-card">
      <div class="dsh-cap">Where the model would click</div>
      ${st.out ? `<img class="dsh-result" src="${st.out.image}" alt="Predicted click">
        <div class="dsh-code">${esc(st.out.code)}</div>
        <div class="dsh-meta"><span>${esc(st.out.coords)}</span><span>${esc(st.out.time)}</span></div>`
      : `<div class="dsh-empty">${st.busy ? '<span class="dsh-spin"></span> Looking at the screen…' : 'The predicted click point shows up here.'}</div>`}
    </div>
  </div>
  <div class="dsh-cap dsh-excap">Or try an example</div>
  <div class="dsh-ex">${EX.map((e, i) => `<button type="button" class="dsh-exi" data-ex="${i}" title="${esc(e.text)}">
     <img src="${esc(e.url)}" alt=""><span>${esc(e.text)}</span></button>`).join('')}</div>`;
}

async function useFile(file) {
  if (!file || !file.type.startsWith('image/')) return;
  st.preview = URL.createObjectURL(file); st.out = null; st.err = ''; st.path = null; render();
  try { const r = await upload(file); st.path = r.path; }
  catch (e) { st.err = 'Upload failed, please try again.'; render(); }
}

async function go() {
  const text = (element.querySelector('.dsh-text') || {}).value || st.text;
  st.text = text;
  if (!st.path) { st.err = st.preview ? 'Still uploading, one moment…' : 'Choose a screenshot first.'; render(); return; }
  if (!text.trim()) { st.err = 'Write an instruction first.'; render(); return; }
  st.busy = true; st.err = ''; st.out = null; render();
  try { st.out = await gpuCall('ground_screenshot', { src: st.path, text, model: st.model }); }
  catch (e) {
    const msg = e && e.message ? e.message : String(e), f = friendlyGpuError(msg);
    st.err = f ? f.title + '. ' + f.text : 'Grounding failed: ' + msg;
  }
  st.busy = false; render();
}

element.addEventListener('change', e => { if (e.target.classList.contains('dsh-file')) useFile(e.target.files[0]); });
element.addEventListener('input', e => { if (e.target.classList.contains('dsh-text')) st.text = e.target.value; });
element.addEventListener('keydown', e => { if (e.target.classList.contains('dsh-text') && e.key === 'Enter') go(); });
element.addEventListener('dragover', e => e.preventDefault());
element.addEventListener('drop', e => { e.preventDefault(); useFile(e.dataTransfer.files[0]); });
element.addEventListener('paste', e => { const f = [...(e.clipboardData || {}).files || []][0]; if (f) useFile(f); });
element.addEventListener('click', e => {
  const t = e.target.closest('[data-model],[data-act],[data-ex]');
  if (!t) return;
  if (t.dataset.model) { st.model = t.dataset.model; render(); }
  else if (t.dataset.act === 'go') go();
  else if (t.dataset.ex) {
    const ex = EX[+t.dataset.ex];
    st.path = 'example:' + ex.name; st.preview = ex.url; st.text = ex.text; st.out = null; st.err = '';
    render(); go();
  }
});
render();
"""

CSS = """
#df-view-shot { display: none; background: #F3EDE2; padding: 22px 26px !important; }
#df-view-shot .dsh { font-family: 'Hanken Grotesk', system-ui, sans-serif; color: #2B2722; }
#df-view-shot .dsh-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
@media (max-width: 1000px) { .dsh-grid { grid-template-columns: 1fr; } }
#df-view-shot .dsh-card { background: #fff; border: 1px solid #E6E3DF; border-radius: 12px; padding: 14px; display: flex; flex-direction: column; gap: 9px;
            box-shadow: 0 1px 2px rgba(40,30,20,.04); }
#df-view-shot .dsh-cap { font: 500 10px 'JetBrains Mono', monospace; letter-spacing: .06em; text-transform: uppercase; color: #9a9389; }
#df-view-shot .dsh-drop { min-height: 230px; border: 1.5px dashed #DCD8D2; border-radius: 10px; display: flex; flex-direction: column; align-items: center;
            justify-content: center; gap: 4px; cursor: pointer; background: #FAF9F7; text-align: center; padding: 14px; overflow: hidden; }
#df-view-shot .dsh-drop:hover { border-color: #D2603A; }
#df-view-shot .dsh-drop.has { padding: 0; border-style: solid; background: #fff; }
#df-view-shot .dsh-drop img { width: 100%; display: block; }
#df-view-shot .dsh-drop-main { font-size: 14px; font-weight: 600; color: #3a332b; }
#df-view-shot .dsh-drop-sub { font-size: 12.5px; color: #8a8378; }
#df-view-shot .dsh-text { font: 14px 'Hanken Grotesk', system-ui, sans-serif; color: #2B2722; background: #fff; border: 1px solid #E6E3DF; border-radius: 9px;
            padding: 9px 11px; outline: none; }
#df-view-shot .dsh-text:focus { border-color: #D2603A; box-shadow: 0 0 0 3px rgba(210,96,58,.10); }
#df-view-shot .dsh-row { display: flex; align-items: center; justify-content: space-between; gap: 10px; flex-wrap: wrap; }
#df-view-shot .dsh-seg { display: inline-flex; border: 1px solid #E6E3DF; border-radius: 9px; overflow: hidden; background: #fff; }
#df-view-shot .dsh-seg button { font: 500 13px 'Hanken Grotesk', system-ui, sans-serif; color: #6f6a62; background: transparent; border: none; padding: 7px 12px; cursor: pointer; }
#df-view-shot .dsh-seg button + button { border-left: 1px solid #E6E3DF; }
#df-view-shot .dsh-seg button.on { background: #FBF3EC; color: #C0552F; font-weight: 600; }
#df-view-shot .dsh-go { font: 600 14px 'Hanken Grotesk', system-ui, sans-serif; color: #fff; background: #D2603A; border: 1px solid #C0552F; border-radius: 9px;
          padding: 8px 16px; cursor: pointer; }
#df-view-shot .dsh-go:disabled { opacity: .6; cursor: default; }
#df-view-shot .dsh-err { font-size: 13px; color: #A8452A; }
#df-view-shot .dsh-result { width: 100%; border-radius: 9px; border: 1px solid #E6E3DF; display: block; }
#df-view-shot .dsh-code { font: 500 12.5px/1.6 'JetBrains Mono', monospace; color: #3a332b; background: #FAF9F7; border: 1px solid #EEECE8; border-radius: 8px;
            padding: 8px 10px; overflow-wrap: anywhere; }
#df-view-shot .dsh-meta { display: flex; justify-content: space-between; font: 500 11.5px 'JetBrains Mono', monospace; color: #9a9389; }
#df-view-shot .dsh-empty { min-height: 230px; display: flex; align-items: center; justify-content: center; gap: 8px; border-radius: 10px; background: #FAF9F7;
             border: 1px solid #EEECE8; font-size: 13.5px; color: #8a8378; }
#df-view-shot .dsh-spin { width: 13px; height: 13px; border-radius: 50%; border: 2px solid #E6E3DF; border-top-color: #D2603A; animation: df-spin .7s linear infinite; }
#df-view-shot .dsh-excap { margin: 18px 0 8px; }
#df-view-shot .dsh-ex { display: grid; grid-template-columns: repeat(auto-fill, minmax(170px, 1fr)); gap: 10px; }
#df-view-shot .dsh-exi { display: flex; flex-direction: column; gap: 6px; background: #fff; border: 1px solid #E6E3DF; border-radius: 10px; padding: 6px; cursor: pointer;
           text-align: left; font: 12.5px/1.35 'Hanken Grotesk', system-ui, sans-serif; color: #3a332b; }
#df-view-shot .dsh-exi:hover { border-color: #D2603A; }
#df-view-shot .dsh-exi img { width: 100%; height: 92px; object-fit: cover; object-position: top left; border-radius: 6px; border: 1px solid #EEECE8; }
#df-view-shot .dsh-exi span { display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; }
"""