deno/index.ts

import PQueue from "https://deno.land/x/p_queue@1.0.1/mod.ts"
import { Application, Router } from "https://deno.land/x/oak@v9.0.0/mod.ts";

import * as CSV from './csv.ts';
import Puppet from './puppet.ts';
import selectors from './selectors.ts';

const puppet = new Puppet();
async function get_logos(page, selector): {}[] {
  const logos = await page.$$(selector) || [];
  for (const i in logos) {
    const bb = await page.evaluate(e => {
      const { x, y, width, height } = e.getBoundingClientRect();
      return {
        x, y, width, height, top: window.screen.top, left: window.screen.left
      }
    }, logos[i])
    logos[i].box = bb;
  }
  return logos;
}

async function fetch_logos(page, id, dest) {
  console.error(`getting logos for: ${id}`)
        try {
          const imgs = await get_logos(page, selectors.img_logo);
          const ids = await get_logos(page, selectors.id_logo);
          const cls = await get_logos(page, selectors.class_logo);
          const logos = [
            ...imgs, ...ids, ...cls
          ]

          let annotations = '';
          for (const i in logos) {
            const bb = logos[i].box
            if (!bb
              || (bb.width < 10)
              || (bb.height < 10)
              || (bb.x + bb.width < 0)
              || (bb.y + bb.height < 0)) continue;
            console.log('got bb',  bb)

            try {
              await logos[i].screenshot({
                path: dest
                  .replace('images', 'logos')
                  .replace('.png', `.${i}.png`)
              })
              annotations +=
                `${id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
            } catch (e) {
              console.error(`couldn't screenshot logo: ${e}`);
            }
          }
          if (logos.length) {
            await Deno.writeTextFile(dest
              .replace('images', 'labels')
              .replace('png', 'txt'),
                                     annotations);
        }
        } catch (err) {
          console.error(`error in screenshot: ${err}`);
        }
}

const app = new Application();
const router = new Router();

const stats = {
  in_flight: 0,
  done: 0
}
router.post('/screenshot', async (ctx) => {
  const {request, response} = ctx;
  const q = await request.body().value;

  stats.in_flight++;
  const ret = await puppet.run(async page => {
    console.error('running', q, stats)
    await page.goto(q.url, {waitUntil: 'networkidle2', timeout: 60000})
    await page.screenshot({ path: q.path, fullPage: true })
    if (q.logos) {
      await fetch_logos(page, q.id, q.logos)
    }
    console.error(`screenshot ok: ${q.path}`)
    return {response: 'ok'}
  })
  stats.in_flight--;
  stats.done++
  response.body = ret
})
router.post('/bco', async (ctx) => {
  const {request, response} = ctx;
  const q = await request.body().value;
  const ret = await process(q)

  console.error(`ret: ${ret}`)
  response.body = ret
});

app.use(router.routes())
app.use(router.allowedMethods())
const addr = '0.0.0.0:8000'
console.error(`listen on ${addr}`)
app.listen(addr)