add-statistics.py

p="crawl-full.ts"

s=open(p).read()

insert = r'''
const statistics = await page.evaluate(() => {

  const text = document.body.innerText;

  const patterns = [
    /\d+(?:,\d{3})*(?:\.\d+)?\+?/g,
    /\$[\d,.]+[TBMK]?/g,
    /\d+(?:\.\d+)?%/g,
    /\d+\s*(?:countries|users|customers|requests|transactions|million|billion|trillion)/gi
  ];

  const found = new Set<string>();

  for (const pattern of patterns) {
    const matches = text.match(pattern) || [];

    for (const item of matches) {

      const value = item.trim();

      if (
        value.length > 1 &&
        value.length < 40
      ) {
        found.add(value);
      }
    }
  }

  return Array.from(found).slice(0,100);

});
'''

if "const statistics" not in s:

    s=s.replace(
        "const vp = page.viewportSize();",
        insert + "\nconst vp = page.viewportSize();"
    )


s=s.replace(
    "design,\n        text:",
    "design,\n        statistics,\n        text:"
)

open(p,"w").write(s)

print("statistics added")