scripts/debug_bing.py

import asyncio
import re

import httpx

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"


async def main():
    async with httpx.AsyncClient(timeout=20, follow_redirects=True) as c:
        r = await c.get(
            "https://www.bing.com/search",
            params={"q": "P/E ratio real estate sector higher than consumer discretionary"},
            headers={"User-Agent": UA},
        )
        print("status:", r.status_code, "| len:", len(r.text))
        links = re.findall(r'<h2><a href="([^"]+)"', r.text)
        titles = re.findall(r"<h2><a[^>]*>(.*?)</a></h2>", r.text)
        print("h2 links:", len(links), "| titles:", len(titles))
        for l, t in list(zip(links, titles))[:5]:
            print("  >", re.sub("<[^>]+>", "", t)[:60], "|", l[:70])


asyncio.run(main())