Spaces:
Running
Running
Add debug logging to match_detail.py to trace scraping issues
Browse files- match_detail.py +12 -13
match_detail.py
CHANGED
|
@@ -127,7 +127,7 @@ def _get_h2h(soup):
|
|
| 127 |
m['date'] = _cl(cells[0].get_text())
|
| 128 |
m['home'] = _cl(cells[1].get_text())
|
| 129 |
m['score'] = _cl(cells[2].get_text())
|
| 130 |
-
if m.get('home')
|
| 131 |
if len(cells) > 3: m['away'] = _cl(cells[3].get_text())
|
| 132 |
h2h['matches'].append(m)
|
| 133 |
if h2h['matches']: break
|
|
@@ -159,14 +159,17 @@ def _get_info(soup):
|
|
| 159 |
return info
|
| 160 |
|
| 161 |
def _scrape(url):
|
|
|
|
| 162 |
try:
|
| 163 |
r = requests.get(url, headers=HH, timeout=15, allow_redirects=True)
|
|
|
|
| 164 |
if r.status_code != 200:
|
| 165 |
return False, {}
|
| 166 |
sp = _sp(r.text)
|
| 167 |
d = {}
|
| 168 |
|
| 169 |
teams = _get_teams(sp)
|
|
|
|
| 170 |
if teams: d['info'] = teams
|
| 171 |
|
| 172 |
mi = _get_info(sp)
|
|
@@ -195,8 +198,12 @@ def _scrape(url):
|
|
| 195 |
if fm.get('home'): d['home_form'] = fm['home']
|
| 196 |
if fm.get('away'): d['away_form'] = fm['away']
|
| 197 |
|
|
|
|
| 198 |
return True, d
|
| 199 |
except Exception as e:
|
|
|
|
|
|
|
|
|
|
| 200 |
return False, {}
|
| 201 |
|
| 202 |
def fetch_match_detail_by_url(url):
|
|
@@ -204,37 +211,33 @@ def fetch_match_detail_by_url(url):
|
|
| 204 |
if not m: return {"error": "Could not extract event_id", "found": False}
|
| 205 |
event_id = int(m.group(1))
|
| 206 |
res = {"event_id": event_id, "found": False, "sections": []}
|
| 207 |
-
|
| 208 |
-
# API
|
| 209 |
_fetch_api(event_id, res)
|
| 210 |
-
# HTML
|
| 211 |
ok, d = _scrape(url)
|
|
|
|
| 212 |
if ok: _merge(res, d)
|
| 213 |
return res
|
| 214 |
|
| 215 |
def fetch_match_detail(event_id):
|
|
|
|
| 216 |
res = {"event_id": event_id, "found": False, "sections": []}
|
| 217 |
-
|
| 218 |
-
# API first
|
| 219 |
_fetch_api(event_id, res)
|
| 220 |
|
| 221 |
-
# Then HTML (centre first, then preview)
|
| 222 |
for pt in ["centre", "preview"]:
|
| 223 |
url = f"https://bongda.com.vn/tran-dau/{event_id}/{pt}/"
|
| 224 |
ok, d = _scrape(url)
|
|
|
|
| 225 |
if ok:
|
| 226 |
_merge(res, d)
|
| 227 |
if res.get("found"): break
|
| 228 |
|
|
|
|
| 229 |
return res
|
| 230 |
|
| 231 |
def _fetch_api(eid, res):
|
| 232 |
-
# pre-match
|
| 233 |
pm = _api("/api/event-standing/pre-match", {"event_id": eid})
|
| 234 |
res["pre_match"] = pm
|
| 235 |
res["pre_match_html"] = pm.get("html","") if pm and pm.get("status")=="success" and len(pm.get("html","").strip())>10 else ""
|
| 236 |
|
| 237 |
-
# h2h matches
|
| 238 |
hm = _api("/api/fixtures/h2h-match", {"event_id": eid})
|
| 239 |
res["h2h_match"] = hm
|
| 240 |
if hm and hm.get("status")=="success":
|
|
@@ -244,7 +247,6 @@ def _fetch_api(eid, res):
|
|
| 244 |
res["sections"].append("h2h")
|
| 245 |
else: res["h2h_html"] = ""
|
| 246 |
|
| 247 |
-
# h2h stats
|
| 248 |
hs = _api("/api/fixtures/h2h-stats", {"event_id": eid})
|
| 249 |
res["h2h_stats"] = hs
|
| 250 |
if hs and hs.get("status")=="success":
|
|
@@ -252,7 +254,6 @@ def _fetch_api(eid, res):
|
|
| 252 |
if len(h.strip())>10:
|
| 253 |
res["h2h_stats_html"] = h
|
| 254 |
res["sections"].append("h2h_stats")
|
| 255 |
-
# Parse stats
|
| 256 |
try:
|
| 257 |
sp = _sp(h)
|
| 258 |
stats = {}
|
|
@@ -265,7 +266,6 @@ def _fetch_api(eid, res):
|
|
| 265 |
except: pass
|
| 266 |
else: res["h2h_stats_html"] = ""
|
| 267 |
|
| 268 |
-
# performance
|
| 269 |
pf = _api("/api/event-standing/player-performance", {"event_id": eid})
|
| 270 |
res["performance"] = pf
|
| 271 |
if pf and pf.get("status")=="success" and len(pf.get("html","").strip())>10:
|
|
@@ -273,7 +273,6 @@ def _fetch_api(eid, res):
|
|
| 273 |
res["sections"].append("stats")
|
| 274 |
else: res["stats_html"] = ""
|
| 275 |
|
| 276 |
-
# commentaries
|
| 277 |
cm = _api("/api/fixtures/commentaries", {"event_id": eid})
|
| 278 |
if cm and cm.get("status")=="success" and len(cm.get("html","").strip())>10:
|
| 279 |
res["commentaries_html"] = cm["html"]
|
|
|
|
| 127 |
m['date'] = _cl(cells[0].get_text())
|
| 128 |
m['home'] = _cl(cells[1].get_text())
|
| 129 |
m['score'] = _cl(cells[2].get_text())
|
| 130 |
+
if m.get('home'):
|
| 131 |
if len(cells) > 3: m['away'] = _cl(cells[3].get_text())
|
| 132 |
h2h['matches'].append(m)
|
| 133 |
if h2h['matches']: break
|
|
|
|
| 159 |
return info
|
| 160 |
|
| 161 |
def _scrape(url):
|
| 162 |
+
print(f"[DEBUG] _scrape: {url[:80]}", flush=True)
|
| 163 |
try:
|
| 164 |
r = requests.get(url, headers=HH, timeout=15, allow_redirects=True)
|
| 165 |
+
print(f"[DEBUG] HTTP={r.status_code}", flush=True)
|
| 166 |
if r.status_code != 200:
|
| 167 |
return False, {}
|
| 168 |
sp = _sp(r.text)
|
| 169 |
d = {}
|
| 170 |
|
| 171 |
teams = _get_teams(sp)
|
| 172 |
+
print(f"[DEBUG] teams={teams}", flush=True)
|
| 173 |
if teams: d['info'] = teams
|
| 174 |
|
| 175 |
mi = _get_info(sp)
|
|
|
|
| 198 |
if fm.get('home'): d['home_form'] = fm['home']
|
| 199 |
if fm.get('away'): d['away_form'] = fm['away']
|
| 200 |
|
| 201 |
+
print(f"[DEBUG] success keys={list(d.keys())}", flush=True)
|
| 202 |
return True, d
|
| 203 |
except Exception as e:
|
| 204 |
+
import traceback
|
| 205 |
+
print(f"[DEBUG] error: {e}", flush=True)
|
| 206 |
+
traceback.print_exc()
|
| 207 |
return False, {}
|
| 208 |
|
| 209 |
def fetch_match_detail_by_url(url):
|
|
|
|
| 211 |
if not m: return {"error": "Could not extract event_id", "found": False}
|
| 212 |
event_id = int(m.group(1))
|
| 213 |
res = {"event_id": event_id, "found": False, "sections": []}
|
|
|
|
|
|
|
| 214 |
_fetch_api(event_id, res)
|
|
|
|
| 215 |
ok, d = _scrape(url)
|
| 216 |
+
print(f"[DEBUG] by_url: ok={ok} d_keys={list(d.keys())}", flush=True)
|
| 217 |
if ok: _merge(res, d)
|
| 218 |
return res
|
| 219 |
|
| 220 |
def fetch_match_detail(event_id):
|
| 221 |
+
print(f"[DEBUG] fetch_match_detail({event_id})", flush=True)
|
| 222 |
res = {"event_id": event_id, "found": False, "sections": []}
|
|
|
|
|
|
|
| 223 |
_fetch_api(event_id, res)
|
| 224 |
|
|
|
|
| 225 |
for pt in ["centre", "preview"]:
|
| 226 |
url = f"https://bongda.com.vn/tran-dau/{event_id}/{pt}/"
|
| 227 |
ok, d = _scrape(url)
|
| 228 |
+
print(f"[DEBUG] {pt}: ok={ok}", flush=True)
|
| 229 |
if ok:
|
| 230 |
_merge(res, d)
|
| 231 |
if res.get("found"): break
|
| 232 |
|
| 233 |
+
print(f"[DEBUG] final: found={res['found']} sections={res['sections']}", flush=True)
|
| 234 |
return res
|
| 235 |
|
| 236 |
def _fetch_api(eid, res):
|
|
|
|
| 237 |
pm = _api("/api/event-standing/pre-match", {"event_id": eid})
|
| 238 |
res["pre_match"] = pm
|
| 239 |
res["pre_match_html"] = pm.get("html","") if pm and pm.get("status")=="success" and len(pm.get("html","").strip())>10 else ""
|
| 240 |
|
|
|
|
| 241 |
hm = _api("/api/fixtures/h2h-match", {"event_id": eid})
|
| 242 |
res["h2h_match"] = hm
|
| 243 |
if hm and hm.get("status")=="success":
|
|
|
|
| 247 |
res["sections"].append("h2h")
|
| 248 |
else: res["h2h_html"] = ""
|
| 249 |
|
|
|
|
| 250 |
hs = _api("/api/fixtures/h2h-stats", {"event_id": eid})
|
| 251 |
res["h2h_stats"] = hs
|
| 252 |
if hs and hs.get("status")=="success":
|
|
|
|
| 254 |
if len(h.strip())>10:
|
| 255 |
res["h2h_stats_html"] = h
|
| 256 |
res["sections"].append("h2h_stats")
|
|
|
|
| 257 |
try:
|
| 258 |
sp = _sp(h)
|
| 259 |
stats = {}
|
|
|
|
| 266 |
except: pass
|
| 267 |
else: res["h2h_stats_html"] = ""
|
| 268 |
|
|
|
|
| 269 |
pf = _api("/api/event-standing/player-performance", {"event_id": eid})
|
| 270 |
res["performance"] = pf
|
| 271 |
if pf and pf.get("status")=="success" and len(pf.get("html","").strip())>10:
|
|
|
|
| 273 |
res["sections"].append("stats")
|
| 274 |
else: res["stats_html"] = ""
|
| 275 |
|
|
|
|
| 276 |
cm = _api("/api/fixtures/commentaries", {"event_id": eid})
|
| 277 |
if cm and cm.get("status")=="success" and len(cm.get("html","").strip())>10:
|
| 278 |
res["commentaries_html"] = cm["html"]
|