bep40 commited on
Commit
fcbc2e3
·
verified ·
1 Parent(s): 0778dc9

Add debug logging to match_detail.py to trace scraping issues

Browse files
Files changed (1) hide show
  1. match_detail.py +12 -13
match_detail.py CHANGED
@@ -127,7 +127,7 @@ def _get_h2h(soup):
127
  m['date'] = _cl(cells[0].get_text())
128
  m['home'] = _cl(cells[1].get_text())
129
  m['score'] = _cl(cells[2].get_text())
130
- if m.get('home') and m.get('away' if len(cells) > 3 else 'home'):
131
  if len(cells) > 3: m['away'] = _cl(cells[3].get_text())
132
  h2h['matches'].append(m)
133
  if h2h['matches']: break
@@ -159,14 +159,17 @@ def _get_info(soup):
159
  return info
160
 
161
  def _scrape(url):
 
162
  try:
163
  r = requests.get(url, headers=HH, timeout=15, allow_redirects=True)
 
164
  if r.status_code != 200:
165
  return False, {}
166
  sp = _sp(r.text)
167
  d = {}
168
 
169
  teams = _get_teams(sp)
 
170
  if teams: d['info'] = teams
171
 
172
  mi = _get_info(sp)
@@ -195,8 +198,12 @@ def _scrape(url):
195
  if fm.get('home'): d['home_form'] = fm['home']
196
  if fm.get('away'): d['away_form'] = fm['away']
197
 
 
198
  return True, d
199
  except Exception as e:
 
 
 
200
  return False, {}
201
 
202
  def fetch_match_detail_by_url(url):
@@ -204,37 +211,33 @@ def fetch_match_detail_by_url(url):
204
  if not m: return {"error": "Could not extract event_id", "found": False}
205
  event_id = int(m.group(1))
206
  res = {"event_id": event_id, "found": False, "sections": []}
207
-
208
- # API
209
  _fetch_api(event_id, res)
210
- # HTML
211
  ok, d = _scrape(url)
 
212
  if ok: _merge(res, d)
213
  return res
214
 
215
  def fetch_match_detail(event_id):
 
216
  res = {"event_id": event_id, "found": False, "sections": []}
217
-
218
- # API first
219
  _fetch_api(event_id, res)
220
 
221
- # Then HTML (centre first, then preview)
222
  for pt in ["centre", "preview"]:
223
  url = f"https://bongda.com.vn/tran-dau/{event_id}/{pt}/"
224
  ok, d = _scrape(url)
 
225
  if ok:
226
  _merge(res, d)
227
  if res.get("found"): break
228
 
 
229
  return res
230
 
231
  def _fetch_api(eid, res):
232
- # pre-match
233
  pm = _api("/api/event-standing/pre-match", {"event_id": eid})
234
  res["pre_match"] = pm
235
  res["pre_match_html"] = pm.get("html","") if pm and pm.get("status")=="success" and len(pm.get("html","").strip())>10 else ""
236
 
237
- # h2h matches
238
  hm = _api("/api/fixtures/h2h-match", {"event_id": eid})
239
  res["h2h_match"] = hm
240
  if hm and hm.get("status")=="success":
@@ -244,7 +247,6 @@ def _fetch_api(eid, res):
244
  res["sections"].append("h2h")
245
  else: res["h2h_html"] = ""
246
 
247
- # h2h stats
248
  hs = _api("/api/fixtures/h2h-stats", {"event_id": eid})
249
  res["h2h_stats"] = hs
250
  if hs and hs.get("status")=="success":
@@ -252,7 +254,6 @@ def _fetch_api(eid, res):
252
  if len(h.strip())>10:
253
  res["h2h_stats_html"] = h
254
  res["sections"].append("h2h_stats")
255
- # Parse stats
256
  try:
257
  sp = _sp(h)
258
  stats = {}
@@ -265,7 +266,6 @@ def _fetch_api(eid, res):
265
  except: pass
266
  else: res["h2h_stats_html"] = ""
267
 
268
- # performance
269
  pf = _api("/api/event-standing/player-performance", {"event_id": eid})
270
  res["performance"] = pf
271
  if pf and pf.get("status")=="success" and len(pf.get("html","").strip())>10:
@@ -273,7 +273,6 @@ def _fetch_api(eid, res):
273
  res["sections"].append("stats")
274
  else: res["stats_html"] = ""
275
 
276
- # commentaries
277
  cm = _api("/api/fixtures/commentaries", {"event_id": eid})
278
  if cm and cm.get("status")=="success" and len(cm.get("html","").strip())>10:
279
  res["commentaries_html"] = cm["html"]
 
127
  m['date'] = _cl(cells[0].get_text())
128
  m['home'] = _cl(cells[1].get_text())
129
  m['score'] = _cl(cells[2].get_text())
130
+ if m.get('home'):
131
  if len(cells) > 3: m['away'] = _cl(cells[3].get_text())
132
  h2h['matches'].append(m)
133
  if h2h['matches']: break
 
159
  return info
160
 
161
  def _scrape(url):
162
+ print(f"[DEBUG] _scrape: {url[:80]}", flush=True)
163
  try:
164
  r = requests.get(url, headers=HH, timeout=15, allow_redirects=True)
165
+ print(f"[DEBUG] HTTP={r.status_code}", flush=True)
166
  if r.status_code != 200:
167
  return False, {}
168
  sp = _sp(r.text)
169
  d = {}
170
 
171
  teams = _get_teams(sp)
172
+ print(f"[DEBUG] teams={teams}", flush=True)
173
  if teams: d['info'] = teams
174
 
175
  mi = _get_info(sp)
 
198
  if fm.get('home'): d['home_form'] = fm['home']
199
  if fm.get('away'): d['away_form'] = fm['away']
200
 
201
+ print(f"[DEBUG] success keys={list(d.keys())}", flush=True)
202
  return True, d
203
  except Exception as e:
204
+ import traceback
205
+ print(f"[DEBUG] error: {e}", flush=True)
206
+ traceback.print_exc()
207
  return False, {}
208
 
209
  def fetch_match_detail_by_url(url):
 
211
  if not m: return {"error": "Could not extract event_id", "found": False}
212
  event_id = int(m.group(1))
213
  res = {"event_id": event_id, "found": False, "sections": []}
 
 
214
  _fetch_api(event_id, res)
 
215
  ok, d = _scrape(url)
216
+ print(f"[DEBUG] by_url: ok={ok} d_keys={list(d.keys())}", flush=True)
217
  if ok: _merge(res, d)
218
  return res
219
 
220
  def fetch_match_detail(event_id):
221
+ print(f"[DEBUG] fetch_match_detail({event_id})", flush=True)
222
  res = {"event_id": event_id, "found": False, "sections": []}
 
 
223
  _fetch_api(event_id, res)
224
 
 
225
  for pt in ["centre", "preview"]:
226
  url = f"https://bongda.com.vn/tran-dau/{event_id}/{pt}/"
227
  ok, d = _scrape(url)
228
+ print(f"[DEBUG] {pt}: ok={ok}", flush=True)
229
  if ok:
230
  _merge(res, d)
231
  if res.get("found"): break
232
 
233
+ print(f"[DEBUG] final: found={res['found']} sections={res['sections']}", flush=True)
234
  return res
235
 
236
  def _fetch_api(eid, res):
 
237
  pm = _api("/api/event-standing/pre-match", {"event_id": eid})
238
  res["pre_match"] = pm
239
  res["pre_match_html"] = pm.get("html","") if pm and pm.get("status")=="success" and len(pm.get("html","").strip())>10 else ""
240
 
 
241
  hm = _api("/api/fixtures/h2h-match", {"event_id": eid})
242
  res["h2h_match"] = hm
243
  if hm and hm.get("status")=="success":
 
247
  res["sections"].append("h2h")
248
  else: res["h2h_html"] = ""
249
 
 
250
  hs = _api("/api/fixtures/h2h-stats", {"event_id": eid})
251
  res["h2h_stats"] = hs
252
  if hs and hs.get("status")=="success":
 
254
  if len(h.strip())>10:
255
  res["h2h_stats_html"] = h
256
  res["sections"].append("h2h_stats")
 
257
  try:
258
  sp = _sp(h)
259
  stats = {}
 
266
  except: pass
267
  else: res["h2h_stats_html"] = ""
268
 
 
269
  pf = _api("/api/event-standing/player-performance", {"event_id": eid})
270
  res["performance"] = pf
271
  if pf and pf.get("status")=="success" and len(pf.get("html","").strip())>10:
 
273
  res["sections"].append("stats")
274
  else: res["stats_html"] = ""
275
 
 
276
  cm = _api("/api/fixtures/commentaries", {"event_id": eid})
277
  if cm and cm.get("status")=="success" and len(cm.get("html","").strip())>10:
278
  res["commentaries_html"] = cm["html"]