bep40 commited on
Commit
1248ff6
·
verified ·
1 Parent(s): 2eb0083

fix: swap xemtivitop primary, remove Dantri/SKDS shorts from yt_scraper_fixed.py

Browse files
Files changed (1) hide show
  1. yt_scraper_fixed.py +16 -62
yt_scraper_fixed.py CHANGED
@@ -1,6 +1,7 @@
1
  """
2
  YouTube Shorts Scraper using yt-dlp (already installed on Space)
3
  Optimized for fast load with long cache + fallback
 
4
  """
5
  import subprocess
6
  import json
@@ -11,10 +12,9 @@ import re as re_mod
11
 
12
  _cache = {}
13
  _lock = threading.Lock()
14
- CACHE_TTL = 1800 # 30 min cache - longer to reduce timeout issues
15
 
16
  def _get_cached(key):
17
- """Get cached data if still valid"""
18
  with _lock:
19
  if key in _cache:
20
  entry = _cache[key]
@@ -23,12 +23,10 @@ def _get_cached(key):
23
  return None
24
 
25
  def _set_cached(key, data):
26
- """Set cache with timestamp"""
27
  with _lock:
28
  _cache[key] = {'t': time.time(), 'd': data}
29
 
30
  def run_yt_dlp(args, timeout=45):
31
- """Run yt-dlp and return parsed JSON lines - with shorter timeout"""
32
  try:
33
  result = subprocess.run(
34
  ["yt-dlp"] + args,
@@ -58,7 +56,7 @@ def run_yt_dlp(args, timeout=45):
58
  return []
59
 
60
  def get_channel_shorts_fast(channel_username, max_count=25):
61
- """Get shorts fast - prioritize /shorts page only to avoid timeout"""
62
  shorts = []
63
 
64
  url = f"https://www.youtube.com/@{channel_username}/shorts"
@@ -68,10 +66,10 @@ def get_channel_shorts_fast(channel_username, max_count=25):
68
  "--no-download",
69
  "--playlist-end", str(max_count),
70
  "--no-check-certificates",
71
- "--quiet", # Reduce output for speed
72
  "--no-warnings",
73
  url
74
- ], timeout=35) # Increased timeout but still reasonable
75
 
76
  seen_ids = set()
77
  for item in items:
@@ -94,69 +92,25 @@ def get_channel_shorts_fast(channel_username, max_count=25):
94
 
95
  return shorts
96
 
97
- def get_dantri_shorts(max_count=25):
98
- """Get Dantri shorts - fast, no fallback needed, cache for 30min"""
99
- cached = _get_cached('dantri_shorts_yt')
100
  if cached is not None:
101
  return cached
102
 
103
- shorts = get_channel_shorts_fast('baodantri7941', max_count)
104
 
105
  if shorts:
106
- _set_cached('dantri_shorts_yt', shorts)
107
  return shorts
108
 
109
  # Fallback to static list if scrape fails
110
- return [
111
- {"id":"Lu_iCQ5YwNM","title":"Công an lập hồ xử lý người phụ nữ chửi bới tát nam tài xế ô tô ở Hà Nội","channel":"baodantri7941"},
112
- {"id":"CwWvijF8BOA","title":"Chú rể Ninh Bình bật khóc nhận món quà bí mật người cha","channel":"baodantri7941"},
113
  ]
114
-
115
- def get_skds_shorts(max_count=25):
116
- """Get SKĐS shorts - fast, no fallback needed, cache for 30min"""
117
- cached = _get_cached('skds_shorts_yt')
118
- if cached is not None:
119
- return cached
120
-
121
- shorts = get_channel_shorts_fast('baosuckhoedoisongboyte', max_count)
122
-
123
- if shorts:
124
- _set_cached('skds_shorts_yt', shorts)
125
- return shorts
126
-
127
- # Fallback to static list if scrape fails
128
- return [
129
- {"id":"7Pd6vZ2Lz1M","title":"Hành động ấm lòng của người đàn ông tìm kiếm 5 học sinh tử vong","channel":"baosuckhoedoisongboyte"},
130
- {"id":"SlHLt_ZyPiE","title":"Xử phạt người đàn ông xóa số điện thoại cứu hộ trên cao tốc Bắc Nam","channel":"baosuckhoedoisongboyte"},
131
- ]
132
-
133
- def get_dantri_skds_shorts(max_count=50):
134
- """Get interleaved Dantri + SKĐS shorts - optimized with separate caching"""
135
- # Get each channel's shorts separately (allows partial fallback)
136
- dantri = get_dantri_shorts(max_count // 2 + 10)
137
- skds = get_skds_shorts(max_count // 2 + 10)
138
-
139
- # Interleave them
140
- result = []
141
- seen = set()
142
- i, j = 0, 0
143
-
144
- while (i < len(dantri) or j < len(skds)) and len(result) < max_count:
145
- if i < len(dantri):
146
- item = dantri[i]
147
- if item.get('id') not in seen:
148
- seen.add(item.get('id'))
149
- result.append(item)
150
- i += 1
151
- if j < len(skds):
152
- item = skds[j]
153
- if item.get('id') not in seen:
154
- seen.add(item.get('id'))
155
- result.append(item)
156
- j += 1
157
-
158
- return result
159
 
160
  # For backward compatibility
161
- get_vtvnambo_shorts = get_dantri_skds_shorts
162
- get_wc_related_shorts = get_dantri_skds_shorts
 
1
  """
2
  YouTube Shorts Scraper using yt-dlp (already installed on Space)
3
  Optimized for fast load with long cache + fallback
4
+ Only VTV Nam Bộ — removed Dantri/SKDS entirely
5
  """
6
  import subprocess
7
  import json
 
12
 
13
  _cache = {}
14
  _lock = threading.Lock()
15
+ CACHE_TTL = 1800 # 30 min cache
16
 
17
  def _get_cached(key):
 
18
  with _lock:
19
  if key in _cache:
20
  entry = _cache[key]
 
23
  return None
24
 
25
  def _set_cached(key, data):
 
26
  with _lock:
27
  _cache[key] = {'t': time.time(), 'd': data}
28
 
29
  def run_yt_dlp(args, timeout=45):
 
30
  try:
31
  result = subprocess.run(
32
  ["yt-dlp"] + args,
 
56
  return []
57
 
58
  def get_channel_shorts_fast(channel_username, max_count=25):
59
+ """Get shorts fast - prioritize /shorts page"""
60
  shorts = []
61
 
62
  url = f"https://www.youtube.com/@{channel_username}/shorts"
 
66
  "--no-download",
67
  "--playlist-end", str(max_count),
68
  "--no-check-certificates",
69
+ "--quiet",
70
  "--no-warnings",
71
  url
72
+ ], timeout=35)
73
 
74
  seen_ids = set()
75
  for item in items:
 
92
 
93
  return shorts
94
 
95
+ def get_vtvnambo_shorts(max_count=50):
96
+ """Get shorts from VTV Nam Bộ only"""
97
+ cached = _get_cached('vtvnambo_shorts_yt')
98
  if cached is not None:
99
  return cached
100
 
101
+ shorts = get_channel_shorts_fast('vtvnambo', max_count)
102
 
103
  if shorts:
104
+ _set_cached('vtvnambo_shorts_yt', shorts)
105
  return shorts
106
 
107
  # Fallback to static list if scrape fails
108
+ fallback = [
109
+ {"id":"nqlLH6chLRo","title":"Tin nóng VTV Nam Bộ | #shorts","channel":"vtvnambo"},
110
+ {"id":"E7Kq0v3hG6w","title":"VTV Nam Bộ - Tin tức miền Nam | #shorts","channel":"vtvnambo"},
111
  ]
112
+ _set_cached('vtvnambo_shorts_yt', fallback)
113
+ return fallback
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
114
 
115
  # For backward compatibility
116
+ get_wc_related_shorts = get_vtvnambo_shorts