Spaces:
Sleeping
Sleeping
Commit ·
4f9adc4
1
Parent(s): deb89c9
simple request
Browse files- pyproject.toml +3 -0
- scripts/test_source_finding.py +213 -0
- src/source_finding/__init__.py +33 -0
- src/source_finding/downloader.py +245 -0
- src/source_finding/models.py +80 -0
- src/source_finding/scraper.py +267 -0
pyproject.toml
CHANGED
|
@@ -12,6 +12,9 @@ dependencies = [
|
|
| 12 |
"pillow>=10.0.0",
|
| 13 |
"pydantic>=2.0.0",
|
| 14 |
"pytesseract>=0.3.10",
|
|
|
|
|
|
|
|
|
|
| 15 |
]
|
| 16 |
|
| 17 |
[dependency-groups]
|
|
|
|
| 12 |
"pillow>=10.0.0",
|
| 13 |
"pydantic>=2.0.0",
|
| 14 |
"pytesseract>=0.3.10",
|
| 15 |
+
"requests>=2.31.0",
|
| 16 |
+
"beautifulsoup4>=4.12.0",
|
| 17 |
+
"yt-dlp>=2024.1.0",
|
| 18 |
]
|
| 19 |
|
| 20 |
[dependency-groups]
|
scripts/test_source_finding.py
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""
|
| 3 |
+
Test script for the source_finding module.
|
| 4 |
+
|
| 5 |
+
This script demonstrates and validates the functionality of:
|
| 6 |
+
1. Searching for games by team name(s)
|
| 7 |
+
2. Generating download commands
|
| 8 |
+
3. Extracting direct video URLs (if yt-dlp is available)
|
| 9 |
+
|
| 10 |
+
Usage:
|
| 11 |
+
# Activate virtual environment first
|
| 12 |
+
source .venv/bin/activate
|
| 13 |
+
|
| 14 |
+
# Run with default test (Ohio State vs Oregon)
|
| 15 |
+
python scripts/test_source_finding.py
|
| 16 |
+
|
| 17 |
+
# Search for a specific team
|
| 18 |
+
python scripts/test_source_finding.py --team-a "Tennessee"
|
| 19 |
+
|
| 20 |
+
# Search for a specific matchup
|
| 21 |
+
python scripts/test_source_finding.py --team-a "Ohio State" --team-b "Oregon"
|
| 22 |
+
|
| 23 |
+
# Search more pages (default is 3)
|
| 24 |
+
python scripts/test_source_finding.py --team-a "Michigan" --max-pages 10
|
| 25 |
+
|
| 26 |
+
# Test URL extraction on a specific result
|
| 27 |
+
python scripts/test_source_finding.py --team-a "Ohio State" --test-download
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
import argparse
|
| 31 |
+
import logging
|
| 32 |
+
import sys
|
| 33 |
+
from pathlib import Path
|
| 34 |
+
|
| 35 |
+
# Add src to path for imports
|
| 36 |
+
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
| 37 |
+
|
| 38 |
+
from source_finding import search_games, get_download_command, extract_direct_video_url, get_suggested_filename
|
| 39 |
+
from source_finding.downloader import is_ytdlp_available, get_video_info
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def setup_logging(verbose: bool = False) -> None:
|
| 43 |
+
"""Configure logging for the test script."""
|
| 44 |
+
level = logging.DEBUG if verbose else logging.INFO
|
| 45 |
+
logging.basicConfig(
|
| 46 |
+
level=level,
|
| 47 |
+
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
| 48 |
+
datefmt="%H:%M:%S",
|
| 49 |
+
)
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def print_separator(title: str = "") -> None:
|
| 53 |
+
"""Print a visual separator with optional title."""
|
| 54 |
+
width = 70
|
| 55 |
+
if title:
|
| 56 |
+
print(f"\n{'=' * width}")
|
| 57 |
+
print(f" {title}")
|
| 58 |
+
print(f"{'=' * width}")
|
| 59 |
+
else:
|
| 60 |
+
print(f"\n{'-' * width}")
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_search(team_a: str, team_b: str | None, max_pages: int) -> None:
|
| 64 |
+
"""Test game search functionality."""
|
| 65 |
+
print_separator(f"Searching for: {team_a}" + (f" vs {team_b}" if team_b else ""))
|
| 66 |
+
|
| 67 |
+
print(f"\nParameters:")
|
| 68 |
+
print(f" team_a: {team_a}")
|
| 69 |
+
print(f" team_b: {team_b or '(any)'}")
|
| 70 |
+
print(f" max_pages: {max_pages}")
|
| 71 |
+
|
| 72 |
+
# Perform the search
|
| 73 |
+
results = search_games(team_a=team_a, team_b=team_b, max_pages=max_pages)
|
| 74 |
+
|
| 75 |
+
print(f"\nSearch Results:")
|
| 76 |
+
print(f" Pages searched: {results.pages_searched}")
|
| 77 |
+
print(f" Total games scanned: {results.total_games_scanned}")
|
| 78 |
+
print(f" Matching games found: {len(results.games)}")
|
| 79 |
+
|
| 80 |
+
if not results.games:
|
| 81 |
+
print("\n No matching games found!")
|
| 82 |
+
return results
|
| 83 |
+
|
| 84 |
+
print_separator("Found Games")
|
| 85 |
+
|
| 86 |
+
for i, game in enumerate(results.games, 1):
|
| 87 |
+
print(f"\n[{i}] {game.title}")
|
| 88 |
+
print(f" Teams: {game.team_a} vs {game.team_b}")
|
| 89 |
+
if game.event:
|
| 90 |
+
print(f" Event: {game.event}")
|
| 91 |
+
if game.year:
|
| 92 |
+
print(f" Year: {game.year}")
|
| 93 |
+
if game.date_str:
|
| 94 |
+
print(f" Date: {game.date_str}")
|
| 95 |
+
print(f" URL: {game.url}")
|
| 96 |
+
if game.thumbnail_url:
|
| 97 |
+
print(f" Thumbnail: {game.thumbnail_url[:60]}...")
|
| 98 |
+
|
| 99 |
+
return results
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def test_download_command(game) -> None:
|
| 103 |
+
"""Test download command generation."""
|
| 104 |
+
print_separator("Download Command Generation")
|
| 105 |
+
|
| 106 |
+
# Generate download command
|
| 107 |
+
cmd = get_download_command(game, output_dir="~/Downloads")
|
| 108 |
+
filename = get_suggested_filename(game)
|
| 109 |
+
|
| 110 |
+
print(f"\nGame: {game}")
|
| 111 |
+
print(f"Suggested filename: {filename}")
|
| 112 |
+
print(f"\nDownload command:")
|
| 113 |
+
print(f" {cmd}")
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def test_url_extraction(game) -> None:
|
| 117 |
+
"""Test direct URL extraction (requires yt-dlp)."""
|
| 118 |
+
print_separator("Direct URL Extraction")
|
| 119 |
+
|
| 120 |
+
if not is_ytdlp_available():
|
| 121 |
+
print("\n yt-dlp is not available. Install with: pip install yt-dlp")
|
| 122 |
+
print(" Skipping URL extraction test.")
|
| 123 |
+
return
|
| 124 |
+
|
| 125 |
+
print(f"\nExtracting direct URL for: {game.title}")
|
| 126 |
+
print(" (This may take 10-30 seconds...)")
|
| 127 |
+
|
| 128 |
+
url = extract_direct_video_url(game, timeout_seconds=60)
|
| 129 |
+
|
| 130 |
+
if url:
|
| 131 |
+
print(f"\n SUCCESS! Direct URL extracted:")
|
| 132 |
+
# Truncate URL for display if very long
|
| 133 |
+
if len(url) > 100:
|
| 134 |
+
print(f" {url[:100]}...")
|
| 135 |
+
else:
|
| 136 |
+
print(f" {url}")
|
| 137 |
+
print("\n This URL can be used for browser-direct download.")
|
| 138 |
+
print(" NOTE: URL may expire after a few minutes!")
|
| 139 |
+
else:
|
| 140 |
+
print("\n FAILED to extract direct URL.")
|
| 141 |
+
print(" The video may use a hosting service not supported by yt-dlp,")
|
| 142 |
+
print(" or there may be extraction issues.")
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
def test_video_info(game) -> None:
|
| 146 |
+
"""Test video info extraction (requires yt-dlp)."""
|
| 147 |
+
print_separator("Video Info Extraction")
|
| 148 |
+
|
| 149 |
+
if not is_ytdlp_available():
|
| 150 |
+
print("\n yt-dlp is not available. Skipping.")
|
| 151 |
+
return
|
| 152 |
+
|
| 153 |
+
print(f"\nGetting video info for: {game.title}")
|
| 154 |
+
print(" (This may take a few seconds...)")
|
| 155 |
+
|
| 156 |
+
info = get_video_info(game, timeout_seconds=30)
|
| 157 |
+
|
| 158 |
+
if info:
|
| 159 |
+
print("\n Video metadata:")
|
| 160 |
+
print(f" Title: {info.get('title', 'N/A')}")
|
| 161 |
+
print(f" Duration: {info.get('duration', 'N/A')} seconds")
|
| 162 |
+
|
| 163 |
+
filesize = info.get("filesize") or info.get("filesize_approx")
|
| 164 |
+
if filesize:
|
| 165 |
+
print(f" File size: {filesize / (1024*1024*1024):.2f} GB")
|
| 166 |
+
|
| 167 |
+
print(f" Format: {info.get('format', 'N/A')}")
|
| 168 |
+
print(f" Resolution: {info.get('resolution', 'N/A')}")
|
| 169 |
+
else:
|
| 170 |
+
print("\n FAILED to get video info.")
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def main():
|
| 174 |
+
parser = argparse.ArgumentParser(description="Test the source_finding module")
|
| 175 |
+
parser.add_argument("--team-a", default="Ohio State", help="Primary team to search for")
|
| 176 |
+
parser.add_argument("--team-b", default=None, help="Optional second team for specific matchup")
|
| 177 |
+
parser.add_argument("--max-pages", type=int, default=3, help="Maximum pages to search")
|
| 178 |
+
parser.add_argument("--test-download", action="store_true", help="Test URL extraction on first result")
|
| 179 |
+
parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
|
| 180 |
+
|
| 181 |
+
args = parser.parse_args()
|
| 182 |
+
|
| 183 |
+
setup_logging(args.verbose)
|
| 184 |
+
|
| 185 |
+
print("\n" + "=" * 70)
|
| 186 |
+
print(" SOURCE FINDING MODULE TEST")
|
| 187 |
+
print("=" * 70)
|
| 188 |
+
|
| 189 |
+
# Check yt-dlp availability
|
| 190 |
+
print(f"\nyt-dlp available: {is_ytdlp_available()}")
|
| 191 |
+
|
| 192 |
+
# Run search test
|
| 193 |
+
results = test_search(args.team_a, args.team_b, args.max_pages)
|
| 194 |
+
|
| 195 |
+
if results and results.games:
|
| 196 |
+
# Test with the first result
|
| 197 |
+
first_game = results.games[0]
|
| 198 |
+
|
| 199 |
+
# Always test download command generation
|
| 200 |
+
test_download_command(first_game)
|
| 201 |
+
|
| 202 |
+
# Optionally test URL extraction
|
| 203 |
+
if args.test_download:
|
| 204 |
+
test_url_extraction(first_game)
|
| 205 |
+
test_video_info(first_game)
|
| 206 |
+
|
| 207 |
+
print_separator("Test Complete")
|
| 208 |
+
print("\nTo test URL extraction, run with --test-download flag:")
|
| 209 |
+
print(f' python scripts/test_source_finding.py --team-a "{args.team_a}" --test-download')
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
if __name__ == "__main__":
|
| 213 |
+
main()
|
src/source_finding/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Source finding module for discovering and downloading college football game videos.
|
| 3 |
+
|
| 4 |
+
This module provides functionality to:
|
| 5 |
+
1. Search nfl-video.com for college football games by team name(s)
|
| 6 |
+
2. Return matching games with metadata (title, teams, date, event)
|
| 7 |
+
3. Generate download commands or extract direct video URLs for downloading
|
| 8 |
+
|
| 9 |
+
Example usage:
|
| 10 |
+
from source_finding import search_games, get_download_command, extract_direct_video_url
|
| 11 |
+
|
| 12 |
+
# Search for games
|
| 13 |
+
results = search_games(team_a="Ohio State", team_b="Oregon", max_pages=3)
|
| 14 |
+
|
| 15 |
+
# Get download command for offline use
|
| 16 |
+
cmd = get_download_command(results.games[0])
|
| 17 |
+
|
| 18 |
+
# Or extract direct URL for browser download
|
| 19 |
+
url = extract_direct_video_url(results.games[0])
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from .models import GameResult, SearchResults
|
| 23 |
+
from .scraper import search_games
|
| 24 |
+
from .downloader import get_download_command, extract_direct_video_url, get_suggested_filename
|
| 25 |
+
|
| 26 |
+
__all__ = [
|
| 27 |
+
"GameResult",
|
| 28 |
+
"SearchResults",
|
| 29 |
+
"search_games",
|
| 30 |
+
"get_download_command",
|
| 31 |
+
"extract_direct_video_url",
|
| 32 |
+
"get_suggested_filename",
|
| 33 |
+
]
|
src/source_finding/downloader.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Video download utilities for college football games.
|
| 3 |
+
|
| 4 |
+
This module provides multiple strategies for downloading videos:
|
| 5 |
+
1. Generate yt-dlp commands for users to run locally (offline mode)
|
| 6 |
+
2. Extract direct video URLs for browser-direct downloads (preferred for apps)
|
| 7 |
+
3. Stream video through a proxy (fallback, uses server bandwidth)
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
import shlex
|
| 12 |
+
import subprocess
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Generator, Optional
|
| 15 |
+
|
| 16 |
+
from .models import GameResult
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def get_suggested_filename(game: GameResult, extension: str = "mp4") -> str:
|
| 22 |
+
"""
|
| 23 |
+
Generate a suggested filename for a game video.
|
| 24 |
+
|
| 25 |
+
Args:
|
| 26 |
+
game: GameResult object
|
| 27 |
+
extension: File extension (default "mp4")
|
| 28 |
+
|
| 29 |
+
Returns:
|
| 30 |
+
Safe filename string like "Ohio_State_vs_Oregon_2024.mp4"
|
| 31 |
+
"""
|
| 32 |
+
return f"{game.get_filename_base()}.{extension}"
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def get_download_command(game: GameResult, output_dir: str = ".", output_filename: Optional[str] = None) -> str:
|
| 36 |
+
"""
|
| 37 |
+
Generate a yt-dlp command string for downloading a game video.
|
| 38 |
+
|
| 39 |
+
This is the "offline mode" - returns a command users can copy and run locally.
|
| 40 |
+
|
| 41 |
+
Args:
|
| 42 |
+
game: GameResult object containing the game URL
|
| 43 |
+
output_dir: Directory to save the video (default current directory)
|
| 44 |
+
output_filename: Optional custom filename; if None, generates from game metadata
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
A complete yt-dlp command string ready to run in a terminal
|
| 48 |
+
|
| 49 |
+
Example:
|
| 50 |
+
>>> cmd = get_download_command(game, output_dir="~/Downloads")
|
| 51 |
+
>>> print(cmd)
|
| 52 |
+
yt-dlp "https://nfl-video.com/ohio-state-vs-oregon..." -o "~/Downloads/Ohio_State_vs_Oregon_2024.mp4"
|
| 53 |
+
"""
|
| 54 |
+
if output_filename is None:
|
| 55 |
+
output_filename = get_suggested_filename(game)
|
| 56 |
+
|
| 57 |
+
# Build the output path
|
| 58 |
+
output_path = Path(output_dir) / output_filename
|
| 59 |
+
|
| 60 |
+
# Quote the URL and path for shell safety
|
| 61 |
+
quoted_url = shlex.quote(game.url)
|
| 62 |
+
quoted_output = shlex.quote(str(output_path))
|
| 63 |
+
|
| 64 |
+
# Build the command
|
| 65 |
+
# Using yt-dlp with common options for best compatibility
|
| 66 |
+
command = f"yt-dlp {quoted_url} -o {quoted_output}"
|
| 67 |
+
|
| 68 |
+
return command
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def extract_direct_video_url(game: GameResult, timeout_seconds: int = 60) -> Optional[str]:
|
| 72 |
+
"""
|
| 73 |
+
Extract the direct video URL from a game page using yt-dlp.
|
| 74 |
+
|
| 75 |
+
This is the preferred method for in-app downloads - extracts the actual video
|
| 76 |
+
URL from the hosting service (mixdrop, streamtape, etc.) so the browser can
|
| 77 |
+
download directly without going through your server.
|
| 78 |
+
|
| 79 |
+
IMPORTANT: The extracted URL may expire after a few minutes. Call this
|
| 80 |
+
function on-demand when the user clicks "Download", not when displaying results.
|
| 81 |
+
|
| 82 |
+
Args:
|
| 83 |
+
game: GameResult object containing the game page URL
|
| 84 |
+
timeout_seconds: Maximum time to wait for URL extraction
|
| 85 |
+
|
| 86 |
+
Returns:
|
| 87 |
+
Direct video URL string, or None if extraction failed
|
| 88 |
+
|
| 89 |
+
Example:
|
| 90 |
+
>>> url = extract_direct_video_url(game)
|
| 91 |
+
>>> # In Shiny: redirect browser to this URL for download
|
| 92 |
+
>>> print(url)
|
| 93 |
+
'https://s1.mixdropjp.pw/v/abc123def456.mp4?...'
|
| 94 |
+
"""
|
| 95 |
+
try:
|
| 96 |
+
logger.info("Extracting direct video URL for: %s", game.url)
|
| 97 |
+
|
| 98 |
+
# Use yt-dlp with --get-url to extract the direct video URL
|
| 99 |
+
# --no-warnings to suppress non-critical warnings
|
| 100 |
+
# -f best to get the best quality single file
|
| 101 |
+
result = subprocess.run(
|
| 102 |
+
["yt-dlp", "--get-url", "--no-warnings", "-f", "best", game.url],
|
| 103 |
+
capture_output=True,
|
| 104 |
+
text=True,
|
| 105 |
+
timeout=timeout_seconds,
|
| 106 |
+
check=False, # We handle return codes manually
|
| 107 |
+
)
|
| 108 |
+
|
| 109 |
+
if result.returncode != 0:
|
| 110 |
+
logger.error("yt-dlp failed with code %d: %s", result.returncode, result.stderr)
|
| 111 |
+
return None
|
| 112 |
+
|
| 113 |
+
# The URL is in stdout
|
| 114 |
+
url = result.stdout.strip()
|
| 115 |
+
|
| 116 |
+
if not url:
|
| 117 |
+
logger.error("yt-dlp returned empty URL")
|
| 118 |
+
return None
|
| 119 |
+
|
| 120 |
+
# Sometimes yt-dlp returns multiple URLs (for video/audio streams)
|
| 121 |
+
# Take the first one which is typically the video
|
| 122 |
+
if "\n" in url:
|
| 123 |
+
urls = url.split("\n")
|
| 124 |
+
url = urls[0]
|
| 125 |
+
logger.debug("Multiple URLs returned, using first: %s", url[:100])
|
| 126 |
+
|
| 127 |
+
logger.info("Extracted direct URL: %s...", url[:100] if len(url) > 100 else url)
|
| 128 |
+
return url
|
| 129 |
+
|
| 130 |
+
except subprocess.TimeoutExpired:
|
| 131 |
+
logger.error("yt-dlp timed out after %d seconds", timeout_seconds)
|
| 132 |
+
return None
|
| 133 |
+
except FileNotFoundError:
|
| 134 |
+
logger.error("yt-dlp not found. Please install it: pip install yt-dlp")
|
| 135 |
+
return None
|
| 136 |
+
except Exception as e: # pylint: disable=broad-exception-caught
|
| 137 |
+
logger.error("Failed to extract video URL: %s", e)
|
| 138 |
+
return None
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def is_ytdlp_available() -> bool:
|
| 142 |
+
"""
|
| 143 |
+
Check if yt-dlp is available on the system.
|
| 144 |
+
|
| 145 |
+
Returns:
|
| 146 |
+
True if yt-dlp is installed and accessible
|
| 147 |
+
"""
|
| 148 |
+
try:
|
| 149 |
+
result = subprocess.run(["yt-dlp", "--version"], capture_output=True, text=True, timeout=5, check=False)
|
| 150 |
+
return result.returncode == 0
|
| 151 |
+
except (FileNotFoundError, subprocess.TimeoutExpired):
|
| 152 |
+
return False
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
def stream_video_proxy(game: GameResult, chunk_size: int = 8192) -> Generator[bytes, None, None]:
|
| 156 |
+
"""
|
| 157 |
+
Stream video content in chunks (fallback method).
|
| 158 |
+
|
| 159 |
+
This is Option B from the architecture - streams video through your server.
|
| 160 |
+
Use this only if extract_direct_video_url() doesn't work reliably (e.g., URLs expire too fast).
|
| 161 |
+
|
| 162 |
+
Memory usage is constant regardless of video size (~chunk_size bytes).
|
| 163 |
+
|
| 164 |
+
NOTE: This is a stub implementation. For production use, you would:
|
| 165 |
+
1. First extract the direct URL using yt-dlp
|
| 166 |
+
2. Stream from that URL using requests with stream=True
|
| 167 |
+
|
| 168 |
+
Args:
|
| 169 |
+
game: GameResult object
|
| 170 |
+
chunk_size: Size of chunks to yield (default 8KB)
|
| 171 |
+
|
| 172 |
+
Yields:
|
| 173 |
+
Bytes chunks of the video file
|
| 174 |
+
|
| 175 |
+
Example:
|
| 176 |
+
>>> for chunk in stream_video_proxy(game):
|
| 177 |
+
... response.write(chunk) # In a web framework
|
| 178 |
+
"""
|
| 179 |
+
# First, get the direct video URL
|
| 180 |
+
direct_url = extract_direct_video_url(game)
|
| 181 |
+
|
| 182 |
+
if direct_url is None:
|
| 183 |
+
logger.error("Cannot stream - failed to extract direct URL")
|
| 184 |
+
return
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
import requests # pylint: disable=import-outside-toplevel
|
| 188 |
+
|
| 189 |
+
logger.info("Starting streaming proxy for: %s", game.title)
|
| 190 |
+
|
| 191 |
+
# Stream the video content
|
| 192 |
+
with requests.get(direct_url, stream=True, timeout=30) as response:
|
| 193 |
+
response.raise_for_status()
|
| 194 |
+
|
| 195 |
+
# Yield chunks as they arrive
|
| 196 |
+
for chunk in response.iter_content(chunk_size=chunk_size):
|
| 197 |
+
if chunk: # Filter out keep-alive chunks
|
| 198 |
+
yield chunk
|
| 199 |
+
|
| 200 |
+
logger.info("Streaming complete for: %s", game.title)
|
| 201 |
+
|
| 202 |
+
except Exception as e: # pylint: disable=broad-exception-caught
|
| 203 |
+
logger.error("Streaming failed: %s", e)
|
| 204 |
+
return
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def get_video_info(game: GameResult, timeout_seconds: int = 30) -> Optional[dict]:
|
| 208 |
+
"""
|
| 209 |
+
Get video metadata without downloading.
|
| 210 |
+
|
| 211 |
+
Useful for showing file size, duration, quality options to users before download.
|
| 212 |
+
|
| 213 |
+
Args:
|
| 214 |
+
game: GameResult object
|
| 215 |
+
timeout_seconds: Maximum time to wait
|
| 216 |
+
|
| 217 |
+
Returns:
|
| 218 |
+
Dictionary with video info (title, duration, filesize, etc.) or None if failed
|
| 219 |
+
"""
|
| 220 |
+
try:
|
| 221 |
+
import json # pylint: disable=import-outside-toplevel
|
| 222 |
+
|
| 223 |
+
logger.info("Getting video info for: %s", game.url)
|
| 224 |
+
|
| 225 |
+
result = subprocess.run(
|
| 226 |
+
["yt-dlp", "--dump-json", "--no-download", "--no-warnings", game.url],
|
| 227 |
+
capture_output=True,
|
| 228 |
+
text=True,
|
| 229 |
+
timeout=timeout_seconds,
|
| 230 |
+
check=False, # We handle return codes manually
|
| 231 |
+
)
|
| 232 |
+
|
| 233 |
+
if result.returncode != 0:
|
| 234 |
+
logger.error("yt-dlp info extraction failed: %s", result.stderr)
|
| 235 |
+
return None
|
| 236 |
+
|
| 237 |
+
info = json.loads(result.stdout)
|
| 238 |
+
return info
|
| 239 |
+
|
| 240 |
+
except subprocess.TimeoutExpired:
|
| 241 |
+
logger.error("yt-dlp info timed out after %d seconds", timeout_seconds)
|
| 242 |
+
return None
|
| 243 |
+
except Exception as e: # pylint: disable=broad-exception-caught
|
| 244 |
+
logger.error("Failed to get video info: %s", e)
|
| 245 |
+
return None
|
src/source_finding/models.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pydantic models for the source finding module.
|
| 3 |
+
|
| 4 |
+
Contains data models for representing game search results and metadata.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from typing import Optional
|
| 8 |
+
|
| 9 |
+
from pydantic import BaseModel, Field
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class GameResult(BaseModel):
|
| 13 |
+
"""
|
| 14 |
+
Represents a single college football game found on nfl-video.com.
|
| 15 |
+
|
| 16 |
+
Attributes:
|
| 17 |
+
title: Full title from the website, e.g. "Ohio State vs Oregon Football 2024 Big Ten Championship Full Game Replay"
|
| 18 |
+
team_a: First team name parsed from the title (appears before "vs")
|
| 19 |
+
team_b: Second team name parsed from the title (appears after "vs")
|
| 20 |
+
url: Full URL to the game's page on nfl-video.com
|
| 21 |
+
thumbnail_url: URL to the game's thumbnail image, if available
|
| 22 |
+
date_str: Date string parsed from the page, e.g. "January 8, 2024"
|
| 23 |
+
event: Event name parsed from the title, e.g. "CFP National Championship", "Big Ten Championship"
|
| 24 |
+
year: Year of the game, parsed from the title
|
| 25 |
+
"""
|
| 26 |
+
|
| 27 |
+
title: str = Field(..., description="Full title of the game from the website")
|
| 28 |
+
team_a: str = Field(..., description="First team name (before 'vs')")
|
| 29 |
+
team_b: str = Field(..., description="Second team name (after 'vs')")
|
| 30 |
+
url: str = Field(..., description="Full URL to the game page")
|
| 31 |
+
thumbnail_url: Optional[str] = Field(default=None, description="URL to thumbnail image")
|
| 32 |
+
date_str: Optional[str] = Field(default=None, description="Date string, e.g. 'January 8, 2024'")
|
| 33 |
+
event: Optional[str] = Field(default=None, description="Event name, e.g. 'CFP National Championship'")
|
| 34 |
+
year: Optional[int] = Field(default=None, description="Year of the game")
|
| 35 |
+
|
| 36 |
+
def __str__(self) -> str:
|
| 37 |
+
"""Human-readable representation of the game."""
|
| 38 |
+
event_str = f" ({self.event})" if self.event else ""
|
| 39 |
+
date_str = f" - {self.date_str}" if self.date_str else ""
|
| 40 |
+
return f"{self.team_a} vs {self.team_b}{event_str}{date_str}"
|
| 41 |
+
|
| 42 |
+
def get_filename_base(self) -> str:
|
| 43 |
+
"""
|
| 44 |
+
Generate a safe filename base for this game (without extension).
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
A filename-safe string like "Ohio_State_vs_Oregon_2024"
|
| 48 |
+
"""
|
| 49 |
+
# Replace spaces with underscores and remove special characters
|
| 50 |
+
# pylint: disable=no-member # False positive: Pydantic fields are strings at runtime
|
| 51 |
+
team_a_safe = self.team_a.replace(" ", "_").replace("(", "").replace(")", "")
|
| 52 |
+
team_b_safe = self.team_b.replace(" ", "_").replace("(", "").replace(")", "")
|
| 53 |
+
year_str = f"_{self.year}" if self.year else ""
|
| 54 |
+
return f"{team_a_safe}_vs_{team_b_safe}{year_str}"
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class SearchResults(BaseModel):
|
| 58 |
+
"""
|
| 59 |
+
Results from searching for games on nfl-video.com.
|
| 60 |
+
|
| 61 |
+
Attributes:
|
| 62 |
+
query_team_a: The team name that was searched for (required)
|
| 63 |
+
query_team_b: Optional second team name that was searched for
|
| 64 |
+
games: List of matching GameResult objects
|
| 65 |
+
pages_searched: Number of pages that were searched
|
| 66 |
+
total_games_scanned: Total number of games scanned across all pages
|
| 67 |
+
"""
|
| 68 |
+
|
| 69 |
+
query_team_a: str = Field(..., description="Primary team name searched for")
|
| 70 |
+
query_team_b: Optional[str] = Field(default=None, description="Optional second team name searched for")
|
| 71 |
+
games: list[GameResult] = Field(default_factory=list, description="List of matching games found")
|
| 72 |
+
pages_searched: int = Field(default=0, description="Number of pages searched")
|
| 73 |
+
total_games_scanned: int = Field(default=0, description="Total games scanned across all pages")
|
| 74 |
+
|
| 75 |
+
def __str__(self) -> str:
|
| 76 |
+
"""Human-readable summary of search results."""
|
| 77 |
+
team_str = self.query_team_a
|
| 78 |
+
if self.query_team_b:
|
| 79 |
+
team_str += f" vs {self.query_team_b}"
|
| 80 |
+
return f"Search for '{team_str}': {len(self.games)} games found (scanned {self.total_games_scanned} across {self.pages_searched} pages)"
|
src/source_finding/scraper.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Web scraper for finding college football games on nfl-video.com.
|
| 3 |
+
|
| 4 |
+
This module handles fetching and parsing game listings from the website,
|
| 5 |
+
filtering by team names, and extracting metadata from game titles.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import logging
|
| 9 |
+
import re
|
| 10 |
+
import time
|
| 11 |
+
from typing import Optional
|
| 12 |
+
|
| 13 |
+
import requests
|
| 14 |
+
from bs4 import BeautifulSoup
|
| 15 |
+
|
| 16 |
+
from .models import GameResult, SearchResults
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
+
|
| 20 |
+
# Base URL for college football full game replays
|
| 21 |
+
BASE_URL = "https://nfl-video.com/cfb/ncaa_college_football_highlights_games_replay/ncaa_college_football_full_game_replays/2"
|
| 22 |
+
|
| 23 |
+
# User agent to avoid being blocked
|
| 24 |
+
HEADERS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
|
| 25 |
+
|
| 26 |
+
# Delay between page requests to be polite to the server
|
| 27 |
+
REQUEST_DELAY_SECONDS = 0.5
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def _get_page_url(page_num: int) -> str:
|
| 31 |
+
"""
|
| 32 |
+
Get the URL for a specific page of game listings.
|
| 33 |
+
|
| 34 |
+
Args:
|
| 35 |
+
page_num: Page number (1-indexed)
|
| 36 |
+
|
| 37 |
+
Returns:
|
| 38 |
+
Full URL for that page
|
| 39 |
+
"""
|
| 40 |
+
if page_num == 1:
|
| 41 |
+
return BASE_URL
|
| 42 |
+
return f"{BASE_URL}-{page_num}"
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _parse_game_title(title: str) -> dict:
|
| 46 |
+
"""
|
| 47 |
+
Parse a game title to extract team names, year, and event.
|
| 48 |
+
|
| 49 |
+
Example titles:
|
| 50 |
+
"Ohio State vs Oregon Football 2024 Big Ten Championship Full Game Replay"
|
| 51 |
+
"Washington vs Michigan Football 2024 CFP National Championship Full Game Replay"
|
| 52 |
+
"Iowa vs Tennessee Football 2024 Citrus Bowl Full Game Replay"
|
| 53 |
+
|
| 54 |
+
Args:
|
| 55 |
+
title: Full game title string
|
| 56 |
+
|
| 57 |
+
Returns:
|
| 58 |
+
Dictionary with keys: team_a, team_b, year, event (some may be None)
|
| 59 |
+
"""
|
| 60 |
+
result = {"team_a": None, "team_b": None, "year": None, "event": None}
|
| 61 |
+
|
| 62 |
+
# Clean up the title
|
| 63 |
+
title = title.strip()
|
| 64 |
+
|
| 65 |
+
# Pattern: "Team A vs Team B Football YYYY [Event] Full Game Replay"
|
| 66 |
+
# Some titles use "vs." with a period, others use "vs" without
|
| 67 |
+
# First, try to extract the "vs" or "vs." split
|
| 68 |
+
vs_match = re.match(r"^(.+?)\s+vs\.?\s+(.+?)\s+Football\s+", title, re.IGNORECASE)
|
| 69 |
+
if vs_match:
|
| 70 |
+
result["team_a"] = vs_match.group(1).strip()
|
| 71 |
+
result["team_b"] = vs_match.group(2).strip()
|
| 72 |
+
|
| 73 |
+
# Extract year (4-digit number)
|
| 74 |
+
year_match = re.search(r"\b(20\d{2})\b", title)
|
| 75 |
+
if year_match:
|
| 76 |
+
result["year"] = int(year_match.group(1))
|
| 77 |
+
|
| 78 |
+
# Extract event - everything between the date and "Full Game Replay"
|
| 79 |
+
# Format: "Football [Month Day, Year] [Event] Full Game Replay"
|
| 80 |
+
# Example: "Football December 20, 2025 CFP First Round Full Game Replay"
|
| 81 |
+
event_match = re.search(r"Football\s+\w+\s+\d{1,2},?\s+20\d{2}\s+(.+?)\s+Full Game Replay", title, re.IGNORECASE)
|
| 82 |
+
if event_match:
|
| 83 |
+
event = event_match.group(1).strip()
|
| 84 |
+
# Clean up common patterns
|
| 85 |
+
if event and event.lower() not in ["full", "game", "replay"]:
|
| 86 |
+
result["event"] = event
|
| 87 |
+
|
| 88 |
+
return result
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _fetch_page(url: str) -> Optional[BeautifulSoup]:
|
| 92 |
+
"""
|
| 93 |
+
Fetch a page and return its parsed HTML.
|
| 94 |
+
|
| 95 |
+
Args:
|
| 96 |
+
url: URL to fetch
|
| 97 |
+
|
| 98 |
+
Returns:
|
| 99 |
+
BeautifulSoup object or None if fetch failed
|
| 100 |
+
"""
|
| 101 |
+
try:
|
| 102 |
+
logger.debug("Fetching URL: %s", url)
|
| 103 |
+
response = requests.get(url, headers=HEADERS, timeout=30)
|
| 104 |
+
response.raise_for_status()
|
| 105 |
+
return BeautifulSoup(response.text, "html.parser")
|
| 106 |
+
except requests.RequestException as e:
|
| 107 |
+
logger.error("Failed to fetch %s: %s", url, e)
|
| 108 |
+
return None
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _extract_games_from_page(soup: BeautifulSoup) -> list[GameResult]:
|
| 112 |
+
"""
|
| 113 |
+
Extract game listings from a parsed page.
|
| 114 |
+
|
| 115 |
+
Args:
|
| 116 |
+
soup: BeautifulSoup object of the page
|
| 117 |
+
|
| 118 |
+
Returns:
|
| 119 |
+
List of GameResult objects found on the page
|
| 120 |
+
"""
|
| 121 |
+
games = []
|
| 122 |
+
|
| 123 |
+
# Find all game links - they typically have h3 headers or are in specific divs
|
| 124 |
+
# Looking for links that contain "Full Game Replay" in the text
|
| 125 |
+
for link in soup.find_all("a", href=True):
|
| 126 |
+
text = link.get_text(strip=True)
|
| 127 |
+
|
| 128 |
+
# Skip links that don't look like game titles
|
| 129 |
+
if "Full Game Replay" not in text:
|
| 130 |
+
continue
|
| 131 |
+
|
| 132 |
+
# Skip if it doesn't have "vs" or "vs." (not a game matchup)
|
| 133 |
+
# Some titles use "vs." with a period, others use "vs" without
|
| 134 |
+
text_lower = text.lower()
|
| 135 |
+
if " vs " not in text_lower and " vs. " not in text_lower:
|
| 136 |
+
continue
|
| 137 |
+
|
| 138 |
+
href = link["href"]
|
| 139 |
+
|
| 140 |
+
# Make sure it's a full URL
|
| 141 |
+
if not href.startswith("http"):
|
| 142 |
+
href = f"https://nfl-video.com{href}"
|
| 143 |
+
|
| 144 |
+
# Skip if we've already seen this URL (duplicates on page)
|
| 145 |
+
if any(g.url == href for g in games):
|
| 146 |
+
continue
|
| 147 |
+
|
| 148 |
+
# Parse the title for metadata
|
| 149 |
+
parsed = _parse_game_title(text)
|
| 150 |
+
|
| 151 |
+
if parsed["team_a"] and parsed["team_b"]:
|
| 152 |
+
# Try to find thumbnail - look for img in parent or sibling elements
|
| 153 |
+
thumbnail_url = None
|
| 154 |
+
parent = link.find_parent()
|
| 155 |
+
if parent:
|
| 156 |
+
img = parent.find("img")
|
| 157 |
+
if img and img.get("src"):
|
| 158 |
+
thumbnail_url = img["src"]
|
| 159 |
+
if not thumbnail_url.startswith("http"):
|
| 160 |
+
thumbnail_url = f"https://nfl-video.com{thumbnail_url}"
|
| 161 |
+
|
| 162 |
+
game = GameResult(
|
| 163 |
+
title=text,
|
| 164 |
+
team_a=parsed["team_a"],
|
| 165 |
+
team_b=parsed["team_b"],
|
| 166 |
+
url=href,
|
| 167 |
+
thumbnail_url=thumbnail_url,
|
| 168 |
+
year=parsed["year"],
|
| 169 |
+
event=parsed["event"],
|
| 170 |
+
)
|
| 171 |
+
games.append(game)
|
| 172 |
+
logger.debug("Found game: %s", game)
|
| 173 |
+
|
| 174 |
+
return games
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _team_matches(game: GameResult, team_name: str) -> bool:
|
| 178 |
+
"""
|
| 179 |
+
Check if a game involves a given team.
|
| 180 |
+
|
| 181 |
+
Performs case-insensitive partial matching to handle variations like:
|
| 182 |
+
- "Ohio State" matches "Ohio State Buckeyes"
|
| 183 |
+
- "OSU" would need to be handled separately if desired
|
| 184 |
+
|
| 185 |
+
Args:
|
| 186 |
+
game: GameResult to check
|
| 187 |
+
team_name: Team name to search for
|
| 188 |
+
|
| 189 |
+
Returns:
|
| 190 |
+
True if the team appears in either team_a or team_b
|
| 191 |
+
"""
|
| 192 |
+
team_lower = team_name.lower()
|
| 193 |
+
return team_lower in game.team_a.lower() or team_lower in game.team_b.lower()
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def search_games(
|
| 197 |
+
team_a: str,
|
| 198 |
+
team_b: Optional[str] = None,
|
| 199 |
+
max_pages: int = 5,
|
| 200 |
+
delay_seconds: float = REQUEST_DELAY_SECONDS,
|
| 201 |
+
) -> SearchResults:
|
| 202 |
+
"""
|
| 203 |
+
Search for college football games on nfl-video.com by team name(s).
|
| 204 |
+
|
| 205 |
+
Args:
|
| 206 |
+
team_a: Primary team name to search for (required)
|
| 207 |
+
team_b: Optional second team name - if provided, only games with BOTH teams are returned
|
| 208 |
+
max_pages: Maximum number of pages to search (default 5)
|
| 209 |
+
delay_seconds: Delay between page requests to avoid rate limiting
|
| 210 |
+
|
| 211 |
+
Returns:
|
| 212 |
+
SearchResults object containing matching games and search metadata
|
| 213 |
+
|
| 214 |
+
Example:
|
| 215 |
+
# Find all Ohio State games
|
| 216 |
+
results = search_games("Ohio State", max_pages=10)
|
| 217 |
+
|
| 218 |
+
# Find Ohio State vs Oregon specifically
|
| 219 |
+
results = search_games("Ohio State", "Oregon")
|
| 220 |
+
"""
|
| 221 |
+
logger.info("Searching for games: team_a='%s', team_b='%s', max_pages=%d", team_a, team_b, max_pages)
|
| 222 |
+
|
| 223 |
+
matching_games: list[GameResult] = []
|
| 224 |
+
total_scanned = 0
|
| 225 |
+
|
| 226 |
+
for page_num in range(1, max_pages + 1):
|
| 227 |
+
url = _get_page_url(page_num)
|
| 228 |
+
logger.info("Searching page %d/%d: %s", page_num, max_pages, url)
|
| 229 |
+
|
| 230 |
+
soup = _fetch_page(url)
|
| 231 |
+
if soup is None:
|
| 232 |
+
logger.warning("Failed to fetch page %d, stopping search", page_num)
|
| 233 |
+
break
|
| 234 |
+
|
| 235 |
+
# Extract all games from this page
|
| 236 |
+
page_games = _extract_games_from_page(soup)
|
| 237 |
+
total_scanned += len(page_games)
|
| 238 |
+
|
| 239 |
+
# Filter by team name(s)
|
| 240 |
+
for game in page_games:
|
| 241 |
+
# Must match team_a
|
| 242 |
+
if not _team_matches(game, team_a):
|
| 243 |
+
continue
|
| 244 |
+
|
| 245 |
+
# If team_b specified, must also match team_b
|
| 246 |
+
if team_b and not _team_matches(game, team_b):
|
| 247 |
+
continue
|
| 248 |
+
|
| 249 |
+
# Avoid duplicates (same URL)
|
| 250 |
+
if not any(g.url == game.url for g in matching_games):
|
| 251 |
+
matching_games.append(game)
|
| 252 |
+
logger.info("Found matching game: %s", game)
|
| 253 |
+
|
| 254 |
+
# Be polite - add delay between requests
|
| 255 |
+
if page_num < max_pages:
|
| 256 |
+
time.sleep(delay_seconds)
|
| 257 |
+
|
| 258 |
+
results = SearchResults(
|
| 259 |
+
query_team_a=team_a,
|
| 260 |
+
query_team_b=team_b,
|
| 261 |
+
games=matching_games,
|
| 262 |
+
pages_searched=max_pages,
|
| 263 |
+
total_games_scanned=total_scanned,
|
| 264 |
+
)
|
| 265 |
+
|
| 266 |
+
logger.info("Search complete: %s", results)
|
| 267 |
+
return results
|