andytaylor-smg commited on
Commit
4f9adc4
·
1 Parent(s): deb89c9

simple request

Browse files
pyproject.toml CHANGED
@@ -12,6 +12,9 @@ dependencies = [
12
  "pillow>=10.0.0",
13
  "pydantic>=2.0.0",
14
  "pytesseract>=0.3.10",
 
 
 
15
  ]
16
 
17
  [dependency-groups]
 
12
  "pillow>=10.0.0",
13
  "pydantic>=2.0.0",
14
  "pytesseract>=0.3.10",
15
+ "requests>=2.31.0",
16
+ "beautifulsoup4>=4.12.0",
17
+ "yt-dlp>=2024.1.0",
18
  ]
19
 
20
  [dependency-groups]
scripts/test_source_finding.py ADDED
@@ -0,0 +1,213 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Test script for the source_finding module.
4
+
5
+ This script demonstrates and validates the functionality of:
6
+ 1. Searching for games by team name(s)
7
+ 2. Generating download commands
8
+ 3. Extracting direct video URLs (if yt-dlp is available)
9
+
10
+ Usage:
11
+ # Activate virtual environment first
12
+ source .venv/bin/activate
13
+
14
+ # Run with default test (Ohio State vs Oregon)
15
+ python scripts/test_source_finding.py
16
+
17
+ # Search for a specific team
18
+ python scripts/test_source_finding.py --team-a "Tennessee"
19
+
20
+ # Search for a specific matchup
21
+ python scripts/test_source_finding.py --team-a "Ohio State" --team-b "Oregon"
22
+
23
+ # Search more pages (default is 3)
24
+ python scripts/test_source_finding.py --team-a "Michigan" --max-pages 10
25
+
26
+ # Test URL extraction on a specific result
27
+ python scripts/test_source_finding.py --team-a "Ohio State" --test-download
28
+ """
29
+
30
+ import argparse
31
+ import logging
32
+ import sys
33
+ from pathlib import Path
34
+
35
+ # Add src to path for imports
36
+ sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
37
+
38
+ from source_finding import search_games, get_download_command, extract_direct_video_url, get_suggested_filename
39
+ from source_finding.downloader import is_ytdlp_available, get_video_info
40
+
41
+
42
+ def setup_logging(verbose: bool = False) -> None:
43
+ """Configure logging for the test script."""
44
+ level = logging.DEBUG if verbose else logging.INFO
45
+ logging.basicConfig(
46
+ level=level,
47
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
48
+ datefmt="%H:%M:%S",
49
+ )
50
+
51
+
52
+ def print_separator(title: str = "") -> None:
53
+ """Print a visual separator with optional title."""
54
+ width = 70
55
+ if title:
56
+ print(f"\n{'=' * width}")
57
+ print(f" {title}")
58
+ print(f"{'=' * width}")
59
+ else:
60
+ print(f"\n{'-' * width}")
61
+
62
+
63
+ def test_search(team_a: str, team_b: str | None, max_pages: int) -> None:
64
+ """Test game search functionality."""
65
+ print_separator(f"Searching for: {team_a}" + (f" vs {team_b}" if team_b else ""))
66
+
67
+ print(f"\nParameters:")
68
+ print(f" team_a: {team_a}")
69
+ print(f" team_b: {team_b or '(any)'}")
70
+ print(f" max_pages: {max_pages}")
71
+
72
+ # Perform the search
73
+ results = search_games(team_a=team_a, team_b=team_b, max_pages=max_pages)
74
+
75
+ print(f"\nSearch Results:")
76
+ print(f" Pages searched: {results.pages_searched}")
77
+ print(f" Total games scanned: {results.total_games_scanned}")
78
+ print(f" Matching games found: {len(results.games)}")
79
+
80
+ if not results.games:
81
+ print("\n No matching games found!")
82
+ return results
83
+
84
+ print_separator("Found Games")
85
+
86
+ for i, game in enumerate(results.games, 1):
87
+ print(f"\n[{i}] {game.title}")
88
+ print(f" Teams: {game.team_a} vs {game.team_b}")
89
+ if game.event:
90
+ print(f" Event: {game.event}")
91
+ if game.year:
92
+ print(f" Year: {game.year}")
93
+ if game.date_str:
94
+ print(f" Date: {game.date_str}")
95
+ print(f" URL: {game.url}")
96
+ if game.thumbnail_url:
97
+ print(f" Thumbnail: {game.thumbnail_url[:60]}...")
98
+
99
+ return results
100
+
101
+
102
+ def test_download_command(game) -> None:
103
+ """Test download command generation."""
104
+ print_separator("Download Command Generation")
105
+
106
+ # Generate download command
107
+ cmd = get_download_command(game, output_dir="~/Downloads")
108
+ filename = get_suggested_filename(game)
109
+
110
+ print(f"\nGame: {game}")
111
+ print(f"Suggested filename: {filename}")
112
+ print(f"\nDownload command:")
113
+ print(f" {cmd}")
114
+
115
+
116
+ def test_url_extraction(game) -> None:
117
+ """Test direct URL extraction (requires yt-dlp)."""
118
+ print_separator("Direct URL Extraction")
119
+
120
+ if not is_ytdlp_available():
121
+ print("\n yt-dlp is not available. Install with: pip install yt-dlp")
122
+ print(" Skipping URL extraction test.")
123
+ return
124
+
125
+ print(f"\nExtracting direct URL for: {game.title}")
126
+ print(" (This may take 10-30 seconds...)")
127
+
128
+ url = extract_direct_video_url(game, timeout_seconds=60)
129
+
130
+ if url:
131
+ print(f"\n SUCCESS! Direct URL extracted:")
132
+ # Truncate URL for display if very long
133
+ if len(url) > 100:
134
+ print(f" {url[:100]}...")
135
+ else:
136
+ print(f" {url}")
137
+ print("\n This URL can be used for browser-direct download.")
138
+ print(" NOTE: URL may expire after a few minutes!")
139
+ else:
140
+ print("\n FAILED to extract direct URL.")
141
+ print(" The video may use a hosting service not supported by yt-dlp,")
142
+ print(" or there may be extraction issues.")
143
+
144
+
145
+ def test_video_info(game) -> None:
146
+ """Test video info extraction (requires yt-dlp)."""
147
+ print_separator("Video Info Extraction")
148
+
149
+ if not is_ytdlp_available():
150
+ print("\n yt-dlp is not available. Skipping.")
151
+ return
152
+
153
+ print(f"\nGetting video info for: {game.title}")
154
+ print(" (This may take a few seconds...)")
155
+
156
+ info = get_video_info(game, timeout_seconds=30)
157
+
158
+ if info:
159
+ print("\n Video metadata:")
160
+ print(f" Title: {info.get('title', 'N/A')}")
161
+ print(f" Duration: {info.get('duration', 'N/A')} seconds")
162
+
163
+ filesize = info.get("filesize") or info.get("filesize_approx")
164
+ if filesize:
165
+ print(f" File size: {filesize / (1024*1024*1024):.2f} GB")
166
+
167
+ print(f" Format: {info.get('format', 'N/A')}")
168
+ print(f" Resolution: {info.get('resolution', 'N/A')}")
169
+ else:
170
+ print("\n FAILED to get video info.")
171
+
172
+
173
+ def main():
174
+ parser = argparse.ArgumentParser(description="Test the source_finding module")
175
+ parser.add_argument("--team-a", default="Ohio State", help="Primary team to search for")
176
+ parser.add_argument("--team-b", default=None, help="Optional second team for specific matchup")
177
+ parser.add_argument("--max-pages", type=int, default=3, help="Maximum pages to search")
178
+ parser.add_argument("--test-download", action="store_true", help="Test URL extraction on first result")
179
+ parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
180
+
181
+ args = parser.parse_args()
182
+
183
+ setup_logging(args.verbose)
184
+
185
+ print("\n" + "=" * 70)
186
+ print(" SOURCE FINDING MODULE TEST")
187
+ print("=" * 70)
188
+
189
+ # Check yt-dlp availability
190
+ print(f"\nyt-dlp available: {is_ytdlp_available()}")
191
+
192
+ # Run search test
193
+ results = test_search(args.team_a, args.team_b, args.max_pages)
194
+
195
+ if results and results.games:
196
+ # Test with the first result
197
+ first_game = results.games[0]
198
+
199
+ # Always test download command generation
200
+ test_download_command(first_game)
201
+
202
+ # Optionally test URL extraction
203
+ if args.test_download:
204
+ test_url_extraction(first_game)
205
+ test_video_info(first_game)
206
+
207
+ print_separator("Test Complete")
208
+ print("\nTo test URL extraction, run with --test-download flag:")
209
+ print(f' python scripts/test_source_finding.py --team-a "{args.team_a}" --test-download')
210
+
211
+
212
+ if __name__ == "__main__":
213
+ main()
src/source_finding/__init__.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Source finding module for discovering and downloading college football game videos.
3
+
4
+ This module provides functionality to:
5
+ 1. Search nfl-video.com for college football games by team name(s)
6
+ 2. Return matching games with metadata (title, teams, date, event)
7
+ 3. Generate download commands or extract direct video URLs for downloading
8
+
9
+ Example usage:
10
+ from source_finding import search_games, get_download_command, extract_direct_video_url
11
+
12
+ # Search for games
13
+ results = search_games(team_a="Ohio State", team_b="Oregon", max_pages=3)
14
+
15
+ # Get download command for offline use
16
+ cmd = get_download_command(results.games[0])
17
+
18
+ # Or extract direct URL for browser download
19
+ url = extract_direct_video_url(results.games[0])
20
+ """
21
+
22
+ from .models import GameResult, SearchResults
23
+ from .scraper import search_games
24
+ from .downloader import get_download_command, extract_direct_video_url, get_suggested_filename
25
+
26
+ __all__ = [
27
+ "GameResult",
28
+ "SearchResults",
29
+ "search_games",
30
+ "get_download_command",
31
+ "extract_direct_video_url",
32
+ "get_suggested_filename",
33
+ ]
src/source_finding/downloader.py ADDED
@@ -0,0 +1,245 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Video download utilities for college football games.
3
+
4
+ This module provides multiple strategies for downloading videos:
5
+ 1. Generate yt-dlp commands for users to run locally (offline mode)
6
+ 2. Extract direct video URLs for browser-direct downloads (preferred for apps)
7
+ 3. Stream video through a proxy (fallback, uses server bandwidth)
8
+ """
9
+
10
+ import logging
11
+ import shlex
12
+ import subprocess
13
+ from pathlib import Path
14
+ from typing import Generator, Optional
15
+
16
+ from .models import GameResult
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+
21
+ def get_suggested_filename(game: GameResult, extension: str = "mp4") -> str:
22
+ """
23
+ Generate a suggested filename for a game video.
24
+
25
+ Args:
26
+ game: GameResult object
27
+ extension: File extension (default "mp4")
28
+
29
+ Returns:
30
+ Safe filename string like "Ohio_State_vs_Oregon_2024.mp4"
31
+ """
32
+ return f"{game.get_filename_base()}.{extension}"
33
+
34
+
35
+ def get_download_command(game: GameResult, output_dir: str = ".", output_filename: Optional[str] = None) -> str:
36
+ """
37
+ Generate a yt-dlp command string for downloading a game video.
38
+
39
+ This is the "offline mode" - returns a command users can copy and run locally.
40
+
41
+ Args:
42
+ game: GameResult object containing the game URL
43
+ output_dir: Directory to save the video (default current directory)
44
+ output_filename: Optional custom filename; if None, generates from game metadata
45
+
46
+ Returns:
47
+ A complete yt-dlp command string ready to run in a terminal
48
+
49
+ Example:
50
+ >>> cmd = get_download_command(game, output_dir="~/Downloads")
51
+ >>> print(cmd)
52
+ yt-dlp "https://nfl-video.com/ohio-state-vs-oregon..." -o "~/Downloads/Ohio_State_vs_Oregon_2024.mp4"
53
+ """
54
+ if output_filename is None:
55
+ output_filename = get_suggested_filename(game)
56
+
57
+ # Build the output path
58
+ output_path = Path(output_dir) / output_filename
59
+
60
+ # Quote the URL and path for shell safety
61
+ quoted_url = shlex.quote(game.url)
62
+ quoted_output = shlex.quote(str(output_path))
63
+
64
+ # Build the command
65
+ # Using yt-dlp with common options for best compatibility
66
+ command = f"yt-dlp {quoted_url} -o {quoted_output}"
67
+
68
+ return command
69
+
70
+
71
+ def extract_direct_video_url(game: GameResult, timeout_seconds: int = 60) -> Optional[str]:
72
+ """
73
+ Extract the direct video URL from a game page using yt-dlp.
74
+
75
+ This is the preferred method for in-app downloads - extracts the actual video
76
+ URL from the hosting service (mixdrop, streamtape, etc.) so the browser can
77
+ download directly without going through your server.
78
+
79
+ IMPORTANT: The extracted URL may expire after a few minutes. Call this
80
+ function on-demand when the user clicks "Download", not when displaying results.
81
+
82
+ Args:
83
+ game: GameResult object containing the game page URL
84
+ timeout_seconds: Maximum time to wait for URL extraction
85
+
86
+ Returns:
87
+ Direct video URL string, or None if extraction failed
88
+
89
+ Example:
90
+ >>> url = extract_direct_video_url(game)
91
+ >>> # In Shiny: redirect browser to this URL for download
92
+ >>> print(url)
93
+ 'https://s1.mixdropjp.pw/v/abc123def456.mp4?...'
94
+ """
95
+ try:
96
+ logger.info("Extracting direct video URL for: %s", game.url)
97
+
98
+ # Use yt-dlp with --get-url to extract the direct video URL
99
+ # --no-warnings to suppress non-critical warnings
100
+ # -f best to get the best quality single file
101
+ result = subprocess.run(
102
+ ["yt-dlp", "--get-url", "--no-warnings", "-f", "best", game.url],
103
+ capture_output=True,
104
+ text=True,
105
+ timeout=timeout_seconds,
106
+ check=False, # We handle return codes manually
107
+ )
108
+
109
+ if result.returncode != 0:
110
+ logger.error("yt-dlp failed with code %d: %s", result.returncode, result.stderr)
111
+ return None
112
+
113
+ # The URL is in stdout
114
+ url = result.stdout.strip()
115
+
116
+ if not url:
117
+ logger.error("yt-dlp returned empty URL")
118
+ return None
119
+
120
+ # Sometimes yt-dlp returns multiple URLs (for video/audio streams)
121
+ # Take the first one which is typically the video
122
+ if "\n" in url:
123
+ urls = url.split("\n")
124
+ url = urls[0]
125
+ logger.debug("Multiple URLs returned, using first: %s", url[:100])
126
+
127
+ logger.info("Extracted direct URL: %s...", url[:100] if len(url) > 100 else url)
128
+ return url
129
+
130
+ except subprocess.TimeoutExpired:
131
+ logger.error("yt-dlp timed out after %d seconds", timeout_seconds)
132
+ return None
133
+ except FileNotFoundError:
134
+ logger.error("yt-dlp not found. Please install it: pip install yt-dlp")
135
+ return None
136
+ except Exception as e: # pylint: disable=broad-exception-caught
137
+ logger.error("Failed to extract video URL: %s", e)
138
+ return None
139
+
140
+
141
+ def is_ytdlp_available() -> bool:
142
+ """
143
+ Check if yt-dlp is available on the system.
144
+
145
+ Returns:
146
+ True if yt-dlp is installed and accessible
147
+ """
148
+ try:
149
+ result = subprocess.run(["yt-dlp", "--version"], capture_output=True, text=True, timeout=5, check=False)
150
+ return result.returncode == 0
151
+ except (FileNotFoundError, subprocess.TimeoutExpired):
152
+ return False
153
+
154
+
155
+ def stream_video_proxy(game: GameResult, chunk_size: int = 8192) -> Generator[bytes, None, None]:
156
+ """
157
+ Stream video content in chunks (fallback method).
158
+
159
+ This is Option B from the architecture - streams video through your server.
160
+ Use this only if extract_direct_video_url() doesn't work reliably (e.g., URLs expire too fast).
161
+
162
+ Memory usage is constant regardless of video size (~chunk_size bytes).
163
+
164
+ NOTE: This is a stub implementation. For production use, you would:
165
+ 1. First extract the direct URL using yt-dlp
166
+ 2. Stream from that URL using requests with stream=True
167
+
168
+ Args:
169
+ game: GameResult object
170
+ chunk_size: Size of chunks to yield (default 8KB)
171
+
172
+ Yields:
173
+ Bytes chunks of the video file
174
+
175
+ Example:
176
+ >>> for chunk in stream_video_proxy(game):
177
+ ... response.write(chunk) # In a web framework
178
+ """
179
+ # First, get the direct video URL
180
+ direct_url = extract_direct_video_url(game)
181
+
182
+ if direct_url is None:
183
+ logger.error("Cannot stream - failed to extract direct URL")
184
+ return
185
+
186
+ try:
187
+ import requests # pylint: disable=import-outside-toplevel
188
+
189
+ logger.info("Starting streaming proxy for: %s", game.title)
190
+
191
+ # Stream the video content
192
+ with requests.get(direct_url, stream=True, timeout=30) as response:
193
+ response.raise_for_status()
194
+
195
+ # Yield chunks as they arrive
196
+ for chunk in response.iter_content(chunk_size=chunk_size):
197
+ if chunk: # Filter out keep-alive chunks
198
+ yield chunk
199
+
200
+ logger.info("Streaming complete for: %s", game.title)
201
+
202
+ except Exception as e: # pylint: disable=broad-exception-caught
203
+ logger.error("Streaming failed: %s", e)
204
+ return
205
+
206
+
207
+ def get_video_info(game: GameResult, timeout_seconds: int = 30) -> Optional[dict]:
208
+ """
209
+ Get video metadata without downloading.
210
+
211
+ Useful for showing file size, duration, quality options to users before download.
212
+
213
+ Args:
214
+ game: GameResult object
215
+ timeout_seconds: Maximum time to wait
216
+
217
+ Returns:
218
+ Dictionary with video info (title, duration, filesize, etc.) or None if failed
219
+ """
220
+ try:
221
+ import json # pylint: disable=import-outside-toplevel
222
+
223
+ logger.info("Getting video info for: %s", game.url)
224
+
225
+ result = subprocess.run(
226
+ ["yt-dlp", "--dump-json", "--no-download", "--no-warnings", game.url],
227
+ capture_output=True,
228
+ text=True,
229
+ timeout=timeout_seconds,
230
+ check=False, # We handle return codes manually
231
+ )
232
+
233
+ if result.returncode != 0:
234
+ logger.error("yt-dlp info extraction failed: %s", result.stderr)
235
+ return None
236
+
237
+ info = json.loads(result.stdout)
238
+ return info
239
+
240
+ except subprocess.TimeoutExpired:
241
+ logger.error("yt-dlp info timed out after %d seconds", timeout_seconds)
242
+ return None
243
+ except Exception as e: # pylint: disable=broad-exception-caught
244
+ logger.error("Failed to get video info: %s", e)
245
+ return None
src/source_finding/models.py ADDED
@@ -0,0 +1,80 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Pydantic models for the source finding module.
3
+
4
+ Contains data models for representing game search results and metadata.
5
+ """
6
+
7
+ from typing import Optional
8
+
9
+ from pydantic import BaseModel, Field
10
+
11
+
12
+ class GameResult(BaseModel):
13
+ """
14
+ Represents a single college football game found on nfl-video.com.
15
+
16
+ Attributes:
17
+ title: Full title from the website, e.g. "Ohio State vs Oregon Football 2024 Big Ten Championship Full Game Replay"
18
+ team_a: First team name parsed from the title (appears before "vs")
19
+ team_b: Second team name parsed from the title (appears after "vs")
20
+ url: Full URL to the game's page on nfl-video.com
21
+ thumbnail_url: URL to the game's thumbnail image, if available
22
+ date_str: Date string parsed from the page, e.g. "January 8, 2024"
23
+ event: Event name parsed from the title, e.g. "CFP National Championship", "Big Ten Championship"
24
+ year: Year of the game, parsed from the title
25
+ """
26
+
27
+ title: str = Field(..., description="Full title of the game from the website")
28
+ team_a: str = Field(..., description="First team name (before 'vs')")
29
+ team_b: str = Field(..., description="Second team name (after 'vs')")
30
+ url: str = Field(..., description="Full URL to the game page")
31
+ thumbnail_url: Optional[str] = Field(default=None, description="URL to thumbnail image")
32
+ date_str: Optional[str] = Field(default=None, description="Date string, e.g. 'January 8, 2024'")
33
+ event: Optional[str] = Field(default=None, description="Event name, e.g. 'CFP National Championship'")
34
+ year: Optional[int] = Field(default=None, description="Year of the game")
35
+
36
+ def __str__(self) -> str:
37
+ """Human-readable representation of the game."""
38
+ event_str = f" ({self.event})" if self.event else ""
39
+ date_str = f" - {self.date_str}" if self.date_str else ""
40
+ return f"{self.team_a} vs {self.team_b}{event_str}{date_str}"
41
+
42
+ def get_filename_base(self) -> str:
43
+ """
44
+ Generate a safe filename base for this game (without extension).
45
+
46
+ Returns:
47
+ A filename-safe string like "Ohio_State_vs_Oregon_2024"
48
+ """
49
+ # Replace spaces with underscores and remove special characters
50
+ # pylint: disable=no-member # False positive: Pydantic fields are strings at runtime
51
+ team_a_safe = self.team_a.replace(" ", "_").replace("(", "").replace(")", "")
52
+ team_b_safe = self.team_b.replace(" ", "_").replace("(", "").replace(")", "")
53
+ year_str = f"_{self.year}" if self.year else ""
54
+ return f"{team_a_safe}_vs_{team_b_safe}{year_str}"
55
+
56
+
57
+ class SearchResults(BaseModel):
58
+ """
59
+ Results from searching for games on nfl-video.com.
60
+
61
+ Attributes:
62
+ query_team_a: The team name that was searched for (required)
63
+ query_team_b: Optional second team name that was searched for
64
+ games: List of matching GameResult objects
65
+ pages_searched: Number of pages that were searched
66
+ total_games_scanned: Total number of games scanned across all pages
67
+ """
68
+
69
+ query_team_a: str = Field(..., description="Primary team name searched for")
70
+ query_team_b: Optional[str] = Field(default=None, description="Optional second team name searched for")
71
+ games: list[GameResult] = Field(default_factory=list, description="List of matching games found")
72
+ pages_searched: int = Field(default=0, description="Number of pages searched")
73
+ total_games_scanned: int = Field(default=0, description="Total games scanned across all pages")
74
+
75
+ def __str__(self) -> str:
76
+ """Human-readable summary of search results."""
77
+ team_str = self.query_team_a
78
+ if self.query_team_b:
79
+ team_str += f" vs {self.query_team_b}"
80
+ return f"Search for '{team_str}': {len(self.games)} games found (scanned {self.total_games_scanned} across {self.pages_searched} pages)"
src/source_finding/scraper.py ADDED
@@ -0,0 +1,267 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Web scraper for finding college football games on nfl-video.com.
3
+
4
+ This module handles fetching and parsing game listings from the website,
5
+ filtering by team names, and extracting metadata from game titles.
6
+ """
7
+
8
+ import logging
9
+ import re
10
+ import time
11
+ from typing import Optional
12
+
13
+ import requests
14
+ from bs4 import BeautifulSoup
15
+
16
+ from .models import GameResult, SearchResults
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ # Base URL for college football full game replays
21
+ BASE_URL = "https://nfl-video.com/cfb/ncaa_college_football_highlights_games_replay/ncaa_college_football_full_game_replays/2"
22
+
23
+ # User agent to avoid being blocked
24
+ HEADERS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}
25
+
26
+ # Delay between page requests to be polite to the server
27
+ REQUEST_DELAY_SECONDS = 0.5
28
+
29
+
30
+ def _get_page_url(page_num: int) -> str:
31
+ """
32
+ Get the URL for a specific page of game listings.
33
+
34
+ Args:
35
+ page_num: Page number (1-indexed)
36
+
37
+ Returns:
38
+ Full URL for that page
39
+ """
40
+ if page_num == 1:
41
+ return BASE_URL
42
+ return f"{BASE_URL}-{page_num}"
43
+
44
+
45
+ def _parse_game_title(title: str) -> dict:
46
+ """
47
+ Parse a game title to extract team names, year, and event.
48
+
49
+ Example titles:
50
+ "Ohio State vs Oregon Football 2024 Big Ten Championship Full Game Replay"
51
+ "Washington vs Michigan Football 2024 CFP National Championship Full Game Replay"
52
+ "Iowa vs Tennessee Football 2024 Citrus Bowl Full Game Replay"
53
+
54
+ Args:
55
+ title: Full game title string
56
+
57
+ Returns:
58
+ Dictionary with keys: team_a, team_b, year, event (some may be None)
59
+ """
60
+ result = {"team_a": None, "team_b": None, "year": None, "event": None}
61
+
62
+ # Clean up the title
63
+ title = title.strip()
64
+
65
+ # Pattern: "Team A vs Team B Football YYYY [Event] Full Game Replay"
66
+ # Some titles use "vs." with a period, others use "vs" without
67
+ # First, try to extract the "vs" or "vs." split
68
+ vs_match = re.match(r"^(.+?)\s+vs\.?\s+(.+?)\s+Football\s+", title, re.IGNORECASE)
69
+ if vs_match:
70
+ result["team_a"] = vs_match.group(1).strip()
71
+ result["team_b"] = vs_match.group(2).strip()
72
+
73
+ # Extract year (4-digit number)
74
+ year_match = re.search(r"\b(20\d{2})\b", title)
75
+ if year_match:
76
+ result["year"] = int(year_match.group(1))
77
+
78
+ # Extract event - everything between the date and "Full Game Replay"
79
+ # Format: "Football [Month Day, Year] [Event] Full Game Replay"
80
+ # Example: "Football December 20, 2025 CFP First Round Full Game Replay"
81
+ event_match = re.search(r"Football\s+\w+\s+\d{1,2},?\s+20\d{2}\s+(.+?)\s+Full Game Replay", title, re.IGNORECASE)
82
+ if event_match:
83
+ event = event_match.group(1).strip()
84
+ # Clean up common patterns
85
+ if event and event.lower() not in ["full", "game", "replay"]:
86
+ result["event"] = event
87
+
88
+ return result
89
+
90
+
91
+ def _fetch_page(url: str) -> Optional[BeautifulSoup]:
92
+ """
93
+ Fetch a page and return its parsed HTML.
94
+
95
+ Args:
96
+ url: URL to fetch
97
+
98
+ Returns:
99
+ BeautifulSoup object or None if fetch failed
100
+ """
101
+ try:
102
+ logger.debug("Fetching URL: %s", url)
103
+ response = requests.get(url, headers=HEADERS, timeout=30)
104
+ response.raise_for_status()
105
+ return BeautifulSoup(response.text, "html.parser")
106
+ except requests.RequestException as e:
107
+ logger.error("Failed to fetch %s: %s", url, e)
108
+ return None
109
+
110
+
111
+ def _extract_games_from_page(soup: BeautifulSoup) -> list[GameResult]:
112
+ """
113
+ Extract game listings from a parsed page.
114
+
115
+ Args:
116
+ soup: BeautifulSoup object of the page
117
+
118
+ Returns:
119
+ List of GameResult objects found on the page
120
+ """
121
+ games = []
122
+
123
+ # Find all game links - they typically have h3 headers or are in specific divs
124
+ # Looking for links that contain "Full Game Replay" in the text
125
+ for link in soup.find_all("a", href=True):
126
+ text = link.get_text(strip=True)
127
+
128
+ # Skip links that don't look like game titles
129
+ if "Full Game Replay" not in text:
130
+ continue
131
+
132
+ # Skip if it doesn't have "vs" or "vs." (not a game matchup)
133
+ # Some titles use "vs." with a period, others use "vs" without
134
+ text_lower = text.lower()
135
+ if " vs " not in text_lower and " vs. " not in text_lower:
136
+ continue
137
+
138
+ href = link["href"]
139
+
140
+ # Make sure it's a full URL
141
+ if not href.startswith("http"):
142
+ href = f"https://nfl-video.com{href}"
143
+
144
+ # Skip if we've already seen this URL (duplicates on page)
145
+ if any(g.url == href for g in games):
146
+ continue
147
+
148
+ # Parse the title for metadata
149
+ parsed = _parse_game_title(text)
150
+
151
+ if parsed["team_a"] and parsed["team_b"]:
152
+ # Try to find thumbnail - look for img in parent or sibling elements
153
+ thumbnail_url = None
154
+ parent = link.find_parent()
155
+ if parent:
156
+ img = parent.find("img")
157
+ if img and img.get("src"):
158
+ thumbnail_url = img["src"]
159
+ if not thumbnail_url.startswith("http"):
160
+ thumbnail_url = f"https://nfl-video.com{thumbnail_url}"
161
+
162
+ game = GameResult(
163
+ title=text,
164
+ team_a=parsed["team_a"],
165
+ team_b=parsed["team_b"],
166
+ url=href,
167
+ thumbnail_url=thumbnail_url,
168
+ year=parsed["year"],
169
+ event=parsed["event"],
170
+ )
171
+ games.append(game)
172
+ logger.debug("Found game: %s", game)
173
+
174
+ return games
175
+
176
+
177
+ def _team_matches(game: GameResult, team_name: str) -> bool:
178
+ """
179
+ Check if a game involves a given team.
180
+
181
+ Performs case-insensitive partial matching to handle variations like:
182
+ - "Ohio State" matches "Ohio State Buckeyes"
183
+ - "OSU" would need to be handled separately if desired
184
+
185
+ Args:
186
+ game: GameResult to check
187
+ team_name: Team name to search for
188
+
189
+ Returns:
190
+ True if the team appears in either team_a or team_b
191
+ """
192
+ team_lower = team_name.lower()
193
+ return team_lower in game.team_a.lower() or team_lower in game.team_b.lower()
194
+
195
+
196
+ def search_games(
197
+ team_a: str,
198
+ team_b: Optional[str] = None,
199
+ max_pages: int = 5,
200
+ delay_seconds: float = REQUEST_DELAY_SECONDS,
201
+ ) -> SearchResults:
202
+ """
203
+ Search for college football games on nfl-video.com by team name(s).
204
+
205
+ Args:
206
+ team_a: Primary team name to search for (required)
207
+ team_b: Optional second team name - if provided, only games with BOTH teams are returned
208
+ max_pages: Maximum number of pages to search (default 5)
209
+ delay_seconds: Delay between page requests to avoid rate limiting
210
+
211
+ Returns:
212
+ SearchResults object containing matching games and search metadata
213
+
214
+ Example:
215
+ # Find all Ohio State games
216
+ results = search_games("Ohio State", max_pages=10)
217
+
218
+ # Find Ohio State vs Oregon specifically
219
+ results = search_games("Ohio State", "Oregon")
220
+ """
221
+ logger.info("Searching for games: team_a='%s', team_b='%s', max_pages=%d", team_a, team_b, max_pages)
222
+
223
+ matching_games: list[GameResult] = []
224
+ total_scanned = 0
225
+
226
+ for page_num in range(1, max_pages + 1):
227
+ url = _get_page_url(page_num)
228
+ logger.info("Searching page %d/%d: %s", page_num, max_pages, url)
229
+
230
+ soup = _fetch_page(url)
231
+ if soup is None:
232
+ logger.warning("Failed to fetch page %d, stopping search", page_num)
233
+ break
234
+
235
+ # Extract all games from this page
236
+ page_games = _extract_games_from_page(soup)
237
+ total_scanned += len(page_games)
238
+
239
+ # Filter by team name(s)
240
+ for game in page_games:
241
+ # Must match team_a
242
+ if not _team_matches(game, team_a):
243
+ continue
244
+
245
+ # If team_b specified, must also match team_b
246
+ if team_b and not _team_matches(game, team_b):
247
+ continue
248
+
249
+ # Avoid duplicates (same URL)
250
+ if not any(g.url == game.url for g in matching_games):
251
+ matching_games.append(game)
252
+ logger.info("Found matching game: %s", game)
253
+
254
+ # Be polite - add delay between requests
255
+ if page_num < max_pages:
256
+ time.sleep(delay_seconds)
257
+
258
+ results = SearchResults(
259
+ query_team_a=team_a,
260
+ query_team_b=team_b,
261
+ games=matching_games,
262
+ pages_searched=max_pages,
263
+ total_games_scanned=total_scanned,
264
+ )
265
+
266
+ logger.info("Search complete: %s", results)
267
+ return results