scripts/fetch_questions.py

#!/usr/bin/env python3
"""
Script to fetch all questions from the API and store them in JSON format.
The questions are organized based on their structure from the API response.
"""

import json
import requests
from datetime import datetime
from pathlib import Path

# API Configuration
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space"
QUESTIONS_ENDPOINT = "/questions"


def fetch_questions():
    """
    Fetch all questions from the API endpoint.
    
    Returns:
        list: List of question objects from the API
    """
    url = f"{DEFAULT_API_URL}{QUESTIONS_ENDPOINT}"
    
    print(f"Fetching questions from: {url}")
    
    try:
        response = requests.get(url, timeout=15)
        response.raise_for_status()
        
        questions_data = response.json()
        
        if not questions_data:
            print("Warning: Fetched questions list is empty.")
            return []
        
        print(f"Successfully fetched {len(questions_data)} questions.")
        return questions_data
    
    except requests.exceptions.RequestException as e:
        print(f"Error fetching questions: {e}")
        return []
    except requests.exceptions.JSONDecodeError as e:
        print(f"Error decoding JSON response: {e}")
        return []
    except Exception as e:
        print(f"An unexpected error occurred: {e}")
        return []


def save_questions_to_json(questions_data, output_dir="data"):
    """
    Save questions to a JSON file with timestamp.
    
    Args:
        questions_data (list): List of question objects
        output_dir (str): Directory to save the JSON file
    """
    # Create output directory if it doesn't exist
    output_path = Path(__file__).parent.parent / output_dir
    output_path.mkdir(exist_ok=True)
    
    # Generate filename with timestamp
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filename = f"questions_{timestamp}.json"
    filepath = output_path / filename
    
    # Prepare data structure
    output_data = {
        "metadata": {
            "fetch_timestamp": datetime.now().isoformat(),
            "api_url": DEFAULT_API_URL,
            "total_questions": len(questions_data)
        },
        "questions": questions_data
    }
    
    # Save to JSON file
    try:
        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(output_data, f, indent=2, ensure_ascii=False)
        
        print(f"\n✅ Questions saved successfully to: {filepath}")
        print(f"   Total questions: {len(questions_data)}")
        
        # Also save a latest version without timestamp
        latest_filepath = output_path / "questions_latest.json"
        with open(latest_filepath, 'w', encoding='utf-8') as f:
            json.dump(output_data, f, indent=2, ensure_ascii=False)
        print(f"   Latest version saved to: {latest_filepath}")
        
        return filepath
    
    except Exception as e:
        print(f"Error saving questions to file: {e}")
        return None


def display_sample_questions(questions_data, num_samples=3):
    """
    Display a sample of questions for verification.
    
    Args:
        questions_data (list): List of question objects
        num_samples (int): Number of sample questions to display
    """
    if not questions_data:
        print("No questions to display.")
        return
    
    print(f"\n{'='*60}")
    print(f"Sample Questions (showing {min(num_samples, len(questions_data))} of {len(questions_data)})")
    print(f"{'='*60}\n")
    
    for i, question in enumerate(questions_data[:num_samples], 1):
        print(f"Question {i}:")
        print(f"  Task ID: {question.get('task_id', 'N/A')}")
        print(f"  Question: {question.get('question', 'N/A')[:100]}...")
        
        # Display any additional properties
        other_props = {k: v for k, v in question.items() if k not in ['task_id', 'question']}
        if other_props:
            print(f"  Additional Properties: {other_props}")
        print()


def main():
    """
    Main function to fetch and save questions.
    """
    print("="*60)
    print("Question Fetcher Script")
    print("="*60)
    
    # Fetch questions
    questions_data = fetch_questions()
    
    if not questions_data:
        print("\n❌ No questions were fetched. Exiting.")
        return
    
    # Display sample questions
    display_sample_questions(questions_data)
    
    # Save to JSON
    saved_file = save_questions_to_json(questions_data)
    
    if saved_file:
        print(f"\n{'='*60}")
        print("✅ Process completed successfully!")
        print(f"{'='*60}")
    else:
        print("\n❌ Failed to save questions.")


if __name__ == "__main__":
    main()