predictive-maintenance-pipeline/scripts/03_verify_data.py

#!/usr/bin/env python3
"""Verify data on Hugging Face"""
import os
import logging
import pandas as pd
from huggingface_hub import hf_hub_download

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

HF_TOKEN = os.getenv("HF_TOKEN")
HF_USERNAME = os.getenv("HF_USERNAME", "SharleyK")
DATASET_NAME = os.getenv("DATASET_NAME", "PredictiveMaintenance")
repo_id = f"{HF_USERNAME}/{DATASET_NAME}"

logger.info(f"Verifying dataset: {repo_id}")

downloaded_file = hf_hub_download(
    repo_id=repo_id,
    repo_type="dataset",
    filename="engine_data.csv",
    token=HF_TOKEN)

df = pd.read_csv(downloaded_file)
logger.info(f"✓ Dataset shape: {df.shape}")
logger.info(f"✓ Columns: {list(df.columns)}")
logger.info("✓ Data verification completed!")