"""NULL / missing value issues (10 issues).""" import numpy as np import pandas as pd from .base import Issue, pick_rows def _null_col(col: str, severity: str = "high", rate: float = 0.02) -> Issue: def inject(df, r, rng): df = df.copy() df.loc[pick_rows(df, r, rng), col] = None return df names = { "event_id": ("null_event_id", "Null event_id (primary key)", "critical"), "user_id": ("null_user_id", "Null user_id", "high"), "session_id": ("null_session_id", "Null session_id", "high"), "timestamp": ("null_timestamp", "Null event timestamp", "critical"), "event_type": ("null_event_type", "Null event_type", "high"), "page_url": ("null_page_url", "Null page URL", "medium"), "ip_address": ("null_ip_address", "Null IP address", "medium"), } id_, name, sev = names[col] return Issue(id=id_, name=name, category="nulls", description=name, severity=sev, default_rate=rate, inject=inject) def _null_revenue_purchase(df, rate, rng): df = df.copy() mask = (df["event_type"] == "purchase") & pick_rows(df, rate, rng) df.loc[mask, "revenue"] = None return df def _null_product_purchase(df, rate, rng): df = df.copy() mask = (df["event_type"].isin(["purchase", "add_to_cart", "checkout"])) & pick_rows(df, rate, rng) df.loc[mask, "product_id"] = None return df def _entire_row_null(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) df.loc[mask] = None return df ISSUES = [ _null_col("event_id"), _null_col("user_id"), _null_col("session_id"), _null_col("timestamp"), _null_col("event_type"), _null_col("page_url", severity="medium"), _null_col("ip_address", severity="medium"), Issue( id="null_revenue_for_purchase", name="Null revenue for purchase events", category="nulls", description="Revenue is null on purchase events where it is required", severity="high", default_rate=0.02, inject=_null_revenue_purchase, ), Issue( id="null_product_id_for_purchase", name="Null product_id for purchase/cart events", category="nulls", description="product_id is null on purchase/add_to_cart/checkout events", severity="high", default_rate=0.02, inject=_null_product_purchase, ), Issue( id="entire_row_null", name="Entire row null", category="nulls", description="All columns in a row are set to null", severity="critical", default_rate=0.005, inject=_entire_row_null, ), ]