Spaces:
Runtime error
Runtime error
| """NULL / missing value issues (10 issues).""" | |
| import numpy as np | |
| import pandas as pd | |
| from .base import Issue, pick_rows | |
| def _null_col(col: str, severity: str = "high", rate: float = 0.02) -> Issue: | |
| def inject(df, r, rng): | |
| df = df.copy() | |
| df.loc[pick_rows(df, r, rng), col] = None | |
| return df | |
| names = { | |
| "event_id": ("null_event_id", "Null event_id (primary key)", "critical"), | |
| "user_id": ("null_user_id", "Null user_id", "high"), | |
| "session_id": ("null_session_id", "Null session_id", "high"), | |
| "timestamp": ("null_timestamp", "Null event timestamp", "critical"), | |
| "event_type": ("null_event_type", "Null event_type", "high"), | |
| "page_url": ("null_page_url", "Null page URL", "medium"), | |
| "ip_address": ("null_ip_address", "Null IP address", "medium"), | |
| } | |
| id_, name, sev = names[col] | |
| return Issue(id=id_, name=name, category="nulls", description=name, | |
| severity=sev, default_rate=rate, inject=inject) | |
| def _null_revenue_purchase(df, rate, rng): | |
| df = df.copy() | |
| mask = (df["event_type"] == "purchase") & pick_rows(df, rate, rng) | |
| df.loc[mask, "revenue"] = None | |
| return df | |
| def _null_product_purchase(df, rate, rng): | |
| df = df.copy() | |
| mask = (df["event_type"].isin(["purchase", "add_to_cart", "checkout"])) & pick_rows(df, rate, rng) | |
| df.loc[mask, "product_id"] = None | |
| return df | |
| def _entire_row_null(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df.loc[mask] = None | |
| return df | |
| ISSUES = [ | |
| _null_col("event_id"), | |
| _null_col("user_id"), | |
| _null_col("session_id"), | |
| _null_col("timestamp"), | |
| _null_col("event_type"), | |
| _null_col("page_url", severity="medium"), | |
| _null_col("ip_address", severity="medium"), | |
| Issue( | |
| id="null_revenue_for_purchase", | |
| name="Null revenue for purchase events", | |
| category="nulls", | |
| description="Revenue is null on purchase events where it is required", | |
| severity="high", | |
| default_rate=0.02, | |
| inject=_null_revenue_purchase, | |
| ), | |
| Issue( | |
| id="null_product_id_for_purchase", | |
| name="Null product_id for purchase/cart events", | |
| category="nulls", | |
| description="product_id is null on purchase/add_to_cart/checkout events", | |
| severity="high", | |
| default_rate=0.02, | |
| inject=_null_product_purchase, | |
| ), | |
| Issue( | |
| id="entire_row_null", | |
| name="Entire row null", | |
| category="nulls", | |
| description="All columns in a row are set to null", | |
| severity="critical", | |
| default_rate=0.005, | |
| inject=_entire_row_null, | |
| ), | |
| ] | |