Spaces:
Runtime error
Runtime error
File size: 7,367 Bytes
4815c43 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | """Structural / schema issues (17 issues to reach 100 total)."""
import numpy as np
import pandas as pd
from .base import Issue, pick_rows
def _extra_column(df, rate, rng):
df = df.copy()
df["unexpected_column"] = "surprise"
return df
def _column_name_case(df, rate, rng):
df = df.copy()
rename = {c: c.upper() for c in list(df.columns)[:3]}
return df.rename(columns=rename)
def _inconsistent_user_id_format(df, rate, rng):
df = df.copy()
mask = pick_rows(df, rate, rng)
df.loc[mask, "user_id"] = rng.integers(100000, 999999, size=mask.sum()).astype(str)
return df
def _inconsistent_event_id_format(df, rate, rng):
"""Some event_ids without hyphens."""
if "event_id" not in df.columns:
return df
df = df.copy()
mask = pick_rows(df, rate, rng)
df.loc[mask, "event_id"] = df.loc[mask, "event_id"].apply(
lambda e: e.replace("-", "") if isinstance(e, str) else e
)
return df
def _whitespace_in_strings(df, rate, rng):
df = df.copy()
mask = pick_rows(df, rate, rng)
df.loc[mask, "city"] = df.loc[mask, "city"].apply(
lambda c: f" {c} " if isinstance(c, str) else c
)
df.loc[mask, "email"] = df.loc[mask, "email"].apply(
lambda e: f" {e} " if isinstance(e, str) else e
)
return df
def _mixed_case_event_type(df, rate, rng):
df = df.copy()
mask = pick_rows(df, rate, rng)
transforms = [str.upper, str.title, str.lower]
df.loc[mask, "event_type"] = df.loc[mask, "event_type"].apply(
lambda e: transforms[int(rng.integers(0, 3))](e) if isinstance(e, str) else e
)
return df
def _empty_string_vs_null(df, rate, rng):
df = df.copy()
mask = pick_rows(df, rate, rng)
for col in ["referrer_url", "search_query", "product_id"]:
df.loc[mask & df[col].isna(), col] = ""
return df
def _schema_drift_new_column(df, rate, rng):
df = df.copy()
df["ab_test_variant"] = None
mask = pick_rows(df, 0.5, rng)
df.loc[mask, "ab_test_variant"] = rng.choice(["control", "variant_a", "variant_b"], size=mask.sum())
return df
def _column_order_changed(df, rate, rng):
cols = list(df.columns)
rng.shuffle(cols)
return df[cols]
def _missing_required_column(df, rate, rng):
"""Drop a required column from the output to simulate schema drift."""
df = df.copy()
return df.drop(columns=["session_id"], errors="ignore")
def _inconsistent_boolean_representation(df, rate, rng):
df = df.copy()
mask = pick_rows(df, rate, rng)
df["is_authenticated"] = df["is_authenticated"].astype(object)
df.loc[mask & (df["is_authenticated"] == True), "is_authenticated"] = 1
df.loc[mask & (df["is_authenticated"] == False), "is_authenticated"] = 0
return df
def _mixed_date_types(df, rate, rng):
"""Some registration dates as string, some as date."""
df = df.copy()
mask = pick_rows(df, rate, rng)
df["user_registration_date"] = df["user_registration_date"].astype(object)
df.loc[mask, "user_registration_date"] = df.loc[mask, "user_registration_date"].apply(
lambda d: str(d) if d is not None else None
)
return df
def _trailing_comma_in_values(df, rate, rng):
df = df.copy()
mask = pick_rows(df, rate, rng)
df.loc[mask, "city"] = df.loc[mask, "city"].apply(
lambda c: f"{c}," if isinstance(c, str) else c
)
return df
def _wrong_column_name(df, rate, rng):
"""Rename a column to a typo version."""
df = df.copy()
return df.rename(columns={"event_type": "evenType"})
def _camel_case_column_names(df, rate, rng):
"""Rename columns to camelCase instead of snake_case."""
mapping = {
"event_id": "eventId",
"user_id": "userId",
"session_id": "sessionId",
}
return df.rename(columns=mapping)
def _repeated_header_row(df, rate, rng):
"""Inject a row where column values equal their column names (header row in data)."""
header_row = {col: col for col in df.columns}
header_df = pd.DataFrame([header_row])
return pd.concat([df, header_df], ignore_index=True)
def _all_columns_string(df, rate, rng):
"""Cast all columns to string, losing type information."""
return df.astype(str)
ISSUES = [
Issue("extra_column", "Unexpected extra column in data", "structural", "Data contains an 'unexpected_column' not in schema", "medium", 1.0, _extra_column),
Issue("column_name_case_inconsistency", "Mixed case column names", "structural", "First 3 column names uppercased (case inconsistency)", "medium", 1.0, _column_name_case),
Issue("inconsistent_user_id_format", "Mixed UUID and integer user_id formats", "structural", "Some user_ids are UUIDs, others are plain integers", "high", 0.02, _inconsistent_user_id_format),
Issue("inconsistent_event_id_format", "event_id with/without hyphens", "structural", "Some event_ids use standard UUID format, others omit hyphens", "low", 0.02, _inconsistent_event_id_format),
Issue("whitespace_in_strings", "Leading/trailing whitespace in strings", "structural", "city and email have leading and trailing spaces", "low", 0.02, _whitespace_in_strings),
Issue("mixed_case_event_type", "Mixed case event_type (click/CLICK/Click)", "structural", "Same event types appear in different cases", "medium", 0.02, _mixed_case_event_type),
Issue("empty_string_vs_null", "Empty string where null expected", "structural", "Optional columns have '' instead of NULL", "low", 0.05, _empty_string_vs_null),
Issue("schema_drift_new_column", "New column added (ab_test_variant)", "structural", "New column not in original schema appears in data", "medium", 1.0, _schema_drift_new_column),
Issue("column_order_changed", "Column order randomized", "structural", "Columns appear in a different order than the schema", "low", 1.0, _column_order_changed),
Issue("missing_required_column", "Required column (session_id) dropped", "structural", "session_id column is missing from the output", "critical", 1.0, _missing_required_column),
Issue("inconsistent_boolean_representation", "Boolean stored as 0/1 integer", "structural", "is_authenticated uses 0/1 instead of True/False", "medium", 0.02, _inconsistent_boolean_representation),
Issue("mixed_date_types", "Registration dates as string and date types mixed", "structural", "user_registration_date is string in some rows, date in others", "high", 0.02, _mixed_date_types),
Issue("trailing_comma_in_values", "Trailing comma in city values", "structural", "city values end with a comma (CSV parsing artifact)", "low", 0.02, _trailing_comma_in_values),
Issue("wrong_column_name", "Column renamed to typo (evenType)", "structural", "event_type renamed to 'evenType' (schema drift typo)", "high", 1.0, _wrong_column_name),
Issue("camel_case_column_names", "Column names in camelCase instead of snake_case", "structural", "Some columns renamed from snake_case to camelCase", "medium", 1.0, _camel_case_column_names),
Issue("repeated_header_row", "Header row repeated as data row", "structural", "A row where column values equal column names exists in data", "medium", 1.0, _repeated_header_row),
Issue("all_columns_string", "All columns cast to string type", "structural", "All columns are string type, losing numeric/bool/date types", "critical", 1.0, _all_columns_string),
]
|