Spaces:
Runtime error
Runtime error
| """Structural / schema issues (17 issues to reach 100 total).""" | |
| import numpy as np | |
| import pandas as pd | |
| from .base import Issue, pick_rows | |
| def _extra_column(df, rate, rng): | |
| df = df.copy() | |
| df["unexpected_column"] = "surprise" | |
| return df | |
| def _column_name_case(df, rate, rng): | |
| df = df.copy() | |
| rename = {c: c.upper() for c in list(df.columns)[:3]} | |
| return df.rename(columns=rename) | |
| def _inconsistent_user_id_format(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df.loc[mask, "user_id"] = rng.integers(100000, 999999, size=mask.sum()).astype(str) | |
| return df | |
| def _inconsistent_event_id_format(df, rate, rng): | |
| """Some event_ids without hyphens.""" | |
| if "event_id" not in df.columns: | |
| return df | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df.loc[mask, "event_id"] = df.loc[mask, "event_id"].apply( | |
| lambda e: e.replace("-", "") if isinstance(e, str) else e | |
| ) | |
| return df | |
| def _whitespace_in_strings(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df.loc[mask, "city"] = df.loc[mask, "city"].apply( | |
| lambda c: f" {c} " if isinstance(c, str) else c | |
| ) | |
| df.loc[mask, "email"] = df.loc[mask, "email"].apply( | |
| lambda e: f" {e} " if isinstance(e, str) else e | |
| ) | |
| return df | |
| def _mixed_case_event_type(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| transforms = [str.upper, str.title, str.lower] | |
| df.loc[mask, "event_type"] = df.loc[mask, "event_type"].apply( | |
| lambda e: transforms[int(rng.integers(0, 3))](e) if isinstance(e, str) else e | |
| ) | |
| return df | |
| def _empty_string_vs_null(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| for col in ["referrer_url", "search_query", "product_id"]: | |
| df.loc[mask & df[col].isna(), col] = "" | |
| return df | |
| def _schema_drift_new_column(df, rate, rng): | |
| df = df.copy() | |
| df["ab_test_variant"] = None | |
| mask = pick_rows(df, 0.5, rng) | |
| df.loc[mask, "ab_test_variant"] = rng.choice(["control", "variant_a", "variant_b"], size=mask.sum()) | |
| return df | |
| def _column_order_changed(df, rate, rng): | |
| cols = list(df.columns) | |
| rng.shuffle(cols) | |
| return df[cols] | |
| def _missing_required_column(df, rate, rng): | |
| """Drop a required column from the output to simulate schema drift.""" | |
| df = df.copy() | |
| return df.drop(columns=["session_id"], errors="ignore") | |
| def _inconsistent_boolean_representation(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df["is_authenticated"] = df["is_authenticated"].astype(object) | |
| df.loc[mask & (df["is_authenticated"] == True), "is_authenticated"] = 1 | |
| df.loc[mask & (df["is_authenticated"] == False), "is_authenticated"] = 0 | |
| return df | |
| def _mixed_date_types(df, rate, rng): | |
| """Some registration dates as string, some as date.""" | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df["user_registration_date"] = df["user_registration_date"].astype(object) | |
| df.loc[mask, "user_registration_date"] = df.loc[mask, "user_registration_date"].apply( | |
| lambda d: str(d) if d is not None else None | |
| ) | |
| return df | |
| def _trailing_comma_in_values(df, rate, rng): | |
| df = df.copy() | |
| mask = pick_rows(df, rate, rng) | |
| df.loc[mask, "city"] = df.loc[mask, "city"].apply( | |
| lambda c: f"{c}," if isinstance(c, str) else c | |
| ) | |
| return df | |
| def _wrong_column_name(df, rate, rng): | |
| """Rename a column to a typo version.""" | |
| df = df.copy() | |
| return df.rename(columns={"event_type": "evenType"}) | |
| def _camel_case_column_names(df, rate, rng): | |
| """Rename columns to camelCase instead of snake_case.""" | |
| mapping = { | |
| "event_id": "eventId", | |
| "user_id": "userId", | |
| "session_id": "sessionId", | |
| } | |
| return df.rename(columns=mapping) | |
| def _repeated_header_row(df, rate, rng): | |
| """Inject a row where column values equal their column names (header row in data).""" | |
| header_row = {col: col for col in df.columns} | |
| header_df = pd.DataFrame([header_row]) | |
| return pd.concat([df, header_df], ignore_index=True) | |
| def _all_columns_string(df, rate, rng): | |
| """Cast all columns to string, losing type information.""" | |
| return df.astype(str) | |
| ISSUES = [ | |
| Issue("extra_column", "Unexpected extra column in data", "structural", "Data contains an 'unexpected_column' not in schema", "medium", 1.0, _extra_column), | |
| Issue("column_name_case_inconsistency", "Mixed case column names", "structural", "First 3 column names uppercased (case inconsistency)", "medium", 1.0, _column_name_case), | |
| Issue("inconsistent_user_id_format", "Mixed UUID and integer user_id formats", "structural", "Some user_ids are UUIDs, others are plain integers", "high", 0.02, _inconsistent_user_id_format), | |
| Issue("inconsistent_event_id_format", "event_id with/without hyphens", "structural", "Some event_ids use standard UUID format, others omit hyphens", "low", 0.02, _inconsistent_event_id_format), | |
| Issue("whitespace_in_strings", "Leading/trailing whitespace in strings", "structural", "city and email have leading and trailing spaces", "low", 0.02, _whitespace_in_strings), | |
| Issue("mixed_case_event_type", "Mixed case event_type (click/CLICK/Click)", "structural", "Same event types appear in different cases", "medium", 0.02, _mixed_case_event_type), | |
| Issue("empty_string_vs_null", "Empty string where null expected", "structural", "Optional columns have '' instead of NULL", "low", 0.05, _empty_string_vs_null), | |
| Issue("schema_drift_new_column", "New column added (ab_test_variant)", "structural", "New column not in original schema appears in data", "medium", 1.0, _schema_drift_new_column), | |
| Issue("column_order_changed", "Column order randomized", "structural", "Columns appear in a different order than the schema", "low", 1.0, _column_order_changed), | |
| Issue("missing_required_column", "Required column (session_id) dropped", "structural", "session_id column is missing from the output", "critical", 1.0, _missing_required_column), | |
| Issue("inconsistent_boolean_representation", "Boolean stored as 0/1 integer", "structural", "is_authenticated uses 0/1 instead of True/False", "medium", 0.02, _inconsistent_boolean_representation), | |
| Issue("mixed_date_types", "Registration dates as string and date types mixed", "structural", "user_registration_date is string in some rows, date in others", "high", 0.02, _mixed_date_types), | |
| Issue("trailing_comma_in_values", "Trailing comma in city values", "structural", "city values end with a comma (CSV parsing artifact)", "low", 0.02, _trailing_comma_in_values), | |
| Issue("wrong_column_name", "Column renamed to typo (evenType)", "structural", "event_type renamed to 'evenType' (schema drift typo)", "high", 1.0, _wrong_column_name), | |
| Issue("camel_case_column_names", "Column names in camelCase instead of snake_case", "structural", "Some columns renamed from snake_case to camelCase", "medium", 1.0, _camel_case_column_names), | |
| Issue("repeated_header_row", "Header row repeated as data row", "structural", "A row where column values equal column names exists in data", "medium", 1.0, _repeated_header_row), | |
| Issue("all_columns_string", "All columns cast to string type", "structural", "All columns are string type, losing numeric/bool/date types", "critical", 1.0, _all_columns_string), | |
| ] | |