"""Structural / schema issues (17 issues to reach 100 total).""" import numpy as np import pandas as pd from .base import Issue, pick_rows def _extra_column(df, rate, rng): df = df.copy() df["unexpected_column"] = "surprise" return df def _column_name_case(df, rate, rng): df = df.copy() rename = {c: c.upper() for c in list(df.columns)[:3]} return df.rename(columns=rename) def _inconsistent_user_id_format(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) df.loc[mask, "user_id"] = rng.integers(100000, 999999, size=mask.sum()).astype(str) return df def _inconsistent_event_id_format(df, rate, rng): """Some event_ids without hyphens.""" if "event_id" not in df.columns: return df df = df.copy() mask = pick_rows(df, rate, rng) df.loc[mask, "event_id"] = df.loc[mask, "event_id"].apply( lambda e: e.replace("-", "") if isinstance(e, str) else e ) return df def _whitespace_in_strings(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) df.loc[mask, "city"] = df.loc[mask, "city"].apply( lambda c: f" {c} " if isinstance(c, str) else c ) df.loc[mask, "email"] = df.loc[mask, "email"].apply( lambda e: f" {e} " if isinstance(e, str) else e ) return df def _mixed_case_event_type(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) transforms = [str.upper, str.title, str.lower] df.loc[mask, "event_type"] = df.loc[mask, "event_type"].apply( lambda e: transforms[int(rng.integers(0, 3))](e) if isinstance(e, str) else e ) return df def _empty_string_vs_null(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) for col in ["referrer_url", "search_query", "product_id"]: df.loc[mask & df[col].isna(), col] = "" return df def _schema_drift_new_column(df, rate, rng): df = df.copy() df["ab_test_variant"] = None mask = pick_rows(df, 0.5, rng) df.loc[mask, "ab_test_variant"] = rng.choice(["control", "variant_a", "variant_b"], size=mask.sum()) return df def _column_order_changed(df, rate, rng): cols = list(df.columns) rng.shuffle(cols) return df[cols] def _missing_required_column(df, rate, rng): """Drop a required column from the output to simulate schema drift.""" df = df.copy() return df.drop(columns=["session_id"], errors="ignore") def _inconsistent_boolean_representation(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) df["is_authenticated"] = df["is_authenticated"].astype(object) df.loc[mask & (df["is_authenticated"] == True), "is_authenticated"] = 1 df.loc[mask & (df["is_authenticated"] == False), "is_authenticated"] = 0 return df def _mixed_date_types(df, rate, rng): """Some registration dates as string, some as date.""" df = df.copy() mask = pick_rows(df, rate, rng) df["user_registration_date"] = df["user_registration_date"].astype(object) df.loc[mask, "user_registration_date"] = df.loc[mask, "user_registration_date"].apply( lambda d: str(d) if d is not None else None ) return df def _trailing_comma_in_values(df, rate, rng): df = df.copy() mask = pick_rows(df, rate, rng) df.loc[mask, "city"] = df.loc[mask, "city"].apply( lambda c: f"{c}," if isinstance(c, str) else c ) return df def _wrong_column_name(df, rate, rng): """Rename a column to a typo version.""" df = df.copy() return df.rename(columns={"event_type": "evenType"}) def _camel_case_column_names(df, rate, rng): """Rename columns to camelCase instead of snake_case.""" mapping = { "event_id": "eventId", "user_id": "userId", "session_id": "sessionId", } return df.rename(columns=mapping) def _repeated_header_row(df, rate, rng): """Inject a row where column values equal their column names (header row in data).""" header_row = {col: col for col in df.columns} header_df = pd.DataFrame([header_row]) return pd.concat([df, header_df], ignore_index=True) def _all_columns_string(df, rate, rng): """Cast all columns to string, losing type information.""" return df.astype(str) ISSUES = [ Issue("extra_column", "Unexpected extra column in data", "structural", "Data contains an 'unexpected_column' not in schema", "medium", 1.0, _extra_column), Issue("column_name_case_inconsistency", "Mixed case column names", "structural", "First 3 column names uppercased (case inconsistency)", "medium", 1.0, _column_name_case), Issue("inconsistent_user_id_format", "Mixed UUID and integer user_id formats", "structural", "Some user_ids are UUIDs, others are plain integers", "high", 0.02, _inconsistent_user_id_format), Issue("inconsistent_event_id_format", "event_id with/without hyphens", "structural", "Some event_ids use standard UUID format, others omit hyphens", "low", 0.02, _inconsistent_event_id_format), Issue("whitespace_in_strings", "Leading/trailing whitespace in strings", "structural", "city and email have leading and trailing spaces", "low", 0.02, _whitespace_in_strings), Issue("mixed_case_event_type", "Mixed case event_type (click/CLICK/Click)", "structural", "Same event types appear in different cases", "medium", 0.02, _mixed_case_event_type), Issue("empty_string_vs_null", "Empty string where null expected", "structural", "Optional columns have '' instead of NULL", "low", 0.05, _empty_string_vs_null), Issue("schema_drift_new_column", "New column added (ab_test_variant)", "structural", "New column not in original schema appears in data", "medium", 1.0, _schema_drift_new_column), Issue("column_order_changed", "Column order randomized", "structural", "Columns appear in a different order than the schema", "low", 1.0, _column_order_changed), Issue("missing_required_column", "Required column (session_id) dropped", "structural", "session_id column is missing from the output", "critical", 1.0, _missing_required_column), Issue("inconsistent_boolean_representation", "Boolean stored as 0/1 integer", "structural", "is_authenticated uses 0/1 instead of True/False", "medium", 0.02, _inconsistent_boolean_representation), Issue("mixed_date_types", "Registration dates as string and date types mixed", "structural", "user_registration_date is string in some rows, date in others", "high", 0.02, _mixed_date_types), Issue("trailing_comma_in_values", "Trailing comma in city values", "structural", "city values end with a comma (CSV parsing artifact)", "low", 0.02, _trailing_comma_in_values), Issue("wrong_column_name", "Column renamed to typo (evenType)", "structural", "event_type renamed to 'evenType' (schema drift typo)", "high", 1.0, _wrong_column_name), Issue("camel_case_column_names", "Column names in camelCase instead of snake_case", "structural", "Some columns renamed from snake_case to camelCase", "medium", 1.0, _camel_case_column_names), Issue("repeated_header_row", "Header row repeated as data row", "structural", "A row where column values equal column names exists in data", "medium", 1.0, _repeated_header_row), Issue("all_columns_string", "All columns cast to string type", "structural", "All columns are string type, losing numeric/bool/date types", "critical", 1.0, _all_columns_string), ]