Upload files from src

src/infer.py

@@ -0,0 +1,92 @@
+"""Inference utilities for salary prediction."""
+
+import pickle
+from pathlib import Path
+
+import pandas as pd
+import yaml
+
+from src.schema import SalaryInput
+from src.preprocessing import prepare_features
+
+# Load model and artifacts at module level
+model_path = Path("models/model.pkl")
+
+if not model_path.exists():
+    raise FileNotFoundError(
+        f"Model file not found at {model_path}. Please run 'python -m src.train' first."
+    )
+
+with open(model_path, "rb") as f:
+    artifacts = pickle.load(f)
+    model = artifacts["model"]
+    feature_columns = artifacts["feature_columns"]
+
+# Load valid categories for input validation
+valid_categories_path = Path("config/valid_categories.yaml")
+
+if not valid_categories_path.exists():
+    raise FileNotFoundError(
+        f"Valid categories file not found at {valid_categories_path}. Please run 'python -m src.train' first."
+    )
+
+with open(valid_categories_path, "r") as f:
+    valid_categories = yaml.safe_load(f)
+
+
+def predict_salary(data: SalaryInput) -> float:
+    """Predict salary based on input features.
+
+    Args:
+        data: SalaryInput model with developer information
+
+    Returns:
+        Predicted annual salary in USD
+
+    Raises:
+        ValueError: If country or education_level is not in valid categories
+    """
+    # Validate input against valid categories from training
+    if data.country not in valid_categories["Country"]:
+        raise ValueError(
+            f"Invalid country: '{data.country}'. "
+            f"Must be one of {len(valid_categories['Country'])} valid countries. "
+            f"Check config/valid_categories.yaml for all valid values."
+        )
+
+    if data.education_level not in valid_categories["EdLevel"]:
+        raise ValueError(
+            f"Invalid education level: '{data.education_level}'. "
+            f"Must be one of {len(valid_categories['EdLevel'])} valid education levels. "
+            f"Check config/valid_categories.yaml for all valid values."
+        )
+
+    if data.dev_type not in valid_categories["DevType"]:
+        raise ValueError(
+            f"Invalid developer type: '{data.dev_type}'. "
+            f"Must be one of {len(valid_categories['DevType'])} valid developer types. "
+            f"Check config/valid_categories.yaml for all valid values."
+        )
+
+    # Create a DataFrame with the input data
+    input_df = pd.DataFrame(
+        {
+            "Country": [data.country],
+            "YearsCodePro": [data.years_code_pro],
+            "EdLevel": [data.education_level],
+            "DevType": [data.dev_type],
+        }
+    )
+
+    # Apply the same preprocessing as training
+    input_encoded = prepare_features(input_df)
+
+    # Ensure all feature columns from training are present and in correct order
+    # Use reindex to add missing columns with 0s and reorder in one operation
+    input_encoded = input_encoded.reindex(columns=feature_columns, fill_value=0)
+
+    # Make prediction
+    prediction = model.predict(input_encoded)[0]
+
+    # Ensure non-negative salary
+    return max(0.0, float(prediction))

src/preprocessing.py

@@ -0,0 +1,108 @@
+"""Data preprocessing utilities for consistent feature engineering."""
+
+from pathlib import Path
+import pandas as pd
+import yaml
+
+# Load configuration once at module level
+_config_path = Path("config/model_parameters.yaml")
+with open(_config_path, "r") as f:
+    _config = yaml.safe_load(f)
+
+
+def reduce_cardinality(
+    series: pd.Series,
+    max_categories: int = None,
+    min_frequency: int = None
+) -> pd.Series:
+    """
+    Reduce cardinality by grouping rare categories into 'Other'.
+
+    Args:
+        series: Pandas Series with categorical values
+        max_categories: Maximum number of categories to keep
+                       (default: from config)
+        min_frequency: Minimum occurrences for a category to be kept
+                      (default: from config)
+
+    Returns:
+        Series with rare categories replaced by 'Other'
+    """
+    # Use config defaults if not provided
+    if max_categories is None:
+        max_categories = _config['features']['cardinality']['max_categories']
+    if min_frequency is None:
+        min_frequency = _config['features']['cardinality']['min_frequency']
+
+    # Count value frequencies
+    value_counts = series.value_counts()
+
+    # Keep only categories that meet both criteria:
+    # 1. In top max_categories by frequency
+    # 2. Have at least min_frequency occurrences
+    top_categories = value_counts.head(max_categories)
+    kept_categories = top_categories[top_categories >= min_frequency].index.tolist()
+
+    # Replace rare categories with 'Other'
+    return series.apply(lambda x: x if x in kept_categories else 'Other')
+
+
+def prepare_features(df: pd.DataFrame) -> pd.DataFrame:
+    """
+    Apply consistent feature transformations for both training and inference.
+
+    This function ensures that the same preprocessing steps are applied
+    during training and inference, preventing data leakage and inconsistencies.
+
+    Args:
+        df: DataFrame with columns: Country, YearsCode (or YearsCodePro), EdLevel, DevType
+            NOTE: During training, cardinality reduction should be applied to df
+            BEFORE calling this function. During inference, valid_categories.yaml
+            ensures only valid (already-reduced) categories are used.
+
+    Returns:
+        DataFrame with one-hot encoded features ready for model input
+
+    Note:
+        - Fills missing values with defaults (0 for numeric, "Unknown" for categorical)
+        - Normalizes Unicode apostrophes to regular apostrophes
+        - Applies one-hot encoding with drop_first=True to avoid multicollinearity
+        - Column names in output will be like: YearsCode, Country_X, EdLevel_Y, DevType_Z
+        - Does NOT apply cardinality reduction (must be done before calling this)
+    """
+    # Create a copy to avoid modifying the original
+    df_processed = df.copy()
+
+    # Normalize Unicode apostrophes to regular apostrophes for consistency
+    # This handles cases where data has \u2019 (') instead of '
+    for col in ["Country", "EdLevel", "DevType"]:
+        if col in df_processed.columns:
+            df_processed[col] = df_processed[col].str.replace('\u2019', "'", regex=False)
+
+    # Handle column name variations (YearsCode vs YearsCodePro)
+    if "YearsCodePro" in df_processed.columns and "YearsCode" not in df_processed.columns:
+        df_processed["YearsCode"] = df_processed["YearsCodePro"]
+
+    # Fill missing values with defaults
+    df_processed["YearsCode"] = df_processed["YearsCode"].fillna(0)
+    df_processed["Country"] = df_processed["Country"].fillna("Unknown")
+    df_processed["EdLevel"] = df_processed["EdLevel"].fillna("Unknown")
+    df_processed["DevType"] = df_processed["DevType"].fillna("Unknown")
+
+    # NOTE: Cardinality reduction is NOT applied here
+    # It should be applied during training BEFORE calling this function
+    # During inference, valid_categories.yaml ensures only valid values are used
+
+    # Select only the features we need
+    feature_cols = ["Country", "YearsCode", "EdLevel", "DevType"]
+    df_features = df_processed[feature_cols]
+
+    # Apply one-hot encoding for categorical variables
+    # For inference (single rows), we need drop_first=False to create columns
+    # The reindex in infer.py will align with training columns
+    # For training (many rows), we use the config value
+    is_inference = len(df_features) == 1
+    drop_first = False if is_inference else _config['features']['encoding']['drop_first']
+    df_encoded = pd.get_dummies(df_features, drop_first=drop_first)
+
+    return df_encoded

src/schema.py

@@ -0,0 +1,26 @@
+"""Pydantic models for input validation."""
+
+from pydantic import BaseModel, Field
+
+
+class SalaryInput(BaseModel):
+    """Input model for salary prediction."""
+
+    country: str = Field(..., description="Developer's country")
+    years_code_pro: float = Field(
+        ..., ge=0, description="Years of professional coding experience"
+    )
+    education_level: str = Field(..., description="Education level")
+    dev_type: str = Field(..., description="Developer type")
+
+    class Config:
+        """Pydantic configuration."""
+
+        json_schema_extra = {
+            "example": {
+                "country": "United States",
+                "years_code_pro": 5.0,
+                "education_level": "Bachelor's degree",
+                "dev_type": "Developer, back-end",
+            }
+        }

src/streamlit_app.py

@@ -1,3 +1,5 @@
+"""Streamlit web app for salary prediction."""
+
 import streamlit as st
 
 from src.infer import predict_salary, valid_categories
@@ -124,4 +126,4 @@ if st.button("🔮 Predict Salary", type="primary", use_container_width=True):
 st.divider()
 st.caption(
     "Built with Streamlit • Data from Stack Overflow Developer Survey • Model: XGBoost"
-)
+)

src/train.py

@@ -0,0 +1,225 @@
+"""Training script for salary prediction model."""
+
+import pickle
+from pathlib import Path
+
+import pandas as pd
+import numpy as np
+import yaml
+from xgboost import XGBRegressor
+from sklearn.model_selection import train_test_split
+
+from src.preprocessing import prepare_features, reduce_cardinality
+
+
+def main():
+    """Train and save the salary prediction model."""
+    # Load configuration
+    print("Loading configuration...")
+    config_path = Path("config/model_parameters.yaml")
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+
+    print("Loading data...")
+    data_path = Path("data/survey_results_public.csv")
+
+    if not data_path.exists():
+        print(f"Error: Data file not found at {data_path}")
+        print("Please download the Stack Overflow Developer Survey CSV and place it in the data/ directory.")
+        print("Download from: https://insights.stackoverflow.com/survey")
+        return
+
+    # Load only required columns to save memory
+    df = pd.read_csv(
+        data_path,
+        usecols=["Country", "YearsCode", "EdLevel", "DevType", "ConvertedCompYearly"],
+    )
+
+    print(f"Loaded {len(df):,} rows")
+
+    print("Removing null, extremely small and large reported salaries")
+    # select main label
+    main_label = "ConvertedCompYearly"
+    # select records with main label more than min_salary threshold
+    min_salary = config['data']['min_salary']
+    df = df[df[main_label] > min_salary]
+    # further exclude outliers based on percentile bounds
+    lower_pct = config['data']['lower_percentile']
+    upper_pct = config['data']['upper_percentile']
+    P = np.percentile(df[main_label], [lower_pct, upper_pct])
+    df = df[(df[main_label] > P[0]) & (df[main_label] < P[1])]
+
+    print(df.shape)
+
+    # Drop rows with missing target
+    df = df.dropna(subset=[main_label])
+    print(f"After removing missing targets: {len(df):,} rows")
+
+    # Apply preprocessing first to get cardinality-reduced categories
+    df_copy = df.copy()
+
+    # Normalize Unicode apostrophes to regular apostrophes for consistency
+    df_copy["Country"] = df_copy["Country"].str.replace('\u2019', "'", regex=False)
+    df_copy["EdLevel"] = df_copy["EdLevel"].str.replace('\u2019', "'", regex=False)
+    df_copy["DevType"] = df_copy["DevType"].str.replace('\u2019', "'", regex=False)
+
+    # Apply cardinality reduction
+    df_copy["Country"] = reduce_cardinality(df_copy["Country"])
+    df_copy["EdLevel"] = reduce_cardinality(df_copy["EdLevel"])
+    df_copy["DevType"] = reduce_cardinality(df_copy["DevType"])
+
+    # Apply cardinality reduction to the actual training data as well
+    # (prepare_features no longer does this internally)
+    df["Country"] = reduce_cardinality(df["Country"])
+    df["EdLevel"] = reduce_cardinality(df["EdLevel"])
+    df["DevType"] = reduce_cardinality(df["DevType"])
+
+    # Now apply full feature transformations for model training
+    X = prepare_features(df)
+    y = df[main_label]
+
+    # Save valid categories after cardinality reduction for validation during inference
+    # Extract unique values from the reduced dataframe
+    country_values = df_copy["Country"].dropna().unique().tolist()
+    edlevel_values = df_copy["EdLevel"].dropna().unique().tolist()
+    devtype_values = df_copy["DevType"].dropna().unique().tolist()
+
+    valid_categories = {
+        "Country": sorted(country_values),
+        "EdLevel": sorted(edlevel_values),
+        "DevType": sorted(devtype_values),
+    }
+
+    valid_categories_path = Path("config/valid_categories.yaml")
+    with open(valid_categories_path, "w") as f:
+        yaml.dump(valid_categories, f, default_flow_style=False, sort_keys=False)
+
+    print(f"\nSaved {len(valid_categories['Country'])} valid countries, {len(valid_categories['EdLevel'])} valid education levels, and {len(valid_categories['DevType'])} valid developer types to {valid_categories_path}")
+
+    print(f"\nFeature matrix shape: {X.shape}")
+    print(f"Total features: {X.shape[1]}")
+
+    # Display feature information for debugging and inference comparison
+    print("\n" + "=" * 60)
+    print("FEATURE ANALYSIS (for comparing with inference)")
+    print("=" * 60)
+
+    # Show top countries in the dataset
+    print("\n📍 Top 10 Countries:")
+    top_countries = df["Country"].value_counts().head(10)
+    for country, count in top_countries.items():
+        print(f"  - {country}: {count:,} ({count/len(df)*100:.1f}%)")
+
+    # Show top education levels
+    print("\n🎓 Top Education Levels:")
+    top_edu = df["EdLevel"].value_counts().head(10)
+    for edu, count in top_edu.items():
+        print(f"  - {edu}: {count:,} ({count/len(df)*100:.1f}%)")
+
+    # Show top developer types
+    print("\n👨‍💻 Top Developer Types:")
+    top_devtype = df["DevType"].value_counts().head(10)
+    for devtype, count in top_devtype.items():
+        print(f"  - {devtype}: {count:,} ({count/len(df)*100:.1f}%)")
+
+    # Show YearsCode statistics
+    print("\n💼 Years of Coding Experience:")
+    print(f"  - Min: {df['YearsCode'].min():.1f}")
+    print(f"  - Max: {df['YearsCode'].max():.1f}")
+    print(f"  - Mean: {df['YearsCode'].mean():.1f}")
+    print(f"  - Median: {df['YearsCode'].median():.1f}")
+    print(f"  - 25th percentile: {df['YearsCode'].quantile(0.25):.1f}")
+    print(f"  - 75th percentile: {df['YearsCode'].quantile(0.75):.1f}")
+
+    # Show most common one-hot encoded features (by frequency)
+    # Separate analysis for each categorical feature
+
+    # Calculate feature frequencies (sum of each column for one-hot encoded)
+    feature_counts = X.sum().sort_values(ascending=False)
+
+    # Exclude numeric features (YearsCode)
+    categorical_features = feature_counts[~feature_counts.index.str.startswith('YearsCode')]
+
+    # Country features
+    print("\n🌍 Top 15 Country Features (most common):")
+    country_features = categorical_features[categorical_features.index.str.startswith('Country_')]
+    for i, (feature, count) in enumerate(country_features.head(15).items(), 1):
+        percentage = (count / len(X)) * 100
+        country_name = feature.replace('Country_', '')
+        print(f"  {i:2d}. {country_name:45s} - {count:6.0f} occurrences ({percentage:5.1f}%)")
+
+    # Education level features
+    print("\n🎓 Top 10 Education Level Features (most common):")
+    edlevel_features = categorical_features[categorical_features.index.str.startswith('EdLevel_')]
+    for i, (feature, count) in enumerate(edlevel_features.head(10).items(), 1):
+        percentage = (count / len(X)) * 100
+        edu_name = feature.replace('EdLevel_', '')
+        print(f"  {i:2d}. {edu_name:45s} - {count:6.0f} occurrences ({percentage:5.1f}%)")
+
+    # Developer type features
+    print("\n👨‍💻 Top 10 Developer Type Features (most common):")
+    devtype_features = categorical_features[categorical_features.index.str.startswith('DevType_')]
+    for i, (feature, count) in enumerate(devtype_features.head(10).items(), 1):
+        percentage = (count / len(X)) * 100
+        devtype_name = feature.replace('DevType_', '')
+        print(f"  {i:2d}. {devtype_name:45s} - {count:6.0f} occurrences ({percentage:5.1f}%)")
+
+    print(f"\n📊 Total one-hot encoded features: {len(X.columns)}")
+    print("   - Numeric: 1 (YearsCode)")
+    print(f"   - Country: {len(country_features)}")
+    print(f"   - Education: {len(edlevel_features)}")
+    print(f"   - DevType: {len(devtype_features)}")
+
+    print("=" * 60 + "\n")
+
+    # Split data
+    X_train, X_test, y_train, y_test = train_test_split(
+        X, y,
+        test_size=config['data']['test_size'],
+        random_state=config['data']['random_state']
+    )
+
+    # Train model
+    print("Training XGBoost model...")
+    model_config = config['model']
+    model = XGBRegressor(
+        n_estimators=model_config['n_estimators'],
+        learning_rate=model_config['learning_rate'],
+        max_depth=model_config['max_depth'],
+        min_child_weight=model_config['min_child_weight'],
+        random_state=model_config['random_state'],
+        n_jobs=model_config['n_jobs'],
+        early_stopping_rounds=model_config['early_stopping_rounds'],
+    )
+    model.fit(
+        X_train,
+        y_train,
+        eval_set=[(X_test, y_test)],
+        verbose=config['training']['verbose'],
+    )
+
+    print(f"Best iteration: {model.best_iteration + 1} (early stopping at {model.n_estimators} max)")
+
+    # Evaluate
+    train_score = model.score(X_train, y_train)
+    test_score = model.score(X_test, y_test)
+    print(f"Training R2 score: {train_score:.4f}")
+    print(f"Test R2 score: {test_score:.4f}")
+
+    # Save model and feature columns for inference
+    model_path = Path(config['training']['model_path'])
+    model_path.parent.mkdir(parents=True, exist_ok=True)  # Ensure directory exists
+
+    artifacts = {
+        "model": model,
+        "feature_columns": list(X.columns),
+    }
+
+    with open(model_path, "wb") as f:
+        pickle.dump(artifacts, f)
+
+    print(f"Model saved to {model_path}")
+
+
+if __name__ == "__main__":
+    main()