import streamlit as st
import time
def go_to(page_name, from_callback=False):
"""
Updates the session_state page and optionally triggers a rerun.
- from_callback=True if called inside an on_click callback
"""
st.session_state.page = page_name
if not from_callback:
st.rerun()
st.stop()
def add_navigation(previous_page, next_page):
col1, col2, col3 = st.columns([1, 4, 1])
if previous_page is not None:
with col1:
if st.button("Previous"):
go_to(previous_page)
if next_page is not None:
with col3:
if st.button("Next"):
go_to(next_page)
st.markdown("---")
def add_fadein_text(paragraphs):
# Define CSS fade-in animation
st.markdown("""
""", unsafe_allow_html=True)
# Create one placeholder for each paragraph
placeholders = [st.empty() for _ in paragraphs]
# Reveal paragraphs sequentially
for i, p in enumerate(paragraphs):
placeholders[i].markdown(f"{p[0]}
", unsafe_allow_html=True)
time.sleep(p[1]) # delay (seconds) before showing next one
# Define pipeline stages
pipeline_data = {
"Data Collection": {
"Data Sources": {
"Identify public datasets": ["Where will you find them?", "Update frequency?", "Licensing constraints?"],
"Acquire proprietary data": ["Who owns it?", "Access method?", "Cost/contract terms?"],
"Integrate APIs": ["Which APIs?", "Rate limits?", "Auth method?"],
"Crowdsourced collection": ["Which platform?", "Quality control?", "Incentive model?"],
"Sensor/IoT data gathering": ["What hardware?", "Sampling rate?", "Data format?"],
},
"Data Licensing & Permissions": {
"Check copyright status": ["Is it copyrighted?", "Fair use applicable?", "Geographic limits?"],
"Review usage terms": ["Commercial use allowed?", "Redistribution permitted?", "Attribution required?"],
"Obtain licenses": ["Cost and renewal?", "Scope of use?", "Termination clauses?"],
"NDA agreements": ["Parties and duration?", "Scope of confidentiality?", "Breach penalties?"],
"Open data validation": ["Truly open?", "Source reliability?", "Ethical concerns?"],
},
"Data Quality Checks": {
"Missing value detection": ["% missing?", "MCAR/MAR/MNAR?", "Critical fields affected?"],
"Duplicate detection": ["Exact vs fuzzy?", "Dedup strategy?", "Impact on metrics?"],
"Noise assessment": ["Noise sources?", "Filtering options?", "Tolerance thresholds?"],
"Format consistency": ["Types and units consistent?", "Datetime/encoding issues?", "Schema validation?"],
"Data freshness review": ["Last update?", "Desired recency?", "Auto-refresh feasible?"],
},
"Data Volume Assessment": {
"Sampling strategy": ["Random/stratified/cluster?", "Sample size?", "Bias risks?"],
"Class balance check": ["Imbalance ratio?", "Oversample/undersample?", "Synthetic data?"],
"Size estimation": ["Rows and file size?", "Memory needs?", "Compute bandwidth?"],
"Incremental updates": ["Append vs merge?", "Versioning plan?", "Conflict handling?"],
"Redundancy removal": ["Detect redundancy?", "Compression options?", "Archive policy?"],
},
"Data Storage Setup": {
"Database schema design": ["Relational or NoSQL?", "Indexing strategy?", "Normalization level?"],
"File format selection": ["CSV/Parquet/JSON?", "Compression?", "Interoperability?"],
"Cloud storage choice": ["AWS/Azure/GCP?", "Cost model?", "Latency region?"],
"Security setup": ["At-rest/in-transit encryption?", "Access control?", "Audit logging?"],
"Backup policy": ["Frequency?", "Retention period?", "Restore testing?"],
},
},
"Preprocessing": {
"Data Cleaning": {
"Handle missing values": ["Impute or drop?", "Method chosen?", "Impact analysis?"],
"Remove duplicates": ["Detection method?", "Tie-breaking rule?", "Logging removals?"],
"Fix formatting errors": ["Standardize types?", "Normalize text?", "Unit conversions?"],
"Normalize text fields": ["Lowercasing/stemming?", "Stopwords?", "Unicode handling?"],
"Remove special characters": ["Allowed charset?", "Regex rules?", "Downstream effects?"],
},
"Feature Selection": {
"Manual selection": ["Domain criteria?", "Baseline subset?", "Rationale recorded?"],
"Statistical selection": ["Correlation/ANOVA/chi²?", "Thresholds?", "Leakage checks?"],
"Model-based selection": ["Which estimator?", "Importance cutoff?", "Stability across folds?"],
"Dimensionality reduction": ["PCA/UMAP?", "Target leakage risk?", "Explained variance?"],
"Domain expert input": ["Who signs off?", "Review cadence?", "Conflict resolution?"],
},
"Feature Engineering": {
"Create new features": ["What transformations?", "Business meaning?", "Overfitting risk?"],
"Combine existing features": ["Ratios/interactions?", "Collinearity?", "Scaling needs?"],
"Polynomial features": ["Max degree?", "Sparsity management?", "Regularization plan?"],
"Temporal features": ["Lags/rolling stats?", "Seasonality?", "Time zones?"],
"Categorical encoding": ["One-hot/target/WOE?", "High-cardinality strategy?", "Leakage prevention?"],
},
"Outlier Handling": {
"Z-score method": ["Threshold used?", "Per-group scaling?", "Robust alternatives?"],
"IQR method": ["Multiplier (1.5/3)?", "Per-feature vs joint?", "Winsorize vs remove?"],
"Winsorization": ["Clip bounds?", "Effect on metrics?", "Documented rationale?"],
"Clustering-based removal": ["Which clustering?", "Distance cutoff?", "Class impact?"],
"Manual inspection": ["Visualization used?", "Reviewer criteria?", "Reproducibility?"],
},
"Scaling & Transformation": {
"Min-Max scaling": ["Range chosen?", "Fit on train only?", "Outlier sensitivity?"],
"Standard scaling": ["Fit scope?", "Pipeline placement?", "Assumed distribution?"],
"Log transformation": ["Which features?", "Shift for zeros?", "Interpretability?"],
"Box-Cox transformation": ["Lambda search?", "Normality gain?", "Constraints?"],
"Quantile transformation": ["Quantiles used?", "Monotonicity preserved?", "Generalization?"],
},
},
"Model Selection": {
"Algorithm Research": {
"Linear models": ["Why suitable?", "Regularization choice?", "Feature assumptions?"],
"Tree-based models": ["Depth/leaf constraints?", "Handling missing?", "Interpretability?"],
"Neural networks": ["Architecture size?", "Training budget?", "Latency target?"],
"Ensemble methods": ["Bagging/boosting/stacking?", "Diversity sources?", "Overfit control?"],
"Probabilistic models": ["Distributional assumptions?", "Calibration needs?", "Uncertainty outputs?"],
},
"Baseline Model Creation": {
"Simple logistic regression": ["Baseline metric?", "Class weighting?", "Regularization?"],
"Decision stump": ["Split criterion?", "Benchmark purpose?", "Handling ties?"],
"Dummy classifier": ["Most frequent/stratified?", "Expected score?", "Sanity check?"],
"KNN baseline": ["K selection?", "Distance metric?", "Scaling requirement?"],
"Majority class predictor": ["Imbalance insight?", "Floor performance?", "Usefulness?"],
},
"Pre-trained Model Exploration": {
"Image models": ["Which backbone?", "Input size?", "Fine-tune vs freeze?"],
"NLP models": ["Tokenizer/vocab?", "Sequence length?", "Adaptation method?"],
"Speech models": ["Sampling rate?", "Feature front-end?", "WER target?"],
"Tabular models": ["CatBoost/FT-Transformer?", "Categorical handling?", "GPU needs?"],
"Multi-modal models": ["Fusion strategy?", "Alignment loss?", "Data requirements?"],
},
"Hyperparameter Strategy": {
"Grid search": ["Search space size?", "CV folds?", "Budget/time limit?"],
"Random search": ["Distributions?", "Trials planned?", "Early stopping?"],
"Bayesian optimization": ["Surrogate model?", "Acquisition function?", "Parallelism?"],
"Hyperband": ["Max resources?", "Reduction factor?", "Stochasticity handling?"],
"Manual tuning": ["Heuristics?", "Logging decisions?", "Reproducibility?"],
},
"Model Complexity Assessment": {
"Parameter count": ["Max allowed?", "Memory footprint?", "Compression options?"],
"FLOPs estimation": ["Target platform?", "Latency budget?", "Batch size effects?"],
"Memory usage": ["Peak RAM/VRAM?", "Streaming feasible?", "Quantization?"],
"Inference latency": ["P50/P95 targets?", "Hardware assumptions?", "Batching strategy?"],
"Deployment constraints": ["Edge vs cloud?", "Throughput goals?", "Cost ceiling?"],
},
},
"Training": {
"Data Splitting": {
"Train-test split": ["Split ratio?", "Stratification?", "Random seed?"],
"Cross-validation": ["K folds?", "Shuffle strategy?", "Leakage prevention?"],
"Stratified split": ["Which strata?", "Min group size?", "Imbalance kept?"],
"Time-series split": ["Gap/embargo?", "Horizon size?", "Leakage checks?"],
"Nested CV": ["Outer/inner folds?", "Compute budget?", "Model selection rule?"],
},
"Loss Function Choice": {
"MSE": ["Why MSE?", "Outlier sensitivity?", "Alternatives considered?"],
"Cross-entropy": ["Label smoothing?", "Class weights?", "Numerical stability?"],
"MAE": ["Robustness need?", "Optimization impact?", "Evaluation alignment?"],
"Huber loss": ["Delta parameter?", "Outlier profile?", "Convergence behavior?"],
"Custom loss": ["Definition and gradients?", "Calibration to metrics?", "Debugging plan?"],
},
"Optimization Method": {
"SGD": ["Momentum/nesterov?", "Learning rate schedule?", "Batch size?"],
"Adam": ["Beta values?", "Weight decay?", "Warmup?"],
"RMSProp": ["Decay rate?", "Centered variant?", "Stability?"],
"Adagrad": ["Learning rate decay?", "Sparsity benefits?", "Reset strategy?"],
"L-BFGS": ["Batching approach?", "Memory limits?", "Convergence criteria?"],
},
"Regularization": {
"L1": ["Sparsity goal?", "Lambda value?", "Feature pruning?"],
"L2": ["Weight decay?", "Overfit control?", "Interaction with optimizer?"],
"Dropout": ["Rates per layer?", "Inference behavior?", "Co-adaptation risk?"],
"Data augmentation": ["Which transforms?", "Label preservation?", "Distribution shift?"],
"Early stopping": ["Patience metric?", "Min delta?", "Checkpoint policy?"],
},
"Training Monitoring": {
"Loss curves": ["Smoothing?", "Train/val gap?", "Anomaly alerts?"],
"Accuracy curves": ["Metric tracked?", "Class-wise trends?", "Plateau detection?"],
"Validation metrics": ["Primary KPI?", "Reporting cadence?", "Confidence intervals?"],
"Learning rate schedule": ["Schedule type?", "Boundaries?", "Warm restarts?"],
"Checkpointing": ["Frequency?", "Best-vs-last?", "Storage budget?"],
},
},
"Evaluation": {
"Metric Selection": {
"Accuracy": ["Is class balance fair?", "Threshold chosen?", "Business relevance?"],
"Precision/Recall/F1": ["Which is primary?", "Threshold tuning?", "Cost of errors?"],
"ROC AUC": ["Calibration issues?", "Class imbalance?", "Interpretation limits?"],
"Log loss": ["Probability quality?", "Overconfidence penalty?", "Label noise?"],
"MSE/RMSE": ["Scale sensitivity?", "Baseline comparison?", "Outlier impact?"],
},
"Test Data Strategy": {
"Hold-out set": ["Size and representativeness?", "Temporal leakage?", "Reuse policy?"],
"External dataset": ["Domain match?", "License/ethics?", "Reproducibility?"],
"Cross-validation results": ["Variance across folds?", "Confidence bands?", "Selection bias?"],
"Leave-one-out": ["Compute cost?", "Variance concerns?", "Use case fit?"],
"Bootstrapping": ["Resample size?", "CI method?", "Stability?"],
},
"Fairness Checks": {
"Demographic parity": ["Protected attributes?", "Gap tolerated?", "Mitigation plan?"],
"Equalized odds": ["TPR/FPR parity?", "Group definitions?", "Trade-offs?"],
"Calibration across groups": ["Expected vs observed?", "Bins and sizes?", "Recalibration?"],
"Bias detection": ["Pre/post metrics?", "Data imbalance role?", "Human review?"],
"Ethical review": ["Stakeholder impact?", "Transparency level?", "Documentation?"],
},
"Robustness Testing": {
"Noisy input tests": ["Noise model?", "Degradation curve?", "Defenses?"],
"Adversarial attacks": ["Threat model?", "Attack types?", "Detection/robustness?"],
"Stress tests": ["Extreme values?", "Load/latency?", "Resource limits?"],
"Distribution shift": ["Which shifts?", "Detection method?", "Adaptation strategy?"],
"Random perturbations": ["Perturbation scale?", "Repeatability?", "Metric sensitivity?"],
},
"Model Interpretability": {
"Feature importance": ["Method used?", "Stability across runs?", "Correlated features?"],
"SHAP values": ["Background data?", "Runtime cost?", "Global vs local?"],
"LIME explanations": ["Kernel width?", "Neighborhood size?", "Faithfulness?"],
"Partial dependence plots": ["Feature interactions?", "ICE vs PDP?", "Monotonicity?"],
"Counterfactual explanations": ["Feasible actions?", "Cost function?", "Recourse policy?"],
},
},
}