Upload GID-Flow project snapshot (deduped: code + key artifacts)

.gitattributes

@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+PDGrapher/fonts/Arial[[:space:]]Bold[[:space:]]Italic.ttf filter=lfs diff=lfs merge=lfs -text
+PDGrapher/fonts/Arial[[:space:]]Bold.ttf filter=lfs diff=lfs merge=lfs -text
+PDGrapher/fonts/Arial[[:space:]]Italic.ttf filter=lfs diff=lfs merge=lfs -text
+PDGrapher/fonts/Arial.ttf filter=lfs diff=lfs merge=lfs -text
+PDGrapher/paper/paper.pdf filter=lfs diff=lfs merge=lfs -text
+references/PDGrapher-NBE教你反向设计.pdf filter=lfs diff=lfs merge=lfs -text
+references/SCALE.pdf filter=lfs diff=lfs merge=lfs -text
+references/science.DrugReflector.pdf filter=lfs diff=lfs merge=lfs -text

PDGrapher/.gitignore

@@ -0,0 +1,186 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+
+# VSCode
+.vscode/
+
+# Data folder
+# Ignore everything in data folder. An exception are Python processing files
+# and bash files, .keep files are there to enforce the directory structure.
+# Use `git add -f` to add something to data folder.
+data/
+
+tests/PDGrapher_test/*
+!tests/PDGrapher_test/.keep
+
+
+#jobs scripts
+job_*
+
+
+#large files
+*ours_to_real.txt
+*random_to_real.txt
+*aggregated_ranking_random.pickle
+*aggregated_ranking.pickle
+
+#irrelevant
+*scgen*

PDGrapher/.pytest_cache/.gitignore

@@ -0,0 +1,2 @@
+# Created by pytest automatically.
+*

PDGrapher/.pytest_cache/CACHEDIR.TAG

@@ -0,0 +1,4 @@
+Signature: 8a477f597d28d172789f06886806bc55
+# This file is a cache directory tag created by pytest.
+# For information about cache directory tags, see:
+#	https://bford.info/cachedir/spec.html

PDGrapher/.pytest_cache/README.md

@@ -0,0 +1,8 @@
+# pytest cache directory #
+
+This directory contains data from the pytest's cache plugin,
+which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
+
+**Do not** commit this to version control.
+
+See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.

PDGrapher/.pytest_cache/v/cache/lastfailed

@@ -0,0 +1,13 @@
+{
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_adjacency_row_sum_le_1": true,
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_adjacency_symmetric_and_positive": true,
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_output_not_all_same": true,
+  "tests/test__models.py": true,
+  "tests/test_pdgrapher.py": true,
+  "tests/test_datasets.py::TestDataset::test_multiple_folds": true,
+  "tests/test_datasets.py::TestDataset::test_single_fold": true,
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_with_protein_features": true,
+  "tests/test_package.py::TestPackage::test_multiple_folds": true,
+  "tests/test_package.py::TestPackage::test_single_fold": true,
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_alpha_beta_weighting": true
+}

PDGrapher/.pytest_cache/v/cache/nodeids

@@ -0,0 +1,246 @@
+[
+  "tests/test__utils.py::TestCalculateLossSampleWeights::test_diseased",
+  "tests/test__utils.py::TestCalculateLossSampleWeights::test_intervention",
+  "tests/test__utils.py::TestCalculateLossSampleWeights::test_output",
+  "tests/test__utils.py::TestCalculateLossSampleWeights::test_treated",
+  "tests/test__utils.py::TestCalculateLossSampleWeights::test_wrong_kind",
+  "tests/test__utils.py::TestDummyWriter::test_create",
+  "tests/test__utils.py::TestDummyWriter::test_writing",
+  "tests/test__utils.py::TestEarlyStopping::test_saving_loading",
+  "tests/test__utils.py::TestGetThresholds::test_output",
+  "tests/test__utils.py::TestGetThresholds::test_output_all",
+  "tests/test__utils.py::TestGetThresholds::test_wrong_kind",
+  "tests/test__utils.py::TestTestCondition::test_assertion_false",
+  "tests/test__utils.py::TestTestCondition::test_assertion_true",
+  "tests/test__utils.py::TestTicToc::test_output_basic_1",
+  "tests/test__utils.py::TestTicToc::test_output_basic_2",
+  "tests/test__utils.py::TestTicToc::test_output_custom",
+  "tests/test__utils.py::TestTicToc::test_output_custom_format",
+  "tests/test__utils.py::TestTicToc::test_output_wrapped",
+  "tests/test__utils.py::TestTicToc::test_output_wrapped_custom",
+  "tests/test__utils.py::TestTicToc::test_output_wrapped_custom_format",
+  "tests/test__utils.py::TestTicToc::test_wrapped_wrong_format",
+  "tests/test__utils.py::TestTicToc::test_wrong_format",
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_alpha_beta_weighting",
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_bce_only",
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_gradient_flows",
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_perfect_prediction_low_loss",
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_sampled_bce_different_neg_counts",
+  "tests/test_causal_estimator.py::TestCausalAlignmentLoss::test_with_effect_magnitude",
+  "tests/test_causal_estimator.py::TestCausalGPU::test_estimator_on_cuda",
+  "tests/test_causal_estimator.py::TestCausalGPU::test_planner_on_cuda",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_build_co_occurrence_graph_basic",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_build_co_occurrence_graph_device",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_build_co_occurrence_graph_symmetric",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_co_occurrence_produces_different_scores",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_deterministic_with_seed",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_different_batch_sizes",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_gradient_flows",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_large_gene_set",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_no_nan_forward",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_output_has_variance",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_output_not_all_same",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_output_range_sigmoid",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_output_shape",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_with_co_occurrence_graph",
+  "tests/test_causal_estimator.py::TestCausalGeneEstimator::test_without_co_occurrence_graph",
+  "tests/test_causal_estimator.py::TestCausalPlanner::test_forward_output_shapes",
+  "tests/test_causal_estimator.py::TestCausalPlanner::test_forward_with_co_occurrence",
+  "tests/test_causal_estimator.py::TestCausalPlanner::test_gradient_flows",
+  "tests/test_causal_estimator.py::TestCausalPlanner::test_no_nan",
+  "tests/test_causal_estimator.py::TestCausalPlanner::test_output_range",
+  "tests/test_causal_estimator.py::TestCausalSparsityLoss::test_gradient_flows",
+  "tests/test_causal_estimator.py::TestCausalSparsityLoss::test_output_scalar",
+  "tests/test_causal_estimator.py::TestCausalSparsityLoss::test_scales_with_mean_score",
+  "tests/test_causal_estimator.py::TestCausalSparsityLoss::test_weight_scaling",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGPU::test_forward_on_cuda",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_forward_keys",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_forward_shapes",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_gradient_flows",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_loss_decreases_over_steps",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_no_nan",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_predict_targets",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpace::test_recon_loss_zero",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDGeneSpaceGPU::test_forward_on_cuda",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_causal_scores_informative",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_different_batch_sizes",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_forward_keys",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_forward_shapes",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_gradient_flows",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_loss_components_nonzero",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_loss_is_finite",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_no_nan",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_predict_targets",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_predict_targets_no_topk",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_without_co_occurrence",
+  "tests/test_causal_flow_gid.py::TestCausalFlowGIDModel::test_without_true_perturbation",
+  "tests/test_datasets.py::TestDataset::test_multiple_folds",
+  "tests/test_datasets.py::TestDataset::test_single_fold",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_de_loss_enabled",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_de_zero_when_weight_zero",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_differentiable",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_gpu_if_available",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_loss_lower_when_pred_equals_target",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_no_nan",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_output_keys",
+  "tests/test_distributional_cycle_loss.py::TestDistributionalCycleLoss::test_with_masks",
+  "tests/test_distributional_cycle_loss.py::TestMMDRBF::test_distant_distributions_high_mmd",
+  "tests/test_distributional_cycle_loss.py::TestMMDRBF::test_identical_distributions_low_mmd",
+  "tests/test_distributional_cycle_loss.py::TestMMDRBF::test_mmd_differentiable",
+  "tests/test_distributional_cycle_loss.py::TestMMDRBF::test_mmd_fixed_sigma",
+  "tests/test_distributional_cycle_loss.py::TestMMDRBF::test_mmd_shape",
+  "tests/test_distributional_cycle_loss.py::TestMMDRBF::test_mmd_with_mask",
+  "tests/test_distributional_cycle_loss.py::TestMaskedStats::test_masked_mean_no_mask",
+  "tests/test_distributional_cycle_loss.py::TestMaskedStats::test_masked_mean_with_mask",
+  "tests/test_distributional_cycle_loss.py::TestMaskedStats::test_masked_var_shape",
+  "tests/test_distributional_cycle_loss.py::TestMaskedStats::test_masked_var_with_mask",
+  "tests/test_drug_modules.py::TestDoseConsistencyLoss::test_consistent_predictions",
+  "tests/test_drug_modules.py::TestDoseConsistencyLoss::test_inconsistent_predictions",
+  "tests/test_drug_modules.py::TestDoseConsistencyLoss::test_no_duplicates",
+  "tests/test_drug_modules.py::TestDrugAlignmentLoss::test_forward_with_targets",
+  "tests/test_drug_modules.py::TestDrugAlignmentLoss::test_forward_without_dose",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_different_emb_dims",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_embedding_cache",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_empty_input",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_morgan_encoding_frozen",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_morgan_encoding_shape",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_morgan_encoding_trainable",
+  "tests/test_drug_modules.py::TestDrugEncoder::test_similarity",
+  "tests/test_drug_modules.py::TestDrugGeneBridge::test_forward_shape_with_gene_emb",
+  "tests/test_drug_modules.py::TestDrugGeneBridge::test_forward_shape_without_gene_emb",
+  "tests/test_drug_modules.py::TestDrugGeneBridge::test_gene_aware_attention",
+  "tests/test_drug_modules.py::TestDrugTargetAlignmentLoss::test_perfect_prediction",
+  "tests/test_drug_modules.py::TestDrugTargetAlignmentLoss::test_wrong_prediction",
+  "tests/test_flow_response.py::TestFlowMatchingGPU::test_forward_on_cuda",
+  "tests/test_flow_response.py::TestFlowMatchingGPU::test_sample_on_cuda",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_different_gene_dims",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_forward_loss_no_nan",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_forward_loss_scalar",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_gradient_flows",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_loss_zero_when_source_equals_target",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_no_encoder_decoder",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_reconstruction_loss_zero",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_sample_is_different_from_source",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_sample_no_nan",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_sample_shape",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpace::test_variable_cell_count",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpaceGPU::test_forward_on_cuda",
+  "tests/test_flow_response.py::TestFlowMatchingGeneSpaceGPU::test_sample_on_cuda",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_batch_size_one",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_deterministic_with_seed",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_different_gene_dims",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_forward_loss_no_nan",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_forward_loss_scalar",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_loss_lower_for_similar_targets",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_loss_zero_when_source_equals_target",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_more_steps_better",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_sample_gradient_does_not_flow",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_sample_no_nan",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_sample_shape",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_t_at_extremes",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_target_cells_required",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_training_gradient_flows",
+  "tests/test_flow_response.py::TestFlowMatchingResponseModel::test_variable_cell_count",
+  "tests/test_flow_response.py::TestSinusoidalTimeEmbedding::test_different_for_different_t",
+  "tests/test_flow_response.py::TestSinusoidalTimeEmbedding::test_differentiable",
+  "tests/test_flow_response.py::TestSinusoidalTimeEmbedding::test_output_shape",
+  "tests/test_flow_response.py::TestSinusoidalTimeEmbedding::test_output_shape_odd_dim",
+  "tests/test_gene_graph_encoder.py::TestGIDModelWithGeneGraph::test_forward_shapes",
+  "tests/test_gene_graph_encoder.py::TestGIDModelWithGeneGraph::test_gcn_off_matches_no_graph",
+  "tests/test_gene_graph_encoder.py::TestGIDModelWithGeneGraph::test_gradient_flows_through_gcn",
+  "tests/test_gene_graph_encoder.py::TestGIDModelWithGeneGraph::test_no_nan",
+  "tests/test_gene_graph_encoder.py::TestGIDModelWithGeneGraph::test_predict_targets_shape",
+  "tests/test_gene_graph_encoder.py::TestGIDModelWithGeneGraph::test_with_protein_features_and_graph",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_different_graphs_give_different_embeddings",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_gradient_through_embedding",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_no_nan",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_output_shape",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_real_pdgrapher_graph",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_set_graph_raises_before_forward",
+  "tests/test_gene_graph_encoder.py::TestGeneGraphEncoder::test_with_protein_features",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_adjacency_nonneg_and_diagonal_positive",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_adjacency_row_sum_le_1",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_adjacency_symmetric_and_positive",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_adjacency_undirected_is_symmetric",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_build_normalized_adjacency_shape",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_extract_subgraph_correct_remapping",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_extract_subgraph_reduces_edges",
+  "tests/test_gene_graph_encoder.py::TestGraphUtils::test_load_pdgrapher_edge_index",
+  "tests/test_package.py::TestPackage::test_multiple_folds",
+  "tests/test_package.py::TestPackage::test_single_fold",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_dataloader_integration",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_mask_values",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_max_cells_truncation",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_output_type",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_padded_shapes",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_padding_is_zero",
+  "tests/test_population_batch.py::TestPopulationCollateFn::test_perturbation_stacked_correctly",
+  "tests/test_population_batch.py::TestPopulationPerturbationBatch::test_optional_fields_default_none",
+  "tests/test_population_batch.py::TestPopulationPerturbationBatch::test_shapes",
+  "tests/test_population_batch.py::TestPopulationPerturbationBatch::test_to_device_cpu",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_cell_counts_in_range",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_different_seeds_differ",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_gene_dim",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_item_keys",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_length",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_perturbation_is_multihot",
+  "tests/test_population_batch.py::TestSyntheticPopulationPerturbationDataset::test_reproducible",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_batch_size_1",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_different_n_per_call",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_gpu_if_available",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_gradient_flows",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_mask_accepted",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_no_nan",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_output_shape",
+  "tests/test_population_response.py::TestSimplePopulationResponseModel::test_perturbation_changes_output",
+  "tests/test_real_data.py::TestGeneSelection::test_hvg_fewer_than_requested",
+  "tests/test_real_data.py::TestGeneSelection::test_hvg_shape",
+  "tests/test_real_data.py::TestGeneSelection::test_normalize_log1p",
+  "tests/test_real_data.py::TestGeneSelection::test_normalize_no_log",
+  "tests/test_real_data.py::TestPDGrapherPseudobulkDataset::test_collate_fn_integration",
+  "tests/test_real_data.py::TestPDGrapherPseudobulkDataset::test_item_shapes",
+  "tests/test_real_data.py::TestPDGrapherPseudobulkDataset::test_length",
+  "tests/test_real_data.py::TestPDGrapherPseudobulkDataset::test_no_nan",
+  "tests/test_real_data.py::TestPDGrapherPseudobulkDataset::test_num_genes",
+  "tests/test_real_data.py::TestPDGrapherPseudobulkDataset::test_perturbation_nonzero",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_collate_fn_integration",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_item_shapes",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_most_perts_in_gene_space",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_num_conditions",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_perturbation_is_multihot",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_reproducibility",
+  "tests/test_real_data.py::TestScPerturbPopulationDataset::test_source_target_differ",
+  "tests/test_sparse_planner.py::TestGapEncoder::test_gradient_flows",
+  "tests/test_sparse_planner.py::TestGapEncoder::test_output_shape",
+  "tests/test_sparse_planner.py::TestMMDProjected::test_different_dist_higher_mmd",
+  "tests/test_sparse_planner.py::TestMMDProjected::test_same_dist_low_mmd",
+  "tests/test_sparse_planner.py::TestMMDProjected::test_shape",
+  "tests/test_sparse_planner.py::TestPerturbationEval::test_compute_all_returns_keys",
+  "tests/test_sparse_planner.py::TestPerturbationEval::test_de_direction_agreement_perfect",
+  "tests/test_sparse_planner.py::TestPerturbationEval::test_pearson_r_good_better_than_random",
+  "tests/test_sparse_planner.py::TestPerturbationEval::test_pearson_r_perfect",
+  "tests/test_sparse_planner.py::TestPerturbationEval::test_pearson_r_topk_de",
+  "tests/test_sparse_planner.py::TestPopulationEncoder::test_gradient_flows",
+  "tests/test_sparse_planner.py::TestPopulationEncoder::test_mask_accepted",
+  "tests/test_sparse_planner.py::TestPopulationEncoder::test_output_shape_no_var",
+  "tests/test_sparse_planner.py::TestPopulationEncoder::test_output_shape_with_var",
+  "tests/test_sparse_planner.py::TestPopulationGIDModel::test_end_to_end_gradient",
+  "tests/test_sparse_planner.py::TestPopulationGIDModel::test_forward_keys",
+  "tests/test_sparse_planner.py::TestPopulationGIDModel::test_no_nan",
+  "tests/test_sparse_planner.py::TestPopulationGIDModel::test_predict_targets_shape",
+  "tests/test_sparse_planner.py::TestPopulationGIDModel::test_shapes",
+  "tests/test_sparse_planner.py::TestPopulationGIDModel::test_teacher_forcing",
+  "tests/test_sparse_planner.py::TestSparsePlanner::test_score_shape",
+  "tests/test_sparse_planner.py::TestSparsePlanner::test_soft_mask_range",
+  "tests/test_sparse_planner.py::TestSparsePlanner::test_ste_mask_is_binary",
+  "tests/test_sparse_planner.py::TestSparsePlanner::test_ste_topk_gradient",
+  "tests/test_sparse_planner.py::TestSparsePlanner::test_topk_mask_binary",
+  "tests/test_sparse_planner.py::TestTargetMetrics::test_compute_all_returns_keys",
+  "tests/test_sparse_planner.py::TestTargetMetrics::test_mrr_perfect",
+  "tests/test_sparse_planner.py::TestTargetMetrics::test_ndcg_perfect_vs_random",
+  "tests/test_sparse_planner.py::TestTargetMetrics::test_recall_at_k_perfect",
+  "tests/test_sparse_planner.py::TestTargetMetrics::test_recall_at_k_random",
+  "tests/test_train.py::TestTrainer::test_trainer",
+  "tests/test_train.py::TestTrainer::test_trainer_args",
+  "tests/test_train.py::TestTrainer::test_trainer_unknown_kwargs"
+]

PDGrapher/.pytest_cache/v/cache/stepwise

@@ -0,0 +1 @@
+[]

PDGrapher/CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog - PDGrapher
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [UNRELEASED] - 2023-09-08
+
+### Added
+- Initial PDGrapher project structure.

PDGrapher/LICENSE

@@ -0,0 +1,29 @@
+MIT License
+
+Copyright (c) 2023 Artificial Intelligence for Medicine and Science @ Harvard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

PDGrapher/README.md

@@ -0,0 +1,121 @@
+# Combinatorial prediction of therapeutic perturbations using causally-inspired neural networks
+[![ProjectPage](https://img.shields.io/badge/Project-PDGrapher-red)](https://zitniklab.hms.harvard.edu/projects/PDGrapher/) [![CodePage](https://img.shields.io/badge/Code-GitHub-orange)](https://github.com/mims-harvard/PDGrapher) [![Paper](https://img.shields.io/badge/Paper-BioRxiv-green)](https://www.biorxiv.org/content/10.1101/2024.01.03.573985v5) [![Paper](https://img.shields.io/badge/Paper-NBME-green)](https://www.nature.com/articles/s41551-025-01481-x) [![Data](https://img.shields.io/badge/Data-Links-purple)](https://github.com/mims-harvard/PDGrapher/tree/main/data) ![License](https://img.shields.io/badge/license-MIT-blue) 
+
+
+
+[Guadalupe Gonzalez*](https://www.guadalupegonzalez.io/), [Xiang Lin*](https://xianglin226.github.io/), [Isuru Herath](https://scholar.google.com/citations?user=F-RC5k0AAAAJ&hl=en), [Kirill Veselkov](https://scholar.google.com/citations?user=0n-5UGYAAAAJ&hl=en),
+[Michael Bronstein](https://scholar.google.com/citations?user=UU3N6-UAAAAJ&hl=en), and [Marinka Zitnik](https://dbmi.hms.harvard.edu/people/marinka-zitnik)
+
+![](https://github.com/mims-harvard/PDGrapher/blob/main/figures/figure1.jpg)
+## Project structure
+
+The project consists of next folders:
+- [data](data/) contains all of the data on which our models were built. On how to obtain this data, refer to [Data](#data) section,
+- [docs](docs/) contains documentation, built with 'sphinx',
+- [src/pdgrapher](src/pdgrapher/) contains the source code for PDGrapher,
+- [tests](tests/) contains unit and integration tests.
+
+## Virtual environment
+
+```
+conda env create -f conda-env.yml
+conda activate pdgrapher
+pip install pip==23.2.1
+pip install -r requirements.txt
+
+pip install torch==1.10.1+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1 -f https://download.pytorch.org/whl/cu111/torch_stable.html
+pip install torch-scatter==2.0.9 -f https://data.pyg.org/whl/torch-1.10.1+cu111.html
+pip install torch-sparse==0.6.12 -f https://data.pyg.org/whl/torch-1.10.1+cu111.html
+pip install torch-cluster==1.5.9 -f https://data.pyg.org/whl/torch-1.10.1+cu111.html
+pip install torch-spline-conv==1.2.1 -f https://data.pyg.org/whl/torch-1.10.1+cu111.html
+pip install torch-geometric==2.0.4 -f https://data.pyg.org/whl/torch-1.10.1+cu111.html
+pip install torchmetrics==0.9.3
+pip install lightning==1.9.5
+
+```
+
+
+## Data
+
+For processed data, download the compressed folders and place them in `data/processed/` with the following commands:
+
+### Download genetic and splits data from Zenodo
+```bash
+cd data/processed
+# Download splits and genetic data
+wget -O splits.tar.gz "https://zenodo.org/api/records/15375990/files/splits.tar.gz/content"
+wget -O torch_data_genetic.tar.gz "https://zenodo.org/api/records/15375990/files/torch_data_genetic.tar.gz/content"
+
+# Extract splits data
+tar -xzvf splits.tar.gz
+
+# Create torch_data directory and extract genetic data into it
+mkdir -p torch_data
+cd torch_data
+tar -xzvf ../torch_data_genetic.tar.gz
+cd ..
+```
+
+### Download chemical data from Zenodo
+```bash
+# Download chemical data (run from data/processed directory)
+wget -O torch_data_chemical.tar.gz "https://zenodo.org/api/records/15390483/files/torch_data_chemical.tar.gz/content"
+
+# Extract chemical torch data into torch_data directory
+cd torch_data
+tar -xzvf ../torch_data_chemical.tar.gz
+cd ..
+```
+
+### Data Sources
+- **Genetic data and splits**: [https://zenodo.org/records/15375990](https://zenodo.org/records/15375990)
+- **Chemical data**: [https://zenodo.org/records/15390483](https://zenodo.org/records/15390483)
+
+
+## Building
+
+This project can be build as a Python library by running `pip install -e .` in the root of this repository.
+
+
+## Documentation
+
+Documentation can be built with the following commands:
+
+### Prerequisites
+First, install the required documentation dependencies:
+```bash
+pip install myst-parser
+```
+
+### Build Documentation
+1. `sphinx-apidoc -fe -o docs/source/ src/pdgrapher/` updates the source files from which the documentation is built
+2. `cd docs && make html` builds the documentation
+
+Then, the documentation can be accessed locally by going to `docs/build/html/index.html`
+All of the settings along with links to instructions can be found and modified in [docs/source/conf.py](docs/source/conf.py). 
+
+## Notebooks
+|  Tutorials       | Links                     |
+|----------------|---------------------------------|
+| Train PDGrapher on chemical dataset         | [notebook](./notebooks/train_chemical.ipynb)            |
+| Train PDGrapher on genetic dataset     | [notebook](./notebooks/train_genetic.ipynb)    | 
+| Test PDGrapher on chemical/genetic dataset  | [notebook](./notebooks/test_PDG.ipynb)      | 
+
+## Additional Resources
+* [Paper](https://www.nature.com/articles/s41551-025-01481-x)  
+* [HMS News & Research](https://hms.harvard.edu/news/new-ai-tool-pinpoints-genes-drug-combos-restore-health-diseased-cells)
+
+@article{gonzalez2025combinatorial,  
+  title={Combinatorial Prediction of Therapeutic Perturbations Using Causally-Inspired Neural Networks},  
+  author={Gonzalez, Guadalupe and Lin, Xiang and Herath, Isuru and Veselkov, Kirill and Bronstein, Michael and Zitnik, Marinka},  
+  journal={Nature Biomedical Engineering},  
+  url={https://www.nature.com/articles/s41551-025-01481-x},  
+  year={2025}  
+}  
+
+## License
+The code in this package is licensed under the MIT License.
+
+## Questions
+Please leave a Github issue or contact [Guadalupe Gonzalez](mailto:ggonzalezp16@gmail.com), [Xiang Lin](mailto:xianglin226@gmail.com), or [Marinka Zitnik](mailto:marinka@zitnik.si)  
+

PDGrapher/TODO.md

@@ -0,0 +1,10 @@
+# PDGrapher: TODO
+
+This file summarizes things that still need to be done and some observations.
+
+## Bugs:
+- There is some CUDA memory leak, but it is only present when we train a lot of models in one go, as in train them with one `python ...` call. An example is [examples/hyperparameter_tuning.py](examples/hyperparameter_tuning.py) script, which crashed due to OOM error in CUDA after training >130 models for 1 epoch. I suspect it is related to lightning.Fabric, which handles moving tensort to and from GPU.
+
+## TODO:
+- [data](data/) folder has no data currently, and the folder structure with 'raw' folders is not consistent with code in scripts folders. Maybe we can rename them to '2022-03-PPI', '2022-02-LINCS_Level3' and so on. There are two such PPI folders, so we need to be careful there.
+- [main README.md](README.md) needs some updating, like add authors, project description.

PDGrapher/conda-env.yml

@@ -0,0 +1,63 @@
+name: pdgrapher
+channels:
+  - conda-forge
+  - conda-canary
+  - defaults
+dependencies:
+  - _libgcc_mutex
+  - _openmp_mutex
+  - blas
+  - boost
+  - boost-cpp
+  - bzip2
+  - ca-certificates
+  - cairo
+  - certifi
+  - cycler
+  - fftw
+  - fontconfig
+  - freetype
+  - glib
+  - icu
+  #- intel-openmp
+  - jpeg
+  - ld_impl_linux-64
+  - libffi
+  - libgcc-ng
+  - libgfortran-ng
+  - libgfortran5
+  - libgomp
+  - libiconv
+  - libopenblas
+  - libpng
+  - libstdcxx-ng
+  - libtiff
+  - libuuid
+  - libxcb
+  - libxml2
+  - lz4-c
+  - mkl
+  - mkl-service
+  - mkl_fft
+  - mkl_random
+  - ncurses
+  #- numpy-base
+  - olefile
+  - openssl
+  - pcre
+  - pixman
+  - pycairo
+  - python=3.8  # Kept the Python version to match your requirement
+  - python-dateutil
+  - python_abi
+  - pytz
+  - rdkit
+  - readline
+  - reportlab
+  - six
+  - sqlite
+  - tk
+  - tornado
+  - xz
+  - zlib
+  - zstd

PDGrapher/configs/causal_flow_drug.yaml

@@ -0,0 +1,119 @@
+seed: 42
+
+# ──────────────────────────────────────────────
+# Dataset configuration
+# ──────────────────────────────────────────────
+dataset:
+  # CRISPRi data (Norman2019)
+  type: scperturb
+  h5ad_path: /data/boom/Protein/regulatory_field/data/raw/scPerturb/rna_protein/NormanWeissman2019_filtered.h5ad
+  n_hvg: 2000
+  min_cells_per_cond: 30
+  max_source_cells: 64
+  max_target_cells: 64
+  control_label: control
+  use_single_pert_only: false
+  include_combos_in_train: true
+  force_include_pert_genes: true
+  target_sum: 10000.0
+
+  # Drug data (SciPlex3) — joint training
+  sciplex:
+    h5ad_path: /data/boom/Protein/regulatory_field/data/raw/scPerturb/rna_protein/SrivatsanTrapnell2020_sciplex3.h5ad
+    preprocessed_path: /data/boom/ICLR/data/processed/sciplex3_k562_24h.pt
+    enabled: true
+    min_cells_per_cond: 30
+    max_source_cells: 64
+    max_target_cells: 64
+    cell_lines: ["K562"]
+    doses: [10.0, 100.0, 1000.0, 10000.0]
+    times: [24.0]
+    drug_emb_dim: 128
+    drug_smiles_csv: data/chembl_smiles.csv
+
+# ──────────────────────────────────────────────
+# Model configuration
+# ──────────────────────────────────────────────
+model:
+  num_genes: 2000
+  # Encoder
+  encoder_hidden: 512
+  encoder_output: 256
+  # Gap
+  gap_hidden: 256
+  gap_output: 256
+  # Causal planner
+  causal_gene_emb_dim: 128
+  causal_n_heads: 4
+  causal_n_layers: 2
+  planner_hidden: 512
+  planner_n_layers: 2
+  planner_topk: 1
+  # Flow matching
+  flow_latent_dim: 256
+  flow_hidden_dim: 512
+  flow_n_layers: 3
+  flow_time_embed_dim: 128
+  flow_pert_emb_dim: 256
+  n_layers: 2
+  # Drug side
+  drug_emb_dim: 128
+  use_drug_encoder: true
+  use_drug_gene_bridge: true
+  # Other
+  use_cooccurrence: true
+  use_latent: false
+  lambda_flow: 1.0
+  lambda_cycle: 0.5
+  lambda_causal: 1.0
+  lambda_sparse: 0.01
+  lambda_recon: 1.0
+  n_neg_samples: 128
+  sparse_variance_weight: 0.1
+  use_causal_infonce: true
+  infonce_neg_samples: 64
+  infonce_temperature: 0.1
+  # Drug losses
+  lambda_drug_target: 5.0
+  lambda_drug_contrastive: 1.0
+  lambda_drug_dose: 0.5
+
+# ──────────────────────────────────────────────
+# Loss configuration
+# ──────────────────────────────────────────────
+loss:
+  mmd_weight: 1.0
+  mean_weight: 1.0
+  var_weight: 0.5
+  de_weight: 0.1
+  de_topk: 50
+
+# ──────────────────────────────────────────────
+# Training configuration
+# ──────────────────────────────────────────────
+train:
+  batch_size: 8
+  epochs: 200
+  lr: 5.0e-5
+  weight_decay: 0.0001
+  lr_min: 0.000001
+  val_split: 0.15
+  val_split_type: gene_disjoint
+  early_stopping_patience: 50
+  device: cuda
+  save_every: 10
+  # Drug training schedule
+  drug_start_epoch: 50           # start drug loss after N epochs of CRISPRi pretraining
+  drug_warmup_epochs: 20         # linearly ramp up drug loss weights
+  # Mixed training
+  crispr_weight: 0.7             # weight for CRISPRi batch in mixed training
+  drug_weight: 0.3               # weight for drug batch in mixed training
+
+# ──────────────────────────────────────────────
+# Output configuration
+# ──────────────────────────────────────────────
+output:
+  dir: outputs/causal_flow_drug
+  checkpoint: outputs/causal_flow_drug/checkpoint.pt
+  metrics_csv: outputs/causal_flow_drug/metrics.csv
+  best_checkpoint: outputs/causal_flow_drug/best_checkpoint.pt

PDGrapher/configs/drug_rank_phase1.yaml

@@ -0,0 +1,36 @@
+seed: 42
+data:
+  sciplex3_h5ad: /data/boom/ICLR/data/raw/sciplex3.h5ad
+  annotation_dir: /data/boom/ICLR/data/annotation
+  splits_dir: /data/boom/ICLR/data/splits
+  split: drug_disjoint
+  max_source_cells: 64
+  max_target_cells: 64
+  batch_size: 32
+
+model:
+  num_genes: 2000
+  encoder_hidden: 256
+  encoder_output: 128
+  gap_hidden: 256
+  gap_output: 256
+  gap_proj_dim: 128
+  num_cell_lines: 0   # disabled: was never trained (training omits cell_line_ids); removes dead footgun
+  drug_emb_dim: 128
+  drug_encoder: morgan
+  bridge_hidden_dim: 256
+  bridge_proj_dim: 128
+  bridge_protein_emb_dim: 64
+
+phase1:
+  epochs: 40
+  lr: 1.0e-3
+  weight_decay: 1.0e-4
+  bce_pos_weight: 20.0
+  lambda_recon: 0.1
+  held_out_annotation_frac: 0.15
+  early_stop_auroc: 0.65
+  early_stop_patience: 5
+  device: cuda
+  checkpoint_dir: /data/boom/ICLR/PDGrapher/outputs/drug_rank
+  checkpoint_every: 5

PDGrapher/configs/drug_rank_phase2.yaml

@@ -0,0 +1,53 @@
+seed: 42
+data:
+  sciplex3_h5ad: /data/boom/ICLR/data/raw/sciplex3.h5ad
+  annotation_dir: /data/boom/ICLR/data/annotation
+  splits_dir: /data/boom/ICLR/data/splits
+  split: drug_disjoint
+  max_source_cells: 64
+  max_target_cells: 64
+  batch_size: 32
+
+model:
+  num_genes: 2000
+  encoder_hidden: 256
+  encoder_output: 128
+  gap_hidden: 256
+  gap_output: 256
+  gap_proj_dim: 128
+  num_cell_lines: 0   # disabled: was never trained (training omits cell_line_ids); removes dead footgun
+  drug_emb_dim: 128
+  drug_encoder: morgan
+  bridge_hidden_dim: 256
+  bridge_proj_dim: 128
+  bridge_protein_emb_dim: 64
+
+phase1:
+  epochs: 40
+  lr: 1.0e-3
+  weight_decay: 1.0e-4
+  bce_pos_weight: 20.0
+  lambda_recon: 0.1
+  held_out_annotation_frac: 0.15
+  early_stop_auroc: 0.65
+  early_stop_patience: 5
+  device: cuda
+  checkpoint_dir: /data/boom/ICLR/PDGrapher/outputs/drug_rank
+  checkpoint_every: 5
+
+phase2:
+  epochs: 200
+  lr: 3.0e-4
+  weight_decay: 1.0e-4
+  temp_start: 0.20
+  temp_end: 0.10
+  lambda_start: 50.0
+  lambda_end: 5.0
+  gallery_update_every: 15
+  # Full-library gallery negatives: empirically HURT (harder objective under-trains
+  # given BCE-dominated lambda schedule). Keep in-batch InfoNCE + same-drug/MOA
+  # masking (which fixes the false-negative bug) by leaving this false.
+  use_gallery_negatives: false
+  device: cuda
+  checkpoint_dir: /data/boom/ICLR/PDGrapher/outputs/drug_rank
+  val_every: 10

PDGrapher/configs/drug_repurposing.yaml

@@ -0,0 +1,45 @@
+# GIDFlow Drug Repurposing Configuration
+# Reference: configs/drug_repurposing.yaml
+
+predictor:
+  device: auto                    # 'cuda', 'cpu', or 'auto'
+  n_bootstrap: 10                 # Bootstrap samples for confidence intervals
+  top_k: 50                       # Number of top drugs to return
+  seed: 42                        # Random seed
+
+  # Similarity scoring weights
+  similarity_weights:
+    pearson_r: 0.35
+    cosine_sim: 0.15
+    de_overlap: 0.20
+    de_precision: 0.10
+    lfc_spearman: 0.15
+    de_direction_agreement: 0.05
+
+drug_library:
+  name: norman2019
+  # Built-in libraries: "norman2019", "replogle"
+  # Or provide path to h5ad file:
+  # h5ad_path: /path/to/your/data.h5ad
+  h5ad_path: /data/boom/Protein/regulatory_field/data/raw/scPerturb/rna_protein/NormanWeissman2019_filtered.h5ad
+  cell_type: null                 # Filter to specific cell type (e.g., "K562")
+  use_single_pert_only: false     # Include combinatorial perturbations
+  min_cells_per_cond: 20          # Minimum cells per condition
+
+similarity:
+  de_topk: 50                     # Top-k DE genes for overlap metrics
+  lfc_epsilon: 1e-6               # Epsilon for log fold change computation
+
+pathway:
+  enabled: true
+  gene_sets: "MSigDB_Hallmark_2020"
+  organism: "human"
+  min_size: 10
+  max_size: 500
+  cutoff: 0.05
+
+output:
+  dir: outputs/drug_repurposing
+  rankings_csv: outputs/drug_repurposing/drug_rankings.csv
+  populations_dir: outputs/drug_repurposing/predicted_populations
+  pathways_dir: outputs/drug_repurposing/pathways

PDGrapher/data/README.md

@@ -0,0 +1,38 @@
+
+
+
+
+# Downloading and expanding processed datasets:
+
+## Download genetic and splits data from Zenodo
+```bash
+mkdir -p processed
+cd processed
+# Download splits and genetic data
+wget -O splits.tar.gz "https://zenodo.org/api/records/15375990/files/splits.tar.gz/content"
+wget -O torch_data_genetic.tar.gz "https://zenodo.org/api/records/15375990/files/torch_data_genetic.tar.gz/content"
+
+# Extract splits data
+tar -xzvf splits.tar.gz
+
+# Create torch_data directory and extract genetic data into it
+mkdir -p torch_data
+cd torch_data
+tar -xzvf ../torch_data_genetic.tar.gz
+cd ..
+```
+
+## Download chemical data from Zenodo
+```bash
+# Download chemical data (run from data/processed directory)
+wget -O torch_data_chemical.tar.gz "https://zenodo.org/api/records/15390483/files/torch_data_chemical.tar.gz/content"
+
+# Extract chemical torch data into torch_data directory
+cd torch_data
+tar -xzvf ../torch_data_chemical.tar.gz
+cd ..
+```
+
+## Data Sources
+- **Genetic data and splits**: [https://zenodo.org/records/15375990](https://zenodo.org/records/15375990)
+- **Chemical data**: [https://zenodo.org/records/15390483](https://zenodo.org/records/15390483)

PDGrapher/data/check_data_and_splits.ipynb

@@ -0,0 +1,374 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cell line: A375\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: A549\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: BT20\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: HA1E\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: HELA\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: HT29\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: MCF7\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: MDAMB231\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: PC3\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: VCAP\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n"
+     ]
+    }
+   ],
+   "source": [
+    "##Chemical perturbations\n",
+    "cell_line = ['A375', 'A549', 'BT20', 'HA1E', 'HELA', 'HT29', 'MCF7', 'MDAMB231', 'PC3', 'VCAP']\n",
+    "\n",
+    "for cl in cell_line:\n",
+    "    split = torch.load('processed/splits/chemical/{}/random/5fold/splits.pt'.format(cl))\n",
+    "\n",
+    "    print('Cell line: {}'.format(cl))\n",
+    "\n",
+    "    #Loads data\n",
+    "    b_data = torch.load('processed/torch_data/chemical/real_lognorm/data_backward_{}.pt'.format(cl))\n",
+    "    f_data = torch.load('processed/torch_data/chemical/real_lognorm/data_forward_{}.pt'.format(cl))\n",
+    "\n",
+    "    for i in range(1, 6):\n",
+    "        print(' Split: {}'.format(i))\n",
+    "\n",
+    "        error = False\n",
+    "        #Backward data\n",
+    "        d = b_data\n",
+    "        if len(d) != len(split[i]['train_index_backward']) + len(split[i]['val_index_backward']) + len(split[i]['test_index_backward']):\n",
+    "            print('length is not the same for splits: {}'.format(cl))\n",
+    "            error = True\n",
+    "        \n",
+    "        #Forward data\n",
+    "        d = f_data\n",
+    "        if len(d) == 0:\n",
+    "            if split[i]['train_index_forward'] is not None or split[i]['val_index_forward'] is not None or split[i]['test_index_forward'] is not None:\n",
+    "                print('split is not None when it should be (no forward data)')\n",
+    "                error = True\n",
+    "        else:\n",
+    "            if len(d) != len(split[i]['train_index_forward']) + len(split[i]['val_index_forward']) + len(split[i]['test_index_forward']):\n",
+    "                print('length is not the same for splits: {}'.format(cl))\n",
+    "                error = True\n",
+    "        \n",
+    "        if not error:\n",
+    "            print(' Passed test')\n",
+    "        else:\n",
+    "            print(' Failed test')\n",
+    "    \n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cell line: A375\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: A549\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: AGS\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: BICR6\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: ES2\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: HT29\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: MCF7\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: PC3\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: U251MG\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n",
+      "Cell line: YAPC\n",
+      " Split: 1\n",
+      " Passed test\n",
+      " Split: 2\n",
+      " Passed test\n",
+      " Split: 3\n",
+      " Passed test\n",
+      " Split: 4\n",
+      " Passed test\n",
+      " Split: 5\n",
+      " Passed test\n"
+     ]
+    }
+   ],
+   "source": [
+    "##Genetic perturbations\n",
+    "cell_line = ['A375', 'A549', 'AGS', 'BICR6', 'ES2', 'HT29', 'MCF7', 'PC3', 'U251MG', 'YAPC']\n",
+    "\n",
+    "for cl in cell_line:\n",
+    "    split = torch.load('processed/splits/genetic/{}/random/5fold/splits.pt'.format(cl))\n",
+    "\n",
+    "    print('Cell line: {}'.format(cl))\n",
+    "\n",
+    "    #Loads data\n",
+    "    b_data = torch.load('processed/torch_data/real_lognorm/data_backward_{}.pt'.format(cl))\n",
+    "    f_data = torch.load('processed/torch_data/real_lognorm/data_forward_{}.pt'.format(cl))\n",
+    "\n",
+    "    for i in range(1, 6):\n",
+    "        print(' Split: {}'.format(i))\n",
+    "\n",
+    "        error = False\n",
+    "        #Backward data\n",
+    "        d = b_data\n",
+    "        if len(d) != len(split[i]['train_index_backward']) + len(split[i]['val_index_backward']) + len(split[i]['test_index_backward']):\n",
+    "            print('length is not the same for splits: {}'.format(cl))\n",
+    "            error = True\n",
+    "        \n",
+    "        #Forward data\n",
+    "        d = f_data\n",
+    "        if len(d) == 0:\n",
+    "            if split[i]['train_index_forward'] is not None or split[i]['val_index_forward'] is not None or split[i]['test_index_forward'] is not None:\n",
+    "                print('split is not None when it should be (no forward data)')\n",
+    "                error = True\n",
+    "        else:\n",
+    "            if len(d) != len(split[i]['train_index_forward']) + len(split[i]['val_index_forward']) + len(split[i]['test_index_forward']):\n",
+    "                print('length is not the same for splits: {}'.format(cl))\n",
+    "                error = True\n",
+    "        \n",
+    "        if not error:\n",
+    "            print(' Passed test')\n",
+    "        else:\n",
+    "            print(' Failed test')\n",
+    "    \n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "myenv",
+   "language": "python",
+   "name": "myenv"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

PDGrapher/data/chembl_smiles.csv

@@ -0,0 +1,167 @@
+chembl_id,smiles
+CHEMBL1078178,N#CCNC(=O)c1ccc(-c2ccnc(Nc3ccc(N4CCOCC4)cc3)n2)cc1
+CHEMBL109,CCCC(CCC)C(=O)O
+CHEMBL1090479,CCn1cc(-c2ccnc3[nH]c(-c4cccc(CN(C)C)c4)cc23)c(-c2ccc(NC(=O)N(C)C)cc2)n1
+CHEMBL1090771,NC(=O)[C@@H](CCC(F)(F)F)N(Cc1ccc(-c2ncon2)cc1F)S(=O)(=O)c1ccc(Cl)cc1
+CHEMBL109480,C=CCNC1=C2C[C@@H](C)C[C@H](OC)[C@H](O)[C@@H](C)/C=C(\C)[C@H](OC(N)=O)[C@@H](OC)/C=C\C=C(/C)C(=O)NC(=CC1=O)C2=O
+CHEMBL1098319,O=S(=O)(O)CCS
+CHEMBL1156461,
+CHEMBL116438,
+CHEMBL1170047,NC(=O)c1ccc(I)c([N+](=O)[O-])c1
+CHEMBL1173055,CNCc1ccc(-c2[nH]c3cc(F)cc4c3c2CCNC4=O)cc1
+CHEMBL118,Cc1ccc(-c2cc(C(F)(F)F)nn2-c2ccc(S(N)(=O)=O)cc2)cc1
+CHEMBL1200485,CNC(=O)c1cc(Oc2ccc(NC(=O)Nc3ccc(Cl)c(C(F)(F)F)c3)cc2)ccn1.Cc1ccc(S(=O)(=O)O)cc1
+CHEMBL1200675,CN(C)CCOc1ccc(/C(=C(/CCCl)c2ccccc2)c2ccccc2)cc1.O=C(O)CC(O)(CC(=O)O)C(=O)O
+CHEMBL1200751,O.S=c1[nH]cnc2nc[nH]c12
+CHEMBL1201129,Nc1ncn([C@H]2C[C@H](O)[C@@H](CO)O2)c(=O)n1
+CHEMBL1201148,C[C@H]1C[C@H]2[C@@H]3CCC4=CC(=O)C=C[C@]4(C)[C@H]3C(=O)C[C@]2(C)[C@@]1(O)C(=O)CO
+CHEMBL1201182,CO[C@H]1C[C@@H]2CC[C@@H](C)[C@@](O)(O2)C(=O)C(=O)N2CCCC[C@H]2C(=O)O[C@H]([C@H](C)C[C@@H]2CC[C@@H](OC(=O)C(C)(CO)CO)[C@H](OC)C2)CC(=O)[C@H](C)/C=C(\C)[C@@H](O)[C@@H](OC)C(=O)[C@H](C)C[C@H](C)/C=C/C=C/C=C/1C
+CHEMBL1201187,Cc1nnc(C(C)C)n1[C@@H]1C[C@H]2CC[C@@H](C1)N2CC[C@H](NC(=O)C1CCC(F)(F)CC1)c1ccccc1
+CHEMBL1201616,
+CHEMBL1213492,CCN(CC)Cc1ccc2cc(COC(=O)Nc3ccc(C(=O)NO)cc3)ccc2c1
+CHEMBL1218,CCC(=O)NCC[C@@H]1CCc2ccc3c(c21)CCO3
+CHEMBL1231124,Cc1cc(Nc2nc(N[C@@H](C)c3ncc(F)cn3)ncc2Cl)[nH]n1
+CHEMBL1231461,Br
+CHEMBL1241855,COc1cc(OC)c(/C=C/S(=O)(=O)Cc2ccc(OC)c(NCC(=O)O)c2)c(OC)c1
+CHEMBL1257042,N#C/C(=C/c1ccc(O)c(O)c1)C(=O)NCc1ccccc1
+CHEMBL1287853,Cc1cnc(Nc2ccc(OCCN3CCCC3)cc2)nc1Nc1cccc(S(=O)(=O)NC(C)(C)C)c1
+CHEMBL1358,C[C@]12CC[C@@H]3c4ccc(O)cc4C[C@@H](CCCCCCCCC[S+]([O-])CCCC(F)(F)C(F)(F)F)[C@H]3[C@@H]1CC[C@@H]2O
+CHEMBL1421,Cc1nc(Nc2ncc(C(=O)Nc3c(C)cccc3Cl)s2)cc(N2CCN(CCO)CC2)n1
+CHEMBL1425,Sc1ncnc2nc[nH]c12
+CHEMBL1455,CN(C)c1nc(N(C)C)nc(N(C)C)n1
+CHEMBL14762,CC[C@H](CO)Nc1nc(NCc2ccccc2)c2ncn(C(C)C)c2n1
+CHEMBL1489,Nc1ncn([C@@H]2O[C@H](CO)[C@@H](O)[C@H]2O)c(=O)n1
+CHEMBL1504,CC1(C)O[C@@H]2C[C@H]3[C@@H]4CCC5=CC(=O)C=C[C@]5(C)[C@@]4(F)[C@@H](O)C[C@]3(C)[C@]2(C(=O)CO)O1
+CHEMBL1551724,NC(=O)c1ncn([C@@H]2O[C@H](CO)[C@@H](O)[C@H]2O)c1N
+CHEMBL1556,O=c1[nH]cnc2c1ncn2[C@@H]1O[C@H](CO)[C@@H](O)[C@H]1O
+CHEMBL1645462,Cc1c(NC(=O)OC[C@@H]2COCCN2)cn2ncnc(Nc3ccc4c(cnn4Cc4cccc(F)c4)c3)c12
+CHEMBL165,Oc1ccc(/C=C/c2cc(O)cc(O)c2)cc1
+CHEMBL1651524,O=C(Nc1cccnc1)N1CCC(Cc2cccc(Oc3ccc(C(F)(F)F)cn3)c2)CC1
+CHEMBL1651906,CN(N=O)C(=O)N[C@H]1C(O)O[C@H](CO)[C@@H](O)[C@@H]1O
+CHEMBL1746,O=C([O-])CCCc1ccccc1.[Na+]
+CHEMBL1773,CCCCCOC(=O)Nc1nc(=O)n([C@@H]2O[C@H](C)[C@@H](O)[C@H]2O)cc1F
+CHEMBL1789941,N#CC[C@H](C1CCCC1)n1cc(-c2ncnc3[nH]ccc23)cn1
+CHEMBL185,O=c1[nH]cc(F)c(=O)[nH]1
+CHEMBL1851943,CCCCc1nc2cc(/C=C/C(=O)NO)ccc2n1CCN(CC)CC
+CHEMBL189584,CCN(CC)CCCCNc1ncc2cc(-c3cc(OC)cc(OC)c3)c(NC(=O)NC(C)(C)C)nc2n1
+CHEMBL1908397,O=C(c1ccc(/C=C/c2n[nH]c3ccccc23)cc1)N1CCNCC1
+CHEMBL191003,Nc1nc(Nc2ccc(S(N)(=O)=O)cc2)nn1C(=O)c1c(F)cccc1F
+CHEMBL191482,CC(C)[C@H](C(=O)Nc1ccc(C(=O)NO)cc1)c1ccccc1
+CHEMBL1923234,C[C@H](NC(=O)/C(C#N)=C/c1cccc(Br)n1)c1ccccc1
+CHEMBL1946170,CNC(=O)c1cc(Oc2ccc(NC(=O)Nc3ccc(Cl)c(C(F)(F)F)c3)c(F)c2)ccn1
+CHEMBL1952329,CN1CCC(CNc2ccc3ncc(-c4cccc(OC(F)(F)F)c4)n3n2)CC1
+CHEMBL202721,COc1cc2c(Nc3ccc(NC(=O)c4ccccc4)cc3)ncnc2cc1OCCCN1CCOCC1
+CHEMBL2062804,CN1CCN(c2cccc(Nc3nc4c(-c5ccc(S(C)(=O)=O)cc5)cccn4n3)c2)CC1
+CHEMBL2103863,CN(C)Cc1c(C(=O)NCCOc2ccc(C(=O)NO)cc2)oc2ccccc12
+CHEMBL2103875,CC(=O)Nc1cccc(-n2c(=O)n(C3CC3)c(=O)c3c(Nc4ccc(I)cc4F)n(C)c(=O)c(C)c32)c1
+CHEMBL2105613,CCCC(CCC)C(=O)O.CCCC(CCC)C(=O)[O-].[Na+]
+CHEMBL2105734,COCC[n+]1c2c(n(Cc3cnccn3)c1C)C(=O)c1ccccc1C2=O.[Br-]
+CHEMBL2105759,CCS(=O)(=O)N1CC(CC#N)(n2cc(-c3ncnc4[nH]ccc34)cn2)C1
+CHEMBL2105763,Cn1cc(CNCC2CCN(c3ncc(C(=O)NO)cn3)CC2)c2ccccc21
+CHEMBL2107823,Cc1cc(Nc2cc(CN3CCOCC3)c3nc(C)c(Cc4ccc(Cl)cc4F)n3n2)n[nH]1
+CHEMBL2110734,COCC[n+]1c2c(n(Cc3cnccn3)c1C)C(=O)c1ccccc1C2=O
+CHEMBL2137530,c1ccc2c(CCNc3ccc(Nc4ccncc4)cc3)c[nH]c2c1
+CHEMBL2140408,Cc1csc(-c2nnc(Nc3ccc(Oc4ncccc4-c4ccnc(N)n4)cc3)c3ccccc23)c1
+CHEMBL215152,CCN(CCO)CCCOc1ccc2c(Nc3cc(CC(=O)Nc4cccc(F)c4)[nH]n3)ncnc2c1
+CHEMBL221959,C[C@@H]1CCN(C(=O)CC#N)C[C@@H]1N(C)c1ncnc2[nH]ccc12
+CHEMBL223360,Cc1ccc(F)c(NC(=O)Nc2ccc(-c3cccc4[nH]nc(N)c34)cc2)c1
+CHEMBL225071,Cc1nc(=O)c2cc(CN(C)c3ccc(C(=O)N[C@@H](CCC(=O)O)C(=O)O)s3)ccc2[nH]1
+CHEMBL2338329,Cc1nc(C(=O)NCC(=O)O)c(O)c2ccc(Oc3ccccc3)cc12
+CHEMBL235191,CC(=O)Nc1ccc(C(=O)Nc2ccccc2N)cc1
+CHEMBL2354444,COc1cccc2c1C(=O)c1c(O)c3c(c(O)c1C2=O)C[C@@](O)(C(=O)CO)C[C@@H]3O[C@H]1C[C@H](N)[C@H](O[C@@H]2CCCCO2)[C@H](C)O1
+CHEMBL243712,CCN1CCCC1CNC(=O)c1cc(S(=O)(=O)CC)c(N)cc1OC
+CHEMBL244948,O=c1ccc2ccc(O)c(O)c2o1
+CHEMBL24828,COc1cc2/c(=N/c3ccc(Br)cc3F)nc[nH]c2cc1OCC1CCN(C)CC1
+CHEMBL252164,CCNC(=O)c1noc(-c2cc(C(C)C)c(O)cc2O)c1-c1ccc(CN2CCOCC2)cc1
+CHEMBL255863,Cc1cn(-c2cc(NC(=O)c3ccc(C)c(Nc4nccc(-c5cccnc5)n4)c3)cc(C(F)(F)F)c2)cn1
+CHEMBL257990,N[C@H]1C[C@@H]1c1ccccc1
+CHEMBL259084,O=C(O)c1ccc(Nc2ncc3c(n2)-c2ccc(Cl)cc2C(c2c(F)cccc2F)=NC3)cc1
+CHEMBL261237,N#C/C(=C(/N)Sc1ccc(N)cc1)c1ccccc1C(F)(F)F
+CHEMBL272980,Nc1ccccc1NC(=O)c1ccc(CNc2nccc(-c3cccnc3)n2)cc1
+CHEMBL27759,Nc1ccccc1NC(=O)c1ccc(CNC(=O)OCc2cccnc2)cc1
+CHEMBL288441,COc1cc(Nc2c(C#N)cnc3cc(OCCCN4CCN(C)CC4)c(OC)cc23)c(Cl)cc1Cl
+CHEMBL299613,COc1cc2c(cc1O)CC[C@@H]1[C@@H]2CC[C@]2(C)[C@@H](O)CC[C@@H]12
+CHEMBL30,CN/C(=N\CCSCc1nc[nH]c1C)NC#N
+CHEMBL300138,Cn1cc(C2=C(c3cn(C4CCN(Cc5ccccn5)CC4)c4ccccc34)C(=O)NC2=O)c2ccccc21
+CHEMBL313833,NC1CC1c1ccccc1
+CHEMBL3182444,
+CHEMBL3233481,CN(C)CC(=O)Nc1ccc2[nH]c(=O)c3ccccc3c2c1.Cl
+CHEMBL329993,C/C(=C\c1csc(C)n1)[C@@H]1C[C@@H]2O[C@@H]2CCC[C@H](C)[C@H](O)[C@@H](C)C(=O)C(C)(C)[C@@H](O)CC(=O)O1
+CHEMBL3301607,O=C(Nc1nc2cccc(-c3ccc(CN4CCS(=O)(=O)CC4)cc3)n2n1)C1CC1
+CHEMBL33778,CCCCCCCCCCCCCCCc1cccc(O)c1C(=O)O
+CHEMBL3414621,CCN(c1cc(-c2ccc(CN3CCOCC3)cc2)cc(C(=O)NCc2c(C)cc(C)[nH]c2=O)c1C)C1CCOCC1
+CHEMBL3544932,CCS(=O)(=O)c1cccc(-c2cc(C(=O)NC3CCN(C)CC3)c(C)c3[nH]c4ncc(C)cc4c23)c1
+CHEMBL3544989,
+CHEMBL3545215,CCn1c(C(=O)N(C2CC2)C2CC2)cc2c3c(ncn3C)c(Nc3cc(C)n(C)n3)nc21
+CHEMBL3545320,
+CHEMBL35482,COc1cccc(-c2cc(=O)c3ccccc3o2)c1N
+CHEMBL356066,O=C(/C=C/c1ccc(CN(CCO)CCc2c[nH]c3ccccc23)cc1)NO
+CHEMBL3621988,Nc1cc(F)ccc1NC(=O)c1ccc(CNC(=O)/C=C/c2cccnc2)cc1
+CHEMBL3622533,COc1ccc(-c2nc(N3CCOCC3)c3sc(CN(C)c4ncc(C(=O)NO)cn4)cc3n2)cn1
+CHEMBL363160,CCc1cccc(CC)c1NC(=O)N1Cc2[nH]nc(NC(=O)c3ccc(N4CCN(C)CC4)cc3)c2C1
+CHEMBL372303,CN(C)CC(=O)Nc1ccc2[nH]c(=O)c3ccccc3c2c1
+CHEMBL376408,CN(C)CC[C@H](CSc1ccccc1)Nc1ccc(S(=O)(=O)NC(=O)c2ccc(N3CCN(Cc4ccccc4-c4ccc(Cl)cc4)CC3)cc2)cc1[N+](=O)[O-]
+CHEMBL380797,CC(NC(=O)c1ccccc1/N=C/c1c(O)ccc2ccccc12)c1ccccc1
+CHEMBL38380,O=S(=O)(c1cccc2cnccc12)N1CCCNCC1
+CHEMBL383824,CO[C@H]1/C=C\C=C(/C)C(=O)NC2=CC(=O)C(NCCN(C)C)=C(C[C@@H](C)C[C@H](OC)[C@H](O)[C@@H](C)/C=C(\C)[C@@H]1OC(N)=O)C2=O
+CHEMBL3989843,N[C@@H]1C[C@H]1c1ccccc1.N[C@H]1C[C@@H]1c1ccccc1
+CHEMBL3989914,COCCNCc1ccc(-c2cc3nccc(Oc4ccc(NC(=S)NC(=O)Cc5ccc(F)cc5)cc4F)c3s2)nc1
+CHEMBL3989958,N#Cc1ccnc(N2C(=O)CC[C@H]2C(=O)N(c2cncc(F)c2)[C@H](C(=O)NC2CC(F)(F)C2)c2ccccc2Cl)c1
+CHEMBL4,CC1COc2c(N3CCN(C)CC3)c(F)cc3c(=O)c(C(=O)O)cn1c23
+CHEMBL401930,Cc1cc(N2CCOCC2)cc2[nH]c(-c3c(NC[C@@H](O)c4cccc(Cl)c4)cc[nH]c3=O)nc12
+CHEMBL402548,CO[C@@H](C(=O)N1Cc2[nH]nc(NC(=O)c3ccc(N4CCN(C)CC4)cc3)c2C1)c1ccccc1
+CHEMBL408194,COC1=CC(c2cc3ccccc3[nH]2)=N/C1=C\c1[nH]c(C)cc1C
+CHEMBL408513,O=C(/C=C/c1cccc(S(=O)(=O)Nc2ccccc2)c1)NO
+CHEMBL4116008,CCS(=O)(=O)N1CCN(c2ccc(Nc3ncc(C(N)=O)c(NC4CC4)n3)cc2)CC1
+CHEMBL4205422,O=C(O)CC[C@H]1CC[C@@](c2cc(F)ccc2F)(S(=O)(=O)c2ccc(Cl)cc2)CC1
+CHEMBL428690,CN1CC[C@H](c2c(O)cc(O)c3c(=O)cc(-c4ccccc4Cl)oc23)[C@H](O)C1
+CHEMBL4296717,CN(C)Cc1ccc(S(=O)(=O)n2ccc(/C=C/C(=O)NO)c2)cc1
+CHEMBL4297436,Cc1nc(-c2cccnc2)sc1C(=O)Nc1ccccc1-c1cn2c(CN3CCOCC3)csc2n1
+CHEMBL4303060,CCOC(=O)[C@H](Cc1ccc(F)cc1)NC(=O)[C@@H](N)Cc1ccc(N(CCCl)CCCl)cc1
+CHEMBL440084,NC(=O)c1ccc(-c2nc(-c3ccccn3)c(-c3ccc4c(c3)OCO4)[nH]2)cc1
+CHEMBL443684,CC1(C)CCC(c2ccc(Cl)cc2)=C(CN2CCN(c3ccc(C(=O)NS(=O)(=O)c4ccc(N[C@H](CCN5CCOCC5)CSc5ccccc5)c(S(=O)(=O)C(F)(F)F)c4)cc3)CC2)C1
+CHEMBL455368,C=CC/C=C\C/C=C\CCCCCCCc1cccc(O)c1C(=O)O
+CHEMBL458875,Cc1cn([C@H]2O[C@@H](CO)[C@H](O)[C@H]2F)c(=O)[nH]c1=O
+CHEMBL460499,CCCCCCNC(=O)n1cc(F)c(=O)[nH]c1=O
+CHEMBL4650286,Nc1nn2cc(F)cnc2c1C(=O)Nc1cncc(F)c1N1CCC(C(=O)N2CCN(C3COC3)CC2)CC1
+CHEMBL468,O=C1CCC(N2C(=O)c3ccccc3C2=O)C(=O)N1
+CHEMBL473773,COc1cc2ncnc(Nc3ccc(O)c(Br)c3)c2cc1OC
+CHEMBL477936,Cc1ccc(S(=O)(=O)OCC(=O)Nc2ccc(C(=O)O)c(O)c2)cc1
+CHEMBL482767,O=C(Nc1cccc(Cl)c1)Nc1ncc(CCNc2ncnc3ccsc23)s1
+CHEMBL482967,Cc1nc(N)sc1-c1ccnc(Nc2ccc(N3CCOCC3)cc2)n1
+CHEMBL482968,Cc1cc(Nc2cc(N3CCN(C)CC3)nc(/C=C/c3ccccc3)n2)n[nH]1
+CHEMBL483158,COc1cc(Nc2ncc3c(n2)-c2ccc(Cl)cc2C(c2c(F)cccc2OC)=NC3)ccc1C(=O)O
+CHEMBL483254,Cc1[nH]c2ccccc2c1CCNCc1ccc(/C=C/C(=O)NO)cc1
+CHEMBL488,CCC1(c2ccc(N)cc2)CCC(=O)NC1=O
+CHEMBL491473,COc1cc2c(Oc3ccc4[nH]c(C)cc4c3F)ncnc2cc1OCCCN1CCCC1
+CHEMBL495727,O=C(Nc1c[nH]nc1-c1nc2ccc(CN3CCOCC3)cc2[nH]1)NC1CC1
+CHEMBL50,O=c1c(O)c(-c2ccc(O)c(O)c2)oc2cc(O)cc(O)c12
+CHEMBL502835,COC(=O)c1ccc2c(c1)NC(=O)/C2=C(\Nc1ccc(N(C)C(=O)CN2CCN(C)CC2)cc1)c1ccccc1
+CHEMBL506871,C[C@]1(c2nc3c(C(N)=O)cccc3[nH]2)CCCN1
+CHEMBL514,O=NN(CCCl)C(=O)NC1CCCCC1
+CHEMBL514409,CCS(=O)(=O)Nc1ccc2c(c1)/C(=C(/Nc1ccc(CN3CCCCC3)cc1)c1ccccc1)C(=O)N2
+CHEMBL52885,Cn1cc(C2=C(c3cn(C)c4cc([N+](=O)[O-])ccc34)C(=O)NC2=O)c2ccccc21
+CHEMBL554,CS(=O)(=O)CCNCc1ccc(-c2ccc3ncnc(Nc4ccc(OCc5cccc(F)c5)c(Cl)c4)c3c2)o1
+CHEMBL572878,Cc1cc(Nc2cc(N3CCN(C)CC3)nc(Sc3ccc(NC(=O)C4CC4)cc3)n2)n[nH]1
+CHEMBL572881,CC1(C)CNc2cc(NC(=O)c3cccnc3NCc3ccncc3)ccc21
+CHEMBL575448,C[C@@]1(C(=O)Nc2ccc(F)nc2)CCCN1c1nc(Nc2cc(C3CC3)n[nH]2)c2cccn2n1
+CHEMBL598797,C#Cc1cccc(Nc2ncnc3cc(OC)c(OCCCCCCC(=O)NO)cc23)c1
+CHEMBL601719,C[C@@H](Oc1cc(-c2cnn(C3CCNCC3)c2)cnc1N)c1c(Cl)ccc(F)c1Cl
+CHEMBL607707,CCOc1cc2ncc(C#N)c(Nc3ccc(F)c(Cl)c3)c2cc1NC(=O)/C=C/CN(C)C
+CHEMBL6246,O=c1oc2c(O)c(O)cc3c(=O)oc4c(O)c(O)cc1c4c23
+CHEMBL6291,Cn1cc(C2=C(c3cn(CCCSC(=N)N)c4ccccc34)C(=O)NC2=O)c2ccccc21
+CHEMBL635,C[C@]12C=CC(=O)C=C1CC[C@@H]1[C@@H]2C(=O)C[C@@]2(C)[C@H]1CC[C@]2(O)C(=O)CO
+CHEMBL65892,CN(C)Cc1ccc(-c2nc3cccc4c3n2CCNC4=O)cc1
+CHEMBL671,S=P(N1CC1)(N1CC1)N1CC1
+CHEMBL675,NCCCC(O)(P(=O)([O-])O)P(=O)(O)O.[Na+]
+CHEMBL81977,NC(=O)c1cccc(N)c1
+CHEMBL820,CS(=O)(=O)OCCCCOS(C)(=O)=O
+CHEMBL84617,CCC(=O)N(c1ccccc1F)C1(c2ccccc2)CCN(CCn2nnn(CC)c2=O)CC1
+CHEMBL848,Nc1cccc2c1CN(C1CCC(=O)NC1=O)C2=O
+CHEMBL86416,C=C1C(=O)O[C@@H]2/C=C(\C)CC/C=C(\C)CC[C@@H]12
+CHEMBL93,CC(c1cc2ccccc2s1)N(O)C(N)=O
+CHEMBL94657,C/C(=C\c1csc(C)n1)[C@@H]1C[C@@H]2O[C@]2(C)CCC[C@H](C)[C@H](O)[C@@H](C)C(=O)C(C)(C)[C@@H](O)CC(=O)O1
+CHEMBL953,CCN(CC)C(=O)/C(C#N)=C/c1cc(O)c(O)c([N+](=O)[O-])c1
+CHEMBL964,CCN(CC)C(=S)SSC(=S)N(CC)CC
+CHEMBL975,O=S(=O)([O-])CCS.[Na+]
+CHEMBL99,CC(/C=C/C(=O)NO)=C\[C@@H](C)C(=O)c1ccc(N(C)C)cc1

PDGrapher/data/process_all_data.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Process COSMIC data
+cd scripts/cosmic
+python3 generate_cell_line_mutations_files.py
+cd ../..
+
+
+# Process COSMIC data
+cd scripts/drugbank
+python3 get_drug_targets.py
+cd ../..
+
+# Process PPI data
+cd scripts/ppi
+python3 export_ppi_all_genes.py
+cd ../..
+
+
+# Process LINCS data
+cd scripts/lincs
+python3 process_data.py
+python3 process_data_healthy.py
+python3 process_data_chemical_1.py
+python3 process_data_chemical_2.py
+python3 process_data_healthy_chemical.py
+cd ../..
+
+
+
+# Data preparation
+cd scripts/rep-learning-approach-3
+python3 export_data_for_torch_geometric.py
+python3 export_data_for_torch_geometric_chemical.py
+cd ../..

PDGrapher/data/processed/sciplex3_k562_24h_gene_map.json

@@ -0,0 +1,1978 @@
+{
+  "ENSG00000000971": "CFH",
+  "ENSG00000001629": "ANKIB1",
+  "ENSG00000001631": "KRIT1",
+  "ENSG00000002822": "MAD1L1",
+  "ENSG00000002834": "LASP1",
+  "ENSG00000003402": "CFLAR",
+  "ENSG00000003436": "TFPI",
+  "ENSG00000004487": "KDM1A",
+  "ENSG00000004534": "RBM6",
+  "ENSG00000004864": "SLC25A13",
+  "ENSG00000004866": "ST7",
+  "ENSG00000004897": "CDC27",
+  "ENSG00000005187": "ACSM3",
+  "ENSG00000005249": "PRKAR2B",
+  "ENSG00000005339": "CREBBP",
+  "ENSG00000005483": "KMT2E",
+  "ENSG00000005700": "IBTK",
+  "ENSG00000005810": "MYCBP2",
+  "ENSG00000005893": "LAMP2",
+  "ENSG00000006125": "AP2B1",
+  "ENSG00000006576": "PHTF2",
+  "ENSG00000006704": "GTF2IRD1",
+  "ENSG00000006831": "ADIPOR2",
+  "ENSG00000007047": "MARK4",
+  "ENSG00000007168": "PAFAH1B1",
+  "ENSG00000007202": "BLTP2",
+  "ENSG00000007392": "LUC7L",
+  "ENSG00000007923": "DNAJC11",
+  "ENSG00000007944": "MYLIP",
+  "ENSG00000008083": "JARID2",
+  "ENSG00000008294": "SPAG9",
+  "ENSG00000009307": "CSDE1",
+  "ENSG00000009335": "UBE3C",
+  "ENSG00000009413": "REV3L",
+  "ENSG00000009694": "TENM1",
+  "ENSG00000009954": "BAZ1B",
+  "ENSG00000010017": "RANBP9",
+  "ENSG00000010244": "ZNF207",
+  "ENSG00000010292": "NCAPD2",
+  "ENSG00000010803": "SCMH1",
+  "ENSG00000011114": "BTBD7",
+  "ENSG00000011198": "ABHD5",
+  "ENSG00000011243": "AKAP8L",
+  "ENSG00000011258": "MBTD1",
+  "ENSG00000011275": "RNF216",
+  "ENSG00000011304": "PTBP1",
+  "ENSG00000011376": "LARS2",
+  "ENSG00000011405": "PIK3C2A",
+  "ENSG00000011454": "RABGAP1",
+  "ENSG00000011566": "MAP4K3",
+  "ENSG00000012048": "BRCA1",
+  "ENSG00000012660": "ELOVL5",
+  "ENSG00000014824": "SLC30A9",
+  "ENSG00000018189": "RUFY3",
+  "ENSG00000018510": "AGPS",
+  "ENSG00000018699": "TTC27",
+  "ENSG00000020577": "SAMD4A",
+  "ENSG00000021776": "AQR",
+  "ENSG00000023228": "NDUFS1",
+  "ENSG00000023287": "RB1CC1",
+  "ENSG00000023909": "GCLM",
+  "ENSG00000024048": "UBR2",
+  "ENSG00000025293": "PHF20",
+  "ENSG00000025796": "SEC63",
+  "ENSG00000025800": "KPNA6",
+  "ENSG00000026025": "VIM",
+  "ENSG00000028203": "VEZT",
+  "ENSG00000029363": "BCLAF1",
+  "ENSG00000029534": "ANK1",
+  "ENSG00000029725": "RABEP1",
+  "ENSG00000030066": "NUP160",
+  "ENSG00000031003": "FAM13B",
+  "ENSG00000031698": "SARS1",
+  "ENSG00000033178": "UBA6",
+  "ENSG00000033327": "GAB2",
+  "ENSG00000033627": "ATP6V0A1",
+  "ENSG00000033800": "PIAS1",
+  "ENSG00000034053": "APBA2",
+  "ENSG00000034510": "TMSB10",
+  "ENSG00000035403": "VCL",
+  "ENSG00000036257": "CUL3",
+  "ENSG00000036549": "ZZZ3",
+  "ENSG00000037637": "FBXO42",
+  "ENSG00000038532": "CLEC16A",
+  "ENSG00000039123": "MTREX",
+  "ENSG00000039560": "RAI14",
+  "ENSG00000040341": "STAU2",
+  "ENSG00000042062": "RIPOR3",
+  "ENSG00000044115": "CTNNA1",
+  "ENSG00000044574": "HSPA5",
+  "ENSG00000046604": "DSG2",
+  "ENSG00000047410": "TPR",
+  "ENSG00000047579": "DTNBP1",
+  "ENSG00000047648": "ARHGAP6",
+  "ENSG00000047849": "MAP4",
+  "ENSG00000048405": "ZNF800",
+  "ENSG00000048471": "SNX29",
+  "ENSG00000048649": "RSF1",
+  "ENSG00000048707": "VPS13D",
+  "ENSG00000048828": "FAM120A",
+  "ENSG00000048991": "R3HDM1",
+  "ENSG00000049323": "LTBP1",
+  "ENSG00000049618": "ARID1B",
+  "ENSG00000049759": "NEDD4L",
+  "ENSG00000050405": "LIMA1",
+  "ENSG00000050426": "LETMD1",
+  "ENSG00000051382": "PIK3CB",
+  "ENSG00000051825": "MPHOSPH9",
+  "ENSG00000052126": "PLEKHA5",
+  "ENSG00000052841": "TTC17",
+  "ENSG00000054118": "THRAP3",
+  "ENSG00000054267": "ARID4B",
+  "ENSG00000054282": "SDCCAG8",
+  "ENSG00000054523": "KIF1B",
+  "ENSG00000054611": "TBC1D22A",
+  "ENSG00000054965": "FAM168A",
+  "ENSG00000055044": "NOP58",
+  "ENSG00000055130": "CUL1",
+  "ENSG00000055208": "TAB2",
+  "ENSG00000055609": "KMT2C",
+  "ENSG00000055917": "PUM2",
+  "ENSG00000056097": "ZFR",
+  "ENSG00000056586": "RC3H2",
+  "ENSG00000057608": "GDI2",
+  "ENSG00000057663": "ATG5",
+  "ENSG00000058063": "ATP11B",
+  "ENSG00000058272": "PPP1R12A",
+  "ENSG00000058668": "ATP2B4",
+  "ENSG00000058804": "NDC1",
+  "ENSG00000059758": "CDK17",
+  "ENSG00000059804": "SLC2A3",
+  "ENSG00000060237": "WNK1",
+  "ENSG00000060339": "CCAR1",
+  "ENSG00000060749": "QSER1",
+  "ENSG00000060982": "BCAT1",
+  "ENSG00000061676": "NCKAP1",
+  "ENSG00000061936": "SFSWAP",
+  "ENSG00000061987": "MON2",
+  "ENSG00000062194": "GPBP1",
+  "ENSG00000062650": "WAPL",
+  "ENSG00000062716": "VMP1",
+  "ENSG00000062725": "APPBP2",
+  "ENSG00000063046": "EIF4B",
+  "ENSG00000063177": "RPL18",
+  "ENSG00000063978": "RNF4",
+  "ENSG00000064313": "TAF2",
+  "ENSG00000064393": "HIPK2",
+  "ENSG00000064419": "TNPO3",
+  "ENSG00000064607": "SUGP2",
+  "ENSG00000064999": "ANKS1A",
+  "ENSG00000065060": "BLTP3A",
+  "ENSG00000065243": "PKN2",
+  "ENSG00000065526": "SPEN",
+  "ENSG00000065833": "ME1",
+  "ENSG00000065883": "CDK13",
+  "ENSG00000066027": "PPP2R5A",
+  "ENSG00000066084": "DIP2B",
+  "ENSG00000066279": "ASPM",
+  "ENSG00000066777": "ARFGEF1",
+  "ENSG00000066933": "MYO9A",
+  "ENSG00000067208": "EVI5",
+  "ENSG00000067225": "PKM",
+  "ENSG00000067369": "TP53BP1",
+  "ENSG00000067560": "RHOA",
+  "ENSG00000067900": "ROCK1",
+  "ENSG00000067955": "CBFB",
+  "ENSG00000068305": "MEF2A",
+  "ENSG00000068366": "ACSL4",
+  "ENSG00000068489": "PRR11",
+  "ENSG00000068650": "ATP11A",
+  "ENSG00000068654": "POLR1A",
+  "ENSG00000068784": "SRBD1",
+  "ENSG00000068796": "KIF2A",
+  "ENSG00000068878": "PSME4",
+  "ENSG00000069275": "NUCKS1",
+  "ENSG00000069702": "TGFBR3",
+  "ENSG00000069849": "ATP1B3",
+  "ENSG00000069869": "NEDD4",
+  "ENSG00000069956": "MAPK6",
+  "ENSG00000069974": "RAB27A",
+  "ENSG00000070010": "UFD1",
+  "ENSG00000070366": "SMG6",
+  "ENSG00000070371": "CLTCL1",
+  "ENSG00000070413": "DGCR2",
+  "ENSG00000070756": "PABPC1",
+  "ENSG00000070785": "EIF2B3",
+  "ENSG00000070882": "OSBPL3",
+  "ENSG00000070950": "RAD18",
+  "ENSG00000070961": "ATP2B1",
+  "ENSG00000071054": "MAP4K4",
+  "ENSG00000071082": "RPL31",
+  "ENSG00000071909": "MYO3B",
+  "ENSG00000072274": "TFRC",
+  "ENSG00000072364": "AFF4",
+  "ENSG00000072518": "MARK2",
+  "ENSG00000072736": "NFATC3",
+  "ENSG00000072803": "FBXW11",
+  "ENSG00000073060": "SCARB1",
+  "ENSG00000073417": "PDE8A",
+  "ENSG00000073614": "KDM5A",
+  "ENSG00000073792": "IGF2BP2",
+  "ENSG00000073849": "ST6GAL1",
+  "ENSG00000073921": "PICALM",
+  "ENSG00000074054": "CLASP1",
+  "ENSG00000074201": "CLNS1A",
+  "ENSG00000074370": "ATP2A3",
+  "ENSG00000074603": "DPP8",
+  "ENSG00000074695": "LMAN1",
+  "ENSG00000074696": "HACD3",
+  "ENSG00000074800": "ENO1",
+  "ENSG00000075151": "EIF4G3",
+  "ENSG00000075292": "ZNF638",
+  "ENSG00000075413": "MARK3",
+  "ENSG00000075420": "FNDC3B",
+  "ENSG00000075539": "FRYL",
+  "ENSG00000075568": "TMEM131",
+  "ENSG00000075624": "ACTB",
+  "ENSG00000075651": "PLD1",
+  "ENSG00000075711": "DLG1",
+  "ENSG00000075785": "RAB7A",
+  "ENSG00000076108": "BAZ2A",
+  "ENSG00000076321": "KLHL20",
+  "ENSG00000076685": "NT5C2",
+  "ENSG00000076706": "MCAM",
+  "ENSG00000077097": "TOP2B",
+  "ENSG00000077157": "PPP1R12B",
+  "ENSG00000077254": "USP33",
+  "ENSG00000077549": "CAPZB",
+  "ENSG00000078114": "NEBL",
+  "ENSG00000078124": "ACER3",
+  "ENSG00000078140": "UBE2K",
+  "ENSG00000078269": "SYNJ2",
+  "ENSG00000078304": "PPP2R5C",
+  "ENSG00000078369": "GNB1",
+  "ENSG00000078403": "MLLT10",
+  "ENSG00000078618": "NRDC",
+  "ENSG00000078674": "PCM1",
+  "ENSG00000078699": "CBFA2T2",
+  "ENSG00000078747": "ITCH",
+  "ENSG00000079246": "XRCC5",
+  "ENSG00000079335": "CDC14A",
+  "ENSG00000079459": "FDFT1",
+  "ENSG00000079805": "DNM2",
+  "ENSG00000080298": "RFX3",
+  "ENSG00000080345": "RIF1",
+  "ENSG00000080608": "PUM3",
+  "ENSG00000080802": "CNOT4",
+  "ENSG00000080824": "HSP90AA1",
+  "ENSG00000080839": "RBL1",
+  "ENSG00000081026": "MAGI3",
+  "ENSG00000081189": "MEF2C",
+  "ENSG00000081237": "PTPRC",
+  "ENSG00000081913": "PHLPP1",
+  "ENSG00000082068": "WDR70",
+  "ENSG00000082269": "FAM135A",
+  "ENSG00000082701": "GSK3B",
+  "ENSG00000082805": "ERC1",
+  "ENSG00000082898": "XPO1",
+  "ENSG00000083168": "KAT6A",
+  "ENSG00000083312": "TNPO1",
+  "ENSG00000083444": "PLOD1",
+  "ENSG00000083535": "PIBF1",
+  "ENSG00000083845": "RPS5",
+  "ENSG00000084070": "SMAP2",
+  "ENSG00000084234": "APLP2",
+  "ENSG00000084623": "EIF3I",
+  "ENSG00000084676": "NCOA1",
+  "ENSG00000084733": "RAB10",
+  "ENSG00000084754": "HADHA",
+  "ENSG00000085224": "ATRX",
+  "ENSG00000085511": "MAP3K4",
+  "ENSG00000085832": "EPS15",
+  "ENSG00000086015": "MAST2",
+  "ENSG00000086102": "NFX1",
+  "ENSG00000086200": "IPO11",
+  "ENSG00000086232": "EIF2AK1",
+  "ENSG00000086666": "ZFAND6",
+  "ENSG00000086758": "HUWE1",
+  "ENSG00000087086": "FTL",
+  "ENSG00000087095": "NLK",
+  "ENSG00000087206": "UIMC1",
+  "ENSG00000087274": "ADD1",
+  "ENSG00000087460": "GNAS",
+  "ENSG00000087589": "CASS4",
+  "ENSG00000087903": "RFX2",
+  "ENSG00000088179": "PTPN4",
+  "ENSG00000088247": "KHSRP",
+  "ENSG00000088325": "TPX2",
+  "ENSG00000088387": "DOCK9",
+  "ENSG00000088854": "DNAAF9",
+  "ENSG00000088888": "MAVS",
+  "ENSG00000089053": "ANAPC5",
+  "ENSG00000089094": "KDM2B",
+  "ENSG00000089154": "GCN1",
+  "ENSG00000089157": "RPLP0",
+  "ENSG00000089280": "FUS",
+  "ENSG00000089597": "GANAB",
+  "ENSG00000089902": "RCOR1",
+  "ENSG00000090006": "LTBP4",
+  "ENSG00000090013": "BLVRB",
+  "ENSG00000090273": "NUDC",
+  "ENSG00000090686": "USP48",
+  "ENSG00000090861": "AARS1",
+  "ENSG00000090863": "GLG1",
+  "ENSG00000090905": "TNRC6A",
+  "ENSG00000091039": "OSBPL8",
+  "ENSG00000091127": "PUS7",
+  "ENSG00000091436": "MAP3K20",
+  "ENSG00000091490": "SEL1L3",
+  "ENSG00000091592": "NLRP1",
+  "ENSG00000092108": "SCFD1",
+  "ENSG00000092148": "HECTD1",
+  "ENSG00000092199": "HNRNPC",
+  "ENSG00000092439": "TRPM7",
+  "ENSG00000092820": "EZR",
+  "ENSG00000092847": "AGO1",
+  "ENSG00000092853": "CLSPN",
+  "ENSG00000092929": "UNC13D",
+  "ENSG00000092931": "MFSD11",
+  "ENSG00000093009": "CDC45",
+  "ENSG00000093010": "COMT",
+  "ENSG00000093167": "LRRFIP2",
+  "ENSG00000094916": "CBX5",
+  "ENSG00000094975": "SUCO",
+  "ENSG00000095002": "MSH2",
+  "ENSG00000095261": "PSMD5",
+  "ENSG00000095564": "BTAF1",
+  "ENSG00000095637": "SORBS1",
+  "ENSG00000095787": "WAC",
+  "ENSG00000095951": "HIVEP1",
+  "ENSG00000096060": "FKBP5",
+  "ENSG00000096063": "SRPK1",
+  "ENSG00000096384": "HSP90AB1",
+  "ENSG00000096968": "JAK2",
+  "ENSG00000097007": "ABL1",
+  "ENSG00000099194": "SCD",
+  "ENSG00000099331": "MYO9B",
+  "ENSG00000099783": "HNRNPM",
+  "ENSG00000099910": "KLHL22",
+  "ENSG00000099917": "MED15",
+  "ENSG00000099940": "SNAP29",
+  "ENSG00000099942": "CRKL",
+  "ENSG00000099968": "BCL2L13",
+  "ENSG00000099991": "CABIN1",
+  "ENSG00000100023": "PPIL2",
+  "ENSG00000100030": "MAPK1",
+  "ENSG00000100034": "PPM1F",
+  "ENSG00000100075": "SLC25A1",
+  "ENSG00000100084": "HIRA",
+  "ENSG00000100154": "TTC28",
+  "ENSG00000100181": "TPTEP1",
+  "ENSG00000100201": "DDX17",
+  "ENSG00000100239": "PPP6R2",
+  "ENSG00000100266": "PACSIN2",
+  "ENSG00000100316": "RPL3",
+  "ENSG00000100320": "RBFOX2",
+  "ENSG00000100354": "TNRC6B",
+  "ENSG00000100372": "SLC25A17",
+  "ENSG00000100376": "SIRAL2",
+  "ENSG00000100393": "EP300",
+  "ENSG00000100485": "SOS2",
+  "ENSG00000100596": "SPTLC2",
+  "ENSG00000100749": "VRK1",
+  "ENSG00000100784": "RPS6KA5",
+  "ENSG00000100811": "YY1",
+  "ENSG00000100813": "ACIN1",
+  "ENSG00000100888": "CHD8",
+  "ENSG00000100991": "TRPC4AP",
+  "ENSG00000101040": "ZMYND8",
+  "ENSG00000101126": "ADNP",
+  "ENSG00000101152": "DNAJC5",
+  "ENSG00000101191": "DIDO1",
+  "ENSG00000101236": "RNF24",
+  "ENSG00000101266": "CSNK2A1",
+  "ENSG00000101294": "HM13",
+  "ENSG00000101337": "TM9SF4",
+  "ENSG00000101557": "USP14",
+  "ENSG00000101558": "VAPA",
+  "ENSG00000101577": "LPIN2",
+  "ENSG00000101596": "SMCHD1",
+  "ENSG00000101639": "CEP192",
+  "ENSG00000101745": "ANKRD12",
+  "ENSG00000101752": "MIB1",
+  "ENSG00000101782": "RIOK3",
+  "ENSG00000101849": "TBL1X",
+  "ENSG00000101868": "POLA1",
+  "ENSG00000101966": "XIAP",
+  "ENSG00000101972": "STAG2",
+  "ENSG00000101974": "ATP11C",
+  "ENSG00000102144": "PGK1",
+  "ENSG00000102145": "GATA1",
+  "ENSG00000102189": "EEA1",
+  "ENSG00000102265": "TIMP1",
+  "ENSG00000102531": "FNDC3A",
+  "ENSG00000102595": "UGGT2",
+  "ENSG00000102763": "VWA8",
+  "ENSG00000102786": "INTS6",
+  "ENSG00000102893": "PHKB",
+  "ENSG00000102908": "NFAT5",
+  "ENSG00000102910": "LONP2",
+  "ENSG00000102974": "CTCF",
+  "ENSG00000103047": "TANGO6",
+  "ENSG00000103091": "WDR59",
+  "ENSG00000103148": "NPRL3",
+  "ENSG00000103194": "USP10",
+  "ENSG00000103222": "ABCC1",
+  "ENSG00000103319": "EEF2K",
+  "ENSG00000103335": "PIEZO1",
+  "ENSG00000103342": "GSPT1",
+  "ENSG00000103365": "GGA2",
+  "ENSG00000103381": "CPPED1",
+  "ENSG00000103485": "QPRT",
+  "ENSG00000103657": "HERC1",
+  "ENSG00000103978": "TMEM87A",
+  "ENSG00000103994": "ZNF106",
+  "ENSG00000103995": "CEP152",
+  "ENSG00000104067": "TJP1",
+  "ENSG00000104093": "DMXL2",
+  "ENSG00000104133": "SPG11",
+  "ENSG00000104218": "CSPP1",
+  "ENSG00000104299": "INTS9",
+  "ENSG00000104341": "LAPTM4B",
+  "ENSG00000104375": "STK3",
+  "ENSG00000104388": "RAB2A",
+  "ENSG00000104447": "TRPS1",
+  "ENSG00000104490": "NCALD",
+  "ENSG00000104517": "UBR5",
+  "ENSG00000104529": "EEF1D",
+  "ENSG00000104549": "SQLE",
+  "ENSG00000104626": "ERI1",
+  "ENSG00000104687": "GSR",
+  "ENSG00000104723": "TUSC3",
+  "ENSG00000104738": "MCM4",
+  "ENSG00000104765": "BNIP3L",
+  "ENSG00000104805": "NUCB1",
+  "ENSG00000104852": "SNRNP70",
+  "ENSG00000104904": "OAZ1",
+  "ENSG00000105176": "URI1",
+  "ENSG00000105186": "ANKRD27",
+  "ENSG00000105193": "RPS16",
+  "ENSG00000105220": "GPI",
+  "ENSG00000105223": "PLD3",
+  "ENSG00000105281": "SLC1A5",
+  "ENSG00000105323": "HNRNPUL1",
+  "ENSG00000105329": "TGFB1",
+  "ENSG00000105355": "PLIN3",
+  "ENSG00000105372": "RPS19",
+  "ENSG00000105426": "PTPRS",
+  "ENSG00000105438": "KDELR1",
+  "ENSG00000105464": "GRIN2D",
+  "ENSG00000105576": "TNPO2",
+  "ENSG00000105738": "SIPA1L3",
+  "ENSG00000105778": "AVL9",
+  "ENSG00000105810": "CDK6",
+  "ENSG00000105887": "MTPN",
+  "ENSG00000105889": "STEAP1B",
+  "ENSG00000105939": "ZC3HAV1",
+  "ENSG00000105953": "OGDH",
+  "ENSG00000105968": "H2AZ2",
+  "ENSG00000105983": "LMBR1",
+  "ENSG00000105993": "DNAJB6",
+  "ENSG00000106034": "CPED1",
+  "ENSG00000106049": "HIBADH",
+  "ENSG00000106052": "TAX1BP1",
+  "ENSG00000106070": "GRB10",
+  "ENSG00000106105": "GARS1",
+  "ENSG00000106153": "CHCHD2",
+  "ENSG00000106211": "HSPB1",
+  "ENSG00000106261": "ZKSCAN1",
+  "ENSG00000106263": "EIF3B",
+  "ENSG00000106299": "WASL",
+  "ENSG00000106327": "TFR2",
+  "ENSG00000106443": "PHF14",
+  "ENSG00000106459": "NRF1",
+  "ENSG00000106462": "EZH2",
+  "ENSG00000106526": "ACTR3C",
+  "ENSG00000106554": "CHCHD3",
+  "ENSG00000106603": "COA1",
+  "ENSG00000106615": "RHEB",
+  "ENSG00000106665": "CLIP2",
+  "ENSG00000106771": "TMEM245",
+  "ENSG00000106868": "SUSD1",
+  "ENSG00000107077": "KDM4C",
+  "ENSG00000107099": "DOCK8",
+  "ENSG00000107164": "FUBP3",
+  "ENSG00000107242": "PIP5K1B",
+  "ENSG00000107263": "RAPGEF1",
+  "ENSG00000107290": "SETX",
+  "ENSG00000107341": "UBE2R2",
+  "ENSG00000107581": "EIF3A",
+  "ENSG00000107625": "DDX50",
+  "ENSG00000107643": "MAPK8",
+  "ENSG00000107745": "MICU1",
+  "ENSG00000107758": "PPP3CB",
+  "ENSG00000107771": "CCSER2",
+  "ENSG00000107779": "BMPR1A",
+  "ENSG00000107862": "GBF1",
+  "ENSG00000107863": "ARHGAP21",
+  "ENSG00000107929": "LARP4B",
+  "ENSG00000108018": "SORCS1",
+  "ENSG00000108021": "TASOR2",
+  "ENSG00000108100": "CCNY",
+  "ENSG00000108107": "RPL28",
+  "ENSG00000108175": "ZMIZ1",
+  "ENSG00000108256": "NUFIP2",
+  "ENSG00000108298": "RPL19",
+  "ENSG00000108306": "FBXL20",
+  "ENSG00000108395": "TRIM37",
+  "ENSG00000108405": "P2RX1",
+  "ENSG00000108424": "KPNB1",
+  "ENSG00000108474": "PIGL",
+  "ENSG00000108510": "MED13",
+  "ENSG00000108518": "PFN1",
+  "ENSG00000108848": "LUC7L3",
+  "ENSG00000108883": "EFTUD2",
+  "ENSG00000108953": "YWHAE",
+  "ENSG00000109046": "WSB1",
+  "ENSG00000109111": "SUPT6H",
+  "ENSG00000109118": "PHF12",
+  "ENSG00000109180": "OCIAD1",
+  "ENSG00000109255": "NMU",
+  "ENSG00000109332": "UBE2D3",
+  "ENSG00000109381": "ELF2",
+  "ENSG00000109475": "RPL34",
+  "ENSG00000109572": "CLCN3",
+  "ENSG00000109586": "GALNT7",
+  "ENSG00000109606": "DHX15",
+  "ENSG00000109670": "FBXW7",
+  "ENSG00000109685": "NSD2",
+  "ENSG00000109756": "RAPGEF2",
+  "ENSG00000109762": "SNX25",
+  "ENSG00000109861": "CTSC",
+  "ENSG00000109911": "ELP4",
+  "ENSG00000109920": "FNBP4",
+  "ENSG00000110048": "OSBP",
+  "ENSG00000110075": "PPP6R3",
+  "ENSG00000110344": "UBE4A",
+  "ENSG00000110367": "DDX6",
+  "ENSG00000110395": "CBL",
+  "ENSG00000110497": "AMBRA1",
+  "ENSG00000110619": "CARS1",
+  "ENSG00000110651": "CD81",
+  "ENSG00000110693": "SOX6",
+  "ENSG00000110713": "NUP98",
+  "ENSG00000110880": "CORO1C",
+  "ENSG00000110917": "MLEC",
+  "ENSG00000110955": "ATP5F1B",
+  "ENSG00000110958": "PTGES3",
+  "ENSG00000111057": "KRT18",
+  "ENSG00000111110": "PPM1H",
+  "ENSG00000111252": "SH2B3",
+  "ENSG00000111300": "NAA25",
+  "ENSG00000111371": "SLC38A1",
+  "ENSG00000111581": "NUP107",
+  "ENSG00000111596": "CNOT2",
+  "ENSG00000111640": "GAPDH",
+  "ENSG00000111642": "CHD4",
+  "ENSG00000111666": "CHPT1",
+  "ENSG00000111669": "TPI1",
+  "ENSG00000111860": "CEP85L",
+  "ENSG00000111880": "RNGTT",
+  "ENSG00000111885": "MAN1A1",
+  "ENSG00000112033": "PPARD",
+  "ENSG00000112062": "MAPK14",
+  "ENSG00000112077": "RHAG",
+  "ENSG00000112078": "KCTD20",
+  "ENSG00000112079": "STK38",
+  "ENSG00000112159": "MDN1",
+  "ENSG00000112182": "BACH2",
+  "ENSG00000112200": "ZNF451",
+  "ENSG00000112242": "E2F3",
+  "ENSG00000112249": "ASCC3",
+  "ENSG00000112306": "RPS12",
+  "ENSG00000112308": "C6ORF62",
+  "ENSG00000112339": "HBS1L",
+  "ENSG00000112419": "PHACTR2",
+  "ENSG00000112531": "QKI",
+  "ENSG00000112576": "CCND3",
+  "ENSG00000112624": "BICRAL",
+  "ENSG00000112699": "GMDS",
+  "ENSG00000112701": "SENP6",
+  "ENSG00000112739": "PRP4K",
+  "ENSG00000112851": "ERBIN",
+  "ENSG00000112893": "MAN2A1",
+  "ENSG00000112972": "HMGCS1",
+  "ENSG00000112977": "DAP",
+  "ENSG00000113013": "HSPA9",
+  "ENSG00000113048": "MRPS27",
+  "ENSG00000113163": "CERT1",
+  "ENSG00000113194": "FAF2",
+  "ENSG00000113269": "RNF130",
+  "ENSG00000113282": "CLINT1",
+  "ENSG00000113318": "MSH3",
+  "ENSG00000113384": "GOLPH3",
+  "ENSG00000113391": "ARB2A",
+  "ENSG00000113441": "LNPEP",
+  "ENSG00000113569": "NUP155",
+  "ENSG00000113580": "NR3C1",
+  "ENSG00000113615": "SEC24A",
+  "ENSG00000113643": "RARS1",
+  "ENSG00000113648": "MACROH2A1",
+  "ENSG00000113649": "TCERG1",
+  "ENSG00000113712": "CSNK1A1",
+  "ENSG00000113732": "ATP6V0E1",
+  "ENSG00000113742": "CPEB4",
+  "ENSG00000113810": "SMC4",
+  "ENSG00000114062": "UBE3A",
+  "ENSG00000114126": "TFDP2",
+  "ENSG00000114127": "XRN1",
+  "ENSG00000114268": "PFKFB4",
+  "ENSG00000114302": "PRKAR2A",
+  "ENSG00000114331": "ACAP2",
+  "ENSG00000114439": "BBX",
+  "ENSG00000114480": "GBE1",
+  "ENSG00000114650": "SCAP",
+  "ENSG00000114770": "ABCC5",
+  "ENSG00000114857": "NKTR",
+  "ENSG00000114861": "FOXP1",
+  "ENSG00000114867": "EIF4G1",
+  "ENSG00000115084": "SLC35F5",
+  "ENSG00000115091": "ACTR3",
+  "ENSG00000115170": "ACVR1",
+  "ENSG00000115233": "PSMD14",
+  "ENSG00000115241": "PPM1G",
+  "ENSG00000115268": "RPS15",
+  "ENSG00000115306": "SPTBN1",
+  "ENSG00000115310": "RTN4",
+  "ENSG00000115355": "CCDC88A",
+  "ENSG00000115464": "USP34",
+  "ENSG00000115484": "CCT4",
+  "ENSG00000115504": "EHBP1",
+  "ENSG00000115677": "HDLBP",
+  "ENSG00000115756": "HPCAL1",
+  "ENSG00000115760": "BIRC6",
+  "ENSG00000115761": "NOL10",
+  "ENSG00000115762": "PLEKHB2",
+  "ENSG00000115808": "STRN",
+  "ENSG00000115839": "RAB3GAP1",
+  "ENSG00000115840": "SLC25A12",
+  "ENSG00000115866": "DARS1",
+  "ENSG00000115896": "PLCL1",
+  "ENSG00000115904": "SOS1",
+  "ENSG00000115934": "LINC02566",
+  "ENSG00000115942": "ORC2",
+  "ENSG00000115966": "ATF2",
+  "ENSG00000115970": "THADA",
+  "ENSG00000115977": "AAK1",
+  "ENSG00000116017": "ARID3A",
+  "ENSG00000116044": "NFE2L2",
+  "ENSG00000116117": "PARD3B",
+  "ENSG00000116199": "FAM20B",
+  "ENSG00000116350": "SRSF4",
+  "ENSG00000116406": "EDEM3",
+  "ENSG00000116473": "RAP1A",
+  "ENSG00000116478": "HDAC1",
+  "ENSG00000116489": "CAPZA1",
+  "ENSG00000116497": "S100PBP",
+  "ENSG00000116539": "ASH1L",
+  "ENSG00000116560": "SFPQ",
+  "ENSG00000116580": "GON4L",
+  "ENSG00000116584": "ARHGEF2",
+  "ENSG00000116675": "DNAJC6",
+  "ENSG00000116698": "SMG7",
+  "ENSG00000116747": "RO60",
+  "ENSG00000116754": "SRSF11",
+  "ENSG00000116786": "PLEKHM2",
+  "ENSG00000116815": "CD58",
+  "ENSG00000116830": "TTF2",
+  "ENSG00000116984": "MTR",
+  "ENSG00000117000": "RLF",
+  "ENSG00000117139": "KDM5B",
+  "ENSG00000117298": "ECE1",
+  "ENSG00000117335": "CD46",
+  "ENSG00000117394": "SLC2A1",
+  "ENSG00000117523": "PRRC2C",
+  "ENSG00000117569": "PTBP2",
+  "ENSG00000117592": "PRDX6",
+  "ENSG00000117616": "RSRP1",
+  "ENSG00000117697": "NSL1",
+  "ENSG00000117713": "ARID1A",
+  "ENSG00000117724": "CENPF",
+  "ENSG00000117859": "OSBPL9",
+  "ENSG00000117868": "ESYT2",
+  "ENSG00000117984": "CTSD",
+  "ENSG00000118007": "STAG1",
+  "ENSG00000118058": "KMT2A",
+  "ENSG00000118193": "KIF14",
+  "ENSG00000118217": "ATF6",
+  "ENSG00000118260": "CREB1",
+  "ENSG00000118412": "CASP8AP2",
+  "ENSG00000118420": "UBE3D",
+  "ENSG00000118454": "ANKRD13C",
+  "ENSG00000118482": "PHF3",
+  "ENSG00000118513": "MYB",
+  "ENSG00000118689": "FOXO3",
+  "ENSG00000118705": "RPN2",
+  "ENSG00000118816": "CCNI",
+  "ENSG00000118873": "RAB3GAP2",
+  "ENSG00000118900": "UBN1",
+  "ENSG00000118961": "LDAH",
+  "ENSG00000119042": "SATB2",
+  "ENSG00000119048": "UBE2B",
+  "ENSG00000119125": "GDA",
+  "ENSG00000119231": "SENP5",
+  "ENSG00000119285": "HEATR1",
+  "ENSG00000119314": "PTBP3",
+  "ENSG00000119318": "RAD23B",
+  "ENSG00000119335": "SET",
+  "ENSG00000119471": "HSDL2",
+  "ENSG00000119487": "MAPKAP1",
+  "ENSG00000119509": "INVS",
+  "ENSG00000119522": "DENND1A",
+  "ENSG00000119715": "ESRRB",
+  "ENSG00000119778": "ATAD2B",
+  "ENSG00000119787": "ATL2",
+  "ENSG00000119820": "YIPF4",
+  "ENSG00000120071": "KANSL1",
+  "ENSG00000120158": "RCL1",
+  "ENSG00000120254": "MTHFD1L",
+  "ENSG00000120265": "PCMT1",
+  "ENSG00000120519": "SLC10A7",
+  "ENSG00000120616": "EPC1",
+  "ENSG00000120690": "ELF1",
+  "ENSG00000120725": "SIL1",
+  "ENSG00000120733": "KDM3B",
+  "ENSG00000120742": "SERP1",
+  "ENSG00000120802": "TMPO",
+  "ENSG00000120899": "PTK2B",
+  "ENSG00000120910": "PPP3CC",
+  "ENSG00000120992": "LYPLA1",
+  "ENSG00000121104": "FAM117A",
+  "ENSG00000121210": "TMEM131L",
+  "ENSG00000121741": "ZMYM2",
+  "ENSG00000121774": "KHDRBS1",
+  "ENSG00000121879": "PIK3CA",
+  "ENSG00000121892": "PDS5A",
+  "ENSG00000121988": "ZRANB3",
+  "ENSG00000122008": "POLK",
+  "ENSG00000122218": "COPA",
+  "ENSG00000122257": "RBBP6",
+  "ENSG00000122376": "SHLD2",
+  "ENSG00000122406": "RPL5",
+  "ENSG00000122482": "ZNF644",
+  "ENSG00000122483": "CCDC18",
+  "ENSG00000122484": "RPAP2",
+  "ENSG00000122545": "SEPTIN7",
+  "ENSG00000122566": "HNRNPA2B1",
+  "ENSG00000122643": "NT5C3A",
+  "ENSG00000122741": "DCAF10",
+  "ENSG00000122779": "TRIM24",
+  "ENSG00000122862": "SRGN",
+  "ENSG00000122884": "P4HA1",
+  "ENSG00000122912": "SLC25A16",
+  "ENSG00000122966": "CIT",
+  "ENSG00000123066": "MED13L",
+  "ENSG00000123106": "CCDC91",
+  "ENSG00000123146": "ADGRE5",
+  "ENSG00000123200": "ZC3H13",
+  "ENSG00000123352": "SPATS2",
+  "ENSG00000123416": "TUBA1B",
+  "ENSG00000123473": "STIL",
+  "ENSG00000123600": "METTL8",
+  "ENSG00000123636": "BAZ2B",
+  "ENSG00000123908": "AGO2",
+  "ENSG00000123983": "ACSL3",
+  "ENSG00000124151": "NCOA3",
+  "ENSG00000124177": "CHD6",
+  "ENSG00000124198": "ARFGEF2",
+  "ENSG00000124207": "CSE1L",
+  "ENSG00000124214": "STAU1",
+  "ENSG00000124486": "USP9X",
+  "ENSG00000124571": "XPO5",
+  "ENSG00000124588": "NQO2",
+  "ENSG00000124767": "GLO1",
+  "ENSG00000124782": "RREB1",
+  "ENSG00000124783": "SSR1",
+  "ENSG00000124789": "NUP153",
+  "ENSG00000124795": "DEK",
+  "ENSG00000124831": "LRRFIP1",
+  "ENSG00000124942": "AHNAK",
+  "ENSG00000125037": "EMC3",
+  "ENSG00000125107": "CNOT1",
+  "ENSG00000125257": "ABCC4",
+  "ENSG00000125354": "SEPTIN6",
+  "ENSG00000125386": "FAM193A",
+  "ENSG00000125505": "MBOAT7",
+  "ENSG00000125676": "THOC2",
+  "ENSG00000125691": "RPL23",
+  "ENSG00000125741": "OPA3",
+  "ENSG00000125944": "HNRNPR",
+  "ENSG00000125970": "RALY",
+  "ENSG00000125977": "EIF2S2",
+  "ENSG00000126070": "AGO3",
+  "ENSG00000126091": "ST3GAL3",
+  "ENSG00000126261": "UBA2",
+  "ENSG00000126561": "STAT5A",
+  "ENSG00000126602": "TRAP1",
+  "ENSG00000126653": "NSRP1",
+  "ENSG00000126777": "KTN1",
+  "ENSG00000126858": "RHOT1",
+  "ENSG00000126870": "DYNC2I1",
+  "ENSG00000126883": "NUP214",
+  "ENSG00000127022": "CANX",
+  "ENSG00000127314": "RAP1B",
+  "ENSG00000127481": "UBR4",
+  "ENSG00000127483": "HP1BP3",
+  "ENSG00000127603": "MACF1",
+  "ENSG00000127616": "SMARCA4",
+  "ENSG00000127663": "KDM4B",
+  "ENSG00000127914": "AKAP9",
+  "ENSG00000127928": "GNGT1",
+  "ENSG00000127946": "HIP1",
+  "ENSG00000127947": "PTPN12",
+  "ENSG00000127948": "POR",
+  "ENSG00000128050": "PAICS",
+  "ENSG00000128191": "DGCR8",
+  "ENSG00000128294": "TPST2",
+  "ENSG00000128563": "PRKRIP1",
+  "ENSG00000128578": "STRIP2",
+  "ENSG00000128585": "MKLN1",
+  "ENSG00000128595": "CALU",
+  "ENSG00000128607": "KLHDC10",
+  "ENSG00000128708": "HAT1",
+  "ENSG00000128731": "HERC2",
+  "ENSG00000128881": "TTBK2",
+  "ENSG00000128908": "INO80",
+  "ENSG00000128918": "ALDH1A2",
+  "ENSG00000128923": "MINDY2",
+  "ENSG00000129003": "VPS13C",
+  "ENSG00000129083": "COPB1",
+  "ENSG00000129084": "PSMA1",
+  "ENSG00000129351": "ILF3",
+  "ENSG00000129515": "SNX6",
+  "ENSG00000129636": "ITFG1",
+  "ENSG00000129682": "FGF13",
+  "ENSG00000129691": "ASH2L",
+  "ENSG00000129933": "MAU2",
+  "ENSG00000129993": "CBFA2T3",
+  "ENSG00000130164": "LDLR",
+  "ENSG00000130175": "PRKCSH",
+  "ENSG00000130202": "NECTIN2",
+  "ENSG00000130227": "XPO7",
+  "ENSG00000130338": "TULP4",
+  "ENSG00000130340": "SNX9",
+  "ENSG00000130396": "AFDN",
+  "ENSG00000130402": "ACTN4",
+  "ENSG00000130449": "ZSWIM6",
+  "ENSG00000130517": "PGPEP1",
+  "ENSG00000130560": "UBAC1",
+  "ENSG00000130695": "CEP85",
+  "ENSG00000130726": "TRIM28",
+  "ENSG00000130741": "EIF2S3",
+  "ENSG00000130766": "SESN2",
+  "ENSG00000130779": "CLIP1",
+  "ENSG00000130816": "DNMT1",
+  "ENSG00000130821": "SLC6A8",
+  "ENSG00000130939": "UBE4B",
+  "ENSG00000131023": "LATS1",
+  "ENSG00000131051": "RBM39",
+  "ENSG00000131069": "ACSS2",
+  "ENSG00000131149": "GSE1",
+  "ENSG00000131236": "CAP1",
+  "ENSG00000131368": "MRPS25",
+  "ENSG00000131374": "TBC1D5",
+  "ENSG00000131389": "SLC6A6",
+  "ENSG00000131504": "DIAPH1",
+  "ENSG00000131508": "UBE2D2",
+  "ENSG00000131558": "EXOC4",
+  "ENSG00000131591": "C1ORF159",
+  "ENSG00000131626": "PPFIA1",
+  "ENSG00000131711": "MAP1B",
+  "ENSG00000131725": "WDR44",
+  "ENSG00000131747": "TOP2A",
+  "ENSG00000132155": "RAF1",
+  "ENSG00000132182": "NUP210",
+  "ENSG00000132199": "ENOSF1",
+  "ENSG00000132204": "LINC00470",
+  "ENSG00000132300": "PTCD3",
+  "ENSG00000132305": "IMMT",
+  "ENSG00000132388": "UBE2G1",
+  "ENSG00000132405": "TBC1D14",
+  "ENSG00000132424": "PNISR",
+  "ENSG00000132466": "ANKRD17",
+  "ENSG00000132475": "H3-3B",
+  "ENSG00000132485": "ZRANB2",
+  "ENSG00000132549": "VPS13B",
+  "ENSG00000132670": "PTPRA",
+  "ENSG00000132676": "DAP3",
+  "ENSG00000132680": "KHDC4",
+  "ENSG00000132780": "NASP",
+  "ENSG00000132842": "AP3B1",
+  "ENSG00000132953": "XPO4",
+  "ENSG00000133026": "MYH10",
+  "ENSG00000133059": "DSTYK",
+  "ENSG00000133112": "TPT1",
+  "ENSG00000133195": "SLC39A11",
+  "ENSG00000133226": "SRRM1",
+  "ENSG00000133315": "MACROD1",
+  "ENSG00000133318": "RTN3",
+  "ENSG00000133454": "MYO18B",
+  "ENSG00000133612": "AGAP3",
+  "ENSG00000133624": "ZNF767P",
+  "ENSG00000133657": "ATP13A3",
+  "ENSG00000133706": "LARS1",
+  "ENSG00000133812": "SBF2",
+  "ENSG00000133816": "MICAL2",
+  "ENSG00000133858": "ZFC3H1",
+  "ENSG00000133961": "NUMB",
+  "ENSG00000134108": "ARL8B",
+  "ENSG00000134138": "MEIS2",
+  "ENSG00000134152": "KATNBL1",
+  "ENSG00000134243": "SORT1",
+  "ENSG00000134278": "SPIRE1",
+  "ENSG00000134283": "PPHLN1",
+  "ENSG00000134294": "SLC38A2",
+  "ENSG00000134313": "KIDINS220",
+  "ENSG00000134318": "ROCK2",
+  "ENSG00000134324": "LPIN1",
+  "ENSG00000134333": "LDHA",
+  "ENSG00000134352": "IL6ST",
+  "ENSG00000134369": "NAV1",
+  "ENSG00000134371": "CDC73",
+  "ENSG00000134452": "FBH1",
+  "ENSG00000134490": "SLC35D4",
+  "ENSG00000134644": "PUM1",
+  "ENSG00000134684": "YARS1",
+  "ENSG00000134744": "TUT4",
+  "ENSG00000134759": "ELP2",
+  "ENSG00000134851": "TMEM165",
+  "ENSG00000134852": "CLOCK",
+  "ENSG00000134882": "UBAC2",
+  "ENSG00000134982": "APC",
+  "ENSG00000135090": "TAOK3",
+  "ENSG00000135250": "SRPK2",
+  "ENSG00000135298": "ADGRB3",
+  "ENSG00000135317": "SNX14",
+  "ENSG00000135336": "ORC3",
+  "ENSG00000135365": "PHF21A",
+  "ENSG00000135387": "CAPRIN1",
+  "ENSG00000135457": "TFCP2",
+  "ENSG00000135486": "HNRNPA1",
+  "ENSG00000135541": "AHI1",
+  "ENSG00000135549": "PKIB",
+  "ENSG00000135605": "TEC",
+  "ENSG00000135655": "USP15",
+  "ENSG00000135766": "EGLN1",
+  "ENSG00000135821": "GLUL",
+  "ENSG00000135829": "DHX9",
+  "ENSG00000135837": "CEP350",
+  "ENSG00000135842": "NIBAN1",
+  "ENSG00000135870": "RC3H1",
+  "ENSG00000135932": "CAB39",
+  "ENSG00000135968": "GCC2",
+  "ENSG00000135999": "EPC2",
+  "ENSG00000136068": "FLNB",
+  "ENSG00000136167": "LCP1",
+  "ENSG00000136231": "IGF2BP3",
+  "ENSG00000136261": "BZW2",
+  "ENSG00000136280": "CCM2",
+  "ENSG00000136381": "IREB2",
+  "ENSG00000136448": "NMT1",
+  "ENSG00000136485": "DCAF7",
+  "ENSG00000136492": "BRIP1",
+  "ENSG00000136536": "MARCHF7",
+  "ENSG00000136542": "GALNT5",
+  "ENSG00000136560": "TANK",
+  "ENSG00000136628": "EPRS1",
+  "ENSG00000136643": "RPS6KC1",
+  "ENSG00000136709": "WDR33",
+  "ENSG00000136731": "UGGT1",
+  "ENSG00000136758": "YME1L1",
+  "ENSG00000136770": "DNAJC1",
+  "ENSG00000136813": "ECPAS",
+  "ENSG00000136819": "C9ORF78",
+  "ENSG00000136842": "TMOD1",
+  "ENSG00000136861": "CDK5RAP2",
+  "ENSG00000136868": "SLC31A1",
+  "ENSG00000136878": "USP20",
+  "ENSG00000136938": "ANP32B",
+  "ENSG00000136942": "RPL35",
+  "ENSG00000137073": "UBAP2",
+  "ENSG00000137074": "APTX",
+  "ENSG00000137075": "RNF38",
+  "ENSG00000137076": "TLN1",
+  "ENSG00000137145": "DENND4C",
+  "ENSG00000137154": "RPS6",
+  "ENSG00000137177": "KIF13A",
+  "ENSG00000137198": "GMPR",
+  "ENSG00000137309": "HMGA1",
+  "ENSG00000137478": "FCHSD2",
+  "ENSG00000137491": "SLCO2B1",
+  "ENSG00000137513": "NARS2",
+  "ENSG00000137601": "NEK1",
+  "ENSG00000137710": "RDX",
+  "ENSG00000137764": "MAP2K5",
+  "ENSG00000137770": "CTDSPL2",
+  "ENSG00000137776": "SLTM",
+  "ENSG00000137804": "NUSAP1",
+  "ENSG00000137812": "KNL1",
+  "ENSG00000137818": "RPLP1",
+  "ENSG00000137845": "ADAM10",
+  "ENSG00000137968": "SLC44A5",
+  "ENSG00000138018": "SELENOI",
+  "ENSG00000138029": "HADHB",
+  "ENSG00000138069": "RAB1A",
+  "ENSG00000138071": "ACTR2",
+  "ENSG00000138081": "FBXO11",
+  "ENSG00000138095": "LRPPRC",
+  "ENSG00000138182": "KIF20B",
+  "ENSG00000138190": "EXOC6",
+  "ENSG00000138303": "ASCC1",
+  "ENSG00000138316": "ADAMTS14",
+  "ENSG00000138326": "RPS24",
+  "ENSG00000138336": "TET1",
+  "ENSG00000138346": "DNA2",
+  "ENSG00000138398": "PPIG",
+  "ENSG00000138399": "FASTKD1",
+  "ENSG00000138430": "OLA1",
+  "ENSG00000138592": "USP8",
+  "ENSG00000138594": "TMOD3",
+  "ENSG00000138600": "SPPL2A",
+  "ENSG00000138639": "ARHGAP24",
+  "ENSG00000138640": "FAM13A",
+  "ENSG00000138668": "HNRNPD",
+  "ENSG00000138674": "SEC31A",
+  "ENSG00000138709": "LARP1B",
+  "ENSG00000138756": "BMP2K",
+  "ENSG00000138757": "G3BP2",
+  "ENSG00000138758": "SEPTIN11",
+  "ENSG00000138768": "USO1",
+  "ENSG00000138778": "CENPE",
+  "ENSG00000138802": "SEC24B",
+  "ENSG00000138814": "PPP3CA",
+  "ENSG00000139083": "ETV6",
+  "ENSG00000139116": "KIF21A",
+  "ENSG00000139163": "ETNK1",
+  "ENSG00000139218": "SCAF11",
+  "ENSG00000139613": "SMARCC2",
+  "ENSG00000139641": "ESYT1",
+  "ENSG00000139644": "TMBIM6",
+  "ENSG00000139668": "WDFY2",
+  "ENSG00000139697": "SBNO1",
+  "ENSG00000139734": "DIAPH3",
+  "ENSG00000139746": "RBM26",
+  "ENSG00000139793": "MBNL2",
+  "ENSG00000140022": "STON2",
+  "ENSG00000140105": "WARS1",
+  "ENSG00000140262": "TCF12",
+  "ENSG00000140350": "ANP32A",
+  "ENSG00000140374": "ETFA",
+  "ENSG00000140396": "NCOA2",
+  "ENSG00000140455": "USP3",
+  "ENSG00000140479": "PCSK6",
+  "ENSG00000140525": "FANCI",
+  "ENSG00000140526": "ABHD2",
+  "ENSG00000140545": "MFGE8",
+  "ENSG00000140575": "IQGAP1",
+  "ENSG00000140612": "SEC11A",
+  "ENSG00000140632": "GLYR1",
+  "ENSG00000140694": "PARN",
+  "ENSG00000140718": "FTO",
+  "ENSG00000140743": "CDR2",
+  "ENSG00000140988": "RPS2",
+  "ENSG00000141002": "TCF25",
+  "ENSG00000141027": "NCOR1",
+  "ENSG00000141068": "KSR1",
+  "ENSG00000141084": "RANBP10",
+  "ENSG00000141252": "VPS53",
+  "ENSG00000141279": "NPEPPS",
+  "ENSG00000141298": "SSH2",
+  "ENSG00000141367": "CLTC",
+  "ENSG00000141376": "BCAS3",
+  "ENSG00000141425": "RPRD1A",
+  "ENSG00000141524": "TMC6",
+  "ENSG00000141556": "TBCD",
+  "ENSG00000141562": "NARF",
+  "ENSG00000141564": "RPTOR",
+  "ENSG00000141568": "FOXK2",
+  "ENSG00000141576": "RNF157",
+  "ENSG00000141627": "DYM",
+  "ENSG00000141867": "BRD4",
+  "ENSG00000141905": "NFIC",
+  "ENSG00000142207": "URB1",
+  "ENSG00000142230": "SAE1",
+  "ENSG00000142534": "RPS11",
+  "ENSG00000142541": "RPL13A",
+  "ENSG00000142599": "RERE",
+  "ENSG00000142655": "PEX14",
+  "ENSG00000142657": "PGD",
+  "ENSG00000142676": "RPL11",
+  "ENSG00000142687": "KIAA0319L",
+  "ENSG00000142798": "HSPG2",
+  "ENSG00000142864": "SERBP1",
+  "ENSG00000142937": "RPS8",
+  "ENSG00000142945": "KIF2C",
+  "ENSG00000143033": "MTF2",
+  "ENSG00000143106": "PSMA5",
+  "ENSG00000143164": "DCAF6",
+  "ENSG00000143190": "POU2F1",
+  "ENSG00000143207": "COP1",
+  "ENSG00000143322": "ABL2",
+  "ENSG00000143324": "XPR1",
+  "ENSG00000143420": "ENSA",
+  "ENSG00000143442": "POGZ",
+  "ENSG00000143476": "DTL",
+  "ENSG00000143493": "INTS7",
+  "ENSG00000143549": "TPM3",
+  "ENSG00000143614": "GATAD2B",
+  "ENSG00000143641": "GALNT2",
+  "ENSG00000143702": "CEP170",
+  "ENSG00000143748": "NVL",
+  "ENSG00000143756": "FBXO28",
+  "ENSG00000143761": "ARF1",
+  "ENSG00000143776": "CDC42BPA",
+  "ENSG00000143797": "MBOAT2",
+  "ENSG00000143815": "LBR",
+  "ENSG00000143851": "PTPN7",
+  "ENSG00000143870": "PDIA6",
+  "ENSG00000143919": "CAMKMT",
+  "ENSG00000143924": "EML4",
+  "ENSG00000143951": "WDPCP",
+  "ENSG00000143952": "VPS54",
+  "ENSG00000143970": "ASXL2",
+  "ENSG00000144036": "EXOC6B",
+  "ENSG00000144040": "SFXN5",
+  "ENSG00000144224": "UBXN4",
+  "ENSG00000144283": "PKP4",
+  "ENSG00000144320": "LNPK",
+  "ENSG00000144357": "UBR3",
+  "ENSG00000144381": "HSPD1",
+  "ENSG00000144407": "PTH2R",
+  "ENSG00000144426": "NBEAL1",
+  "ENSG00000144455": "SUMF1",
+  "ENSG00000144554": "FANCD2",
+  "ENSG00000144566": "RAB5A",
+  "ENSG00000144674": "GOLGA4",
+  "ENSG00000144713": "RPL32",
+  "ENSG00000144791": "LIMD1",
+  "ENSG00000145348": "TBCK",
+  "ENSG00000145375": "AFG2A",
+  "ENSG00000145495": "MARCHF6",
+  "ENSG00000145592": "RPL37",
+  "ENSG00000145687": "SSBP2",
+  "ENSG00000145703": "IQGAP2",
+  "ENSG00000145715": "RASA1",
+  "ENSG00000145725": "PPIP5K2",
+  "ENSG00000145734": "BDP1",
+  "ENSG00000145740": "SLC30A5",
+  "ENSG00000145743": "FBXL17",
+  "ENSG00000145779": "TNFAIP8",
+  "ENSG00000145781": "COMMD10",
+  "ENSG00000145833": "DDX46",
+  "ENSG00000145907": "G3BP1",
+  "ENSG00000145982": "FARS2",
+  "ENSG00000145990": "GFOD1",
+  "ENSG00000145996": "CDKAL1",
+  "ENSG00000146143": "PRIM2",
+  "ENSG00000146223": "RPL7L1",
+  "ENSG00000146247": "PHIP",
+  "ENSG00000146263": "MMS22L",
+  "ENSG00000146373": "RNF217",
+  "ENSG00000146376": "ARHGAP18",
+  "ENSG00000146414": "SHPRH",
+  "ENSG00000146416": "AIG1",
+  "ENSG00000146426": "TIAM2",
+  "ENSG00000146433": "TMEM181",
+  "ENSG00000146463": "ZMYM4",
+  "ENSG00000146530": "VWDE",
+  "ENSG00000146776": "ATXN7L1",
+  "ENSG00000146842": "TMEM209",
+  "ENSG00000146872": "TLK2",
+  "ENSG00000146918": "NCAPG2",
+  "ENSG00000147050": "KDM6A",
+  "ENSG00000147065": "MSN",
+  "ENSG00000147099": "HDAC8",
+  "ENSG00000147133": "TAF1",
+  "ENSG00000147162": "OGT",
+  "ENSG00000147251": "DOCK11",
+  "ENSG00000147274": "RBMX",
+  "ENSG00000147316": "MCPH1",
+  "ENSG00000147403": "RPL10",
+  "ENSG00000147421": "HMBOX1",
+  "ENSG00000147454": "SLC25A37",
+  "ENSG00000147526": "TACC1",
+  "ENSG00000147548": "NSD3",
+  "ENSG00000147649": "MTDH",
+  "ENSG00000147650": "LRP12",
+  "ENSG00000147677": "EIF3H",
+  "ENSG00000147689": "SACK1A",
+  "ENSG00000147905": "ZCCHC7",
+  "ENSG00000148120": "AOPEP",
+  "ENSG00000148200": "NR6A1",
+  "ENSG00000148303": "RPL7A",
+  "ENSG00000148358": "GPR107",
+  "ENSG00000148429": "USP6NL",
+  "ENSG00000148488": "ST8SIA6",
+  "ENSG00000148498": "PARD3",
+  "ENSG00000148634": "HERC4",
+  "ENSG00000148700": "ADD3",
+  "ENSG00000148773": "MKI67",
+  "ENSG00000148842": "CNNM2",
+  "ENSG00000148843": "PDCD11",
+  "ENSG00000148950": "IMMP1L",
+  "ENSG00000149177": "PTPRJ",
+  "ENSG00000149179": "CSTPP1",
+  "ENSG00000149187": "CELF1",
+  "ENSG00000149262": "INTS4",
+  "ENSG00000149273": "RPS3",
+  "ENSG00000149308": "NPAT",
+  "ENSG00000149311": "ATM",
+  "ENSG00000149485": "FADS1",
+  "ENSG00000149639": "MTCL2",
+  "ENSG00000149806": "FAU",
+  "ENSG00000149925": "ALDOA",
+  "ENSG00000150093": "ITGB1",
+  "ENSG00000150477": "KIAA1328",
+  "ENSG00000150867": "PIP4K2A",
+  "ENSG00000150961": "SEC24D",
+  "ENSG00000150991": "UBC",
+  "ENSG00000150995": "ITPR1",
+  "ENSG00000151012": "SLC7A11",
+  "ENSG00000151090": "THRB",
+  "ENSG00000151092": "NGLY1",
+  "ENSG00000151150": "ANK3",
+  "ENSG00000151208": "DLG5",
+  "ENSG00000151229": "SLC2A13",
+  "ENSG00000151247": "EIF4E",
+  "ENSG00000151276": "MAGI1",
+  "ENSG00000151338": "MIPOL1",
+  "ENSG00000151414": "NEK7",
+  "ENSG00000151422": "FER",
+  "ENSG00000151461": "UPF2",
+  "ENSG00000151466": "SCLT1",
+  "ENSG00000151532": "VTI1A",
+  "ENSG00000151694": "ADAM17",
+  "ENSG00000151718": "WWC2",
+  "ENSG00000151746": "BICD1",
+  "ENSG00000151779": "NBAS",
+  "ENSG00000151914": "DST",
+  "ENSG00000152061": "RABGAP1L",
+  "ENSG00000152102": "FAM168B",
+  "ENSG00000152127": "MGAT5",
+  "ENSG00000152256": "PDK1",
+  "ENSG00000152270": "PDE3B",
+  "ENSG00000152291": "TGOLN2",
+  "ENSG00000152348": "ATG10",
+  "ENSG00000152520": "PAN3",
+  "ENSG00000152601": "MBNL1",
+  "ENSG00000152818": "UTRN",
+  "ENSG00000153046": "CDYL",
+  "ENSG00000153113": "CAST",
+  "ENSG00000153179": "RASSF3",
+  "ENSG00000153187": "HNRNPU",
+  "ENSG00000153201": "RANBP2",
+  "ENSG00000153207": "AHCTF1",
+  "ENSG00000153310": "CYRIB",
+  "ENSG00000153317": "ASAP1",
+  "ENSG00000153339": "TRAPPC8",
+  "ENSG00000153721": "CNKSR3",
+  "ENSG00000153774": "CFDP1",
+  "ENSG00000153815": "CMIP",
+  "ENSG00000153827": "TRIP12",
+  "ENSG00000153922": "CHD1",
+  "ENSG00000153936": "HS2ST1",
+  "ENSG00000153944": "MSI2",
+  "ENSG00000153976": "HS3ST3A1",
+  "ENSG00000154001": "PPP2R5E",
+  "ENSG00000154188": "ANGPT1",
+  "ENSG00000154229": "PRKCA",
+  "ENSG00000154269": "ENPP3",
+  "ENSG00000154310": "TNIK",
+  "ENSG00000154654": "NCAM2",
+  "ENSG00000154743": "TSEN2",
+  "ENSG00000154845": "PPP4R1",
+  "ENSG00000155111": "CDK19",
+  "ENSG00000155307": "SAMSN1",
+  "ENSG00000155313": "USP25",
+  "ENSG00000155380": "SLC16A1",
+  "ENSG00000155506": "LARP1",
+  "ENSG00000155561": "NUP205",
+  "ENSG00000155660": "PDIA4",
+  "ENSG00000155846": "PPARGC1B",
+  "ENSG00000156026": "MCU",
+  "ENSG00000156052": "GNAQ",
+  "ENSG00000156110": "ADK",
+  "ENSG00000156140": "ADAMTS3",
+  "ENSG00000156206": "CFAP161",
+  "ENSG00000156273": "BACH1",
+  "ENSG00000156304": "SCAF4",
+  "ENSG00000156482": "RPL30",
+  "ENSG00000156515": "HK1",
+  "ENSG00000156639": "ZFAND3",
+  "ENSG00000156650": "KAT6B",
+  "ENSG00000156671": "SAMD8",
+  "ENSG00000156675": "RAB11FIP1",
+  "ENSG00000156802": "ATAD2",
+  "ENSG00000156831": "NSMCE2",
+  "ENSG00000156931": "VPS8",
+  "ENSG00000156958": "GALK2",
+  "ENSG00000157077": "ZFYVE9",
+  "ENSG00000157106": "SMG1",
+  "ENSG00000157216": "SSBP3",
+  "ENSG00000157350": "ST3GAL2",
+  "ENSG00000157450": "RNF111",
+  "ENSG00000157540": "DYRK1A",
+  "ENSG00000157600": "TMEM164",
+  "ENSG00000157741": "UBN2",
+  "ENSG00000157764": "BRAF",
+  "ENSG00000157837": "SPPL3",
+  "ENSG00000157985": "AGAP1",
+  "ENSG00000158019": "BABAM2",
+  "ENSG00000158161": "EYA3",
+  "ENSG00000158169": "FANCC",
+  "ENSG00000158195": "WASF2",
+  "ENSG00000158201": "ABHD3",
+  "ENSG00000158417": "EIF5B",
+  "ENSG00000158467": "AHCYL2",
+  "ENSG00000158470": "B4GALT5",
+  "ENSG00000158636": "EMSY",
+  "ENSG00000158710": "TAGLN2",
+  "ENSG00000158985": "CDC42SE2",
+  "ENSG00000159023": "EPB41",
+  "ENSG00000159131": "GART",
+  "ENSG00000159140": "SON",
+  "ENSG00000159216": "RUNX1",
+  "ENSG00000159217": "IGF2BP1",
+  "ENSG00000159256": "MORC3",
+  "ENSG00000159267": "HLCS",
+  "ENSG00000159459": "UBR1",
+  "ENSG00000159592": "GPBP1L1",
+  "ENSG00000159733": "ZFYVE28",
+  "ENSG00000160007": "ARHGAP35",
+  "ENSG00000160049": "DFFA",
+  "ENSG00000160194": "NDUFV3",
+  "ENSG00000160211": "G6PD",
+  "ENSG00000160216": "AGPAT3",
+  "ENSG00000160218": "TRAPPC10",
+  "ENSG00000160285": "LSS",
+  "ENSG00000160294": "MCM3AP",
+  "ENSG00000160299": "PCNT",
+  "ENSG00000160305": "DIP2A",
+  "ENSG00000160551": "TAOK1",
+  "ENSG00000160563": "MED27",
+  "ENSG00000160584": "SIK3",
+  "ENSG00000160633": "SAFB",
+  "ENSG00000160746": "ANO10",
+  "ENSG00000160752": "FDPS",
+  "ENSG00000160789": "LMNA",
+  "ENSG00000160799": "CCDC12",
+  "ENSG00000160991": "ORAI2",
+  "ENSG00000161011": "SQSTM1",
+  "ENSG00000161016": "RPL8",
+  "ENSG00000161040": "FBXL13",
+  "ENSG00000161638": "ITGA5",
+  "ENSG00000161813": "LARP4",
+  "ENSG00000162039": "MEIOB",
+  "ENSG00000162378": "ZYG11B",
+  "ENSG00000162402": "USP24",
+  "ENSG00000162408": "NOL9",
+  "ENSG00000162434": "JAK1",
+  "ENSG00000162521": "RBBP4",
+  "ENSG00000162599": "NFIA",
+  "ENSG00000162852": "CNST",
+  "ENSG00000162909": "CAPN2",
+  "ENSG00000162923": "WDR26",
+  "ENSG00000162980": "ARL5A",
+  "ENSG00000163006": "CCDC138",
+  "ENSG00000163029": "SMC6",
+  "ENSG00000163110": "PDLIM5",
+  "ENSG00000163125": "RPRD2",
+  "ENSG00000163162": "RNF149",
+  "ENSG00000163297": "ANTXR2",
+  "ENSG00000163399": "ATP1A1",
+  "ENSG00000163468": "CCT3",
+  "ENSG00000163479": "SSR2",
+  "ENSG00000163527": "STT3B",
+  "ENSG00000163539": "CLASP2",
+  "ENSG00000163558": "PRKCI",
+  "ENSG00000163602": "RYBP",
+  "ENSG00000163655": "GMPS",
+  "ENSG00000163681": "SLMAP",
+  "ENSG00000163714": "U2SURP",
+  "ENSG00000163811": "WDR43",
+  "ENSG00000163814": "CDCP1",
+  "ENSG00000163848": "ZNF148",
+  "ENSG00000163872": "YEATS2",
+  "ENSG00000163939": "PBRM1",
+  "ENSG00000163960": "UBXN7",
+  "ENSG00000163961": "RNF168",
+  "ENSG00000164074": "ABHD18",
+  "ENSG00000164134": "NAA15",
+  "ENSG00000164190": "NIPBL",
+  "ENSG00000164199": "ADGRV1",
+  "ENSG00000164237": "CMBL",
+  "ENSG00000164292": "RHOBTB3",
+  "ENSG00000164327": "RICTOR",
+  "ENSG00000164329": "TENT2",
+  "ENSG00000164466": "SFXN1",
+  "ENSG00000164494": "PDSS2",
+  "ENSG00000164506": "STXBP5",
+  "ENSG00000164548": "TRA2A",
+  "ENSG00000164574": "GALNT10",
+  "ENSG00000164587": "RPS14",
+  "ENSG00000164597": "COG5",
+  "ENSG00000164659": "ELAPOR2",
+  "ENSG00000164715": "LMTK2",
+  "ENSG00000164733": "CTSB",
+  "ENSG00000164808": "SPIDR",
+  "ENSG00000164916": "FOXK1",
+  "ENSG00000164924": "YWHAZ",
+  "ENSG00000164944": "VIRMA",
+  "ENSG00000165185": "KIAA1958",
+  "ENSG00000165209": "STRBP",
+  "ENSG00000165219": "GAPVD1",
+  "ENSG00000165280": "VCP",
+  "ENSG00000165322": "ARHGAP12",
+  "ENSG00000165392": "WRN",
+  "ENSG00000165476": "REEP3",
+  "ENSG00000165525": "NEMF",
+  "ENSG00000165632": "TAF3",
+  "ENSG00000165650": "PDZD8",
+  "ENSG00000165671": "NSD1",
+  "ENSG00000165675": "ENOX2",
+  "ENSG00000165732": "DDX21",
+  "ENSG00000165914": "TTC7B",
+  "ENSG00000166128": "RAB8B",
+  "ENSG00000166233": "ARIH1",
+  "ENSG00000166266": "CUL5",
+  "ENSG00000166295": "ANAPC16",
+  "ENSG00000166326": "TRIM44",
+  "ENSG00000166435": "XRRA1",
+  "ENSG00000166441": "RPL27A",
+  "ENSG00000166471": "TMEM41B",
+  "ENSG00000166501": "PRKCB",
+  "ENSG00000166575": "TMEM135",
+  "ENSG00000166598": "HSP90B1",
+  "ENSG00000166669": "ATF7IP2",
+  "ENSG00000166734": "GOLM2",
+  "ENSG00000166747": "AP1G1",
+  "ENSG00000166794": "PPIB",
+  "ENSG00000166913": "YWHAB",
+  "ENSG00000166949": "SMAD3",
+  "ENSG00000167004": "PDIA3",
+  "ENSG00000167110": "GOLGA2",
+  "ENSG00000167193": "CRK",
+  "ENSG00000167258": "CDK12",
+  "ENSG00000167323": "STIM1",
+  "ENSG00000167460": "TPM4",
+  "ENSG00000167491": "GATAD2A",
+  "ENSG00000167522": "ANKRD11",
+  "ENSG00000167526": "RPL13",
+  "ENSG00000167632": "TRAPPC9",
+  "ENSG00000167635": "ZNF146",
+  "ENSG00000167645": "YIF1B",
+  "ENSG00000167658": "EEF2",
+  "ENSG00000167766": "ZNF83",
+  "ENSG00000167972": "ABCA3",
+  "ENSG00000167978": "SRRM2",
+  "ENSG00000167996": "FTH1",
+  "ENSG00000168003": "SLC3A2",
+  "ENSG00000168028": "RPSA",
+  "ENSG00000168036": "CTNNB1",
+  "ENSG00000168092": "PAFAH1B2",
+  "ENSG00000168137": "SETD5",
+  "ENSG00000168172": "HOOK3",
+  "ENSG00000168214": "RBPJ",
+  "ENSG00000168259": "DNAJC7",
+  "ENSG00000168385": "SEPTIN2",
+  "ENSG00000168411": "RFWD3",
+  "ENSG00000168421": "RHOH",
+  "ENSG00000168461": "RAB31",
+  "ENSG00000168575": "SLC20A2",
+  "ENSG00000168610": "STAT3",
+  "ENSG00000168710": "AHCYL1",
+  "ENSG00000168754": "FAM178B",
+  "ENSG00000169032": "MAP2K1",
+  "ENSG00000169057": "MECP2",
+  "ENSG00000169071": "ROR2",
+  "ENSG00000169180": "XPO6",
+  "ENSG00000169372": "CRADD",
+  "ENSG00000169398": "PTK2",
+  "ENSG00000169432": "SCN9A",
+  "ENSG00000169504": "CLIC4",
+  "ENSG00000169519": "METTL15",
+  "ENSG00000169554": "ZEB2",
+  "ENSG00000169635": "HIC2",
+  "ENSG00000169679": "BUB1",
+  "ENSG00000169710": "FASN",
+  "ENSG00000169756": "LIMS1",
+  "ENSG00000169813": "HNRNPF",
+  "ENSG00000169871": "TRIM56",
+  "ENSG00000169905": "TOR1AIP2",
+  "ENSG00000169946": "ZFPM2",
+  "ENSG00000169967": "MAP3K2",
+  "ENSG00000170027": "YWHAG",
+  "ENSG00000170142": "UBE2E1",
+  "ENSG00000170144": "HNRNPA3",
+  "ENSG00000170145": "SIK2",
+  "ENSG00000170242": "USP47",
+  "ENSG00000170248": "PDCD6IP",
+  "ENSG00000170315": "UBB",
+  "ENSG00000170421": "KRT8",
+  "ENSG00000170515": "PA2G4",
+  "ENSG00000170522": "ELOVL6",
+  "ENSG00000170606": "HSPA4",
+  "ENSG00000170634": "ACYP2",
+  "ENSG00000170734": "POLH",
+  "ENSG00000170759": "KIF5B",
+  "ENSG00000170832": "USP32",
+  "ENSG00000170871": "KIAA0232",
+  "ENSG00000170889": "RPS9",
+  "ENSG00000170921": "TANC2",
+  "ENSG00000171310": "CHST11",
+  "ENSG00000171316": "CHD7",
+  "ENSG00000171345": "KRT19",
+  "ENSG00000171456": "ASXL1",
+  "ENSG00000171490": "RSL1D1",
+  "ENSG00000171530": "TBCA",
+  "ENSG00000171552": "BCL2L1",
+  "ENSG00000171634": "BPTF",
+  "ENSG00000171723": "GPHN",
+  "ENSG00000171735": "CAMTA1",
+  "ENSG00000171853": "TRAPPC12",
+  "ENSG00000171858": "RPS21",
+  "ENSG00000171988": "JMJD1C",
+  "ENSG00000172262": "ZBTB35",
+  "ENSG00000172270": "BSG",
+  "ENSG00000172292": "CERS6",
+  "ENSG00000172340": "SUCLG2",
+  "ENSG00000172493": "AFF1",
+  "ENSG00000172534": "HCFC1",
+  "ENSG00000172613": "RAD9A",
+  "ENSG00000172757": "CFL1",
+  "ENSG00000172765": "TMCC1",
+  "ENSG00000172795": "DCP2",
+  "ENSG00000172809": "RPL38",
+  "ENSG00000172845": "SP3",
+  "ENSG00000172869": "DMXL1",
+  "ENSG00000172893": "DHCR7",
+  "ENSG00000172954": "LCLAT1",
+  "ENSG00000172965": "MIR4435-2HG",
+  "ENSG00000172967": "XKR3",
+  "ENSG00000172992": "DCAKD",
+  "ENSG00000173064": "HECTD4",
+  "ENSG00000173065": "FAM222B",
+  "ENSG00000173120": "KDM2A",
+  "ENSG00000173163": "COMMD1",
+  "ENSG00000173230": "GOLGB1",
+  "ENSG00000173273": "TNKS",
+  "ENSG00000173473": "SMARCC1",
+  "ENSG00000173517": "PEAK1",
+  "ENSG00000173542": "MOB1B",
+  "ENSG00000173575": "CHD2",
+  "ENSG00000173611": "SCAI",
+  "ENSG00000173660": "UQCRH",
+  "ENSG00000173692": "PSMD1",
+  "ENSG00000173715": "TOP6BL",
+  "ENSG00000173726": "TOMM20",
+  "ENSG00000173727": "FAUP4",
+  "ENSG00000173744": "AGFG1",
+  "ENSG00000173757": "STAT5B",
+  "ENSG00000173821": "RNF213",
+  "ENSG00000173889": "PHC3",
+  "ENSG00000173926": "MARCHF3",
+  "ENSG00000173960": "UBXN2A",
+  "ENSG00000174197": "MGA",
+  "ENSG00000174231": "PRPF8",
+  "ENSG00000174437": "ATP2A2",
+  "ENSG00000174444": "RPL4",
+  "ENSG00000174469": "CNTNAP2",
+  "ENSG00000174485": "DENND4A",
+  "ENSG00000174574": "AKIRIN1",
+  "ENSG00000174718": "RESF1",
+  "ENSG00000174748": "RPL15",
+  "ENSG00000174780": "SRP72",
+  "ENSG00000174891": "RSRC1",
+  "ENSG00000175029": "CTBP2",
+  "ENSG00000175048": "ZDHHC14",
+  "ENSG00000175054": "ATR",
+  "ENSG00000175061": "SNHG29",
+  "ENSG00000175115": "PACS1",
+  "ENSG00000175161": "CADM2",
+  "ENSG00000175164": "ABO",
+  "ENSG00000175175": "PPM1E",
+  "ENSG00000175198": "PCCA",
+  "ENSG00000175216": "CKAP5",
+  "ENSG00000175582": "RAB6A",
+  "ENSG00000175727": "MLXIP",
+  "ENSG00000176014": "TUBB6",
+  "ENSG00000176095": "IP6K1",
+  "ENSG00000176105": "YES1",
+  "ENSG00000176124": "DLEU1",
+  "ENSG00000176155": "CCDC57",
+  "ENSG00000176208": "ATAD5",
+  "ENSG00000176261": "ZBTB8OS",
+  "ENSG00000176340": "COX8A",
+  "ENSG00000176407": "KCMF1",
+  "ENSG00000176658": "MYO1D",
+  "ENSG00000176953": "NFATC2IP",
+  "ENSG00000177000": "MTHFR",
+  "ENSG00000177119": "ANO6",
+  "ENSG00000177156": "TALDO1",
+  "ENSG00000177189": "RPS6KA3",
+  "ENSG00000177200": "CHD9",
+  "ENSG00000177311": "ZBTB38",
+  "ENSG00000177463": "NR2C2",
+  "ENSG00000177469": "CAVIN1",
+  "ENSG00000177479": "ARIH2",
+  "ENSG00000177565": "TBL1XR1",
+  "ENSG00000177600": "RPLP2",
+  "ENSG00000177853": "ZNF518A",
+  "ENSG00000177885": "GRB2",
+  "ENSG00000178105": "DDX10",
+  "ENSG00000178177": "LCORL",
+  "ENSG00000178209": "PLEC",
+  "ENSG00000178234": "GALNT11",
+  "ENSG00000178538": "CA8",
+  "ENSG00000178691": "SUZ12",
+  "ENSG00000178974": "FBXO34",
+  "ENSG00000179134": "SAMD4B",
+  "ENSG00000179218": "CALR",
+  "ENSG00000179295": "PTPN11",
+  "ENSG00000179399": "GPC5",
+  "ENSG00000179406": "LINC00174",
+  "ENSG00000179715": "PCED1B",
+  "ENSG00000179820": "MYADM",
+  "ENSG00000179912": "R3HDM2",
+  "ENSG00000180098": "TRNAU1AP",
+  "ENSG00000180198": "RCC1",
+  "ENSG00000180263": "FGD6",
+  "ENSG00000180357": "ZNF609",
+  "ENSG00000180370": "PAK2",
+  "ENSG00000180385": "EMC3-AS1",
+  "ENSG00000180530": "NRIP1",
+  "ENSG00000180573": "H2AC6",
+  "ENSG00000181090": "EHMT1",
+  "ENSG00000181163": "NPM1",
+  "ENSG00000181555": "SETD2",
+  "ENSG00000181722": "ZBTB20",
+  "ENSG00000181827": "RFX7",
+  "ENSG00000182095": "TNRC18",
+  "ENSG00000182158": "CREB3L2",
+  "ENSG00000182185": "RAD51B",
+  "ENSG00000182256": "GABRG3",
+  "ENSG00000182446": "NPLOC4",
+  "ENSG00000182606": "TRAK1",
+  "ENSG00000182621": "PLCB1",
+  "ENSG00000182628": "SKA2",
+  "ENSG00000182670": "TTC3",
+  "ENSG00000182774": "RPS17",
+  "ENSG00000182827": "ACBD3",
+  "ENSG00000182899": "RPL35A",
+  "ENSG00000182973": "CNOT10",
+  "ENSG00000183032": "SLC25A21",
+  "ENSG00000183117": "CSMD1",
+  "ENSG00000183166": "CALN1",
+  "ENSG00000183291": "SELENOF",
+  "ENSG00000183337": "BCOR",
+  "ENSG00000183431": "SF3A3",
+  "ENSG00000183495": "EP400",
+  "ENSG00000183530": "PRR14L",
+  "ENSG00000183597": "TANGO2",
+  "ENSG00000183742": "MACC1",
+  "ENSG00000184007": "PTP4A2",
+  "ENSG00000184009": "ACTG1",
+  "ENSG00000184014": "DENND5A",
+  "ENSG00000184029": "DSCR4",
+  "ENSG00000184178": "SCFD2",
+  "ENSG00000184220": "CMSS1",
+  "ENSG00000184226": "PCDH9",
+  "ENSG00000184384": "MAML2",
+  "ENSG00000184445": "KNTC1",
+  "ENSG00000184640": "SEPTIN9",
+  "ENSG00000184787": "UBE2G2",
+  "ENSG00000184838": "PRR16",
+  "ENSG00000184863": "RBM33",
+  "ENSG00000184903": "IMMP2L",
+  "ENSG00000184992": "BRI3BP",
+  "ENSG00000185033": "SEMA4B",
+  "ENSG00000185104": "FAF1",
+  "ENSG00000185219": "ZNF445",
+  "ENSG00000185238": "PRMT3",
+  "ENSG00000185305": "ARL15",
+  "ENSG00000185344": "ATP6V0A2",
+  "ENSG00000185420": "SMYD3",
+  "ENSG00000185551": "NR2F2",
+  "ENSG00000185624": "P4HB",
+  "ENSG00000185651": "UBE2L3",
+  "ENSG00000185658": "BRWD1",
+  "ENSG00000185686": "PRAME",
+  "ENSG00000185760": "KCNQ5",
+  "ENSG00000185811": "IKZF1",
+  "ENSG00000185842": "DNAH14",
+  "ENSG00000186001": "LRCH3",
+  "ENSG00000186153": "WWOX",
+  "ENSG00000186187": "ZNRF1",
+  "ENSG00000186260": "MRTFB",
+  "ENSG00000186314": "PRELID2",
+  "ENSG00000186432": "KPNA4",
+  "ENSG00000186468": "RPS23",
+  "ENSG00000186480": "INSIG1",
+  "ENSG00000186566": "GPATCH8",
+  "ENSG00000186591": "UBE2H",
+  "ENSG00000186716": "BCR",
+  "ENSG00000186908": "ZDHHC17",
+  "ENSG00000187079": "TEAD1",
+  "ENSG00000187109": "NAP1L1",
+  "ENSG00000187172": "BAGE2",
+  "ENSG00000187239": "FNBP1",
+  "ENSG00000187257": "RSBN1L",
+  "ENSG00000187605": "TET3",
+  "ENSG00000187741": "FANCA",
+  "ENSG00000187772": "LIN28B",
+  "ENSG00000187951": "LOC100288637",
+  "ENSG00000188312": "CENPP",
+  "ENSG00000188342": "GTF2F2",
+  "ENSG00000188511": "MIR3667HG",
+  "ENSG00000188529": "SRSF10",
+  "ENSG00000188735": "TMEM120B",
+  "ENSG00000188846": "RPL14",
+  "ENSG00000188994": "ZNF292",
+  "ENSG00000189056": "RELN",
+  "ENSG00000189079": "ARID2",
+  "ENSG00000189091": "SF3B3",
+  "ENSG00000189229": "LOC105376944",
+  "ENSG00000189283": "FHIT",
+  "ENSG00000189308": "LIN54",
+  "ENSG00000189337": "KAZN",
+  "ENSG00000196182": "STK40",
+  "ENSG00000196187": "TMEM63A",
+  "ENSG00000196230": "TUBB",
+  "ENSG00000196233": "LCOR",
+  "ENSG00000196284": "SUPT3H",
+  "ENSG00000196305": "IARS1",
+  "ENSG00000196312": "MFSD14CP",
+  "ENSG00000196313": "POM121",
+  "ENSG00000196365": "LONP1",
+  "ENSG00000196367": "TRRAP",
+  "ENSG00000196396": "PTPN1",
+  "ENSG00000196405": "EVL",
+  "ENSG00000196418": "ZNF124",
+  "ENSG00000196419": "XRCC6",
+  "ENSG00000196428": "TSC22D2",
+  "ENSG00000196498": "NCOR2",
+  "ENSG00000196504": "PRPF40A",
+  "ENSG00000196565": "HBG2",
+  "ENSG00000196584": "XRCC2",
+  "ENSG00000196588": "MRTFA",
+  "ENSG00000196628": "TCF4",
+  "ENSG00000196642": "RABL6",
+  "ENSG00000196705": "ZNF431",
+  "ENSG00000196712": "NF1",
+  "ENSG00000196715": "VKORC1L1",
+  "ENSG00000196730": "DAPK1",
+  "ENSG00000196792": "STRN3",
+  "ENSG00000196821": "ILRUN",
+  "ENSG00000196914": "ARHGEF12",
+  "ENSG00000196924": "FLNA",
+  "ENSG00000197081": "IGF2R",
+  "ENSG00000197102": "DYNC1H1",
+  "ENSG00000197147": "LRRC8B",
+  "ENSG00000197157": "SND1",
+  "ENSG00000197183": "NOL4L",
+  "ENSG00000197312": "DDI2",
+  "ENSG00000197323": "TRIM33",
+  "ENSG00000197410": "DCHS2",
+  "ENSG00000197442": "MAP3K5",
+  "ENSG00000197471": "SPN",
+  "ENSG00000197548": "ATG7",
+  "ENSG00000197555": "SIPA1L1",
+  "ENSG00000197721": "CR1L",
+  "ENSG00000197746": "PSAP",
+  "ENSG00000197756": "RPL37A",
+  "ENSG00000197892": "KIF13B",
+  "ENSG00000197958": "RPL12",
+  "ENSG00000197965": "MPZL1",
+  "ENSG00000197969": "VPS13A",
+  "ENSG00000198087": "CD2AP",
+  "ENSG00000198160": "MIER1",
+  "ENSG00000198162": "MAN1A2",
+  "ENSG00000198218": "QRICH1",
+  "ENSG00000198242": "RPL23A",
+  "ENSG00000198265": "HELZ",
+  "ENSG00000198286": "CARD11",
+  "ENSG00000198363": "ASPH",
+  "ENSG00000198369": "SPRED2",
+  "ENSG00000198373": "WWP2",
+  "ENSG00000198380": "GFPT1",
+  "ENSG00000198382": "UVRAG",
+  "ENSG00000198399": "ITSN2",
+  "ENSG00000198408": "OGA",
+  "ENSG00000198431": "TXNRD1",
+  "ENSG00000198492": "YTHDF2",
+  "ENSG00000198586": "TLK1",
+  "ENSG00000198589": "LRBA",
+  "ENSG00000198604": "BAZ1A",
+  "ENSG00000198646": "NCOA6",
+  "ENSG00000198663": "C6ORF89",
+  "ENSG00000198677": "SKIC3",
+  "ENSG00000198695": "MT-ND6",
+  "ENSG00000198700": "IPO9",
+  "ENSG00000198712": "MT-CO2",
+  "ENSG00000198727": "MT-CYB",
+  "ENSG00000198740": "ZNF652",
+  "ENSG00000198742": "SMURF1",
+  "ENSG00000198755": "RPL10A",
+  "ENSG00000198763": "MT-ND2",
+  "ENSG00000198771": "RCSD1",
+  "ENSG00000198786": "MT-ND5",
+  "ENSG00000198793": "MTOR",
+  "ENSG00000198804": "MT-CO1",
+  "ENSG00000198815": "FOXJ3",
+  "ENSG00000198836": "OPA1",
+  "ENSG00000198840": "MT-ND3",
+  "ENSG00000198874": "TYW1",
+  "ENSG00000198879": "SFMBT2",
+  "ENSG00000198886": "MT-ND4",
+  "ENSG00000198888": "MT-ND1",
+  "ENSG00000198898": "CAPZA2",
+  "ENSG00000198899": "MT-ATP6",
+  "ENSG00000198900": "TOP1",
+  "ENSG00000198911": "SREBF2",
+  "ENSG00000198938": "MT-CO3",
+  "ENSG00000198947": "DMD",
+  "ENSG00000198964": "SGMS1",
+  "ENSG00000203666": "DRC8",
+  "ENSG00000203867": "RBM20",
+  "ENSG00000204084": "INPP5B",
+  "ENSG00000204120": "GIGYF2",
+  "ENSG00000204130": "RUFY2",
+  "ENSG00000204138": "PHACTR4",
+  "ENSG00000204178": "MACO1",
+  "ENSG00000204217": "BMPR2",
+  "ENSG00000204394": "VARS1",
+  "ENSG00000204406": "MBD5",
+  "ENSG00000204442": "NALF1",
+  "ENSG00000204568": "MRPS18B",
+  "ENSG00000204628": "RACK1",
+  "ENSG00000204713": "TRIM27",
+  "ENSG00000204842": "ATXN2",
+  "ENSG00000205268": "PDE7A",
+  "ENSG00000205339": "IPO7",
+  "ENSG00000205531": "NAP1L4",
+  "ENSG00000205628": "LINC01446",
+  "ENSG00000206560": "ANKRD28",
+  "ENSG00000211459": "MT-RNR1",
+  "ENSG00000212907": "MT-ND4L",
+  "ENSG00000213047": "DENND1B",
+  "ENSG00000213079": "SCAF8",
+  "ENSG00000213551": "DNAJC9",
+  "ENSG00000213719": "CLIC1",
+  "ENSG00000213741": "RPS29",
+  "ENSG00000213931": "HBE1",
+  "ENSG00000213934": "HBG1",
+  "ENSG00000214113": "LYRM4",
+  "ENSG00000214827": "MTCP1",
+  "ENSG00000214960": "CRPPA",
+  "ENSG00000215417": "MIR17HG",
+  "ENSG00000215908": "CROCCP2",
+  "ENSG00000221914": "PPP2R2A",
+  "ENSG00000221983": "UBA52",
+  "ENSG00000222041": "CYTOR",
+  "ENSG00000223482": "NUTM2A-AS1",
+  "ENSG00000223745": "CCDC18-AS1",
+  "ENSG00000223812": "PYDC2-AS1",
+  "ENSG00000224063": "CALCRL-AS1",
+  "ENSG00000224141": "MIR548XHG",
+  "ENSG00000224184": "MIR3681HG",
+  "ENSG00000224271": "EPIC1",
+  "ENSG00000224699": "LAMTOR5-AS1",
+  "ENSG00000225470": "JPX",
+  "ENSG00000225484": "NUTM2B-AS1",
+  "ENSG00000225546": "LINC02476",
+  "ENSG00000226383": "LINC01876",
+  "ENSG00000226419": "SLC16A1-AS1",
+  "ENSG00000226688": "ENTPD1-AS1",
+  "ENSG00000226752": "CUTALP",
+  "ENSG00000226985": "LINC01203",
+  "ENSG00000227070": "EPS15-AS1",
+  "ENSG00000227398": "KIF9-AS1",
+  "ENSG00000227706": "ERVH-3",
+  "ENSG00000228253": "MT-ATP8",
+  "ENSG00000228782": "MRPL45P2",
+  "ENSG00000229140": "CCDC26",
+  "ENSG00000229180": "RABGEF1P1",
+  "ENSG00000229807": "XIST",
+  "ENSG00000230124": "ACBD6",
+  "ENSG00000230590": "FTX",
+  "ENSG00000230876": "LINC00486",
+  "ENSG00000231185": "LINC01844",
+  "ENSG00000231304": "SGO1-AS1",
+  "ENSG00000231312": "MAP4K3-DT",
+  "ENSG00000231500": "RPS18",
+  "ENSG00000231607": "DLEU2",
+  "ENSG00000231689": "LINC01090",
+  "ENSG00000231721": "LINC-PINT",
+  "ENSG00000231925": "TAPBP",
+  "ENSG00000232040": "SCAND3",
+  "ENSG00000232053": "LOC105375523",
+  "ENSG00000233098": "CCDC144NL-AS1",
+  "ENSG00000233766": "CAVIN2-AS1",
+  "ENSG00000234062": "TM9SF5P",
+  "ENSG00000234129": "HCCS-DT",
+  "ENSG00000234722": "LINC01287",
+  "ENSG00000234948": "LINC01524",
+  "ENSG00000235335": "B3GALT1-AS1",
+  "ENSG00000237978": "KCNMB2-AS1",
+  "ENSG00000241743": "XACT",
+  "ENSG00000241956": "LOC102546299",
+  "ENSG00000241973": "PI4KA",
+  "ENSG00000242086": "SDHAP2",
+  "ENSG00000243156": "MICAL3",
+  "ENSG00000244754": "N4BP2L2",
+  "ENSG00000244879": "GABPB1-AS1",
+  "ENSG00000245532": "NEAT1",
+  "ENSG00000246067": "RAB30-DT",
+  "ENSG00000247809": "NR2F2-AS1",
+  "ENSG00000247828": "TMEM161B-DT",
+  "ENSG00000248049": "UBA6-DT",
+  "ENSG00000248131": "LINC01194",
+  "ENSG00000248685": "LINC02484",
+  "ENSG00000248858": "FLJ46284",
+  "ENSG00000249307": "LINC01088",
+  "ENSG00000249592": "PCGF3-AS1",
+  "ENSG00000249859": "PVT1",
+  "ENSG00000250903": "GMDS-DT",
+  "ENSG00000250954": "LOC101928622",
+  "ENSG00000251003": "ZFPM2-AS1",
+  "ENSG00000251129": "LINC02506",
+  "ENSG00000251138": "LINC02882",
+  "ENSG00000251209": "LINC00923",
+  "ENSG00000251381": "LINC00958",
+  "ENSG00000251513": "LIX1-AS1",
+  "ENSG00000251562": "MALAT1",
+  "ENSG00000251574": "LOC105379109",
+  "ENSG00000253394": "LINC00534",
+  "ENSG00000253729": "PRKDC",
+  "ENSG00000253877": "LINC01608",
+  "ENSG00000254087": "LYN",
+  "ENSG00000255679": "JRKL-AS1",
+  "ENSG00000257103": "LSM14A",
+  "ENSG00000257261": "SLC38A4-AS1",
+  "ENSG00000257923": "CUX1",
+  "ENSG00000258038": "LINC02327",
+  "ENSG00000258667": "HIF1A-AS3",
+  "ENSG00000258932": "RPS27AP4",
+  "ENSG00000259240": "MIR4713HG",
+  "ENSG00000259471": "LINC01169",
+  "ENSG00000259692": "LINC01418",
+  "ENSG00000260032": "NORAD",
+  "ENSG00000261008": "LINC01572",
+  "ENSG00000261609": "GAN",
+  "ENSG00000263001": "GTF2I",
+  "ENSG00000263711": "LINC02864",
+  "ENSG00000264538": "SUZ12P1",
+  "ENSG00000265491": "RNF115",
+  "ENSG00000265843": "LINC01029",
+  "ENSG00000265972": "TXNIP",
+  "ENSG00000266412": "NCOA4",
+  "ENSG00000266976": "ERVE-5",
+  "ENSG00000267265": "GP6-AS1",
+  "ENSG00000269821": "KCNQ1OT1",
+  "ENSG00000270647": "TAF15",
+  "ENSG00000272325": "NUDT3",
+  "ENSG00000272391": "POM121C",
+  "ENSG00000272693": "NUPR2P1",
+  "ENSG00000272886": "DCP1A",
+  "ENSG00000275066": "SYNRG",
+  "ENSG00000276476": "LINC00540",
+  "ENSG00000277149": "TYW1B",
+  "ENSG00000277654": "NOP35P1",
+  "ENSG00000278259": "MYO19",
+  "ENSG00000278311": "GGNBP2",
+  "ENSG00000278540": "ACACA",
+  "ENSG00000281131": "SCHLAP1",
+  "ENSG00000283117": "MGC4859"
+}

PDGrapher/data/raw/cosmic/2022-10-COSMIC/README.md

@@ -0,0 +1,17 @@
+# COSMIC
+
+## Sources
+Cosmic CLP: https://cancer.sanger.ac.uk/cell_lines/download (downloaded 15th September 2022)
+
+Curated gene list: https://cancer.sanger.ac.uk/cell_lines/curation (downloaded on 18th March 2024)
+
+## Date retrieved
+
+November 15, 2022
+March 18, 2024
+
+## Contacts
+
+Guadalupe Gonzalez
+<ggonzalezp16@gmail.com>
+

PDGrapher/data/raw/drugbank/2022-11-DrugBank/README.md

@@ -0,0 +1,24 @@
+# DrugBank
+
+## Sources
+
+https://www.drugbank.ca
+
+See also an older version of the database with Python scripts in 2020-08-DrugBank directory.
+
+## References
+
+* Wishart DS, Feunang YD, Guo AC, Lo EJ, Marcu A, Grant JR, Sajed T, Johnson D, Li C, Sayeeda Z, Assempour N. DrugBank 5.0: a major update to the DrugBank database for 2018. Nucleic acids research. 2018 Jan 4;46(D1):D1074-82.
+
+* Wishart DS, Wu A. Using DrugBank for in silico drug exploration and discovery. Current Protocols in Bioinformatics. 2016 Jun;54(1):14-4.
+
+* Probst D, Reymond JL. Exploring drugbank in virtual reality chemical space. Journal of chemical information and modeling. 2018 Aug 16;58(9):1731-5.
+
+## Date retrieved
+
+November 4, 2022
+
+## Contacts
+
+Marinka Zitnik
+<marinka@hms.harvard.edu>

PDGrapher/data/raw/lincs/2022-02-LINCS_Level3/README.md

@@ -0,0 +1,21 @@
+# LINCS
+
+## Sources
+
+https://clue.io/releases/data-dashboard 
+cellinfo_beta.txt
+compoundinfo_beta.txt
+geneinfo_beta.txt
+instinfo_beta.txt
+level3_beta_ctl_n188708x12328.gctx
+level3_beta_trt_sh_n453175x12328.gctx
+level3_beta_trt_xpr_n420583x12328.gctx
+
+## Date retrieved
+
+Devember 16, 2020
+
+## Contacts
+
+Guadalupe Gonzalez
+<ggonzalezp16@gmail.com>

PDGrapher/data/raw/ppi/2022-03-PPI/README.md

@@ -0,0 +1,31 @@
+# PPI 
+
+## Sources
+
+
+https://downloads.thebiogrid.org/File/BioGRID/Release-Archive/BIOGRID-4.4.207/BIOGRID-MV-Physical-4.4.207.tab3.zip
+
+https://science.sciencemag.org/highwire/filestream/628238/field_highwire_adjunct_files/1/Datasets_S1-S4.zip
+
+http://www.interactome-atlas.org/data/HuRI.tsv  
+
+## Auxiliary files
+hgnc2map.txt: gene identity map  
+geneinfo_beta.txt: gene information of C-map data used for training PDGrapher
+
+## References
+
+* Stark C, Breitkreutz BJ, Reguly T, Boucher L, Breitkreutz A, Tyers M. Biogrid: A General Repository for Interaction Datasets. Nucleic Acids Res. Jan 1, 2006; 34:D535-9. 
+
+* Menche, Jörg, et al. "Uncovering disease-disease relationships through the incomplete interactome." Science 347.6224 (2015).
+
+* Luck, Katja, et al. "A reference map of the human binary protein interactome." Nature 580.7803 (2020): 402-408.
+
+## Date retrieved
+
+March 14, 2022
+
+## Contacts
+
+Guadalupe Gonzalez
+<ggonzalezp16@gmail.com>

PDGrapher/data/scripts/GRN/GENIE3.py

@@ -0,0 +1,345 @@
+from sklearn.tree import BaseDecisionTree
+from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor
+from numpy import *
+import time
+from operator import itemgetter
+from multiprocessing import Pool
+
+
+def compute_feature_importances(estimator):
+    if isinstance(estimator, BaseDecisionTree):
+        return estimator.tree_.compute_feature_importances(normalize=False)
+    else:
+        importances = [e.tree_.compute_feature_importances(normalize=False)
+                       for e in estimator.estimators_]
+        importances = array(importances)
+        return sum(importances,axis=0) / len(estimator)
+        
+
+
+def get_link_list(VIM,gene_names=None,regulators='all',maxcount='all',file_name=None):
+    
+    """Gets the ranked list of (directed) regulatory links.
+    
+    Parameters
+    ----------
+    
+    VIM: numpy array
+        Array as returned by the function GENIE3(), in which the element (i,j) is the score of the edge directed from the i-th gene to the j-th gene. 
+        
+    gene_names: list of strings, optional
+        List of length p, where p is the number of rows/columns in VIM, containing the names of the genes. The i-th item of gene_names must correspond to the i-th row/column of VIM. When the gene names are not provided, the i-th gene is named Gi.
+        default: None
+        
+    regulators: list of strings, optional
+        List containing the names of the candidate regulators. When a list of regulators is provided, the names of all the genes must be provided (in gene_names), and the returned list contains only edges directed from the candidate regulators. When regulators is set to 'all', any gene can be a candidate regulator.
+        default: 'all'
+        
+    maxcount: 'all' or positive integer, optional
+        Writes only the first maxcount regulatory links of the ranked list. When maxcount is set to 'all', all the regulatory links are written.
+        default: 'all'
+        
+    file_name: string, optional
+        Writes the ranked list of regulatory links to the file file_name.
+        default: None
+        
+        
+    
+    Returns
+    -------
+    
+    The list of regulatory links, ordered according to the edge score. Auto-regulations do not appear in the list. Regulatory links with a score equal to zero are randomly permuted. In the ranked list of edges, each line has format:
+        
+        regulator   target gene     score of edge
+    """
+    
+    # Check input arguments      
+    if not isinstance(VIM,ndarray):
+        raise ValueError('VIM must be a square array')
+    elif VIM.shape[0] != VIM.shape[1]:
+        raise ValueError('VIM must be a square array')
+        
+    ngenes = VIM.shape[0]
+        
+    if gene_names is not None:
+        if not isinstance(gene_names,(list,tuple)):
+            raise ValueError('input argument gene_names must be a list of gene names')
+        elif len(gene_names) != ngenes:
+            raise ValueError('input argument gene_names must be a list of length p, where p is the number of columns/genes in the expression data')
+        
+    if regulators != 'all':
+        if not isinstance(regulators,(list,tuple)):
+            raise ValueError('input argument regulators must be a list of gene names')
+
+        if gene_names is None:
+            raise ValueError('the gene names must be specified (in input argument gene_names)')
+        else:
+            sIntersection = set(gene_names).intersection(set(regulators))
+            if not sIntersection:
+                raise ValueError('The genes must contain at least one candidate regulator')
+        
+    if maxcount != 'all' and not isinstance(maxcount,int):
+        raise ValueError('input argument maxcount must be "all" or a positive integer')
+        
+    if file_name is not None and not isinstance(file_name,str):
+        raise ValueError('input argument file_name must be a string')
+    
+    
+
+    # Get the indices of the candidate regulators
+    if regulators == 'all':
+        input_idx = range(ngenes)
+    else:
+        input_idx = [i for i, gene in enumerate(gene_names) if gene in regulators]
+    
+    # Get the non-ranked list of regulatory links
+    vInter = [(i,j,score) for (i,j),score in ndenumerate(VIM) if i in input_idx and i!=j]
+    
+    # Rank the list according to the weights of the edges        
+    vInter_sort = sorted(vInter,key=itemgetter(2),reverse=True)
+    nInter = len(vInter_sort)
+    
+    # Random permutation of edges with score equal to 0
+    flag = 1
+    i = 0
+    while flag and i < nInter:
+        (TF_idx,target_idx,score) = vInter_sort[i]
+        if score == 0:
+            flag = 0
+        else:
+            i += 1
+            
+    if not flag:
+        items_perm = vInter_sort[i:]
+        items_perm = random.permutation(items_perm)
+        vInter_sort[i:] = items_perm
+        
+    # Write the ranked list of edges
+    nToWrite = nInter
+    if isinstance(maxcount,int) and maxcount >= 0 and maxcount < nInter:
+        nToWrite = maxcount
+    
+    edge_list = []
+    if file_name:
+    
+        outfile = open(file_name,'w')
+    
+        if gene_names is not None:
+            for i in range(nToWrite):
+                (TF_idx,target_idx,score) = vInter_sort[i]
+                TF_idx = int(TF_idx)
+                target_idx = int(target_idx)
+                outfile.write('%s\t%s\t%.6f\n' % (gene_names[TF_idx],gene_names[target_idx],score))
+                edge_list.append((gene_names[TF_idx],gene_names[target_idx],score))
+        else:
+            for i in range(nToWrite):
+                (TF_idx,target_idx,score) = vInter_sort[i]
+                TF_idx = int(TF_idx)
+                target_idx = int(target_idx)
+                outfile.write('G%d\tG%d\t%.6f\n' % (TF_idx+1,target_idx+1,score))
+                edge_list.append((TF_idx+1,target_idx+1,score))
+        
+        outfile.close()
+        
+    else:
+        
+        if gene_names is not None:
+            for i in range(nToWrite):
+                (TF_idx,target_idx,score) = vInter_sort[i]
+                TF_idx = int(TF_idx)
+                target_idx = int(target_idx)
+                #print('%s\t%s\t%.6f' % (gene_names[TF_idx],gene_names[target_idx],score))
+                edge_list.append((gene_names[TF_idx],gene_names[target_idx],score))
+        else:
+            for i in range(nToWrite):
+                (TF_idx,target_idx,score) = vInter_sort[i]
+                TF_idx = int(TF_idx)
+                target_idx = int(target_idx)
+                #print('G%d\tG%d\t%.6f' % (TF_idx+1,target_idx+1,score))
+                edge_list.append((TF_idx+1,target_idx+1,score))
+                
+    return edge_list    
+                
+                
+
+
+
+def GENIE3(expr_data,gene_names=None,regulators='all',tree_method='RF',K='sqrt',ntrees=1000,nthreads=1):
+    
+    '''Computation of tree-based scores for all putative regulatory links.
+    
+    Parameters
+    ----------
+    
+    expr_data: numpy array
+        Array containing gene expression values. Each row corresponds to a condition and each column corresponds to a gene.
+        
+    gene_names: list of strings, optional
+        List of length p, where p is the number of columns in expr_data, containing the names of the genes. The i-th item of gene_names must correspond to the i-th column of expr_data.
+        default: None
+        
+    regulators: list of strings, optional
+        List containing the names of the candidate regulators. When a list of regulators is provided, the names of all the genes must be provided (in gene_names). When regulators is set to 'all', any gene can be a candidate regulator.
+        default: 'all'
+        
+    tree-method: 'RF' or 'ET', optional
+        Specifies which tree-based procedure is used: either Random Forest ('RF') or Extra-Trees ('ET')
+        default: 'RF'
+        
+    K: 'sqrt', 'all' or a positive integer, optional
+        Specifies the number of selected attributes at each node of one tree: either the square root of the number of candidate regulators ('sqrt'), the total number of candidate regulators ('all'), or any positive integer.
+        default: 'sqrt'
+         
+    ntrees: positive integer, optional
+        Specifies the number of trees grown in an ensemble.
+        default: 1000
+    
+    nthreads: positive integer, optional
+        Number of threads used for parallel computing
+        default: 1
+        
+        
+    Returns
+    -------
+
+    An array in which the element (i,j) is the score of the edge directed from the i-th gene to the j-th gene. All diagonal elements are set to zero (auto-regulations are not considered). When a list of candidate regulators is provided, the scores of all the edges directed from a gene that is not a candidate regulator are set to zero.
+        
+    '''
+    
+    time_start = time.time()
+    
+    # Check input arguments
+    if not isinstance(expr_data,ndarray):
+        raise ValueError('expr_data must be an array in which each row corresponds to a condition/sample and each column corresponds to a gene')
+        
+    ngenes = expr_data.shape[1]
+    
+    if gene_names is not None:
+        if not isinstance(gene_names,(list,tuple)):
+            raise ValueError('input argument gene_names must be a list of gene names')
+        elif len(gene_names) != ngenes:
+            raise ValueError('input argument gene_names must be a list of length p, where p is the number of columns/genes in the expr_data')
+        
+    if regulators != 'all':
+        if not isinstance(regulators,(list,tuple)):
+            raise ValueError('input argument regulators must be a list of gene names')
+
+        if gene_names is None:
+            raise ValueError('the gene names must be specified (in input argument gene_names)')
+        else:
+            sIntersection = set(gene_names).intersection(set(regulators))
+            if not sIntersection:
+                raise ValueError('the genes must contain at least one candidate regulator')        
+        
+    if tree_method != 'RF' and tree_method != 'ET':
+        raise ValueError('input argument tree_method must be "RF" (Random Forests) or "ET" (Extra-Trees)')
+        
+    if K != 'sqrt' and K != 'all' and not isinstance(K,int): 
+        raise ValueError('input argument K must be "sqrt", "all" or a stricly positive integer')
+        
+    if isinstance(K,int) and K <= 0:
+        raise ValueError('input argument K must be "sqrt", "all" or a stricly positive integer')
+    
+    if not isinstance(ntrees,int):
+        raise ValueError('input argument ntrees must be a stricly positive integer')
+    elif ntrees <= 0:
+        raise ValueError('input argument ntrees must be a stricly positive integer')
+        
+    if not isinstance(nthreads,int):
+        raise ValueError('input argument nthreads must be a stricly positive integer')
+    elif nthreads <= 0:
+        raise ValueError('input argument nthreads must be a stricly positive integer')
+        
+        
+    print('Tree method: ' + str(tree_method))
+    print('K: ' + str(K))
+    print('Number of trees: ' + str(ntrees))
+    print('\n')
+        
+    
+    # Get the indices of the candidate regulators
+    if regulators == 'all':
+        input_idx = list(range(ngenes))
+    else:
+        input_idx = [i for i, gene in enumerate(gene_names) if gene in regulators]
+
+    
+    # Learn an ensemble of trees for each target gene, and compute scores for candidate regulators
+    VIM = zeros((ngenes,ngenes))
+    
+    if nthreads > 1:
+        print('running jobs on %d threads' % nthreads)
+
+        input_data = list()
+        for i in range(ngenes):
+            input_data.append( [expr_data,i,input_idx,tree_method,K,ntrees] )
+
+        pool = Pool(nthreads)
+        alloutput = pool.map(wr_GENIE3_single, input_data)
+    
+        for (i,vi) in alloutput:
+            VIM[i,:] = vi
+
+    else:
+        print('running single threaded jobs')
+        for i in range(ngenes):
+            print('Gene %d/%d...' % (i+1,ngenes))
+            
+            vi = GENIE3_single(expr_data,i,input_idx,tree_method,K,ntrees)
+            VIM[i,:] = vi
+
+   
+    VIM = transpose(VIM)
+ 
+    time_end = time.time()
+    print("Elapsed time: %.2f seconds" % (time_end - time_start))
+
+    return VIM
+    
+    
+    
+def wr_GENIE3_single(args):
+    return([args[1], GENIE3_single(args[0], args[1], args[2], args[3], args[4], args[5])])
+    
+
+
+def GENIE3_single(expr_data,output_idx,input_idx,tree_method,K,ntrees):
+    
+    ngenes = expr_data.shape[1]
+    
+    # Expression of target gene
+    output = expr_data[:,output_idx]
+    
+    # Normalize output data
+    output = output / std(output)
+    
+    # Remove target gene from candidate regulators
+    input_idx = input_idx[:]
+    if output_idx in input_idx:
+        input_idx.remove(output_idx)
+
+    expr_data_input = expr_data[:,input_idx]
+    
+    # Parameter K of the tree-based method
+    if (K == 'all') or (isinstance(K,int) and K >= len(input_idx)):
+        max_features = "auto"
+    else:
+        max_features = K
+    
+    if tree_method == 'RF':
+        treeEstimator = RandomForestRegressor(n_estimators=ntrees,max_features=max_features)
+    elif tree_method == 'ET':
+        treeEstimator = ExtraTreesRegressor(n_estimators=ntrees,max_features=max_features)
+
+    # Learn ensemble of trees
+    treeEstimator.fit(expr_data_input,output)
+    
+    # Compute importance scores
+    feature_importances = compute_feature_importances(treeEstimator)
+    vi = zeros(ngenes)
+    vi[input_idx] = feature_importances
+       
+    return vi
+        
+        
+        

PDGrapher/data/scripts/GRN/GENIEppi-run.py

@@ -0,0 +1,53 @@
+from GENIE3 import *
+import networkx as nx
+import numpy as np
+import os
+import time
+root = 'data/raw/'
+import argparse
+parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('--cell_line', default=None, nargs='+')
+parser.add_argument('--data_type', default='cmp', type=str) #cmp or gen for chemical or genetic
+args = parser.parse_args()
+data_type = args.data_type
+celllines = args.cell_line
+
+def run_GENIE():
+    for cn in celllines:
+        startTime=time.time()
+        d = root + 'xpr_matrices/{}_xpr_matrix_{}_nonpertsubset.txt'.format(cn, data_type)
+        data=loadtxt(d, skiprows=1)
+        f=open(d)
+        gene_names=f.readline()
+        f.close()
+        gene_names = gene_names.rstrip('\n').split('\t')
+        VIM = GENIE3(data, ntrees=100, nthreads=20)
+        outdir = root + 'processed/'
+        os.makedirs(outdir, exist_ok=True)
+        with open(outdir + "{}_{}_GENIE3arr.npy".format(cn, data_type), 'wb') as f:
+            np.save(f, VIM)
+        print("Cell Line: {} - ".format(cn)+str(time.time()-startTime))
+
+def get_edgelist(): 
+    for cn in celllines:
+        outdir = root + 'processed/'
+        startTime=time.time()
+        d = root + 'xpr_matrices/{}_xpr_matrix_{}_nonpertsubset.txt'.format(cn, data_type)
+        arr = np.load(outdir + "{}_{}_GENIE3arr.npy".format(cn, data_type))
+        f=open(d)
+        gene_names=f.readline()
+        f.close()
+        gene_names = gene_names.rstrip('\n').split('\t')
+        
+        reg_link_list=get_link_list(arr, gene_names=gene_names, file_name=outdir+"{}_{}_edgelist.txt".format(cn, data_type))
+        g = nx.DiGraph((x,y,{'weight': v}) for (x, y, v) in reg_link_list)
+        nx.write_weighted_edgelist(g, outdir+'{}_{}_nxEdgelist.txt'.format(cn, data_type), delimiter='    ')
+        #print(reg_link_list[0], type(reg_link_list[0]))
+        print("Cell Line: {} - ".format(cn)+str(time.time()-startTime))
+
+    
+if __name__ == "__main__":
+    #startTime=time.time()
+    run_GENIE()
+    get_edgelist()
+    #print(time.time()-startTime)

PDGrapher/data/scripts/GRN/README.md

@@ -0,0 +1,31 @@
+## Scripts to build GRN for PDGrapher
+---
+
+### 1. Generate Expression Matrices
+Use `xpr_matrix_generator_datatypesplit.py` to build the `xpr_matrices` files.  
+
+**Inputs:**  
+- `level3_beta_ctl_n188708x12328.gctx` (download from [C-map](https://clue.io/releases/data-dashboard))  
+- `ppi_all_genes_edgelist.txt` (see `data/scripts/ppi` for instructions on generating this file)  
+
+**Command Example:**  
+python xpr_matrix_generator_datatypesplit.py
+
+
+### 2. Generate GENIE3 edge list
+Use `GENIEppi-run.py` to build the `{cell_line}_{pert_type}_edgelist.txt` files.  
+
+**Inputs:**  
+- `xpr_matrices` (from step 1)  
+
+**Command Example:**  
+python GENIEppi-run.py --data_type cmp --cell_line A549
+
+### 3. Filter edge lists
+Use `filter_edge_list.py` to build the `{cell_line}_{pert_type}_edgelist_filtered.txt` files.  
+
+**Inputs:**  
+- `{cell_line}_{pert_type}_edgelist.txt` files (from step 2)  
+
+**Command Example:**  
+python filter_edge_list.py --data_type cmp --cell_line A549

PDGrapher/data/scripts/GRN/filter_edge_list.py

@@ -0,0 +1,89 @@
+import networkx as nx
+import csrgraph as cg
+from collections import Counter
+import pandas as pd
+import os
+import os.path as osp
+import numpy as np
+import sys
+import argparse
+parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('--cell_line', default=None, nargs='+')
+parser.add_argument('--data_type', default='cross', type=str)
+args = parser.parse_args()
+data_type = args.data_type
+celllines = args.cell_line
+
+#chemical
+if data_type == "chemical":
+    print("Chemical")
+    gene_info = pd.read_csv('data/raw/lincs/geneinfo_beta.txt', sep="\t", low_memory=False)
+    outdir = 'data/raw/grn/filter/'
+    for i in celllines:
+        log_handle = open(f'log_grn_all_genes_genetic_{i}.txt', 'w')
+        path_edge_list = 'data/raw/grn/processed/{}_cmp_edgelist.txt'.format(i)
+        print("Do: ", i)
+        grn_0 = nx.read_edgelist(path_edge_list, nodetype=str, data=(('weight', float),))
+        grn_1 = [(u, v, d['weight']) for u, v, d in grn_0.edges(data=True)]
+        grn_2 = sorted(grn_1, key=lambda x: x[2])
+        grn = nx.Graph()
+        grn.add_weighted_edges_from(grn_2)
+        wl = np.array([float(weight) for u, v, weight in grn_2])
+        k = np.quantile(wl, 0.99)
+        edges = np.array(grn_2)
+        print(f"Do cutoff {k}...")
+        edges_filter = edges[wl < k]
+        grn.remove_edges_from(edges_filter)
+        #Loads gene info LINCS
+        log_handle.write('Cutoff:{}, removed edges {}\n'.format(k, len(edges_filter)))
+        log_handle.write('Overlap of genes from LINCS to grn:{}/{}\n'.format(len(set(grn.nodes()).intersection(set(gene_info['gene_symbol']))), len(gene_info)))
+        print("Flitering...")
+        #Filter nodes from grn to keep only the ones in LINCS
+        grn = grn.subgraph(gene_info['gene_symbol'].tolist())
+        log_handle.write('Keeping only grn nodes that are in LINCS:{}\n'.format(grn.number_of_nodes()))
+        print("Find connected components...")
+        ccs = [len(c) for c in sorted(nx.connected_components(grn), key=len, reverse=True)]
+        log_handle.write('Number of connected componens:\t{}\n'.format(len(ccs)))
+        Gcc = sorted(nx.connected_components(grn), key=len, reverse=True)
+        grn = grn.subgraph(Gcc[0])
+        log_handle.write('After keeping only biggest CC:\n')
+        log_handle.write('stats: {} nodes, {} edges, {} density, {} diameter\n\n\n'.format(grn.number_of_nodes(), grn.number_of_edges(), nx.density(grn), nx.diameter(grn)))
+        print("Done and save...")
+        grn_f = osp.join(outdir, f'{i}_cmp_edgelist_filtered.txt')
+        nx.write_edgelist(grn, grn_f, data=False)
+
+elif data_type == "genetic":
+    gene_info = pd.read_csv('data/raw/lincs/geneinfo_beta.txt', sep="\t", low_memory=False)
+    outdir = 'data/raw/grn/filter/'
+    for i in celllines:
+        log_handle = open(f'log_grn_all_genes_genetic_{i}.txt', 'w')
+        path_edge_list = 'data/raw/grn/processed/{}_gen_edgelist.txt'.format(i)
+        print("Do: ", i)
+        grn_0 = nx.read_edgelist(path_edge_list, nodetype=str, data=(('weight', float),))
+        grn_1 = [(u, v, d['weight']) for u, v, d in grn_0.edges(data=True)]
+        grn_2 = sorted(grn_1, key=lambda x: x[2])
+        grn = nx.Graph()
+        grn.add_weighted_edges_from(grn_2)
+        wl = np.array([float(weight) for u, v, weight in grn_2])
+        k = np.quantile(wl, 0.99)
+        edges = np.array(grn_2)
+        print(f"Do cutoff {k}...")
+        edges_filter = edges[wl < k]
+        grn.remove_edges_from(edges_filter)
+        #Loads gene info LINCS
+        log_handle.write('Cutoff:{}, removed edges {}\n'.format(k, len(edges_filter)))
+        log_handle.write('Overlap of genes from LINCS to grn:{}/{}\n'.format(len(set(grn.nodes()).intersection(set(gene_info['gene_symbol']))), len(gene_info)))
+        print("Flitering...")
+        #Filter nodes from grn to keep only the ones in LINCS
+        grn = grn.subgraph(gene_info['gene_symbol'].tolist())
+        log_handle.write('Keeping only grn nodes that are in LINCS:{}\n'.format(grn.number_of_nodes()))
+        print("Find connected components...")
+        ccs = [len(c) for c in sorted(nx.connected_components(grn), key=len, reverse=True)]
+        log_handle.write('Number of connected componens:\t{}\n'.format(len(ccs)))
+        Gcc = sorted(nx.connected_components(grn), key=len, reverse=True)
+        grn = grn.subgraph(Gcc[0])
+        log_handle.write('After keeping only biggest CC:\n')
+        log_handle.write('stats: {} nodes, {} edges, {} density, {} diameter\n\n\n'.format(grn.number_of_nodes(), grn.number_of_edges(), nx.density(grn), nx.diameter(grn)))
+        print("Done and save...")
+        grn_f = osp.join(outdir, f'{i}_gen_edgelist_filtered.txt')
+        nx.write_edgelist(grn, grn_f, data=False)

PDGrapher/data/scripts/GRN/xpr_matrix_generator_datatypesplit.py

@@ -0,0 +1,82 @@
+import networkx as nx
+import pandas as pd
+import numpy as np
+import os
+import os.path as osp
+import h5py
+
+#Loading Data
+DATA_ROOT = "data/raw/lincs/"
+#log_handle = open(osp.join(outdir, 'process_data_lognorm_log.txt'), 'w')
+#log_handle = open(osp.join(outdir, 'test_log.txt'), 'w')
+
+inst_info = pd.read_csv(os.path.join(DATA_ROOT, 'instinfo_beta.txt'), sep="\t", low_memory=False)
+inst_info_xpr = inst_info[np.logical_and(inst_info['pert_type'] == 'trt_xpr', inst_info['failure_mode'].isna())].reset_index(inplace=False, drop=True) 
+inst_info_ctl = inst_info[np.logical_and(np.logical_or(inst_info['pert_type'] == 'ctl_vector',inst_info['pert_type'] == 'ctl_vehicle'), inst_info['failure_mode'].isna())].reset_index(inplace=False, drop=True)
+gene_info = pd.read_csv(os.path.join(DATA_ROOT, 'geneinfo_beta.txt'), sep="\t", low_memory=False)
+df_xpr=pd.DataFrame(inst_info_xpr[['sample_id','pert_id', 'pert_dose','pert_dose_unit','pert_time','cell_iname']])
+
+
+f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_ctl_n188708x12328.gctx'), 'r')
+matrix_xpr = f['0']['DATA']['0']['matrix'][:].transpose()
+gene_ids_xpr = f['0']['META']['ROW']['id'][:]
+sample_ids_xpr = f['0']['META']['COL']['id'][:]
+matrix_xpr = pd.DataFrame(matrix_xpr, columns = sample_ids_xpr.astype(str), index = gene_ids_xpr.astype(int))
+del f
+
+#Decoding sample_ids_xpr array
+decode=np.vectorize(np.char.decode)
+sample_ids_xpr=decode(sample_ids_xpr)
+
+#Creating dict of cell_line:sample_ids
+cell_line_sids={}
+for cn in inst_info_ctl.cell_iname.unique():
+	cell_line_sids[cn]=list(inst_info_ctl.loc[inst_info_ctl['cell_iname']==cn]['sample_id'])
+
+#Lists of sample_ids for ctl_vector and ctl_vehicle samples
+xpr_samples_unique = list(inst_info_ctl.loc[inst_info_ctl['pert_type']=="ctl_vector"].sample_id.unique())
+cmp_samples_unique = list(inst_info_ctl.loc[inst_info_ctl['pert_type']=="ctl_vehicle"].sample_id.unique())
+    
+    
+
+#Renaming columns of matrix_xpr_trans to gene symbols
+matrix_xpr_trans=matrix_xpr.T
+new_cols = []
+for i in matrix_xpr_trans.columns: 
+	new_cols.append(gene_info.loc[gene_info['gene_id']==i]['gene_symbol'].item())
+matrix_xpr_trans.columns = new_cols
+
+
+#Subsetting matrix_xpr_trans column genes to include only those in the ppi, but not those that are perturbed.
+path_edge_list = 'data/raw/ppi/ppi_all_genes_edgelist.txt'
+ppi = nx.read_edgelist(path_edge_list)
+ppi = ppi.subgraph(gene_info['gene_symbol'].tolist())
+
+
+#pert_genes = dict()
+#for cell_line in list(set(inst_info_xpr['cell_iname'])):
+	#pert_genes[cell_line] = set(inst_info_xpr[inst_info_xpr['cell_iname']==cell_line]['cmap_name'].tolist())
+
+outdir = 'data/raw/grn/xpr_matrices/'
+os.makedirs(outdir, exist_ok=True)
+
+#chemical cell lines
+for cn in cell_line_sids:
+	if cn in ['A549', 'MCF7', 'PC3', 'VCAP', 'MDAMB231', 'BT20', 'HT29', 'A375', 'HELA']:
+		print(cn)
+		cellline_xpr_matrix_chem = matrix_xpr_trans[np.logical_and(matrix_xpr_trans.index.isin(cell_line_sids[cn]),matrix_xpr_trans.index.isin(cmp_samples_unique))]
+		gene_intersection_list=list(set(ppi.nodes).intersection(set(list(gene_info.gene_symbol.unique()))))
+		cellline_xpr_matrix_chem=cellline_xpr_matrix_chem[gene_intersection_list]
+		cellline_xpr_matrix_chem.to_csv(osp.join(outdir,'{}_xpr_matrix_cmp_nonpertsubset.txt'.format(cn)), sep='	', index=False)
+
+#genetic cell lines
+for cn in cell_line_sids:
+	if cn in ['BICR6', 'YAPC', 'AGS', 'U251MG', 'ES2', 'MCF7', 'PC3', 'A375', 'HT29', 'A549']:
+		print(cn)
+		cellline_xpr_matrix_gen = matrix_xpr_trans[np.logical_and(matrix_xpr_trans.index.isin(cell_line_sids[cn]),matrix_xpr_trans.index.isin(xpr_samples_unique))]
+		gene_intersection_list=list(set(ppi.nodes).intersection(set(list(gene_info.gene_symbol.unique()))))
+		cellline_xpr_matrix_gen=cellline_xpr_matrix_gen[gene_intersection_list]
+		cellline_xpr_matrix_gen.to_csv(osp.join(outdir,'{}_xpr_matrix_gen_nonpertsubset.txt'.format(cn)), sep='	', index=False)
+
+
+    

PDGrapher/data/scripts/cosmic/generate_cell_line_mutations_files.py

@@ -0,0 +1,60 @@
+'''
+Processes CosmicCLP_MutantExport.tsv (15th Sept 2022) to create a file with cell line: mutations 
+Expert curated list obtained from:https://cancer.sanger.ac.uk/cell_lines/curation#  (18th March 2024)
+
+'''
+
+import pandas as pd
+from collections import Counter
+import os
+import os.path as osp
+
+#Creates out dir
+outdir = '../../processed/cosmic'
+os.makedirs(outdir, exist_ok=True)
+
+
+cell_lines = ['A549', 'PC-3', 'MCF7', 'BT-20', 'MDA-MB-231', 'VCaP', 'A375', 'HT-29', 'ES-2', 'BICR6', 'YAPC', 'AGS', 'U251MG']
+
+
+
+#Loads data 
+data = pd.read_csv('../../raw/cosmic/2022-10-COSMIC/data/CosmicCLP_MutantExport.tsv', sep='\t', encoding="ISO-8859-1")
+#Filter to include only the 5 cell lines of interest
+mask = [e in cell_lines for e in data['Sample name']]
+data = data[mask]
+
+
+
+#Explore data
+log_handle = open('log_stats.txt','w')
+columns = ['Mutation Description', 'Mutation somatic status', 'Mutation verification status']
+for cell_line in cell_lines:
+	data_i = data[data['Sample name']==cell_line]
+	log_handle.write('\nCELL LINE:\t{}\n'.format(cell_line))
+	for column in columns:
+		log_handle.write(column+'\n')
+		log_handle.write(str(Counter(data_i[column])) +'\n\n')
+	log_handle.write('Total genes mutated:\t{}\n\n\n\n'.format(len(data_i['Gene name'])))
+
+
+
+
+
+
+#As agreed with Marinka, take an overlap of the 'verified' mutations and expert curated genes, for each cell line
+	#Filter to keep only verified genes
+data = data[data['Mutation verification status'] == 'Verified']
+
+
+	#Filter to keep only curated genes
+curated_genes = pd.read_csv('../../raw/cosmic/2022-10-COSMIC/data/expert_curated_genes_cosmic_2024.csv', sep='\t')['Genes'].tolist()
+mask = [gene in curated_genes for gene in data['Gene name']]
+data_curated = data[mask]
+data_curated['Sample name'] = [e.replace('-','').upper() for e in data_curated['Sample name']]
+
+
+#Save file
+data_curated.to_csv(osp.join(outdir, 'CosmicCLP_MutantExport_only_verified_and_curated.csv'))
+log_handle.write(str(data_curated['Sample name'].value_counts()))
+log_handle.close()

PDGrapher/data/scripts/drugbank/get_drug_targets.py

@@ -0,0 +1,59 @@
+'''
+Processess DrugBank full database to get txt file with drug names and targets
+'''
+from bs4 import BeautifulSoup
+import pandas as pd
+import os
+import os.path as osp
+
+outdir = '../../processed/drugbank/'
+os.makedirs(outdir, exist_ok=True)
+
+soup = BeautifulSoup(open("../../raw/drugbank/2022-11-DrugBank/data/all-full-database.xml"),"xml")
+
+# sep = ","
+# with open('../processed/targets.txt', 'w') as f:
+
+df = []
+for drug in soup.find_all("drug"):
+    drug_id = drug.find("drugbank-id").text 
+    drug_name = drug.find("name").text
+    targets = drug.find_all("target")
+    synonyms = drug.find("synonyms")
+    if synonyms is None:
+        synonyms = '-'
+    else:
+        synonyms = synonyms.find_all("synonym")
+        synonyms = "||".join([e.text for e in synonyms])
+    if not targets:
+        continue
+    for i in targets:
+        identifiers = i.find_all("external-identifier")
+        identifiers = '||'.join(['|'.join([e.resource.text, e.identifier.text]) for e in identifiers])
+        if i.find("id") is not None:
+            idd = i.find("id").text
+        else:
+            idd = '-'
+        if i.find("name") is not None:
+            name = i.find("name").text 
+        else:
+            name == "-"
+        if i.find("gene-name") is not None:
+            gene_name = i.find("gene-name").text
+        else:
+            gene_name = "-"
+        if i.find("organism") is not None:
+            organism = i.find("organism").text
+        else:
+            organism = "-"
+        if i.find_all('synonym') is not None:
+            synonyms = '||'.join(synonym.text for synonym in i.find_all('synonym'))
+        else:
+            synonyms = '-'
+
+
+        df.append([drug_id, drug_name, synonyms, idd, name, gene_name, synonyms, identifiers, organism])
+        
+
+df = pd.DataFrame(df, columns= ['DrugBank_ID', 'drug_name', 'drug_synonyms', 'target_id', 'target_name', 'gene_name', 'gene_synonyms', 'gene_identifiers', 'organism']).to_csv(osp.join(outdir, 'targets.txt'), sep = ',', index=False)
+

PDGrapher/data/scripts/lincs/process_data.py

@@ -0,0 +1,881 @@
+''' 
+Normalizes perturbed (treated) and control (diseased) data of genetic interventions
+Saves data as npz
+
+1. Reads data (load_data)
+2. Filters to keep only the cell lines that we use in experiments (filter_cell_lines_custom)
+3. Filters to keep only the samples in metadata (filter_data_metadata)
+4. Filters out samples for which the drug targets are not in LINCS genes (filter_samples_with_unknown_perturbed_genes)
+5. Normalizes data between (0,1) and save (normalize_and_save)
+'''
+import pandas as pd
+import h5py
+import os
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os.path as osp
+import numpy as np
+from collections import Counter
+import matplotlib.pyplot as plt
+import math
+from random import sample
+from sklearn.preprocessing import MinMaxScaler
+
+
+outdir = '../../processed/lincs'
+os.makedirs(outdir, exist_ok=True)
+
+	#LOG
+	
+
+################
+# Data loading
+################
+def stats_data(inst_info_xpr, matrix_xpr, matrix_ctl, gene_info):
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+
+		####Data exploration -- GE values of genes that are perturbed (!)
+	#Get GE value for each gene perturbed by CRISPR
+	values_pert = {}
+	values_control = {}
+	for i in range(len(inst_info_xpr)):
+		gene_symbol = inst_info_xpr.at[i, 'cmap_name']
+		if gene_symbol in dict_symbol_id:					#if the cmap_name of gene is in the gene_info
+			sample_id = inst_info_xpr.at[i, 'sample_id']
+			gene_id = dict_symbol_id[gene_symbol]
+			if gene_id in values_pert:
+				values_pert[gene_id].append(matrix_xpr.at[gene_id, sample_id])
+			else:
+				values_pert[gene_id] = [matrix_xpr.at[gene_id, sample_id]]
+
+
+	for gene_symbol in list(set(inst_info_xpr['cmap_name'])):
+		if gene_symbol in dict_symbol_id:				#if the cmap_name of gene is in the gene_info
+			gene_id = dict_symbol_id[gene_symbol]
+			values_control[gene_id] = [matrix_ctl.loc[gene_id]]
+
+	for key in values_pert:
+		values_pert[key] = np.mean(values_pert[key])
+
+	for key in values_control:
+		values_control[key] = np.mean(values_control[key])
+
+	fig, (ax1, ax2) = plt.subplots(2, figsize=(16,6))
+	ax1.hist(values_pert.values())
+	ax2.hist(values_control.values())
+	ax1.set_title('Values of perturbed genes (avg) - CRISPR')
+	ax2.set_title('Values of genes in control (avg)')
+	fig.savefig(osp.join(outdir,'exploration_ge_crispr.png'))
+
+
+	return
+
+
+
+def loads_data(DATA_ROOT, log_handle):
+	#Loads metadata
+	inst_info = pd.read_csv(os.path.join(DATA_ROOT, 'instinfo_beta.txt'), sep="\t", low_memory=False)
+	inst_info_xpr = inst_info[np.logical_and(inst_info['pert_type'] == 'trt_xpr',  inst_info['failure_mode'].isna())].reset_index(inplace=False, drop=True)
+	inst_info_ctl = inst_info[np.logical_and(inst_info['pert_type'] == 'ctl_vector', inst_info['failure_mode'].isna()) ].reset_index(inplace=False, drop=True)
+	gene_info = pd.read_csv(os.path.join(DATA_ROOT, 'geneinfo_beta.txt'), sep="\t", low_memory=False)
+
+
+	####################
+	#Loads data matrices
+	### CRISPR
+	f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_trt_xpr_n420583x12328.gctx'), 'r')
+	matrix_xpr = f['0']['DATA']['0']['matrix'][:].transpose()
+	gene_ids_xpr = f['0']['META']['ROW']['id'][:]
+	sample_ids_xpr = f['0']['META']['COL']['id'][:]
+	matrix_xpr = pd.DataFrame(matrix_xpr, columns = sample_ids_xpr.astype(str), index = gene_ids_xpr.astype(int))
+	del f
+
+
+	#re-order gene_info based on the order in gene_ids_xpr (rows of data)
+	gene_info.index = gene_info['gene_id']
+	gene_info = gene_info.loc[gene_ids_xpr.astype(int)].reset_index(inplace=False, drop=True)
+	gene_info.to_csv(osp.join(outdir, 'gene_info.txt'), index=False)
+
+	#Stats
+	log_handle.write('CRISPR\n------\n')
+	log_handle.write('CRISPR entries in inst_info metadata:\t{}\n'.format(len(inst_info_xpr)))
+	log_handle.write('CRISPR entries in data matrix:\t{}\n'.format(len(sample_ids_xpr)))
+	log_handle.write('Overlap between inst_info metadata and sample ids in data matrix:\t{}\n'.format(len(set(inst_info_xpr['sample_id']).intersection(set(sample_ids_xpr.astype(str))))))
+
+
+
+	### Control data
+	f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_ctl_n188708x12328.gctx'), 'r')
+	matrix_ctl = f['0']['DATA']['0']['matrix'][:].transpose()
+	gene_ids_ctl = f['0']['META']['ROW']['id'][:]					#not in the same order as gene_ids_xpr
+	sample_ids_ctl = f['0']['META']['COL']['id'][:]
+	matrix_ctl = pd.DataFrame(matrix_ctl, columns = sample_ids_ctl.astype(str), index = gene_ids_ctl.astype(int))
+	del f
+
+
+	#Stats
+	log_handle.write('CONTROL\n------\n')
+	log_handle.write('Control entries in inst_info metadata:\t{}\n'.format(len(inst_info_ctl)))
+	log_handle.write('Control entries in data matrix:\t{}\n'.format(len(sample_ids_ctl)))
+	log_handle.write('Overlap between inst_info metadata and sample ids in data matrix:\t{}\n'.format(len(set(inst_info_ctl['sample_id']).intersection(set(sample_ids_ctl.astype(str))))))
+	log_handle.write('\n------\n')
+
+
+	stats_data(inst_info_xpr, matrix_xpr, matrix_ctl, gene_info)
+
+	return inst_info_xpr, inst_info_ctl, gene_info, matrix_xpr, matrix_ctl
+
+
+
+
+################
+# Processing
+################
+
+################
+#1. Filter column metadata and data matrix to keep only those in metadata
+
+def filter_data_metadata(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only those in metadata\n------\n')
+	#CRISPR
+	list_ids = list(inst_info_xpr['sample_id'])	#in metadata
+	matrix_xpr = matrix_xpr[list_ids]	#Filtered data matrix
+	log_handle.write('CRISPR:\t{} datapoints\n'.format(matrix_xpr.shape[1]))
+	#CONTROL
+	list_ids = list(inst_info_ctl['sample_id'])	#in metadata
+	#extra steps
+	#--
+	list_ids = list(set(list_ids).intersection(set(matrix_ctl.columns.astype(str))))	#in metadata and in data matrix (some of metadata are not in data matrix)
+	inst_info_ctl.index = inst_info_ctl['sample_id']; inst_info_ctl = inst_info_ctl.loc[list_ids].reset_index(inplace=False, drop=True) #remove entries from metadata that are not in data matrix
+	#--
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n\n\n'.format(matrix_ctl.shape[1]))
+	return inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl
+
+
+
+################
+#2. Filter to keep only cell lines with more perturbations
+
+def stats_control(inst_info_ctl, log_handle):
+	#Stats unique cell lines
+	log_handle.write('Unique cell lines:\t{}:\n'.format(len(set(inst_info_ctl['cell_iname']))))
+	for c in list(set(inst_info_ctl['cell_iname'])):
+		log_handle.write('\t{}\n'.format(c))
+
+	log_handle.write('\n\n')
+
+	#Stats on dosages
+	df_ctl = pd.DataFrame(inst_info_ctl[['cmap_name', 'cell_iname', 'pert_idose']].groupby(['cmap_name', 'cell_iname']).apply(lambda x: x['pert_idose'].unique()))
+	df_ctl = pd.DataFrame([(i, len(df_ctl.loc[i][0])) for i in df_ctl.index], columns =['cmap_name-cell_line', 'n_doses'])
+	log_handle.write('Stats on dosages and timepoints\n')
+	log_handle.write('\n------\nHOW MANY DOSES ARE THERE FOR CMAP_NAME-CELL LINE PAIRS?\n------\n')
+	for index,value in pd.Series.iteritems(pd.DataFrame(df_ctl['n_doses'])['n_doses'].describe()):
+	    log_handle.write('{}:\t{}\n'.format(index, value))
+
+	log_handle.write('\n')
+	log_handle.write('Number of pairs with more than 1 dose:\t{}/{}\n'.format(sum(df_ctl['n_doses']>1), len(df_ctl)))
+	log_handle.write('Number of pairs with more than 2 doses:\t{}/{}\n\n'.format(sum(df_ctl['n_doses']>2), len(df_ctl)))
+
+	#Stats on timepoints
+	df_ctl = pd.DataFrame(inst_info_ctl[['cmap_name', 'cell_iname', 'pert_time']].groupby(['cmap_name', 'cell_iname']).apply(lambda x: x['pert_time'].unique()))
+	df_ctl = pd.DataFrame([(i, len(df_ctl.loc[i][0])) for i in df_ctl.index], columns =['cmap_name-cell_line', 'n_times'])
+
+	log_handle.write('\n------\nHOW MANY TIMEPOINTS ARE THERE FOR CMAP_NAME-CELL LINE PAIRS?\n------\n')
+	for index,value in pd.Series.iteritems(pd.DataFrame(df_ctl['n_times'])['n_times'].describe()):
+	    log_handle.write('{}:\t{}\n'.format(index, value))
+
+	log_handle.write('\n')
+	log_handle.write('Number of pairs with more than 1 timepoint:\t{}/{}\n'.format(sum(df_ctl['n_times']>1), len(df_ctl['n_times'])))
+	log_handle.write('Number of pairs with more than 2 timepoints:\t{}/{}\n\n'.format(sum(df_ctl['n_times']>2), len(df_ctl['n_times'])))
+
+
+	log_handle.write('\nUSING THEM ALL FOR NOW\n')
+
+	#Types of vectors
+	log_handle.write('Number of vectors:\t{}:\n'.format(len(set(inst_info_ctl['cmap_name']))))
+	df=pd.DataFrame.from_dict(Counter(inst_info_ctl['cmap_name']), orient='index')
+	df = df.sort_values(by=0)
+	for i, v in enumerate(zip(df.index, df[0])):
+		log_handle.write('{}:\t{}\n'.format(v[0], v[1]))
+
+	#Number of controls per cell line
+	replicates = inst_info_ctl.groupby(['cell_iname']).size()
+	log_handle.write('\n\n------\nNUMBER OF REPLICATES PER CELL LINE (different doses, times, vectors)\n-----------\n')
+	df=pd.DataFrame.from_dict(Counter(inst_info_ctl['cell_iname']), orient='index')
+	df = df.sort_values(by=0)
+	for i, v in enumerate(zip(df.index, df[0])):
+		log_handle.write('{}:\t{}\n'.format(v[0], v[1]))
+
+	return
+
+
+def filter_cell_lines(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only cell lines with highest mumber of perturbed genes\n------\n')
+
+	#####CRISPR
+	#Obtain cell lines with the most perturbations (> 90th-percentile)
+	df_xpr = pd.DataFrame(inst_info_xpr[['cmap_name', 'cell_iname']].groupby('cell_iname', as_index=True).apply(lambda x: x['cmap_name'].unique()))
+	df_xpr = pd.DataFrame([(i, len(df_xpr.loc[i][0])) for i in df_xpr.index], columns =['cell_line', 'n_cmap_names'])
+	df_xpr = df_xpr.sort_values(by='n_cmap_names')
+	keep_cell_lines = df_xpr[df_xpr['n_cmap_names']>np.percentile(df_xpr['n_cmap_names'], 60)]['cell_line'].tolist()
+
+	#Find indices of samples that are on the desired cell lines
+	keep_index = []
+	for i in range(len(inst_info_xpr)):
+		if inst_info_xpr.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+
+
+	inst_info_xpr = inst_info_xpr.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_xpr['sample_id'])	#obtain sample ID from metadata
+	matrix_xpr = matrix_xpr[list_ids]	#Filtered data matrix
+	log_handle.write('CRISPR:\t{} datapoints\n'.format(matrix_xpr.shape[1]))
+
+
+
+	#####CONTROL
+	keep_index = []
+	for i in range(len(inst_info_ctl)):
+		if inst_info_ctl.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+
+
+	inst_info_ctl = inst_info_ctl.loc[keep_index].reset_index(inplace=False, drop=True)  #filter from metadata
+	list_ids = list(inst_info_ctl['sample_id'])	#obtain sample ID from metadata
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n'.format(matrix_ctl.shape[1]))
+
+	# Stats
+	# stats_control(inst_info_ctl, log_handle)
+
+	return inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, keep_cell_lines
+
+
+def filter_cell_lines_custom(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only cell lines: A549, PC3, MCF7\n------\n')
+	#####CRISPR
+	#Obtain cell lines with the most perturbations (> 4K genes perturbed)
+	keep_cell_lines = ['A549', 'PC3', 'MCF7', 'A375', 'HT29', 'ES2', 'BICR6', 'YAPC', 'AGS', 'U251MG']
+	#Find indices of samples that are on the desired cell lines
+	keep_index = []
+	for i in range(len(inst_info_xpr)):
+		if inst_info_xpr.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+	inst_info_xpr = inst_info_xpr.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_xpr['sample_id'])	#obtain sample ID from metadata
+	matrix_xpr = matrix_xpr[list_ids]	#Filtered data matrix
+	log_handle.write('CRISPR:\t{} datapoints\n'.format(matrix_xpr.shape[1]))
+	#####CONTROL
+	keep_index = []
+	for i in range(len(inst_info_ctl)):
+		if inst_info_ctl.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+	inst_info_ctl = inst_info_ctl.loc[keep_index].reset_index(inplace=False, drop=True)  #filter from metadata
+	list_ids = list(inst_info_ctl['sample_id'])	#obtain sample ID from metadata
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n'.format(matrix_ctl.shape[1]))
+	# Stats
+	#stats_control(inst_info_ctl, log_handle)
+	return inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, keep_cell_lines
+
+################
+#3. Concatenate perturbation and control data, normalize (binarize), and save
+def filter_samples_with_unknown_perturbed_genes(inst_info_xpr, matrix_xpr, gene_info, log_handle):
+	########################################################################################
+	#First filter perturbation samples to remove those with genes not mapping to genes_info
+	#Will need to remove this once I get the mapping file from CLUE
+	known_genes = list(set(gene_info['gene_symbol']))
+	keep_index = []
+	for i in range(len(inst_info_xpr)):
+		if inst_info_xpr.at[i, 'cmap_name'] in known_genes:
+			keep_index.append(i)
+
+
+	inst_info_xpr = inst_info_xpr.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_xpr['sample_id'])	#obtain sample ID from metadata
+	matrix_xpr = matrix_xpr[list_ids]	#Filtered data matrix
+	log_handle.write('Filtering samples with perturbed genes not mapped to gene_info -- TEMPORARY STEP THAT SHOULD BE REMOVED AFTER MAPPING GENES\n')
+	log_handle.write('CRISPR:\t{} datapoints\n'.format(matrix_xpr.shape[1]))
+	return inst_info_xpr, matrix_xpr
+
+
+def binarize_genewise_ranking_all(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE RANKING ALL SAMPLES\n------\n')
+	outdir = osp.join(outdir, 'binarize_genewise_ranking_all')
+	os.makedirs(outdir, exist_ok= True)
+	########################################################################################
+	#All data
+	metadata = pd.concat([inst_info_xpr, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	matrix = pd.concat([matrix_xpr, matrix_ctl], 1)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+
+
+
+	#Normalization 
+	#0. create dictionary of gene_id: sample_id
+		#gene symbol -> gene id
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+		#gene symbol -> sample id
+	dict_gene_id_sample_id= dict()
+	for i in range(len(inst_info_xpr)):
+		gene_symbol = inst_info_xpr.at[i, 'cmap_name']
+		gene_id = dict_symbol_id[gene_symbol]
+		if gene_id in dict_gene_id_sample_id:
+			dict_gene_id_sample_id[gene_id].append(inst_info_xpr.at[i, 'sample_id'])
+		else:
+			dict_gene_id_sample_id[gene_id] = [inst_info_xpr.at[i, 'sample_id']]
+
+
+	#1. Iterate through each gene, mask out the samples in which it was perturbed, and normalize --> take top 2% of samples as = 1
+	#for the perturbed genes = 0
+	#Create matrix of NGenes x NExperiments (add column name as sample_id)
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+	i = 1
+	higher_than_thr = []
+	higher_than_thr_ids = []
+	for gene_id in list(matrix.index):
+		print('{}/{}'.format(i, len(matrix)))
+		#mask of elements to ignore (perturbed samples --> =0)
+		if gene_id in dict_gene_id_sample_id:
+			mask_pert = dict_gene_id_sample_id[gene_id]
+			mask_norm = list(set(list(matrix.columns)) - set(mask_pert))
+		else:
+			mask_pert = None
+			mask_norm = list(matrix.columns)
+		#normalize
+		threshold = np.percentile(matrix.loc[gene_id][mask_norm].values, 98)
+		matrix_binarized.loc[gene_id][mask_norm] = (matrix.loc[gene_id][mask_norm] >= threshold).astype(int).values
+		i+=1
+		#some stats
+		if mask_pert is not None:
+			higher_than_thr += (matrix.loc[gene_id][mask_pert] >= threshold).values.astype(int).tolist()
+			higher_than_thr_ids += matrix.loc[gene_id][mask_pert].index[np.where(matrix.loc[gene_id][mask_pert] >= threshold)].tolist()
+
+	###Filter columns (samples) in which the perturbed gene has an expression value that is >= the threshold used to binarize
+	log_handle.write('Filtering:\t{} columns/samples because the perturbed gene has an expression value >= the threshold used to binarize -- TEMPORARY STEP THAT SHOULD BE REMOVED LATER ON\n'.format(len(higher_than_thr_ids)))
+	for c in higher_than_thr_ids:
+		del matrix_binarized[c]
+
+
+	#plot higher than thr
+	fig, ax1 = plt.subplots(1, figsize=(16,6))
+	ax1.hist(higher_than_thr)
+	ax1.set_title('Perturbed genes values >= threshold')
+	fig.savefig(osp.join(outdir,'exploration_ge_crispr_higher_than_thr.png'))
+	log_handle.write('After binarizing, stats, perturbed gene values would be higher than threshold in :\t{} samples\n'.format(np.sum(higher_than_thr)))
+
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_xpr', 'ctl_vector']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			data_i = matrix_binarized[metadata_i.index]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+def binarize_genewise_comparing_to_control_all_controls_joint(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log):
+
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_all_controls_joint_lognorm')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_all_controls_joint')
+	os.makedirs(outdir, exist_ok= True)
+
+	########################################################################################
+	#All data
+	metadata = pd.concat([inst_info_xpr, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	matrix = pd.concat([matrix_xpr, matrix_ctl], 1)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+
+
+
+	#Normalization 
+	#0. create dictionary of gene_id: sample_id
+		#gene symbol -> gene id
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+		#gene id -> sample id
+	dict_gene_id_sample_id= dict()
+	for i in range(len(inst_info_xpr)):
+		gene_symbol = inst_info_xpr.at[i, 'cmap_name']
+		gene_id = dict_symbol_id[gene_symbol]
+		if gene_id in dict_gene_id_sample_id:
+			dict_gene_id_sample_id[gene_id].append(inst_info_xpr.at[i, 'sample_id'])
+		else:
+			dict_gene_id_sample_id[gene_id] = [inst_info_xpr.at[i, 'sample_id']]
+
+
+	#1. Iterate through each gene, mask out the samples in which it was perturbed, and normalize --> set to 1 those genes that have value of average + 2std higher than control
+	#for the perturbed genes = 0
+	#Create matrix of NGenes x NExperiments (add column name as sample_id)
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+	i = 1
+	higher_than_thr = []
+	higher_than_thr_ids = []
+
+
+	mask_norm = list(matrix_ctl.columns) #mask_norm is controls only
+
+	# averages = np.mean(matrix[mask_norm], 1)
+	# stds = np.std(matrix[mask_norm], 1)
+	# thresholds = averages + (2*stds)
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	averages = np.mean(matrix[mask_norm], 1)
+	stds = np.std(matrix[mask_norm], 1)
+	thresholds = averages + (2*stds)
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram.png'))
+	plt.close()
+
+
+	for gene_id in list(matrix.index):
+		print('{}/{}'.format(i, len(matrix)))
+		#mask of elements to use for normalization: the ones in control samples only
+		#mask of elements to binarize to 0 (samples in which gene_id is perturbed)
+		if gene_id in dict_gene_id_sample_id:
+			mask_pert = dict_gene_id_sample_id[gene_id]		
+		else:
+			mask_pert = None
+		#normalize
+		threshold = thresholds.loc[gene_id]
+		matrix_binarized.loc[gene_id] = (matrix.loc[gene_id] >= threshold).astype(int).values
+		matrix_binarized.loc[gene_id][mask_pert] = 0
+		i+=1
+		#some stats
+		if mask_pert is not None:
+			gte = matrix.loc[gene_id][mask_pert] >= threshold
+			higher_than_thr += gte.values.astype(int).tolist()
+			higher_than_thr_ids += matrix.loc[gene_id][mask_pert].index[np.where(gte)].tolist()
+
+
+
+
+	###Filter columns (samples) in which the perturbed gene has an expression value that is >= the threshold used to binarize
+	# log_handle.write('Filtering:\t{} columns/samples because the perturbed gene has an expression value >= the threshold used to binarize -- TEMPORARY STEP THAT SHOULD BE REMOVED LATER ON\n'.format(len(higher_than_thr_ids)))
+
+	# print('Filtering columns from data...')
+	# keep_columns = list(set(matrix_binarized.columns) - set(higher_than_thr_ids))
+	# matrix_binarized = matrix_binarized[keep_columns]
+
+
+
+	#plot higher than thr
+	fig, ax1 = plt.subplots(1, figsize=(16,6))
+	ax1.hist(higher_than_thr)
+	ax1.set_title('Perturbed genes values >= threshold')
+	fig.savefig(osp.join(outdir,'exploration_ge_crispr_higher_than_thr.png'))
+	log_handle.write('After binarizing, stats, perturbed gene values would be higher than threshold in :\t{} samples\n'.format(np.sum(higher_than_thr)))
+
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_xpr', 'ctl_vector']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			data_i = matrix_binarized[metadata_i.index]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+def binarize_genewise_comparing_to_control(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control')
+	os.makedirs(outdir, exist_ok= True)
+
+	########################################################################################
+	#All data
+	metadata = pd.concat([inst_info_xpr, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+	matrix = pd.concat([matrix_xpr, matrix_ctl], 1)
+
+
+	#0. create dictionary of gene_id: sample_id
+		#gene symbol -> gene id
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+		#gene id -> sample id
+	dict_gene_id_sample_id= dict()
+	for i in range(len(inst_info_xpr)):
+		gene_symbol = inst_info_xpr.at[i, 'cmap_name']
+		gene_id = dict_symbol_id[gene_symbol]
+		if gene_id in dict_gene_id_sample_id:
+			dict_gene_id_sample_id[gene_id].append(inst_info_xpr.at[i, 'sample_id'])
+		else:
+			dict_gene_id_sample_id[gene_id] = [inst_info_xpr.at[i, 'sample_id']]
+
+
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+
+	i = 1
+	higher_than_thr = []
+	higher_than_thr_ids = []
+	control_corrected = []
+	pert_corrected = []
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram.png'))
+	plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		matrix_i = matrix[metadata[metadata['cell_iname']==cell_line]['sample_id']]
+		#Normalization 
+		#1. Iterate through each gene, mask out the samples in which it was perturbed, and normalize --> set to 1 those genes that have value of average + 2std higher than control
+		#for the perturbed genes = 0
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		mask_norm = list(set(matrix_ctl.columns).intersection(set(matrix_i.columns))) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+		for gene_id in list(matrix_i.index):
+			#mask of elements to use for normalization: the ones in control samples only
+			#mask of elements to binarize to 0 (samples in which gene_id is perturbed)
+			if gene_id in dict_gene_id_sample_id:
+				mask_pert = list(set(dict_gene_id_sample_id[gene_id]).intersection(set(matrix_i.columns)))	#mask_pert for specific cell line 'cell_line'
+				pert_corrected += mask_pert
+			else:
+				mask_pert = []
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+			matrix_binarized.loc[gene_id][mask_pert] = 0
+			#some stats
+			if mask_pert != []:
+				gte = matrix_i.loc[gene_id][mask_pert] >= threshold
+				higher_than_thr += gte.values.astype(int).tolist()
+				higher_than_thr_ids += matrix_i.loc[gene_id][mask_pert].index[np.where(gte)].tolist()
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+	print('Controls covered:{}/{}\n'.format(len(control_corrected), len(inst_info_ctl)))
+	print('Perturbed covered:{}/{}\n'.format(len(pert_corrected), len(inst_info_xpr)))
+
+	###Filter columns (samples) in which the perturbed gene has an expression value that is >= the threshold used to binarize
+	# log_handle.write('Filtering:\t{} columns/samples because the perturbed gene has an expression value >= the threshold used to binarize -- TEMPORARY STEP THAT SHOULD BE REMOVED LATER ON\n'.format(len(higher_than_thr_ids)))
+
+	# print('Filtering columns from data...')
+	# keep_columns = list(set(matrix_binarized.columns) - set(higher_than_thr_ids))
+	# matrix_binarized = matrix_binarized[keep_columns]
+
+
+
+	#plot higher than thr
+	fig, ax1 = plt.subplots(1, figsize=(16,6))
+	ax1.hist(higher_than_thr)
+	ax1.set_title('Perturbed genes values >= threshold')
+	fig.savefig(osp.join(outdir,'exploration_ge_crispr_higher_than_thr.png'))
+	log_handle.write('After binarizing, stats, perturbed gene values would be higher than threshold in :\t{} samples\n'.format(np.sum(higher_than_thr)))
+
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_xpr', 'ctl_vector']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			data_i = matrix_binarized[metadata_i.index]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+
+
+def binarize_genewise_comparing_to_control_augmented(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm/augmented')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control/augmented')
+	os.makedirs(outdir, exist_ok= True)
+
+	########################################################################################
+	#All data
+	metadata = pd.concat([inst_info_xpr, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+	matrix = pd.concat([matrix_xpr, matrix_ctl], 1)
+
+
+	#0. create dictionary of gene_id: sample_id
+		#gene symbol -> gene id
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+		#gene id -> sample id
+	dict_gene_id_sample_id= dict()
+	for i in range(len(inst_info_xpr)):
+		gene_symbol = inst_info_xpr.at[i, 'cmap_name']
+		gene_id = dict_symbol_id[gene_symbol]
+		if gene_id in dict_gene_id_sample_id:
+			dict_gene_id_sample_id[gene_id].append(inst_info_xpr.at[i, 'sample_id'])
+		else:
+			dict_gene_id_sample_id[gene_id] = [inst_info_xpr.at[i, 'sample_id']]
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	matrix_augmented = matrix.copy()
+	###Data augmentation using Gaussian noise
+	AUG_PROPORTION = 5
+	columns = matrix.columns
+	for i in range(AUG_PROPORTION):
+		columns_i = [e+'___{}'.format(i) for e in columns]
+		noise = np.random.normal(0,1,matrix.shape)
+		to_add = pd.DataFrame(matrix.values + noise, columns = columns_i, index = matrix.index)
+		matrix_augmented = pd.concat([matrix_augmented, to_add], 1)
+
+
+	matrix = matrix_augmented
+
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+
+	i = 1
+	higher_than_thr = []
+	higher_than_thr_ids = []
+	control_corrected = []
+	pert_corrected = []
+
+
+
+	#hist of values
+	# mv = matrix.values.flatten()
+	# sampling = sample(range(len(mv)), int(0.005*len(mv)))
+	# mv = mv[sampling]
+
+	# fig, ax = plt.subplots(figsize=(16,6))
+	# ax.hist(mv)
+	# ax.set_title('Histogram of values')
+	# fig.savefig(osp.join(outdir,'histogram.png'))
+	# plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		columns = metadata[metadata['cell_iname']==cell_line]['sample_id'].tolist()
+		columns_augmented = [e+'___{}'.format(i) for i in range(AUG_PROPORTION) for e in columns] + columns
+		columns = columns_augmented
+		matrix_i = matrix[columns]
+		#Normalization 
+		#1. Iterate through each gene, mask out the samples in which it was perturbed, and normalize --> set to 1 those genes that have value of average + 2std higher than control
+		#for the perturbed genes = 0
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		columns_control = list(matrix_ctl.columns)
+		columns_control = [e+'___{}'.format(i) for i in range(AUG_PROPORTION) for e in columns_control] + columns_control
+		columns_i = list(matrix_i.columns)
+		mask_norm = list(set(columns_control).intersection(set(columns_i))) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+
+		for gene_id in list(matrix_i.index):
+			#mask of elements to use for normalization: the ones in control samples only
+			#mask of elements to binarize to 0 (samples in which gene_id is perturbed)
+			if gene_id in dict_gene_id_sample_id:
+				samples_perturbed = dict_gene_id_sample_id[gene_id]
+				samples_perturbed = [e+'___{}'.format(i) for i in range(AUG_PROPORTION) for e in samples_perturbed] + samples_perturbed
+				mask_pert = list(set(samples_perturbed).intersection(set(matrix_i.columns)))	#mask_pert for specific cell line 'cell_line'
+				pert_corrected += mask_pert
+			else:
+				mask_pert = []
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+			matrix_binarized.loc[gene_id][mask_pert] = 0
+			#some stats
+			if mask_pert != []:
+				gte = matrix_i.loc[gene_id][mask_pert] >= threshold
+				higher_than_thr += gte.values.astype(int).tolist()
+				higher_than_thr_ids += matrix_i.loc[gene_id][mask_pert].index[np.where(gte)].tolist()
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+	# print('Controls covered:{}/{}\n'.format(len(control_corrected), len(inst_info_ctl)))
+	# print('Perturbed covered:{}/{}\n'.format(len(pert_corrected), len(inst_info_xpr)))
+
+	###Filter columns (samples) in which the perturbed gene has an expression value that is >= the threshold used to binarize
+	# log_handle.write('Filtering:\t{} columns/samples because the perturbed gene has an expression value >= the threshold used to binarize -- TEMPORARY STEP THAT SHOULD BE REMOVED LATER ON\n'.format(len(higher_than_thr_ids)))
+
+	# print('Filtering columns from data...')
+	# keep_columns = list(set(matrix_binarized.columns) - set(higher_than_thr_ids))
+	# matrix_binarized = matrix_binarized[keep_columns]
+
+
+
+	#plot higher than thr
+	fig, ax1 = plt.subplots(1, figsize=(16,6))
+	ax1.hist(higher_than_thr)
+	ax1.set_title('Perturbed genes values >= threshold')
+	fig.savefig(osp.join(outdir,'exploration_ge_crispr_higher_than_thr.png'))
+	log_handle.write('After binarizing, stats, perturbed gene values would be higher than threshold in :\t{} samples\n'.format(np.sum(higher_than_thr)))
+
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tAUGMENTED SIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	# metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_xpr', 'ctl_vector']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			columns = list(metadata_i.index)
+			to_add = []
+			for i in range(AUG_PROPORTION):
+				to_add += [e+'___{}'.format(i) for e in columns]
+			columns = columns + to_add
+			data_i = matrix_binarized[columns]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), data_i.shape[1], len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+
+def normalize_and_save(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nNORMALIZE DATA AND SAVE\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'real_lognorm')
+	else:
+		outdir = osp.join(outdir, 'real')
+	os.makedirs(outdir, exist_ok= True)
+	
+
+	#All data
+	metadata = pd.concat([inst_info_xpr, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+	matrix = pd.concat([matrix_xpr, matrix_ctl], 1)
+	del(matrix_xpr) 
+
+	#hist of values before lognorm
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.001*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_raw.png'))
+	plt.close()
+
+	#NORMALIZATION
+	#lognorm + minmax
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	scaler = MinMaxScaler((0,1))
+	matrix = matrix.transpose()
+	matrix = pd.DataFrame(scaler.fit_transform(matrix), columns = matrix.columns, index = matrix.index)
+	matrix = matrix.transpose()
+
+
+
+	#hist of values after lognorm
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.001*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_lognorm.png'))
+	plt.close()
+
+	#2. Save data and metadata for each condition and cell line
+	#Compound + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_xpr', 'ctl_vector']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			data_i = matrix[metadata_i.index]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['pert_id'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+
+def main():  
+
+	DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+	log_handle = open(osp.join(outdir, 'process_data_lognorm.txt'), 'w')
+	inst_info_xpr, inst_info_ctl, gene_info, matrix_xpr, matrix_ctl = loads_data(DATA_ROOT, log_handle)
+	inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl = filter_data_metadata(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, log_handle)
+	inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, keep_cell_lines = filter_cell_lines_custom(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, log_handle)
+	# inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, keep_cell_lines = filter_cell_lines(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, log_handle)
+	inst_info_xpr, matrix_xpr = filter_samples_with_unknown_perturbed_genes(inst_info_xpr, matrix_xpr, gene_info, log_handle)
+	use_log=True
+	normalize_and_save(inst_info_xpr, matrix_xpr, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log)
+	log_handle.close()
+
+
+if __name__ == "__main__":
+    main()
+
+

PDGrapher/data/scripts/lincs/process_data_chemical_1.py

@@ -0,0 +1,249 @@
+'''
+Find drug targets in DrugBank, maps drugs to drugs in LINCS
+'''
+
+
+import pandas as pd
+from rdkit import Chem
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+
+import os
+import csv
+import gzip
+import collections
+import re
+import io
+import json
+import os.path as osp
+import xml.etree.ElementTree as ET
+
+import requests
+from bs4 import BeautifulSoup
+import pickle
+
+outdir = '../../processed/lincs/chemical'
+os.makedirs(outdir, exist_ok=True)
+outdir_df = '../../processed/lincs/chemical/dataframes/'
+os.makedirs(outdir_df, exist_ok=True)
+
+
+def load_data(DATA_ROOT,log_handle):
+
+	
+	###Load LINCS Data
+	df_lincs=pd.read_csv(os.path.join(DATA_ROOT, 'instinfo_beta.txt'), sep="\t", low_memory=False)
+	df_cpmeta=pd.read_csv(os.path.join(DATA_ROOT, 'compoundinfo_beta.txt'), sep="\t", low_memory=False) 
+
+	df_trtcp=df_lincs.loc[np.logical_and(df_lincs['pert_type'] == 'trt_cp',  df_lincs['failure_mode'].isna())]
+	unique_cp=df_trtcp.pert_id.unique()
+
+	log_handle.write('LINCS Compound Data\n------\n')
+	log_handle.write('Compound treatments in inst_info:\t{}\n'.format(df_trtcp.shape[0]))
+	log_handle.write('Unique pert_ids:\t{}\n'.format(df_cpmeta.pert_id.unique().shape[0]))
+	log_handle.write('Unique inchi_keys:\t{}\n'.format(df_cpmeta.inchi_key.unique().shape[0]))
+
+	###Load DrugBank Data
+	DATAROOT= '../../raw/drugbank/2022-11-DrugBank/data/'
+	xml_path = os.path.join(DATAROOT, 'all-full-database.xml')
+	with open(xml_path) as xml_file:
+		tree = ET.parse(xml_file)
+	root = tree.getroot()
+
+
+	ns = '{http://www.drugbank.ca}'
+	inchikey_template = "{ns}calculated-properties/{ns}property[{ns}kind='InChIKey']/{ns}value"
+	inchi_template = "{ns}calculated-properties/{ns}property[{ns}kind='InChI']/{ns}value"
+	SMILES_template = "{ns}calculated-properties/{ns}property[{ns}kind='SMILES']/{ns}value"
+
+
+	rows = list()
+	
+
+
+	for i, drug in enumerate(root):
+		row = collections.OrderedDict()
+		assert drug.tag == ns + 'drug'
+		row['type'] = drug.get('type')
+		row['drugbank_id'] = drug.findtext(ns + "drugbank-id[@primary='true']")
+		row['name'] = drug.findtext(ns + "name")
+		row['description'] = drug.findtext(ns + "description")
+		row['groups'] = [group.text for group in
+	    drug.findall("{ns}groups/{ns}group".format(ns = ns))]
+		row['atc_codes'] = [code.get('code') for code in
+	    drug.findall("{ns}atc-codes/{ns}atc-code".format(ns = ns))]
+		
+		row['categories'] = [x.findtext(ns + 'category') for x in
+	    drug.findall("{ns}categories/{ns}category".format(ns = ns))]
+
+		row['inchi'] = drug.findtext(inchi_template.format(ns = ns))
+		row['inchi_key'] = drug.findtext(inchikey_template.format(ns = ns))
+		row['SMILES']=drug.findtext(SMILES_template.format(ns=ns))
+	
+	# Add drug aliases
+		aliases = {
+			elem.text for elem in 
+			drug.findall("{ns}international-brands/{ns}international-brand".format(ns = ns)) +
+			drug.findall("{ns}synonyms/{ns}synonym[@language='English']".format(ns = ns)) +
+			drug.findall("{ns}international-brands/{ns}international-brand".format(ns = ns)) +
+			drug.findall("{ns}products/{ns}product/{ns}name".format(ns = ns))
+
+		}
+		aliases.add(row['name'])
+		row['aliases'] = sorted(aliases)
+
+		rows.append(row)
+		
+	columns = ['drugbank_id', 'name', 'type', 'groups', 'atc_codes', 'categories', 'inchi_key', 'inchi','SMILES', 'description']
+	drugbank_df = pd.DataFrame.from_dict(rows)[columns]
+	drugbank_slim_df = drugbank_df[
+		drugbank_df.inchi.map(lambda x: x is not None) &
+		drugbank_df.SMILES.map(lambda x: x is not None) 
+	]
+	return drugbank_slim_df, unique_cp, df_cpmeta
+
+
+
+
+
+
+### Matching pert_ids to DrugBankIDs
+
+def pert_id2inchikey(pertid, df): 
+	"""Returns InChIKey of the corresponding pert_id"""
+	
+	
+	pertid_index=df.index[df['pert_id']==pertid][0]
+	return (pertid_index, df.at[pertid_index,'inchi_key'])
+
+	
+def df_pert_id2inchikey(pert_idarr, df_cpmeta):
+	""" Returns a DataFrame with corresponding InChIKeys of each pert_id in pert_idarr"""
+
+	d = {'pert_id': [], 'compoundinfo_index': [], 'inchi_key': []}
+	for i in range(len(pert_idarr)): 
+		d['pert_id'].append(pert_idarr[i])
+		index, inchikey = pert_id2inchikey(pert_idarr[i], df_cpmeta)
+		d['compoundinfo_index'].append(index)
+		d['inchi_key'].append(inchikey)
+	inchikey_df=pd.DataFrame(data=d)
+	return inchikey_df
+
+
+def df_DrugBankCol_inchi(df,drugbank_slim_df):
+	"""Adds a column to df with InChIKeys mapped to DrugBank IDs from drugbank_canSmiles_df"""
+	in_drugbank = set(list(drugbank_slim_df['inchi_key']))
+	assert 'inchi_key' in df.columns.values
+	for i in range(df.shape[0]):
+		inchikey = df['inchi_key'][i]
+		if type(inchikey) ==float:
+			df.at[i,"DrugBank_ID"] = "None"
+		else: 
+			if inchikey not in in_drugbank: 
+				df.at[i,"DrugBank_ID"] = "Not in DrugBank"
+			else:
+				ik_index=drugbank_slim_df.index[drugbank_slim_df['inchi_key']==inchikey][0]
+				x= (ik_index, drugbank_slim_df.at[ik_index,'drugbank_id'])
+				df.at[i,"DrugBank_ID"] = x[1]
+	return df
+
+
+
+def createMappedDF(df):
+	"""Returns a copy of the df that were mapped to DrugBankIDs"""
+	df_new = df.copy(deep=True)
+	df_new=df_new.loc[(df_new['DrugBank_ID']!='Not in DrugBank')&(df_new['DrugBank_ID']!= "None")]
+	df_new.reset_index(drop=True, inplace=True)
+	return df_new
+
+
+
+
+
+### Load DrugBank targets
+def load_targets_drugbank(path):
+	return pd.read_csv(path)
+
+
+def summarize_drugbank_targets(mapped_DrugBankDF, df_targets):
+	mapped_DrugBankDF_new = mapped_DrugBankDF.copy(deep=True)
+	mapped_DrugBankDF_new['targets'] = ''
+	for i in range(len(mapped_DrugBankDF)):
+		dbid = mapped_DrugBankDF['DrugBank_ID'].tolist()[i]
+		targets = df_targets[df_targets['DrugBank_ID']==dbid]['idd'].tolist()
+		mapped_DrugBankDF_new.at[i, 'targets'] = targets
+	return mapped_DrugBankDF_new
+
+
+
+
+def countTargets(df): 
+	#Counts number of DrugBank Targets and adds column called "num_targets"
+	df['num_targets']=0
+	for i in range(df.shape[0]):
+		if df.notna().at[i,'targets']:
+			str_list=df.at[i,'targets']
+			df.at[i,'num_targets']=len(df.at[i,'targets'])
+	return df
+	
+
+
+def target_stats(df_targets, log_handle): 
+	#Calculates statistics on number of DrugBank targets and plots distribution    
+	log_handle.write('DrugBank Target Stats\n------\n')
+	for index,value in pd.Series.iteritems(pd.DataFrame(df_targets['num_targets']).describe()):
+		log_handle.write('{}:\t{}\n'.format(index, value))
+	x = list(df_targets['num_targets'])
+	fig, ax1 = plt.subplots()
+	ax1.hist(np.clip(x,0,30), bins=60)
+	ax1.set_xlabel("# of Targets")
+	ax1.set_ylabel("# of Compounds")
+	ax1.set_title('Number of DrugBank Targets')
+	fig.savefig(osp.join(outdir,'num_target_distribution.png'))
+
+
+
+def main():
+	DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+	log_handle = open(osp.join(outdir, 'log_process_data_chemical_1.txt'), 'w') 
+	drugbank_slim_df, unique_cp, df_cpmeta = load_data(DATA_ROOT, log_handle)
+
+	#Creating DataFrame mapping pert_id to InChIKeys from compoundinfo
+	pert_id_inchikeyDF=df_pert_id2inchikey(unique_cp, df_cpmeta)
+
+
+	#Adding the column with corresponding DrugBank IDs
+	pert_id_DrugBankDF=df_DrugBankCol_inchi(pert_id_inchikeyDF,drugbank_slim_df)
+	mapped_DrugBankDF=createMappedDF(pert_id_DrugBankDF)
+
+	log_handle.write('Fraction of compounds found in DrugBank:\t{}/{}\n'.format(mapped_DrugBankDF.shape[0],pert_id_DrugBankDF.shape[0]))
+
+
+	#Loads targets from DrugBank
+
+	df_targets = load_targets_drugbank('../../processed/drugbank/targets.txt')
+	df_targets.columns = ['DrugBank_ID', 'DrugBank_name', 'synonyms', 'idd', 'name', 'gene_name', 'gene_synonyms', 'identifiers', 'organism']
+
+	#Summarizes DrugBank targets in mapped_DrugBankDF
+
+	mapped_DrugBankDF = summarize_drugbank_targets(mapped_DrugBankDF, df_targets)
+	mapped_DrugBankDF=countTargets(mapped_DrugBankDF) 
+
+
+	target_stats(mapped_DrugBankDF, log_handle)
+	no_targets=mapped_DrugBankDF.loc[mapped_DrugBankDF['num_targets']==0].shape[0]
+	log_handle.write('Fraction of compounds without DrugBank targets:\t{}/{}\n'.format(no_targets,mapped_DrugBankDF.shape[0]))
+
+
+	with open(osp.join(outdir_df,"df_targets.pickle"), 'wb') as f:
+		pickle.dump(mapped_DrugBankDF, f)
+	mapped_DrugBankDF.to_csv(osp.join(outdir_df,"df_targets.csv"))
+
+
+
+if __name__ == "__main__":
+	main()
+
+
+

PDGrapher/data/scripts/lincs/process_data_chemical_2.py

@@ -0,0 +1,697 @@
+''' 
+Normalizes perturbed (treated) and control (diseased) data of chemical interventions
+Saves data as npz
+
+1. Reads data (load_data)
+2. Filters to keep only the cell lines that we use in experiments (filter_cell_lines_custom)
+3. Filters to keep only the samples in metadata (filter_data_metadata)
+4. Maps drug gene targets to LINCS genes (map_gene_targets_to_lincs)
+5. Filters out samples for which the drug targets are not in LINCS genes (filter_samples_with_unknown_perturbed_genes)
+6. Normalizes data between (0,1) and save (normalize_and_save)
+'''
+import pandas as pd
+import h5py
+import os
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os.path as osp
+import numpy as np
+from collections import Counter
+import matplotlib.pyplot as plt
+from sklearn.preprocessing import MinMaxScaler
+import math
+from random import sample
+
+
+import networkx as nx
+import csrgraph as cg
+import os
+import itertools
+import json
+import operator
+import swifter
+from tqdm import tqdm
+import time
+import pickle
+
+
+	#LOG
+	
+
+################
+# Data loading
+################
+
+def get_gene_names(x, dict_id_names):
+	# Returns a list of gene names for each protein ID (node ID of protein). 
+	# If there is more than one gene name for a given ID, a list of all of them are given.
+	gene_names = []
+	for gene_id in x:
+		gname = dict_id_names[gene_id]
+		if gname ==[]:
+			continue
+		else:
+			gene_names.append(dict_id_names[gene_id])
+	return gene_names
+
+
+
+def targetList(pid,dict_pid_target_names):
+	if pid not in dict_pid_target_names:
+		return []  
+	else:
+		return dict_pid_target_names[pid]
+		
+
+
+
+def loads_data(DATA_ROOT, log_handle):
+	
+	#Loads metadata
+	inst_info = pd.read_csv(os.path.join(DATA_ROOT, 'instinfo_beta.txt'), sep="\t", low_memory=False)
+	inst_info_cp = inst_info[np.logical_and(inst_info['pert_type'] == 'trt_cp',  inst_info['failure_mode'].isna())].reset_index(inplace=False, drop=True)
+	inst_info_ctl = inst_info[np.logical_and(inst_info['pert_type'] == 'ctl_vehicle', inst_info['failure_mode'].isna()) ].reset_index(inplace=False, drop=True)
+	gene_info = pd.read_csv(os.path.join(DATA_ROOT, 'geneinfo_beta.txt'), sep="\t", low_memory=False)
+	
+	with open('../../processed/lincs/chemical/dataframes/df_targets.pickle', 'rb') as f:
+		df_targets = pickle.load(f)
+	
+
+	
+	drugbank_targets=pd.read_csv("../../processed/drugbank/targets.txt")
+	
+	dict_id_names = dict()
+	for i in range(len(drugbank_targets)):
+		name = drugbank_targets.at[i, 'gene_name']
+		if name != '-':
+			dict_id_names[drugbank_targets.at[i,'target_id']] = [drugbank_targets.at[i,'gene_name']]
+		else:
+			dict_id_names[drugbank_targets.at[i,'target_id']] = []
+		synonyms = drugbank_targets.at[i, 'gene_synonyms']
+		if synonyms != '-' and str(synonyms) != 'nan':
+			dict_id_names[drugbank_targets.at[i,'target_id']] += synonyms.split('||')
+
+	
+	df_targets['target_names']=df_targets['targets'].apply(lambda x: get_gene_names(x, dict_id_names))
+	dict_pid_target_names = dict(zip(df_targets['pert_id'], df_targets['target_names']))
+	inst_info_cp['drugbank_target_names']=inst_info_cp['pert_id'].apply(lambda x: targetList(x,dict_pid_target_names))
+
+	
+	####################
+	#Loads data matrices
+	### Compound
+	f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_trt_cp_n1805898x12328.gctx'), 'r')
+	matrix_cp = f['0']['DATA']['0']['matrix'][:].transpose()
+	gene_ids_cp = f['0']['META']['ROW']['id'][:]
+	sample_ids_cp = f['0']['META']['COL']['id'][:]
+	matrix_cp = pd.DataFrame(matrix_cp, columns = sample_ids_cp.astype(str), index = gene_ids_cp.astype(int))
+	del f
+
+	
+	#re-order gene_info based on the order in gene_ids_cp (rows of data)
+	gene_info.index = gene_info['gene_id']
+	gene_info = gene_info.loc[gene_ids_cp.astype(int)].reset_index(inplace=False, drop=True)
+	gene_info.to_csv(osp.join(outdir, 'gene_info.txt'), index=False)
+
+	#Stats
+	log_handle.write('Compounds\n------\n')
+	log_handle.write('Compound entries in inst_info metadata:\t{}\n'.format(len(inst_info_cp)))
+	log_handle.write('Compound entries in data matrix:\t{}\n'.format(len(sample_ids_cp)))
+	log_handle.write('Overlap between inst_info metadata and sample ids in data matrix:\t{}\n'.format(len(set(inst_info_cp['sample_id']).intersection(set(sample_ids_cp.astype(str))))))
+
+
+
+	### Control data
+	f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_ctl_n188708x12328.gctx'), 'r')
+	matrix_ctl = f['0']['DATA']['0']['matrix'][:].transpose()
+	gene_ids_ctl = f['0']['META']['ROW']['id'][:]					#not in the same order as gene_ids_cp
+	sample_ids_ctl = f['0']['META']['COL']['id'][:]
+	matrix_ctl = pd.DataFrame(matrix_ctl, columns = sample_ids_ctl.astype(str), index = gene_ids_ctl.astype(int))
+	del f
+
+
+	#Stats
+	log_handle.write('CONTROL\n------\n')
+	log_handle.write('Control entries in inst_info metadata:\t{}\n'.format(len(inst_info_ctl)))
+	log_handle.write('Control entries in data matrix:\t{}\n'.format(len(sample_ids_ctl)))
+	log_handle.write('Overlap between inst_info metadata and sample ids in data matrix:\t{}\n'.format(len(set(inst_info_ctl['sample_id']).intersection(set(sample_ids_ctl.astype(str))))))
+	log_handle.write('\n------\n')
+
+
+	# stats_data(inst_info_cp, matrix_cp, matrix_ctl, gene_info, df_targets)
+
+	return sample_ids_cp, inst_info_cp, inst_info_ctl, gene_info, matrix_cp, matrix_ctl, df_targets
+
+
+
+
+
+def stats_data(inst_info_cp, matrix_cp, matrix_ctl, gene_info, df_targets):
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	dict_pert_id = dict(zip(df_targets['pert_id'], df_targets['target_names']))
+
+		####Data exploration -- GE values of genes that are perturbed (!)
+	#Get GE value for each gene perturbed by Compound
+	values_pert = {}
+	values_control = {}
+	for i in range(len(inst_info_cp)):
+		pert_id = inst_info_cp.at[i, 'pert_id']
+		if pert_id in dict_pert_id:
+			gene_symbols=dict_pert_id[pert_id]
+			if type(gene_symbols)==list:
+				for j in range(len(gene_symbols)): 
+					
+					if gene_symbols[j] in dict_symbol_id:					#if the cmap_name of gene is in the gene_info
+						sample_id = inst_info_cp.at[i, 'sample_id']
+						gene_id = dict_symbol_id[gene_symbols[j]]
+						if gene_id in values_pert:
+							values_pert[gene_id].append(matrix_cp.at[gene_id, sample_id])
+						else:
+							values_pert[gene_id] = [matrix_cp.at[gene_id, sample_id]]        
+				
+
+
+	for pert_id in list(set(inst_info_cp['pert_id'])):
+		if pert_id in dict_pert_id:
+			gene_symbols=dict_pert_id[pert_id]
+			if type(gene_symbols)==list:
+				for z in range(len(gene_symbols)):
+		
+				
+				
+					if gene_symbols[z] in dict_symbol_id:				#if the cmap_name of gene is in the gene_info
+						gene_id = dict_symbol_id[gene_symbols[z]]
+						values_control[gene_id] = [matrix_ctl.loc[gene_id]]
+	
+	for key in values_pert:
+		values_pert[key] = np.mean(values_pert[key])
+
+	for key in values_control:
+		values_control[key] = np.mean(values_control[key])
+	
+	fig, (ax1, ax2) = plt.subplots(2, figsize=(16,6))
+	ax1.hist(values_pert.values())
+	ax2.hist(values_control.values())
+	ax1.set_title('Values of perturbed genes (avg) - Compounds')
+	ax2.set_title('Values of genes in control (avg)')
+	fig.savefig(osp.join(outdir,'exploration_ge_compounds.png'))
+
+
+	return
+
+
+
+
+################
+# Processing
+################
+
+
+################
+#1. Filter to keep only cell lines with more perturbations
+
+def stats_control(inst_info_ctl, log_handle):
+	log_handle.write('STATS CONTROL DATA\n***********************************\n')
+	#Stats unique cell lines
+	log_handle.write('Unique cell lines:\t{}:\n'.format(len(set(inst_info_ctl['cell_iname']))))
+	for c in list(set(inst_info_ctl['cell_iname'])):
+		log_handle.write('\t{}\n'.format(c))
+
+	log_handle.write('\n\n')
+
+	#Stats on dosages
+	df_ctl = pd.DataFrame(inst_info_ctl[['cmap_name', 'cell_iname', 'pert_idose']].groupby(['cmap_name', 'cell_iname']).apply(lambda x: x['pert_idose'].unique()))
+	df_ctl = pd.DataFrame([(i, len(df_ctl.loc[i][0])) for i in df_ctl.index], columns =['cmap_name-cell_line', 'n_doses'])
+	log_handle.write('Stats on dosages and timepoints\n')
+	log_handle.write('\n------\nHOW MANY DOSES ARE THERE FOR CMAP_NAME-CELL LINE PAIRS?\n------\n')
+	for index,value in pd.Series.iteritems(pd.DataFrame(df_ctl['n_doses'])['n_doses'].describe()):
+		log_handle.write('{}:\t{}\n'.format(index, value))
+
+	log_handle.write('\n')
+	log_handle.write('Number of pairs with more than 1 dose:\t{}/{}\n'.format(sum(df_ctl['n_doses']>1), len(df_ctl)))
+	log_handle.write('Number of pairs with more than 2 doses:\t{}/{}\n\n'.format(sum(df_ctl['n_doses']>2), len(df_ctl)))
+
+	#Stats on timepoints
+	df_ctl = pd.DataFrame(inst_info_ctl[['cmap_name', 'cell_iname', 'pert_time']].groupby(['cmap_name', 'cell_iname']).apply(lambda x: x['pert_time'].unique()))
+	df_ctl = pd.DataFrame([(i, len(df_ctl.loc[i][0])) for i in df_ctl.index], columns =['cmap_name-cell_line', 'n_times'])
+
+	log_handle.write('\n------\nHOW MANY TIMEPOINTS ARE THERE FOR CMAP_NAME-CELL LINE PAIRS?\n------\n')
+	for index,value in pd.Series.iteritems(pd.DataFrame(df_ctl['n_times'])['n_times'].describe()):
+		log_handle.write('{}:\t{}\n'.format(index, value))
+
+	log_handle.write('\n')
+	log_handle.write('Number of pairs with more than 1 timepoint:\t{}/{}\n'.format(sum(df_ctl['n_times']>1), len(df_ctl['n_times'])))
+	log_handle.write('Number of pairs with more than 2 timepoints:\t{}/{}\n\n'.format(sum(df_ctl['n_times']>2), len(df_ctl['n_times'])))
+
+
+	log_handle.write('\nUSING THEM ALL FOR NOW\n')
+
+	#Types of vectors
+	log_handle.write('Number of vectors:\t{}:\n'.format(len(set(inst_info_ctl['cmap_name']))))
+	df=pd.DataFrame.from_dict(Counter(inst_info_ctl['cmap_name']), orient='index')
+	df = df.sort_values(by=0)
+	for i, v in enumerate(zip(df.index, df[0])):
+		log_handle.write('{}:\t{}\n'.format(v[0], v[1]))
+
+	#Number of controls per cell line
+	replicates = inst_info_ctl.groupby(['cell_iname']).size()
+	log_handle.write('\n\n------\nNUMBER OF REPLICATES PER CELL LINE (different doses, times, vectors)\n-----------\n')
+	df=pd.DataFrame.from_dict(Counter(inst_info_ctl['cell_iname']), orient='index')
+	df = df.sort_values(by=0)
+	for i, v in enumerate(zip(df.index, df[0])):
+		log_handle.write('{}:\t{}\n'.format(v[0], v[1]))
+
+	log_handle.write('\n***********************************\n')
+
+	return
+
+
+def filter_cell_lines(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only cell lines with highest mumber of perturbed genes\n------\n')
+
+	#####Compound
+	#Obtain cell lines with the most perturbations (> 4K genes perturbed)
+	df_cp = pd.DataFrame(inst_info_cp[['cmap_name', 'cell_iname']].groupby('cell_iname', as_index=True).apply(lambda x: x['cmap_name'].unique()))
+	df_cp = pd.DataFrame([(i, len(df_cp.loc[i][0])) for i in df_cp.index], columns =['cell_line', 'n_cmap_names'])
+	df_cp = df_cp.sort_values(by='n_cmap_names')
+
+	keep_cell_lines = df_cp[df_cp['n_cmap_names']>np.percentile(df_cp['n_cmap_names'], 90)]['cell_line'].tolist()
+
+	#Find indices of samples that are on the desired cell lines
+	keep_index = []
+	for i in range(len(inst_info_cp)):
+		if inst_info_cp.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+
+
+	inst_info_cp = inst_info_cp.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_cp['sample_id'])	#obtain sample ID from metadata
+	matrix_cp = matrix_cp[list_ids]	#Filtered data matrix
+	log_handle.write('Compounds:\t{} datapoints\n'.format(matrix_cp.shape[1]))
+
+
+
+	#####CONTROL
+	keep_index = []
+	for i in range(len(inst_info_ctl)):
+		if inst_info_ctl.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+
+
+	inst_info_ctl = inst_info_ctl.loc[keep_index].reset_index(inplace=False, drop=True)  #filter from metadata
+	list_ids = list(inst_info_ctl['sample_id'])	#obtain sample ID from metadata
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n'.format(matrix_ctl.shape[1]))
+
+	# Stats
+	#stats_control(inst_info_ctl, log_handle)
+
+	return inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, keep_cell_lines
+
+
+
+
+def filter_cell_lines_custom(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep custom list of cell lines\n------\n')
+
+	#####Compound
+	keep_cell_lines = ['A549', 'PC3', 'MCF7', 'VCAP', 'MDAMB231', 'BT20', 'HA1E', 'HT29', 'A375', 'HELA', 'YAPC']
+
+	#Find indices of samples that are on the desired cell lines
+	keep_index = []
+	for i in range(len(inst_info_cp)):
+		if inst_info_cp.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+
+
+	inst_info_cp = inst_info_cp.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_cp['sample_id'])	#obtain sample ID from metadata
+	matrix_cp = matrix_cp[list_ids]	#Filtered data matrix
+	log_handle.write('Compounds:\t{} datapoints\n'.format(matrix_cp.shape[1]))
+
+
+
+	#####CONTROL
+	keep_index = []
+	for i in range(len(inst_info_ctl)):
+		if inst_info_ctl.at[i, 'cell_iname'] in keep_cell_lines:
+			keep_index.append(i)
+
+
+	inst_info_ctl = inst_info_ctl.loc[keep_index].reset_index(inplace=False, drop=True)  #filter from metadata
+	list_ids = list(inst_info_ctl['sample_id'])	#obtain sample ID from metadata
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n'.format(matrix_ctl.shape[1]))
+
+	# Stats
+	#stats_control(inst_info_ctl, log_handle)
+
+	return inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, keep_cell_lines
+
+
+def maxSample(w,x,y,z, max_list):
+	if (w,x,y,z) in max_list: 
+		return 1
+	else: 
+		return 0
+
+
+def filter_dosage_timepoints(sample_ids_cp, inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only samples with the highest dosage and longest timepoint per drug-cell line combination\n------\n')
+	#####Compounds   
+
+	df_cp=pd.DataFrame(inst_info_cp[['sample_id','pert_id', 'pert_dose','pert_dose_unit','pert_time','cell_iname']])
+	df_cp_max=df_cp.sort_values(by=['pert_dose','pert_time']).drop_duplicates(["pert_id","cell_iname"],keep="last")
+	max_list=list(zip(df_cp_max['pert_dose'], df_cp_max['pert_time'],df_cp_max['pert_id'],df_cp_max['cell_iname']))
+
+	#Decoding sample_ids_cp array
+	decode=np.vectorize(np.char.decode)
+	sample_ids_cp_dec=decode(sample_ids_cp)
+	tqdm.pandas()
+	df_cp['max'] = df_cp.swifter.apply(lambda row : maxSample(row['pert_dose'],row['pert_time'], row['pert_id'],row['cell_iname'], max_list), axis = 1)
+	df_cp_filtered=df_cp.loc[df_cp['max']== 1]
+	df_cp_filtered.to_csv("../../processed/chemical/dataframes/df_cp_filtered.csv")
+
+	sid_index=np.intersect1d(sample_ids_cp_dec,df_cp_filtered.sample_id.to_numpy(), return_indices=True)[1]
+	f1 = operator.itemgetter(*sid_index)
+	sample_ids=f1(sample_ids_cp_dec)
+	keep_index=inst_info_cp[inst_info_cp['sample_id'].isin(sample_ids)].index.tolist()
+
+	inst_info_cp = inst_info_cp.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_cp['sample_id'])	#obtain sample ID from metadata
+	matrix_cp = matrix_cp[list_ids]	#Filtered data matrix
+	log_handle.write('Compounds:\t{} datapoints\n'.format(matrix_cp.shape[1]))
+	return inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl
+
+
+
+
+################
+#2. Filter column metadata and data matrix to keep only those in metadata
+
+def filter_data_metadata(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only those in metadata\n------\n')
+	#Compund
+	list_ids = list(inst_info_cp['sample_id'])	#in metadata
+	matrix_cp = matrix_cp[list_ids]	#Filtered data matrix
+	log_handle.write('Compounds:\t{} datapoints\n'.format(matrix_cp.shape[1]))
+	#CONTROL
+	list_ids = list(inst_info_ctl['sample_id'])	#in metadata
+	#extra steps
+	#--
+	list_ids = list(set(list_ids).intersection(set(matrix_ctl.columns.astype(str))))	#in metadata and in data matrix (some of metadata are not in data matrix)
+	inst_info_ctl.index = inst_info_ctl['sample_id']; inst_info_ctl = inst_info_ctl.loc[list_ids].reset_index(inplace=False, drop=True) #remove entries from metadata that are not in data matrix
+	#--
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n\n\n'.format(matrix_ctl.shape[1]))
+	return inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl
+
+
+
+
+################
+#3. Map drug target names to symbols from LINCS
+def map_gene_targets_to_lincs(inst_info_cp, gene_info):
+	genes_in_lincs = set(gene_info.gene_symbol)
+	inst_info_cp['target_names'] = ''
+	for i in range(len(inst_info_cp)):
+		drugbank_target_names = inst_info_cp.at[i, 'drugbank_target_names']
+		if drugbank_target_names ==[]:
+			continue
+		else:
+			target_names = []
+			for target_list in drugbank_target_names:
+				target_name = list(set(target_list).intersection(genes_in_lincs))
+				if len(target_name) == 1:
+					target_names.append(target_name[0])
+			inst_info_cp.at[i, 'target_names'] = target_names
+
+	return inst_info_cp
+
+
+################
+#3. Filter samples with unknown perturbed genes (keeping those with at least 1 known perturbed gene)
+def filter_samples_with_unknown_perturbed_genes(inst_info_cp, matrix_cp, gene_info, log_handle):
+	########################################################################################
+	#First filter perturbation samples to remove those with genes not mapping to genes_info
+	keep_index = []
+	for i in range(len(inst_info_cp)):
+		if len(inst_info_cp.at[i, 'target_names']) > 0:
+			keep_index.append(i)
+
+	inst_info_cp = inst_info_cp.loc[keep_index].reset_index(inplace=False, drop=True) #filter from metadata
+	list_ids = list(inst_info_cp['sample_id'])	#obtain sample ID from metadata
+	matrix_cp = matrix_cp[list_ids]	#Filtered data matrix
+	log_handle.write('Filtering samples without protein targets in drugbank and lincs\n')
+	log_handle.write('Compounds:\t{} datapoints\n'.format(matrix_cp.shape[1]))
+	return inst_info_cp, matrix_cp
+
+
+
+
+
+################
+#4. Cretes a dictionary of gene_symbol:sample_ids
+def genesymb2sampleiddict(inst_info_cp):
+	#Returns a dictionary of gene_symbol: [sampleids]
+	genesymb_list=list(inst_info_cp.target_names)
+	genesymb_list=list(itertools.chain(*genesymb_list))
+	genesymb_list_unique=list(set(genesymb_list))
+			
+	l=[ [] for _ in range(len(genesymb_list_unique)) ]
+	d=dict(zip(genesymb_list_unique,l))
+	
+	for sample in range(len(inst_info_cp)):
+		sid=inst_info_cp.at[sample,'sample_id']
+		for target in inst_info_cp.at[sample,'target_names']:
+			if target in d:
+				d[target].append(sid)
+	return d
+
+
+
+def binarize_genewise_comparing_to_control(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control')
+	os.makedirs(outdir, exist_ok= True)
+	
+
+	#All data
+	metadata = pd.concat([inst_info_cp, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+	matrix = pd.concat([matrix_cp, matrix_ctl], 1)
+	del(matrix_cp) 
+
+	#Normalization 
+	#0. create dictionary of gene_id: sample_id
+		#gene symbol -> gene id
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+		#gene symbol -> sample id
+	#Passing in dictionary from genesymb2sampleiddict
+	d=genesymb2sampleiddict(inst_info_cp)
+	#Switching keys to be gene_ids instead of gene_symbols
+	dict_gene_id_sample_id= dict((dict_symbol_id[key],value) for (key,value) in d.items())
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+	del(d)
+	del(dict_symbol_id)
+	del(gene_info)    
+	i = 1
+	higher_than_thr = []
+	higher_than_thr_ids = []
+	control_corrected = []
+	pert_corrected = []
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+	
+	
+	#hist of values
+	# mv_shape=matrix.values.shape
+	# num_elements = mv_shape[0]*mv_shape[1]
+	# chosenCols=np.random.randint(0, mv_shape[1], size=int(0.001*num_elements))
+	# chosenRows=np.random.randint(0, mv_shape[0], size=int(0.001*num_elements))
+	# filter_ind=np.array(np.array(list(zip(chosenRows,chosenCols))))
+	# mv=list(matrix.values[filter_ind[:,0],filter_ind[:,1]])
+	# fig, ax = plt.subplots(figsize=(16,6))
+	# ax.hist(mv)
+	# ax.set_title('Histogram of values')
+	# fig.savefig(osp.join(outdir,'histogram.png'))
+	# plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		matrix_i = matrix[metadata[metadata['cell_iname']==cell_line]['sample_id']]
+		#Normalization 
+		#1. Iterate through each gene, mask out the samples in which it was perturbed, and normalize --> set to 1 those genes that have value of average + 2std higher than control
+		#for the perturbed genes = 0
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		mask_norm = list(set(matrix_ctl.columns).intersection(set(matrix_i.columns))) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+		for gene_id in list(matrix_i.index):
+			#mask of elements to use for normalization: the ones in control samples only
+			#mask of elements to binarize to 0 (samples in which gene_id is perturbed)
+			if gene_id in dict_gene_id_sample_id:
+				mask_pert = list(set(dict_gene_id_sample_id[gene_id]).intersection(set(matrix_i.columns)))	#mask_pert for specific cell line 'cell_line'
+				pert_corrected += mask_pert
+			else:
+				mask_pert = []
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+			# matrix_binarized.loc[gene_id][mask_pert] = 0  #not setting to zero here -- it's chemical perturbations not KO
+			#some stats
+			if mask_pert != []:
+				gte = matrix_i.loc[gene_id][mask_pert] >= threshold
+				higher_than_thr += gte.values.astype(int).tolist()
+				higher_than_thr_ids += matrix_i.loc[gene_id][mask_pert].index[np.where(gte)].tolist()
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+	print('Controls covered:{}/{}\n'.format(len(control_corrected), len(inst_info_ctl)))
+	print('Perturbed covered:{}/{}\n'.format(len(set(pert_corrected)), len(inst_info_cp)))
+
+
+
+	#plot higher than thr
+	fig, ax1 = plt.subplots(1, figsize=(16,6))
+	ax1.hist(higher_than_thr)
+	ax1.set_title('Perturbed genes values >= threshold')
+	fig.savefig(osp.join(outdir,'exploration_ge_compounds_higher_than_thr.png'))
+	log_handle.write('After binarizing, stats, perturbed gene values would be higher than threshold in :\t{} samples\n'.format(np.sum(higher_than_thr)))
+
+
+	#2. Save data and metadata for each condition and cell line
+	#Compound + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_cp', 'ctl_vehicle']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			data_i = matrix_binarized[metadata_i.index]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['pert_id'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+
+
+
+def normalize_and_save(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nNORMALIZE DATA AND SAVE\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'real_lognorm')
+	else:
+		outdir = osp.join(outdir, 'real')
+	os.makedirs(outdir, exist_ok= True)
+	
+
+	#All data
+	metadata = pd.concat([inst_info_cp, inst_info_ctl], axis=0).reset_index(inplace=False, drop=True)
+	metadata.to_csv(osp.join(outdir, 'all_metadata.txt'))
+	matrix = pd.concat([matrix_cp, matrix_ctl], 1)
+	del(matrix_cp) 
+
+	#hist of values before lognorm
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.001*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_raw.png'))
+	plt.close()
+
+	#NORMALIZATION
+	#lognorm + minmax
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	scaler = MinMaxScaler((0,1))
+	matrix = matrix.transpose()
+	matrix = pd.DataFrame(scaler.fit_transform(matrix), columns = matrix.columns, index = matrix.index)
+	matrix = matrix.transpose()
+
+
+
+	#hist of values after lognorm
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.001*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_lognorm.png'))
+	plt.close()
+
+	#2. Save data and metadata for each condition and cell line
+	#Compound + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line in keep_cell_lines:
+		for pert_type in ['trt_cp', 'ctl_vehicle']:
+			metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+			data_i = matrix[metadata_i.index]
+			metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+			filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+			np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+			log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['pert_id'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+outdir = '../../processed/lincs/chemical/nofilter_dose_timepoint'
+os.makedirs(outdir, exist_ok=True)
+
+
+def main():  
+
+	DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+	log_handle = open(osp.join(outdir, 'log_process_data_real_lognorm.txt'), 'w')
+
+	sample_ids_cp, inst_info_cp, inst_info_ctl, gene_info, matrix_cp, matrix_ctl, df_targets = loads_data(DATA_ROOT, log_handle)
+	print('loaded data - lognorm', inst_info_cp.shape)
+
+	# inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, keep_cell_lines = filter_cell_lines(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle)
+	inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, keep_cell_lines = filter_cell_lines_custom(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle)
+	print('filtered cell lines - lognorm', inst_info_cp.shape)
+
+
+	inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl = filter_data_metadata(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, log_handle)
+	print('filtered data - lognorm', inst_info_cp.shape)
+
+	inst_info_cp = map_gene_targets_to_lincs(inst_info_cp, gene_info)
+	inst_info_cp, matrix_cp = filter_samples_with_unknown_perturbed_genes(inst_info_cp, matrix_cp, gene_info, log_handle)
+	print('filtered unknown perturbed genes - lognorm', inst_info_cp.shape)
+	
+	use_log=True
+	normalize_and_save(inst_info_cp, matrix_cp, inst_info_ctl, matrix_ctl, gene_info, keep_cell_lines, log_handle, outdir, use_log)
+	print('real values - lognorm')
+	log_handle.close()
+
+
+if __name__ == "__main__":
+	startTime=time.time()
+	main()
+	print(time.time()-startTime)
+

PDGrapher/data/scripts/lincs/process_data_healthy.py

@@ -0,0 +1,362 @@
+'''
+Process LINCS data of healhty cell lines
+MCF10A, NL20, RWPE1
+Will do some processing first and then rely on the functions in process_data.py
+'''
+
+import pandas as pd
+import h5py
+import os
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os.path as osp
+import numpy as np
+from collections import Counter
+import matplotlib.pyplot as plt
+import math
+from random import sample
+from sklearn.preprocessing import MinMaxScaler
+
+outdir = '../../processed/lincs'
+os.makedirs(outdir, exist_ok=True)
+DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+
+
+################
+# Data loading
+################
+
+#function updated from the one in process_data to load the cell lines of interest
+def loads_data(DATA_ROOT, log_handle):
+	healhty_cell_lines = ['MCF10A', 'NL20', 'RWPE1']
+
+	#Loads metadata
+	inst_info = pd.read_csv(os.path.join(DATA_ROOT, 'instinfo_beta.txt'), sep="\t", low_memory=False)
+
+	inst_info_ctl_mcf10a = inst_info[np.logical_and(inst_info['cell_iname'] == 'MCF10A',np.logical_and(inst_info['pert_type'] == 'ctl_untrt', inst_info['failure_mode'].isna())) ].reset_index(inplace=False, drop=True)
+	inst_info_ctl_nl20 = inst_info[np.logical_and(inst_info['cell_iname'] == 'NL20',np.logical_and(inst_info['pert_type'] == 'ctl_vehicle', inst_info['failure_mode'].isna())) ].reset_index(inplace=False, drop=True)
+	inst_info_ctl_rwpe1 = inst_info[np.logical_and(inst_info['cell_iname'] == 'RWPE1',np.logical_and(inst_info['pert_type'] == 'ctl_vector', inst_info['failure_mode'].isna())) ].reset_index(inplace=False, drop=True)
+
+
+	inst_info_ctl = pd.concat([inst_info_ctl_mcf10a, inst_info_ctl_nl20, inst_info_ctl_rwpe1])
+
+	gene_info = pd.read_csv(os.path.join(DATA_ROOT, 'geneinfo_beta.txt'), sep="\t", low_memory=False)
+
+
+	####################
+	#Loads data matrices
+
+
+	### Control data -- filter to keep only those in my metadata
+	f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_ctl_n188708x12328.gctx'), 'r')
+	matrix_ctl = f['0']['DATA']['0']['matrix'][:].transpose()
+	gene_ids_ctl = f['0']['META']['ROW']['id'][:]					#not in the same order as gene_ids_xpr
+	sample_ids_ctl = f['0']['META']['COL']['id'][:]
+	matrix_ctl = pd.DataFrame(matrix_ctl, columns = sample_ids_ctl.astype(str), index = gene_ids_ctl.astype(int))
+
+	del f
+
+
+	#Stats
+	log_handle.write('CONTROL\n------\n')
+	log_handle.write('Control entries in inst_info metadata:\t{}\n'.format(len(inst_info_ctl)))
+	log_handle.write('Control entries in data matrix:\t{}\n'.format(len(sample_ids_ctl)))
+	log_handle.write('Overlap between inst_info metadata and sample ids in data matrix:\t{}\n'.format(len(set(inst_info_ctl['sample_id']).intersection(set(sample_ids_ctl.astype(str))))))
+	log_handle.write('\n------\n')
+
+	return inst_info_ctl, gene_info, matrix_ctl
+
+
+
+
+################
+# Processing
+################
+
+################
+#1. Filter column metadata and data matrix to keep only those in metadata
+
+def filter_data_metadata(inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only those in metadata\n------\n')
+	#CONTROL
+	list_ids = list(inst_info_ctl['sample_id'])	#in metadata
+	#extra steps
+	#--
+	list_ids = list(set(list_ids).intersection(set(matrix_ctl.columns.astype(str))))	#in metadata and in data matrix (some of metadata are not in data matrix)
+	inst_info_ctl.index = inst_info_ctl['sample_id']; inst_info_ctl = inst_info_ctl.loc[list_ids].reset_index(inplace=False, drop=True) #remove entries from metadata that are not in data matrix
+	#--
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n\n\n'.format(matrix_ctl.shape[1]))
+	return inst_info_ctl, matrix_ctl
+
+
+################
+#3. Normalize (binarize), and save
+
+
+def binarize_genewise_comparing_to_control(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control')
+	os.makedirs(outdir, exist_ok= True)
+
+
+
+
+	########################################################################################
+	#All data
+	metadata = inst_info_ctl
+	metadata.to_csv(osp.join(outdir, 'all_metadata_healthy.txt'))
+	matrix = matrix_ctl
+
+
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+
+	i = 1
+
+	control_corrected = []
+
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_healthy.png'))
+	plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		matrix_i = matrix[metadata[metadata['cell_iname']==cell_line]['sample_id']]
+		#Normalization 
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		mask_norm = list(set(matrix_ctl.columns).intersection(set(matrix_i.columns))) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+		for gene_id in list(matrix_i.index):
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+			
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+
+
+
+	print('Controls covered:{}/{}\n'.format(len(control_corrected), len(inst_info_ctl)))
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line, pert_type in zip(['MCF10A', 'NL20', 'RWPE1'],['ctl_untrt', 'ctl_vehicle', 'ctl_vector'] ):
+		metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+		data_i = matrix_binarized[metadata_i.index]
+		metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+		filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+		np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+		log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+def binarize_genewise_comparing_to_control_augmented(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm/augmented')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control/augmented')
+	os.makedirs(outdir, exist_ok= True)
+
+
+
+
+	########################################################################################
+	#All data
+	metadata = inst_info_ctl
+	metadata.to_csv(osp.join(outdir, 'all_metadata_healthy.txt'))
+	matrix = matrix_ctl
+	
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+
+	matrix_augmented = matrix.copy()
+	###Data augmentation using Gaussian noise
+	AUG_PROPORTION = 10
+	columns = matrix.columns
+	for i in range(AUG_PROPORTION):
+		columns_i = [e+'___{}'.format(i) for e in columns]
+		noise = np.random.normal(0,1,matrix.shape)
+		to_add = pd.DataFrame(matrix.values + noise, columns = columns_i, index = matrix.index)
+		matrix_augmented = pd.concat([matrix_augmented, to_add], 1)
+
+
+	matrix = matrix_augmented
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+
+	i = 1
+
+	control_corrected = []
+
+
+
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_healthy.png'))
+	plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		columns = metadata[metadata['cell_iname']==cell_line]['sample_id'].tolist()
+		columns_augmented = [e+'___{}'.format(i) for i in range(AUG_PROPORTION) for e in columns] + columns
+		columns = columns_augmented
+		matrix_i = matrix[columns]
+		#Normalization 
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		#Binarization
+		mask_norm = list(set(matrix_i.columns)) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+		for gene_id in list(matrix_i.index):
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+
+
+
+	print('Controls covered:{}/{}\n'.format(len(control_corrected), matrix.shape[1]))
+
+	#2. Save data and metadata for each condition and cell line
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tAUGMENTED SIZE\t\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	# metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line, pert_type in zip(['MCF10A', 'NL20', 'RWPE1'],['ctl_untrt', 'ctl_vehicle', 'ctl_vector'] ):
+		metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+		columns = list(metadata_i.index)
+		to_add = []
+		for i in range(AUG_PROPORTION):
+			to_add += [e+'___{}'.format(i) for e in columns]
+		columns = columns + to_add
+		data_i = matrix_binarized[columns]
+		metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+		filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+		np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+		log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), data_i.shape[1], len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+def normalize_and_save(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'real_lognorm')
+	else:
+		outdir = osp.join(outdir, 'real')
+	os.makedirs(outdir, exist_ok= True)
+
+
+	########################################################################################
+	#All data
+	metadata = inst_info_ctl
+	metadata.to_csv(osp.join(outdir, 'all_metadata_healthy.txt'))
+	matrix = matrix_ctl
+
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+	scaler = MinMaxScaler((0,1))
+	matrix = matrix.transpose()
+	matrix = pd.DataFrame(scaler.fit_transform(matrix), columns = matrix.columns, index = matrix.index)
+	matrix = matrix.transpose()
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_healthy.png'))
+	plt.close()
+
+
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line, pert_type in zip(['MCF10A', 'NL20', 'RWPE1'],['ctl_untrt', 'ctl_vehicle', 'ctl_vector'] ):
+		metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+		data_i = matrix[metadata_i.index]
+		metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+		filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+		np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+		log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+
+
+def main():  
+	from process_data import stats_control
+
+
+	DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+	log_handle = open(osp.join(outdir, 'process_data_healthy_lognorm.txt'), 'w')
+	inst_info_ctl, gene_info, matrix_ctl = loads_data(DATA_ROOT, log_handle)
+	inst_info_ctl, matrix_ctl = filter_data_metadata(inst_info_ctl, matrix_ctl, log_handle)
+	#stats_control(inst_info_ctl, log_handle)
+	use_log=True
+	normalize_and_save(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log)
+	log_handle.close()
+
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

PDGrapher/data/scripts/lincs/process_data_healthy_chemical.py

@@ -0,0 +1,364 @@
+'''
+Process LINCS data of healhty cell lines
+
+-- MCF10A, NL20, RWPE1
+
+Will do some processing first and then rely on the functions in process_data.py
+######ONLY DIFFERENCE WITH CRISPR FILE IS THAT I USE CTL_VEHICLE FOR MCF10A INSTEAD OF CTL_UNTRT
+'''
+
+import pandas as pd
+import h5py
+import os
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os.path as osp
+import numpy as np
+from collections import Counter
+import matplotlib.pyplot as plt
+import math
+from random import sample
+from sklearn.preprocessing import MinMaxScaler
+
+outdir = '../../processed/lincs/chemical'
+os.makedirs(outdir, exist_ok=True)
+DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+
+
+################
+# Data loading
+################
+
+#function updated from the one in process_data to load the cell lines of interest
+def loads_data(DATA_ROOT, log_handle):
+
+	#Loads metadata
+	inst_info = pd.read_csv(os.path.join(DATA_ROOT, 'instinfo_beta.txt'), sep="\t", low_memory=False)
+
+
+	inst_info_ctl_mcf10a = inst_info[np.logical_and(inst_info['cell_iname'] == 'MCF10A',np.logical_and(inst_info['pert_type'] == 'ctl_vehicle', inst_info['failure_mode'].isna())) ].reset_index(inplace=False, drop=True)
+	inst_info_ctl_nl20 = inst_info[np.logical_and(inst_info['cell_iname'] == 'NL20',np.logical_and(inst_info['pert_type'] == 'ctl_vehicle', inst_info['failure_mode'].isna())) ].reset_index(inplace=False, drop=True)
+	inst_info_ctl_rwpe1 = inst_info[np.logical_and(inst_info['cell_iname'] == 'RWPE1',np.logical_and(inst_info['pert_type'] == 'ctl_vector', inst_info['failure_mode'].isna())) ].reset_index(inplace=False, drop=True)
+
+
+	inst_info_ctl = pd.concat([inst_info_ctl_mcf10a, inst_info_ctl_nl20, inst_info_ctl_rwpe1])
+
+	gene_info = pd.read_csv(os.path.join(DATA_ROOT, 'geneinfo_beta.txt'), sep="\t", low_memory=False)
+
+
+	####################
+	#Loads data matrices
+
+
+	### Control data -- filter to keep only those in my metadata
+	f = h5py.File(os.path.join(DATA_ROOT, 'level3_beta_ctl_n188708x12328.gctx'), 'r')
+	matrix_ctl = f['0']['DATA']['0']['matrix'][:].transpose()
+	gene_ids_ctl = f['0']['META']['ROW']['id'][:]					#not in the same order as gene_ids_xpr
+	sample_ids_ctl = f['0']['META']['COL']['id'][:]
+	matrix_ctl = pd.DataFrame(matrix_ctl, columns = sample_ids_ctl.astype(str), index = gene_ids_ctl.astype(int))
+
+	del f
+
+
+	#Stats
+	log_handle.write('CONTROL\n------\n')
+	log_handle.write('Control entries in inst_info metadata:\t{}\n'.format(len(inst_info_ctl)))
+	log_handle.write('Control entries in data matrix:\t{}\n'.format(len(sample_ids_ctl)))
+	log_handle.write('Overlap between inst_info metadata and sample ids in data matrix:\t{}\n'.format(len(set(inst_info_ctl['sample_id']).intersection(set(sample_ids_ctl.astype(str))))))
+	log_handle.write('\n------\n')
+
+	return inst_info_ctl, gene_info, matrix_ctl
+
+
+
+
+################
+# Processing
+################
+
+################
+#1. Filter column metadata and data matrix to keep only those in metadata
+
+def filter_data_metadata(inst_info_ctl, matrix_ctl, log_handle):
+	log_handle.write('Filtering to keep only those in metadata\n------\n')
+	#CONTROL
+	list_ids = list(inst_info_ctl['sample_id'])	#in metadata
+	#extra steps
+	#--
+	list_ids = list(set(list_ids).intersection(set(matrix_ctl.columns.astype(str))))	#in metadata and in data matrix (some of metadata are not in data matrix)
+	inst_info_ctl.index = inst_info_ctl['sample_id']; inst_info_ctl = inst_info_ctl.loc[list_ids].reset_index(inplace=False, drop=True) #remove entries from metadata that are not in data matrix
+	#--
+	matrix_ctl = matrix_ctl[list_ids]	#Filtered data matrix
+	log_handle.write('CONTROL:\t{} datapoints\n\n\n'.format(matrix_ctl.shape[1]))
+	return inst_info_ctl, matrix_ctl
+
+
+################
+#3. Normalize (binarize), and save
+
+
+def binarize_genewise_comparing_to_control(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control')
+	os.makedirs(outdir, exist_ok= True)
+
+
+
+
+	########################################################################################
+	#All data
+	metadata = inst_info_ctl
+	metadata.to_csv(osp.join(outdir, 'all_metadata_healthy.txt'))
+	matrix = matrix_ctl
+
+
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+
+	i = 1
+
+	control_corrected = []
+
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_healthy.png'))
+	plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		matrix_i = matrix[metadata[metadata['cell_iname']==cell_line]['sample_id']]
+		#Normalization 
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		mask_norm = list(set(matrix_ctl.columns).intersection(set(matrix_i.columns))) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+		for gene_id in list(matrix_i.index):
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+			
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+
+
+
+	print('Controls covered:{}/{}\n'.format(len(control_corrected), len(inst_info_ctl)))
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line, pert_type in zip(['MCF10A', 'NL20', 'RWPE1'],['ctl_vehicle', 'ctl_vehicle', 'ctl_vector'] ):
+		metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+		data_i = matrix_binarized[metadata_i.index]
+		metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+		filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+		np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+		log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+def binarize_genewise_comparing_to_control_augmented(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control_lognorm/augmented')
+	else:
+		outdir = osp.join(outdir, 'binarize_genewise_comparing_to_control/augmented')
+	os.makedirs(outdir, exist_ok= True)
+
+
+
+
+	########################################################################################
+	#All data
+	metadata = inst_info_ctl
+	metadata.to_csv(osp.join(outdir, 'all_metadata_healthy.txt'))
+	matrix = matrix_ctl
+	
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+
+
+	matrix_augmented = matrix.copy()
+	###Data augmentation using Gaussian noise
+	AUG_PROPORTION = 10
+	columns = matrix.columns
+	for i in range(AUG_PROPORTION):
+		columns_i = [e+'___{}'.format(i) for e in columns]
+		noise = np.random.normal(0,1,matrix.shape)
+		to_add = pd.DataFrame(matrix.values + noise, columns = columns_i, index = matrix.index)
+		matrix_augmented = pd.concat([matrix_augmented, to_add], 1)
+
+
+	matrix = matrix_augmented
+	matrix_binarized = pd.DataFrame(np.zeros_like(matrix), index = matrix.index, columns = matrix.columns)
+
+	i = 1
+
+	control_corrected = []
+
+
+
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_healthy.png'))
+	plt.close()
+
+	for cell_line in list(set(metadata['cell_iname'])):
+		columns = metadata[metadata['cell_iname']==cell_line]['sample_id'].tolist()
+		columns_augmented = [e+'___{}'.format(i) for i in range(AUG_PROPORTION) for e in columns] + columns
+		columns = columns_augmented
+		matrix_i = matrix[columns]
+		#Normalization 
+		#Create matrix of NGenes x NExperiments (add column name as sample_id)
+		#Binarization
+		mask_norm = list(set(matrix_i.columns)) #mask_norm is controls only (for specific cell line 'cell_line')
+		control_corrected += mask_norm
+		averages = np.mean(matrix[mask_norm], 1)
+		stds = np.std(matrix[mask_norm], 1)
+		thresholds = averages + (2*stds)
+		for gene_id in list(matrix_i.index):
+			#normalize
+			threshold = thresholds.loc[gene_id]
+			matrix_binarized.loc[gene_id][matrix_i.columns] = (matrix_i.loc[gene_id] >= threshold).astype(int).values
+		print('{}/{}'.format(i, len(list(set(metadata['cell_iname'])))))
+		i+=1
+
+
+
+
+	print('Controls covered:{}/{}\n'.format(len(control_corrected), matrix.shape[1]))
+
+	#2. Save data and metadata for each condition and cell line
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tAUGMENTED SIZE\t\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	# metadata = metadata.loc[matrix_binarized.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line, pert_type in zip(['MCF10A', 'NL20', 'RWPE1'],['ctl_untrt', 'ctl_vehicle', 'ctl_vector'] ):
+		metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+		columns = list(metadata_i.index)
+		to_add = []
+		for i in range(AUG_PROPORTION):
+			to_add += [e+'___{}'.format(i) for e in columns]
+		columns = columns + to_add
+		data_i = matrix_binarized[columns]
+		metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+		filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+		np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+		log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), data_i.shape[1], len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+
+
+
+
+def normalize_and_save(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log):
+	log_handle.write('\n\n------\nBINARIZING GENEWISE COMPARING TO CONTROL\n------\n')
+	if use_log:
+		outdir = osp.join(outdir, 'real_lognorm')
+	else:
+		outdir = osp.join(outdir, 'real')
+	os.makedirs(outdir, exist_ok= True)
+
+
+
+
+	########################################################################################
+	#All data
+	metadata = inst_info_ctl
+	metadata.to_csv(osp.join(outdir, 'all_metadata_healthy.txt'))
+	matrix = matrix_ctl
+
+
+	#lognorm
+	if use_log:
+		matrix = np.log2(matrix + 1)
+	scaler = MinMaxScaler((0,1))
+	matrix = matrix.transpose()
+	matrix = pd.DataFrame(scaler.fit_transform(matrix), columns = matrix.columns, index = matrix.index)
+	matrix = matrix.transpose()
+
+	#hist of values
+	mv = matrix.values.flatten()
+	sampling = sample(range(len(mv)), int(0.1*len(mv)))
+	mv = mv[sampling]
+
+	fig, ax = plt.subplots(figsize=(16,6))
+	ax.hist(mv)
+	ax.set_title('Histogram of values')
+	fig.savefig(osp.join(outdir,'histogram_healthy.png'))
+	plt.close()
+
+
+
+	#2. Save data and metadata for each condition and cell line
+	#CRISPR + cell lines
+	#Control + cell lines
+	log_handle.write('----------------\n----------------\nDATA MATRICES\n')
+	log_handle.write('CELL\tPERT\t\tSIZE\tUNIQUE GENES/VECTORS\tUNIQUE CELL LINES\tAVG NUMBER OF 1\'s\n')
+	metadata.index = metadata['sample_id']
+	metadata = metadata.loc[matrix.columns]	#sort metadata given by column order in data matrix (and filter samples that have been filtered out from matrix during binarization)
+	for cell_line, pert_type in zip(['MCF10A', 'NL20', 'RWPE1'],['ctl_vehicle', 'ctl_vehicle', 'ctl_vector'] ):
+		metadata_i = metadata[np.logical_and(metadata['cell_iname'] == cell_line, metadata['pert_type'] == pert_type)]
+		data_i = matrix[metadata_i.index]
+		metadata_i.to_csv(osp.join(outdir, 'cell_line_{}_pert_{}_metadata.txt'.format(cell_line, pert_type)), index=False)
+		filename = 'cell_line_{}_pert_{}'.format(cell_line, pert_type)
+		np.savez_compressed(osp.join(outdir, filename), data=data_i.values, row_ids = data_i.index, col_ids=data_i.columns)
+		log_handle.write('{}\t{}\t\t{}\t{}\t{}\t{}\n'.format(cell_line, pert_type, len(metadata_i), len(set(metadata_i['cmap_name'])),  len(set(metadata_i['cell_iname'])), np.mean(np.sum(data_i, 0))))
+	log_handle.write('\n\n------\nSTATS\n------\n')		
+
+	return
+
+def main():  
+	from process_data import stats_control
+
+	DATA_ROOT = "../../raw/lincs/2022-02-LINCS_Level3/data/"
+	log_handle = open(osp.join(outdir, 'log_process_data_healthy_lognorm.txt'), 'w')
+	inst_info_ctl, gene_info, matrix_ctl = loads_data(DATA_ROOT, log_handle)
+	inst_info_ctl, matrix_ctl = filter_data_metadata(inst_info_ctl, matrix_ctl, log_handle)
+	#stats_control(inst_info_ctl, log_handle)
+	use_log=True
+	normalize_and_save(inst_info_ctl, matrix_ctl, gene_info, log_handle, outdir, use_log)
+	log_handle.close()
+
+if __name__ == "__main__":
+    main()
+
+
+
+
+
+

PDGrapher/data/scripts/ppi/README.md

@@ -0,0 +1,3 @@
+## Builld PPI for PDGrapher  
+1. Using hgnc2map.txt (in data/raw/ppi/2022-03-PPI) with the raw PPI files (links are listed in the same folder), run union_ppi.py to generate ppi_edgelist.txt  
+2. Using geneinfo_beta.txt and ppi_edgelist.txt, run export_ppi_all_genes.py to build ppi_all_genes_edgelist.txt

PDGrapher/data/scripts/ppi/export_ppi_all_genes.py

@@ -0,0 +1,70 @@
+'''
+Export PPI with all genes in LINCS (those that overlap)
+'''
+
+#Generate a subset of the PPI
+#First checks the overlap between genes in PPI (BRIOGRID) and genes in LINCS
+#Removes all genes from PPI that are *not* in LINCS (no GE data available for them)
+
+
+import networkx as nx
+import csrgraph as cg
+from collections import Counter
+import pandas as pd
+import os
+import os.path as osp
+
+path_edge_list = '../../raw/ppi/2022-03-PPI/processed/ppi_edgelist.txt'
+log_handle = open('log_ppi_all_genes.txt', 'w')
+
+
+#Loads dataset PPI (BIOGRID)
+ppi = nx.read_edgelist(path_edge_list)
+#Loads gene info LINCS
+gene_info = pd.read_csv('../../raw/lincs/2022-02-LINCS_Level3/data/geneinfo_beta.txt', sep="\t", low_memory=False)
+
+
+
+log_handle.write('Overlap of genes from LINCS to PPI:{}/{}\n'.format(len(set(ppi.nodes()).intersection(set(gene_info['gene_symbol']))), len(gene_info)))
+
+#Filter nodes from PPI to keep only the ones in LINCS
+ppi = ppi.subgraph(gene_info['gene_symbol'].tolist())
+log_handle.write('Keeping only PPI nodes that are in LINCS:{}\n'.format(ppi.number_of_nodes()))
+
+ccs = [len(c) for c in sorted(nx.connected_components(ppi), key=len, reverse=True)]
+log_handle.write('Number of connected componens:\t{}\n'.format(len(ccs)))
+Gcc = sorted(nx.connected_components(ppi), key=len, reverse=True)
+ppi = ppi.subgraph(Gcc[0])
+log_handle.write('After keeping only biggest CC:\n')
+log_handle.write('stats: {} nodes, {} edges, {} density, {} diameter\n\n\n'.format(ppi.number_of_nodes(), ppi.number_of_edges(), nx.density(ppi), nx.diameter(ppi)))
+
+#Saves ppi
+outdir = '../../processed/ppi'
+os.makedirs(outdir, exist_ok=True)
+ppi_f = osp.join(outdir, 'ppi_all_genes_edgelist.txt')
+
+nx.write_edgelist(ppi, ppi_f, data=False) 
+
+
+
+
+log_handle.close()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

PDGrapher/data/scripts/ppi/union_ppi.py

@@ -0,0 +1,143 @@
+# Union BioGRID + Menche et al. 2015 + HuRI 
+import numpy as np
+import networkx as nx
+import pandas as pd
+maxscore = 19
+ref_score_list = {'Affinity Capture-MS':17,
+                'Affinity Capture-Western':14,
+                'Two-hybrid':1,
+                'Reconstituted Complex':2,
+                'Proximity Label-MS':7,
+                'Co-fractionation':5,
+                'Biochemical Activity':11,
+                'Affinity Capture-RNA':13,
+                'Co-localization':4,
+                'Co-purification':6,
+                'PCA':9,
+                'Co-crystal Structure':18,
+                'FRET':10,
+                'Protein-peptide':16,
+                'Affinity Capture-Luminescence':12,
+                'Far Western':8,
+                'Protein-RNA':3}
+
+def read_biogrid(f, mapping):
+    edges = []
+    data = pd.read_csv(f, sep="\t")
+    source_ = data['Entrez Gene Interactor A'].values
+    target_ = data['Entrez Gene Interactor B'].values
+    ref = data['Experimental System'].values
+    score_ = np.array([ref_score_list[i] for i in ref])
+    for i in range(len(source_)):
+        source = str(source_[i])
+        target = str(target_[i])
+        score = score_[i]
+        if source in mapping and target in mapping:
+                source = mapping[source] 
+                target = mapping[target]
+                temp = tuple(sorted((source, target)))
+                #if score == "-":
+                #    score = maxscore
+                temp = temp + (score,)
+                edges.append(temp) 
+    G = nx.Graph() 
+    G.add_weighted_edges_from(edges) 
+    print("BioGRID")
+    #print(nx.info(G))
+    for n in G.nodes:
+        if "," in n: print(n)
+    return G 
+
+
+def read_menche(f, mapping):
+    edges = []
+    with open(f) as fin:
+        for line in fin: 
+            if line.startswith("#"): continue 
+            source = line.split()[0]
+            target = line.split()[1] 
+            if source in mapping and target in mapping:
+                source = mapping[source]
+                target = mapping[target]
+                temp = tuple(sorted((source, target)))
+                temp = temp + (maxscore,)
+                edges.append(temp)             
+    G = nx.Graph()
+    G.add_weighted_edges_from(edges)
+    print("Menche et al. 2015")
+    #print(nx.info(G))
+    for n in G.nodes:
+        if "," in n: print(n) 
+    return G
+
+
+def read_huri(f, mapping):
+    edges = pd.read_csv(f, sep="\t", header=None)
+    edges['weight'] = maxscore
+    edges.columns = ["source", "target", "weight"]
+    G = nx.from_pandas_edgelist(edges, source='source', target='target', edge_attr='weight')
+    G = nx.relabel_nodes(G, mapping)
+    for n in G.nodes:
+        if "," in n: print(n)
+    nodes_to_remove = []
+    for n in G.nodes:
+        if n.startswith("ENS"): nodes_to_remove.append(n) 
+    G.remove_nodes_from(nodes_to_remove)
+
+    print("HuRI")
+    #print(nx.info(G))
+    return G
+
+
+def read_mapping(f):
+    entrez2hgnc = dict()
+    ensembl2hgnc = dict()
+    with open(f) as fin:
+        for line in fin:
+            hgnc_id = line.split("\t")[1].strip()
+            entrez_id = line.split("\t")[2].strip()
+            ensembl_id = line.split("\t")[3].strip()
+            if hgnc_id != "" and entrez_id != "":
+                if entrez_id in entrez2hgnc: 
+                    print(entrez_id, hgnc_id, entrez2hgnc[entrez_id])  
+                    if hgnc_id != entrez2hgnc[entrez_id]: hgnc_id = ",".join([entrez2hgnc[entrez_id], hgnc_id]) 
+                entrez2hgnc[entrez_id] = hgnc_id 
+            if hgnc_id != "" and ensembl_id != "":
+                if ensembl_id in ensembl2hgnc: 
+                    print(ensembl_id, hgnc_id, ensembl2hgnc[ensembl_id]) 
+                    if hgnc_id != ensembl2hgnc[ensembl_id]: hgnc_id = ",".join([ensembl2hgnc[ensembl_id], hgnc_id]) 
+                ensembl2hgnc[ensembl_id] = hgnc_id 
+    print("Num mapping entrez2hgnc", len(entrez2hgnc)) 
+    print("Num mapping ensembl2hgnc", len(ensembl2hgnc)) 
+    return entrez2hgnc, ensembl2hgnc
+
+
+def union_G(biogrid_G, menche_G, huri_G, ppi_f):
+    ppi = nx.Graph() 
+    ppi.add_edges_from(biogrid_G.edges(data=True))
+    ppi.add_edges_from(menche_G.edges(data=True))
+    ppi.add_edges_from(huri_G.edges(data=True))
+    print("Overlap with PPI + BioGRID:", len(set(list(ppi.nodes)).intersection(set(list(biogrid_G.nodes))))) 
+    print("Overlap with PPI + Menche:", len(set(list(ppi.nodes)).intersection(set(list(menche_G.nodes))))) 
+    print("Overlap with PPI + HuRI:", len(set(list(ppi.nodes)).intersection(set(list(huri_G.nodes))))) 
+    print("Overlap with BioGRID + Menche:", len(set(list(biogrid_G.nodes)).intersection(set(list(menche_G.nodes))))) 
+    print("Overlap with BioGRID + HuRI:", len(set(list(biogrid_G.nodes)).intersection(set(list(huri_G.nodes))))) 
+    print("Overlap with HuRI + Menche:", len(set(list(huri_G.nodes)).intersection(set(list(menche_G.nodes))))) 
+    print("Overlap with BioGRID + HuRI + Menche:", len(set(list(biogrid_G.nodes)).intersection(set(list(huri_G.nodes)), set(list(menche_G.nodes))))) 
+    print("Full PPI")
+    #print(nx.info(ppi)) 
+    nx.write_edgelist(ppi, ppi_f, data=True)
+    return ppi 
+
+
+def main():  
+    entrez2hgnc, ensembl2hgnc = read_mapping("../../data/ppi/2022-03-PPI/hgnc2map.txt")
+    biogrid_G = read_biogrid("../../data/ppi/2022-03-PPI/BIOGRID-MV-Physical-4.4.207.tab3.txt", entrez2hgnc)
+    menche_G = read_menche("../../data/ppi/2022-03-PPI/DataS1_interactome.tsv", entrez2hgnc) 
+    huri_G = read_huri("../../data/ppi/2022-03-PPI/HuRI.tsv", ensembl2hgnc) 
+    ppi = union_G(biogrid_G, menche_G, huri_G, "../../data/ppi/2022-03-PPI/ppi_edgelist.txt")
+
+
+if __name__ == "__main__":
+    main()
+

PDGrapher/data/scripts/random-baseline-cancer-targets/get_drugs_and_targets_cancer_cell_lines.py

@@ -0,0 +1,147 @@
+'''
+Builds data table with relevant cancer cell lines and corresponding drugs and targets
+Log how many map to targets in DrugBank
+'''
+
+import pandas as pd
+import os
+import os.path as osp
+
+
+#Outdir
+outdir ='../../processed/nci'
+os.makedirs(outdir, exist_ok=True)
+
+
+#Cell lines and cancer
+cancer_cell_mapping = {'breast': ['MCF7', 'BT20', 'MDAMB231'],
+						'lung': ['A549'],
+						'prostate': ['PC3', 'VCAP'],
+      					'gastric': ['AGS'],
+            			'skin': ['A375'],
+               			'cervical': ['HELA'],
+                  		'colorectal': ['HT29'],
+                    	'head': ['BICR6'],
+                     	'ovarian': ['ES2'],
+                    	'brain': ['U251MG'],
+                     	'pancreatic': ['YAPC']}
+
+
+#Loads data
+cancer_drugs = pd.read_csv('../../raw/nci/2022-11-NCI/data/cancer_drugs_2024.csv')
+cancer_drugs['drug'] =[e.lower() for e in cancer_drugs['drug']]
+drug_targets = pd.read_csv('../../processed/drugbank/targets.txt', sep=',', low_memory=False)
+
+drug_targets.columns = ['drug_id', 'drug_name', 'drug_synonyms', 'target_id', 'target_name', 'gene_name', 'gene_synonyms', 'identifiers', 'organism']
+drug_targets['drug_name'] = [e.lower() for e in drug_targets['drug_name']]
+drug_targets['drug_synonyms'] = [str(e).lower() for e in drug_targets['drug_synonyms']]
+
+#Create dictionary of synonym-->drug
+dict_syn_drug = dict()
+for i in range(len(drug_targets)):
+	name = drug_targets.at[i, 'drug_name']
+	synonyms = drug_targets.at[i, 'drug_synonyms'].split('||') + [name]
+	for s in synonyms:
+		dict_syn_drug[s] = name
+
+
+#Create dictionary of drug-->targets
+dict_drug_targets = dict()
+for i in range(len(drug_targets)):
+	name = drug_targets.at[i, 'drug_name']
+	targets = drug_targets[drug_targets['drug_name']==name]['gene_name'].tolist()
+	dict_drug_targets[name] = targets
+
+
+
+#For each cancer, compile targets of approved drugs
+#Manual mappings
+manual_mappings = {'lapatinib ditosylate': 'lapatinib',
+					'osimertinib mesylate': 'osimertinib',
+					'abiraterone': 'abiraterone',
+					'radium 223 dichloride': 'radium ra 223 dichloride',
+					'rucaparib camsylate': 'rucaparib',
+					'talazoparib tosylate': 'talazoparib',
+					'tepotinib hydrochloride': 'tepotinib',
+					'mobocertinib succinate': 'mobocertinib',
+					'dabrafenib mesylate': 'dabrafenib',
+					'afatinib dimaleate': 'afatinib dimaleate',
+					'trametinib dimethyl sulfoxide': 'trametinib',
+					'tamoxifen citrate': 'tamoxifen',
+					'abiraterone acetate': 'abiraterone',
+					'erlotinib hydrochloride': 'erlotinib',
+					'neratinib maleate': 'neratinib',
+					'lutetium lu 177 vipivotide tetraxetan': 'lutetium lu-177 vipivotide tetraxetan',
+					'capmatinib hydrochloride': 'capmatinib',
+					'afatinib dimaleate' : 'afatinib',
+					'capmatinib hydrochloride': 'capmatinib',
+     				'toripalimab-tpzi': 'toripalimab',
+         			'amivantamab-vmjw': 'amivantamab',
+            		'cemiplimab-rwlc': 'cemiplimab',
+              		'fam-trastuzumab deruxtecan-nxki': 'Trastuzumab deruxtecan',
+                	'tarlatamab-dlle': 'tarlatamab',
+                 	'tremelimumab-actl': 'tremelimumab',
+                  	'sacituzumab govitecan-hziy':'Sacituzumab govitecan',
+                   	'ado-trastuzumab emtansine': 'Trastuzumab emtansine',
+                    'margetuximab-cmkb':'Margetuximab',
+                    'sacituzumab govitecan-hziy':'Sacituzumab govitecan',
+                    'cobimetinib fumarate': 'cobimetinib',
+                    'retifanlimab-dlwr': 'retifanlimab',
+                    'tisotumab vedotin-tftv': 'tisotumab vedotin'
+                    }
+
+in_drugbank = set(dict_syn_drug.keys())
+table_cancer_drugs_and_targets = []
+
+
+log = open('log.txt', 'w')
+for cancer in list(set(cancer_drugs['cancer_type'])):
+	not_mapped = []
+	drugs = cancer_drugs[cancer_drugs['cancer_type'] == cancer]['drug'].tolist()
+	drugs = [e.replace('\xa0', ' ').split(' (')[0] for e in drugs]
+	drugs = [manual_mappings[e] if e in manual_mappings else e for e in drugs]
+	for e in drugs:
+		if e =='pertuzumab, trastuzumab, and hyaluronidase-zzxf':
+			drugs.remove('pertuzumab, trastuzumab, and hyaluronidase-zzxf')
+			drugs.append('pertuzumab')
+			drugs.append('trastuzumab')
+			drugs.append('hyaluronidase')
+	cells = cancer_cell_mapping[cancer]
+	for cell in cells:
+		for d in drugs:
+			if d in dict_syn_drug:
+				name = dict_syn_drug[d]
+			else:
+				not_mapped.append(d)
+			targets = dict_drug_targets[name]
+			targets = ','.join(targets).replace('-,','')
+			table_cancer_drugs_and_targets.append([cell, name, targets])
+	log.write('CANCER:\t{},\tCELL LINE:\t{}\n'.format(cancer, cancer_cell_mapping[cancer]))
+	log.write('Number of approved drugs from NCI:\t{}\n'.format(len(set(drugs))))
+	log.write('Mapped drugs from NCI to DrugBank:\t{}/{}\n'.format(len(set(drugs).intersection(in_drugbank)) , len(set(drugs))))
+	log.write('Drugs not mapped:\n')
+	for d in not_mapped:
+		log.write('{}\n'.format(d))
+	log.write('\n----------\n')
+
+log.close()
+
+df = pd.DataFrame(table_cancer_drugs_and_targets, columns=['cell_line', 'drug', 'targets'])
+df.to_csv(osp.join(outdir, 'drugs_and_targets.csv'), sep='\t', index = False)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

PDGrapher/data/scripts/rep-learning-approach-3-all-genes/export_data_for_scgen_chemical.py

@@ -0,0 +1,55 @@
+
+
+import torch
+import os.path as osp
+import h5py
+import pandas as pd
+import numpy as np
+import anndata
+
+base_path = '../../processed/torch_data/chemical/real_lognorm'
+
+cell_lines = ["A549","A375","BT20","HA1E","HELA","HT29","MCF7","MDAMB231","PC3","VCAP"]
+
+for cell_line in cell_lines:
+    print("Processing cell line: ", cell_line)
+    #Loads datasets
+    data_backward = torch.load(osp.join(base_path, 'data_backward_' + cell_line + '.pt'))
+    #Builds datasets
+    data_treated = []
+    data_perturbagen = []
+    data_control = []
+    for data in data_backward:
+        data_control.append(data.diseased.numpy().tolist())
+        data_treated.append(data.treated.numpy().tolist())
+        data_perturbagen.append(data.perturbagen_name)
+    #Transforms datasets into pandas
+    data_control = pd.DataFrame(np.array(list(data_control)))
+    data_treated = pd.DataFrame(np.array(data_treated))
+    print(data_control.shape)
+    print(data_treated.shape)
+    #Creating obs
+    cellline = [cell_line for i in range(len(data_control) + len(data_treated))]
+    condition = ['control' for i in range(len(data_control))] + data_perturbagen
+    #Creates annotated data
+    X = pd.concat([data_control, data_treated], axis=0).reset_index(inplace=False, drop=True)
+    obs = pd.DataFrame([cellline, condition]).transpose()
+    obs.columns = ['cell_type', 'condition']
+    var = pd.DataFrame(data_backward[0].gene_symbols, columns = ['gene_symbols'])
+    train = anndata.AnnData(X, obs,var)
+    ind = torch.load(f"../../processed/splits/chemical/{cell_line}/random/5fold/splits.pt")
+    for j in range(1,6):
+        train_ = ind[j]["train_index_backward"]
+        test_ = ind[j]["test_index_backward"]
+        val_ = ind[j]["val_index_backward"]
+        print(len(train_) + len(test_) + len(val_))
+        assert len(train_) + len(test_) + len(val_) == len(train) / 2
+        print("Pass checking")
+        temp = np.array(['data0' for _ in range(len(data_treated))])
+        temp[train_] = 'train'
+        temp[test_] = 'test'
+        temp[val_] = 'val'
+        train.obs['split'+str(j)] = np.concatenate([temp, temp], axis=0) # same split for treated and control
+        train.write(
+        osp.join(base_path, 'data_split_' + cell_line +'.h5ad')
+    )

PDGrapher/data/scripts/rep-learning-approach-3-all-genes/export_data_for_scgen_genetic.py

@@ -0,0 +1,70 @@
+
+
+
+import torch
+import os.path as osp
+import h5py
+
+
+
+base_path = '../processed/real_lognorm'
+
+#Loads datasets
+#MCF7
+# mcf7_forward = torch.load(osp.join(base_path, 'data_forward_MCF7.pt'))
+mcf7_backward = torch.load(osp.join(base_path, 'data_backward_MCF7.pt'))
+
+#A549
+# a549_forward = torch.load(osp.join(base_path, 'data_forward_A549.pt'))
+a549_backward = torch.load(osp.join(base_path, 'data_backward_A549.pt'))
+
+
+#Builds datasets
+mcf7_control = []
+mcf7_treated = []
+mcf7_perturbagen = []
+for data in mcf7_backward:
+    mcf7_control.append(data.diseased.numpy().tolist())
+    mcf7_treated.append(data.treated.numpy().tolist())
+    mcf7_perturbagen.append(data.perturbagen_name)
+    
+
+    
+a549_treated = []
+a549_perturbagen = []
+a549_control = []
+for data in a549_backward:
+    a549_control.append(data.diseased.numpy().tolist())
+    a549_treated.append(data.treated.numpy().tolist())
+    a549_perturbagen.append(data.perturbagen_name)
+
+
+#Transforms datasets into pandas
+import numpy as np
+import pandas as pd
+mcf7_control = pd.DataFrame(np.array(list(mcf7_control)))
+mcf7_treated = pd.DataFrame(np.array(mcf7_treated))
+a549_control = pd.DataFrame(np.array(list(a549_control)))
+a549_treated = pd.DataFrame(np.array(a549_treated))
+
+
+#Creating obs
+cell_line = ['MCF7' for i in range(len(mcf7_control) + len(mcf7_treated))] + ['A549' for i in range(len(a549_control) + len(a549_treated))]
+condition = ['control' for i in range(len(mcf7_control))] + mcf7_perturbagen +  ['control' for i in range(len(a549_control))] + a549_perturbagen
+
+
+#Creates annotated data
+X = pd.concat([mcf7_control, mcf7_treated, a549_control, a549_treated], 0).reset_index(inplace=False, drop=True)
+obs = pd.DataFrame([cell_line, condition]).transpose()
+obs.columns = ['cell_type', 'condition']
+var = pd.DataFrame(a549_backward[0].gene_symbols, columns = ['gene_symbols'])
+
+import anndata
+train = anndata.AnnData(X, obs,var)
+
+import hdf5plugin
+train.write(
+    osp.join(base_path, 'data_scgen.h5ad')
+)
+
+

PDGrapher/data/scripts/rep-learning-approach-3-all-genes/export_data_for_torch_geometric.py

@@ -0,0 +1,344 @@
+'''
+Exports data for proof of concept model 
+Code taken from rep-learning-approach
+changed to adapt to new data sources (healthy cell lines + COSMIC)
+
+'''
+
+
+import pandas as pd
+import networkx as nx
+import numpy as np
+import os
+import os.path as osp
+import math
+import torch
+from torch_geometric.data import Data
+from torch_geometric.utils import add_remaining_self_loops, to_undirected
+
+############
+#Data loading
+############
+def load_ppi(path_edge_list, log_handle):
+	#Loads PPI
+	ppi = nx.read_edgelist(path_edge_list)
+	log_handle.write('----------------\nNumber of nodes in PPI:\t{}\n'.format(ppi.number_of_nodes()))
+	log_handle.write('Number of edges in PPI:\t{}\n'.format(ppi.number_of_edges()))
+	return ppi
+
+
+def load_gene_metadata(file, log_handle):
+	#Loads gene metadata
+	gene_info = pd.read_csv(file)
+	# dict_symbol_index = dict(zip(gene_info['gene_symbol'], range(len(gene_info))))	#genes are ordered with the same ordering as rows in data matrices
+	dict_entrez_symbol = dict(zip(gene_info['gene_id'], gene_info['gene_symbol']))
+	dict_symbol_entrez = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	return gene_info, dict_entrez_symbol, dict_symbol_entrez
+
+def load_cosmic(path_cosmic_file, log_handle):
+	data = pd.read_csv(path_cosmic_file)
+	log_handle.write('Loading COSMIC data. Number of cell lines:\t{}\n'.format(len(set(data['Sample name']))))
+	return data
+
+
+def map_cosmic_to_lincs(cosmic_data, cell_line, gene_info, dict_symbol_entrez, log_handle):
+	cosmic_data = cosmic_data[cosmic_data['Sample name']==cell_line]
+	log_handle.write('Mapping cosmic genes to lincs. Mapped: {}/{}\n'.format(len(set(cosmic_data['Gene name']).intersection(gene_info['gene_symbol'])), len(set(cosmic_data['Gene name']))))
+	#Filter genes not mapped to LINCS
+	cosmic_data = cosmic_data[[gene_symbol in dict_symbol_entrez for gene_symbol in cosmic_data['Gene name']]]
+	#Save COSMIC mutations as entrez id (dataframe index)
+	try:
+		cosmic_mutations = list(set([dict_symbol_entrez[symbol] for symbol in cosmic_data['Gene name'].tolist()]))
+	except:
+		import pdb; pdb.set_trace()
+	return cosmic_mutations
+
+
+
+def load_healthy_data(data_root_dir, healthy, log_handle):
+	healthy_data_path = osp.join(data_root_dir, 'cell_line_{}_pert_{}.npz'.format(healthy[0], healthy[1]))
+	healthy_metadata_path = osp.join(data_root_dir, 'cell_line_{}_pert_{}_metadata.txt'.format(healthy[0], healthy[1]))
+	#Loads metadata
+	healthy_metadata = pd.read_csv(healthy_metadata_path)
+	#Loads data
+	with np.load(healthy_data_path, allow_pickle=True) as arr:
+		healthy_data =arr['data']
+		col_ids = arr['col_ids']
+		row_ids = arr['row_ids']
+	healthy_data = pd.DataFrame(healthy_data, columns= col_ids, index=row_ids)
+	log_handle.write('Loading healthy cell line:\t{} Number of samples:\t{}\n'.format(healthy[0], healthy_data.shape[1]))
+	return healthy_data, healthy_metadata
+
+
+
+
+
+def load_data(cell_line, data_root_dir, log_handle):
+	#Loads data matrix (observational)
+	file = osp.join(data_root_dir, 'cell_line_{}_pert_ctl_vector.npz'.format(cell_line))
+	file_metadata = osp.join(data_root_dir, 'cell_line_{}_pert_ctl_vector_metadata.txt'.format(cell_line))
+	obs_metadata = pd.read_csv(file_metadata)
+	with np.load(file, allow_pickle=True) as arr:
+		obs_data =arr['data']
+		col_ids = arr['col_ids']
+		row_ids = arr['row_ids']
+	obs_data = pd.DataFrame(obs_data, columns= col_ids, index=row_ids)
+	log_handle.write('Number of observational datapoints:\t{}\n'.format(len(obs_metadata)))
+	#Loads data matrix (interventional)
+	file = osp.join(data_root_dir, 'cell_line_{}_pert_trt_xpr.npz'.format(cell_line))
+	file_metadata = osp.join(data_root_dir, 'cell_line_{}_pert_trt_xpr_metadata.txt'.format(cell_line))
+	int_metadata = pd.read_csv(file_metadata)
+	with np.load(file, allow_pickle=True) as arr:
+		int_data =arr['data']
+		col_ids = arr['col_ids']
+		row_ids = arr['row_ids']
+	int_data = pd.DataFrame(int_data, columns= col_ids, index=row_ids)
+	log_handle.write('Number of interventional datapoints:\t{}\n'.format(len(int_metadata)))
+	return obs_metadata, obs_data, int_metadata, int_data
+
+
+
+
+############
+#Processing
+############
+
+def filter_data(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle):
+	#1.Filter out obs and int data to keep only genes that are in the PPI
+	gene_symbols_in_ppi = list(ppi.nodes())
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	gene_ids_in_ppi = [dict_symbol_id[i] for i in gene_symbols_in_ppi]
+	gene_info.index = gene_info['gene_id']; gene_info = gene_info.loc[gene_ids_in_ppi].reset_index(inplace=False, drop=True)
+	obs_data = obs_data.loc[gene_ids_in_ppi]
+	int_data = int_data.loc[gene_ids_in_ppi]
+	if healthy_data is not None:
+		healthy_data = healthy_data.loc[gene_ids_in_ppi]
+	if cosmic_mutations is not None:
+		cosmic_mutations = pd.DataFrame(cosmic_mutations)[[e in gene_ids_in_ppi for e in cosmic_mutations]][0].tolist()
+	#2. Filter out samples whose interventions are not in the remaining genes (those in the PPI)
+	keep = []
+	for i, gene_symbol in enumerate(int_metadata['cmap_name']):
+		if gene_symbol in gene_symbols_in_ppi:
+			keep.append(int_metadata.at[i, 'sample_id'])
+	int_metadata.index = int_metadata['sample_id']; int_metadata = int_metadata.loc[keep].reset_index(inplace=False, drop=True)
+	int_data = int_data[keep]
+	log_handle.write('Number of interventional datapoints after keeping only those with perturbed genes in PPI:\t{}\n'.format(len(int_metadata)))
+	return healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, gene_info
+
+
+
+
+
+############
+#Asembling the data
+############
+
+def assemble_data_list(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle):
+	log_handle.write('Assembling data...\n')
+
+	#First, we re-index genes in PPI and data
+		#Gene symbol to index to ordered index
+	gene_symbol_to_index = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	gene_index_to_ordered_index = dict(zip(gene_info['gene_id'], range(len(gene_info))))
+	gene_info['ordered_index'] = [gene_index_to_ordered_index[i] for i in gene_info['gene_id']]
+
+
+		#Reindex genes in PPI, obs_data, int_data, healty_data, and cosmic_mutations
+	ppi = nx.relabel_nodes(ppi, gene_symbol_to_index)
+	ppi = nx.relabel_nodes(ppi, gene_index_to_ordered_index)
+	int_data.index = [gene_index_to_ordered_index[i] for i in int_data.index]
+	int_data = int_data.sort_index(inplace=False)
+	obs_data.index = [gene_index_to_ordered_index[i] for i in obs_data.index]
+	obs_data = obs_data.sort_index(inplace=False)
+	if healthy_data is not None:
+		healthy_data.index = [gene_index_to_ordered_index[i] for i in healthy_data.index]
+		healthy_data = healthy_data.sort_index(inplace=False)
+	if cosmic_mutations is not None:
+		cosmic_mutations = [gene_index_to_ordered_index[i] for i in cosmic_mutations]
+		cosmic_vector = np.zeros(len(healthy_data))
+		cosmic_vector[cosmic_mutations] = 1
+
+
+	#Assembling samples
+	edge_index = torch.LongTensor(np.array(ppi.edges()).transpose())
+	edge_index = add_remaining_self_loops(edge_index)[0]
+	edge_index = to_undirected(edge_index)
+	number_of_nodes = ppi.number_of_nodes()
+
+	#remove incoming edges to perturbed nodes (mutated nodes)
+	# mask = [e not in cosmic_mutations for e in edge_index[1,:]]
+	# edge_index_mutilated = edge_index[:, mask]
+	# edge_index_mutilated = add_remaining_self_loops(edge_index_mutilated)[0]
+
+	dict_forward_sample_and_mutations = dict() #saves the mutation vector used in forward
+
+
+
+	forward_data_list = []
+	if healthy_data is not None: #we only build forward data when we have healthy samples
+		#FORWARD DATA - healthy_data, cosmic_vector, obs_data
+			#each Data object will be a pairing of a random healthy_data column, the cosmic mutations, and a obs_data column
+			#will have as many as obs_data columns
+		i = 0
+		order = np.array(range(healthy_data.shape[1]))
+		np.random.shuffle(order)
+		for sample_id in obs_data.columns:
+			#sample a random healthy GE vector
+			i = i % healthy_data.shape[1]
+			sample_index = order[i]
+			healthy_sample = healthy_data[healthy_data.columns[i]].values
+			healthy = torch.Tensor(healthy_sample)
+			#mutation
+			#randomize mutations. First select the percentage of mutations to include, then select the mutations
+			perc_to_include = np.random.choice([0.25, 0.50, 0.75, 1], 1).item()
+			if int_metadata['cell_mfc_name'][0].split('.')[0] == 'PC3':
+				perc_to_include = 1
+			cosmic_mutations_i = np.random.choice(cosmic_mutations, int(len(cosmic_mutations)* perc_to_include))
+			cosmic_vector = np.zeros(len(healthy_data))
+			cosmic_vector[cosmic_mutations_i] = 1
+			mutations = torch.Tensor(cosmic_vector)
+			#diseased
+			diseased = torch.Tensor(obs_data[sample_id])
+			data = Data(healthy = healthy, mutations=mutations, diseased=diseased, gene_symbols = gene_info['gene_symbol'].tolist())
+			data.num_nodes = number_of_nodes
+			forward_data_list.append(data)
+			#Save 
+			i +=1
+			dict_forward_sample_and_mutations[sample_id] = mutations
+		print('finished data forward')
+
+
+	#BACKWARD DATA - obs_data, int_data
+	#dict sample id: perturbed gene ordered index
+	dict_sample_id_perturbed_gene_ordered_index = dict()
+	for sample_id, cmap_name in zip(int_metadata['sample_id'], int_metadata['cmap_name']):
+		dict_sample_id_perturbed_gene_ordered_index[sample_id] = gene_index_to_ordered_index[gene_symbol_to_index[cmap_name]]
+
+
+		#these are helpers to sample from obs_data
+	order = np.array(range(obs_data.shape[1]))
+	np.random.shuffle(order)
+	i = 0
+	#shuffle obs data columns
+	backward_data_list = []
+	unique_names_pert = set()
+	for sample_id in int_data.columns:
+		binary_indicator_perturbation = np.zeros(len(int_data))
+		binary_indicator_perturbation[dict_sample_id_perturbed_gene_ordered_index[sample_id]] = 1
+		#Get a random pre-intervention sample
+		i = i % obs_data.shape[1]
+		sample_index = order[i]
+		obs_sample_id = obs_data.columns[i]
+		obs_sample = obs_data[obs_data.columns[i]].values
+		#concat initial node features and perturbation indicator
+		diseased = torch.Tensor(obs_sample)
+		intervention = torch.Tensor(binary_indicator_perturbation)
+		if healthy_data is not None:
+			mutations = dict_forward_sample_and_mutations[obs_sample_id]
+		else:
+			mutations = torch.Tensor(np.zeros(len(diseased)))
+		# torch.Tensor(np.stack([obs_sample, binary_indicator_perturbation], 1))
+		#post-intervention
+		treated = torch.Tensor(int_data[sample_id])
+		#remove incoming edges to perturbed node
+		# perturbed_node = dict_sample_id_perturbed_gene_ordered_index[sample_id]
+		# edge_index_mutilated = edge_index[:, edge_index[1,:] != perturbed_node]
+		
+		gene_name = int_metadata[int_metadata['sample_id'] == sample_id]['cmap_name'].item()
+		unique_names_pert.add(gene_name)
+		data = Data(perturbagen_name = gene_name, diseased = diseased, intervention=intervention, treated = treated, gene_symbols = gene_info['gene_symbol'].tolist(), mutations = mutations)
+		data.num_nodes = number_of_nodes
+		backward_data_list.append(data)
+		i +=1
+
+	log_handle.write('Samples forward:\t{}\n'.format(len(forward_data_list)))
+	log_handle.write('Samples backward:\t{}\n'.format(len(backward_data_list)))
+	log_handle.write('Unique perturbagens:\t{}\n'.format(len(unique_names_pert)))
+	
+	return forward_data_list, backward_data_list, edge_index
+
+
+
+
+def save_data(forward_data_list, backward_data_list, edge_index, cell_line, log_handle):
+	log_handle.write('Saving data {} ...\n\n\n'.format(cell_line))
+	torch.save(forward_data_list, osp.join(outdir, 'data_forward_{}.pt'.format(cell_line)))
+	torch.save(backward_data_list, osp.join(outdir, 'data_backward_{}.pt'.format(cell_line)))
+	torch.save(edge_index, osp.join(outdir, 'edge_index_{}.pt'.format(cell_line)))
+	return
+
+
+binarization = 'real_lognorm'
+outdir = '../../processed/torch_data/{}/'.format(binarization)
+os.makedirs(outdir, exist_ok=True)
+
+
+
+
+def main():  
+	#cell-line wise
+	log_handle = open(osp.join(outdir, 'log_export_data.txt'), 'w')
+	data_root_dir = '../../processed/lincs/{}'.format(binarization)
+
+	#Samples with healthy counterparts
+	for cell_line, healthy in zip(['A549', 'MCF7', 'PC3'], [('NL20', 'ctl_vehicle'), ('MCF10A', 'ctl_untrt'), ('RWPE1', 'ctl_vector')]):
+		log_handle.write('----------------\n\nCELL LINE:{}\n----------------\n'.format(cell_line))
+		#PPI
+		ppi = load_ppi('../../processed/ppi/ppi_all_genes_edgelist.txt', log_handle)
+		#gene info
+		gene_info, dict_entrez_symbol, dict_symbol_entrez = load_gene_metadata('../../processed/lincs/gene_info.txt', log_handle)
+		#FORWARD DATA
+			#healthy GE data
+		healthy_data, healthy_metadata = load_healthy_data(data_root_dir, healthy, log_handle)
+			#COSMIC
+		cosmic_data = load_cosmic('../../processed/cosmic/CosmicCLP_MutantExport_only_verified_and_curated.csv', log_handle)
+		cosmic_mutations = map_cosmic_to_lincs(cosmic_data, cell_line, gene_info, dict_symbol_entrez, log_handle)
+		#BACKWARD DATA
+			#LINCS
+		obs_metadata, obs_data, int_metadata, int_data = load_data(cell_line, data_root_dir, log_handle)
+		healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, gene_info = filter_data(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+
+
+		forward_data_list, backward_data_list, edge_index = assemble_data_list(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+		# save_data(forward_data_list, backward_data_list, edge_index, cell_line, log_handle)
+
+
+
+
+	#Samples without healthy counterparts
+	for cell_line, healthy in zip(['A375', 'HT29', 'ES2', 'BICR6', 'YAPC', 'AGS', 'U251MG'], [None, None, None, None, None, None, None]):
+		log_handle.write('----------------\n\nCELL LINE:{}\n----------------\n'.format(cell_line))
+		#PPI
+		ppi = load_ppi('../../processed/ppi/ppi_all_genes_edgelist.txt', log_handle)
+		#gene info
+		gene_info, dict_entrez_symbol, dict_symbol_entrez = load_gene_metadata('../../processed/lincs/gene_info.txt', log_handle)
+
+
+		#FORWARD DATA
+			#healthy GE data
+		# healthy_data, healthy_metadata = load_healthy_data(data_root_dir, healthy, log_handle)
+			#COSMIC
+		# cosmic_data = load_cosmic('../../processed/cosmic/CosmicCLP_MutantExport_only_verified_and_curated.csv', log_handle)
+		# cosmic_mutations = map_cosmic_to_lincs(cosmic_data, cell_line, gene_info, dict_symbol_entrez, log_handle)
+		#BACKWARD DATA
+			#LINCS
+		obs_metadata, obs_data, int_metadata, int_data = load_data(cell_line, data_root_dir, log_handle)
+
+
+		healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, gene_info = filter_data(None, None, None, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+
+
+		forward_data_list, backward_data_list, edge_index = assemble_data_list(None, None, None, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+		# save_data(forward_data_list, backward_data_list, edge_index, cell_line, log_handle)
+
+
+	log_handle.close()
+
+
+
+
+
+
+
+if __name__ == "__main__":
+    main()

PDGrapher/data/scripts/rep-learning-approach-3-all-genes/export_data_for_torch_geometric_chemical.py

@@ -0,0 +1,391 @@
+'''
+Exports data for proof of concept model 
+Code taken from rep-learning-approach
+changed to adapt to new data sources (healthy cell lines + COSMIC)
+
+'''
+
+
+import pandas as pd
+import networkx as nx
+import numpy as np
+import os
+import os.path as osp
+import math
+import torch
+from torch_geometric.data import Data
+from torch_geometric.utils import add_remaining_self_loops, to_undirected
+
+
+
+
+############
+#Data loading
+############
+def load_ppi(path_edge_list, log_handle):
+	#Loads PPI
+	ppi = nx.read_edgelist(path_edge_list)
+	log_handle.write('----------------\nNumber of nodes in PPI:\t{}\n'.format(ppi.number_of_nodes()))
+	log_handle.write('Number of edges in PPI:\t{}\n'.format(ppi.number_of_edges()))
+	return ppi
+
+
+def load_gene_metadata(file, log_handle):
+	#Loads gene metadata
+	gene_info = pd.read_csv(file)
+	# dict_symbol_index = dict(zip(gene_info['gene_symbol'], range(len(gene_info))))	#genes are ordered with the same ordering as rows in data matrices
+	dict_entrez_symbol = dict(zip(gene_info['gene_id'], gene_info['gene_symbol']))
+	dict_symbol_entrez = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	return gene_info, dict_entrez_symbol, dict_symbol_entrez
+
+def load_cosmic(path_cosmic_file, log_handle):
+	data = pd.read_csv(path_cosmic_file)
+	log_handle.write('Loading COSMIC data. Number of cell lines:\t{}\n'.format(len(set(data['Sample name']))))
+	return data
+
+
+def map_cosmic_to_lincs(cosmic_data, cell_line, gene_info, dict_symbol_entrez, log_handle):
+	cosmic_data = cosmic_data[cosmic_data['Sample name']==cell_line]
+	log_handle.write('Mapping cosmic genes to lincs. Mapped: {}/{}\n'.format(len(set(cosmic_data['Gene name']).intersection(gene_info['gene_symbol'])), len(set(cosmic_data['Gene name']))))
+	#Filter genes not mapped to LINCS
+	cosmic_data = cosmic_data[[gene_symbol in dict_symbol_entrez for gene_symbol in cosmic_data['Gene name']]]
+	#Save COSMIC mutations as entrez id (dataframe index)
+	cosmic_mutations = list(set([dict_symbol_entrez[symbol] for symbol in cosmic_data['Gene name']]))
+	return cosmic_mutations
+
+
+
+def load_healthy_data(data_root_dir, healthy, log_handle):
+	healthy_data_path = osp.join(data_root_dir, 'cell_line_{}_pert_{}.npz'.format(healthy[0], healthy[1]))
+	healthy_metadata_path = osp.join(data_root_dir, 'cell_line_{}_pert_{}_metadata.txt'.format(healthy[0], healthy[1]))
+	#Loads metadata
+	healthy_metadata = pd.read_csv(healthy_metadata_path)
+	#Loads data
+	with np.load(healthy_data_path, allow_pickle=True) as arr:
+		healthy_data =arr['data']
+		col_ids = arr['col_ids']
+		row_ids = arr['row_ids']
+	healthy_data = pd.DataFrame(healthy_data, columns= col_ids, index=row_ids)
+	log_handle.write('Loading healthy cell line:\t{} Number of samples:\t{}\n'.format(healthy[0], healthy_data.shape[1]))
+	return healthy_data, healthy_metadata
+
+
+
+
+
+def load_data(cell_line, data_root_dir, log_handle):
+	#Loads data matrix (observational)
+	file = osp.join(data_root_dir, 'cell_line_{}_pert_ctl_vehicle.npz'.format(cell_line))
+	file_metadata = osp.join(data_root_dir, 'cell_line_{}_pert_ctl_vehicle_metadata.txt'.format(cell_line))
+	obs_metadata = pd.read_csv(file_metadata)
+	with np.load(file, allow_pickle=True) as arr:
+		obs_data =arr['data']
+		col_ids = arr['col_ids']
+		row_ids = arr['row_ids']
+	obs_data = pd.DataFrame(obs_data, columns= col_ids, index=row_ids)
+	log_handle.write('Number of observational datapoints:\t{}\n'.format(len(obs_metadata)))
+	#Loads data matrix (interventional)
+	file = osp.join(data_root_dir, 'cell_line_{}_pert_trt_cp.npz'.format(cell_line))
+	file_metadata = osp.join(data_root_dir, 'cell_line_{}_pert_trt_cp_metadata.txt'.format(cell_line))
+	int_metadata = pd.read_csv(file_metadata)
+	with np.load(file, allow_pickle=True) as arr:
+		int_data =arr['data']
+		col_ids = arr['col_ids']
+		row_ids = arr['row_ids']
+	int_data = pd.DataFrame(int_data, columns= col_ids, index=row_ids)
+	log_handle.write('Number of interventional datapoints:\t{}\n'.format(len(int_metadata)))
+	return obs_metadata, obs_data, int_metadata, int_data
+
+
+
+
+############
+#Processing
+############
+
+def filter_data(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle):
+	#1.Filter out obs and int data to keep only genes that are in the PPI
+	gene_symbols_in_ppi = list(ppi.nodes())
+	dict_symbol_id = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	gene_ids_in_ppi = [dict_symbol_id[i] for i in gene_symbols_in_ppi]
+	gene_info.index = gene_info['gene_id']; gene_info = gene_info.loc[gene_ids_in_ppi].reset_index(inplace=False, drop=True)
+	obs_data = obs_data.loc[gene_ids_in_ppi]
+	int_data = int_data.loc[gene_ids_in_ppi]
+	if healthy_data is not None:
+		healthy_data = healthy_data.loc[gene_ids_in_ppi]
+	if cosmic_mutations is not None:
+		cosmic_mutations = pd.DataFrame(cosmic_mutations)[[e in gene_ids_in_ppi for e in cosmic_mutations]][0].tolist()
+		log_handle.write('Mutations remaining in PPI:\t{}\n'.format(len(cosmic_mutations)))
+	#2. Filter out samples whose interventions are not in the remaining genes (those in the PPI)
+	keep = []
+	for i, gene_symbols in enumerate(int_metadata['target_names']):
+		gene_symbols = gene_symbols.replace("[\'", "").replace("\']", "").replace(' ','').replace('\'','').split(',')
+		if len(set(gene_symbols).intersection(set(gene_symbols_in_ppi))) > 0:
+			keep.append(int_metadata.at[i, 'sample_id'])
+	int_metadata.index = int_metadata['sample_id']; int_metadata = int_metadata.loc[keep].reset_index(inplace=False, drop=True)
+	int_data = int_data[keep]
+	log_handle.write('Number of interventional datapoints after keeping only those with perturbed genes in PPI:\t{}\n'.format(len(int_metadata)))
+	return healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, gene_info
+
+
+
+
+
+############
+#Asembling the data
+############
+
+def assemble_data_list(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle):
+	log_handle.write('Assembling data...\n')
+
+	#First, we re-index genes in PPI and data
+		#Gene symbol to index to ordered index
+	gene_symbol_to_index = dict(zip(gene_info['gene_symbol'], gene_info['gene_id']))
+	gene_index_to_ordered_index = dict(zip(gene_info['gene_id'], range(len(gene_info))))
+	gene_info['ordered_index'] = [gene_index_to_ordered_index[i] for i in gene_info['gene_id']]
+
+
+		#Reindex genes in PPI, obs_data, int_data, healty_data, and cosmic_mutations
+	ppi = nx.relabel_nodes(ppi, gene_symbol_to_index)
+	ppi = nx.relabel_nodes(ppi, gene_index_to_ordered_index)
+	int_data.index = [gene_index_to_ordered_index[i] for i in int_data.index]
+	int_data = int_data.sort_index(inplace=False)
+	obs_data.index = [gene_index_to_ordered_index[i] for i in obs_data.index]
+	obs_data = obs_data.sort_index(inplace=False)
+	if healthy_data is not None:
+		healthy_data.index = [gene_index_to_ordered_index[i] for i in healthy_data.index]
+		healthy_data = healthy_data.sort_index(inplace=False)
+	if cosmic_mutations is not None:
+		cosmic_mutations = [gene_index_to_ordered_index[i] for i in cosmic_mutations]
+		cosmic_vector = np.zeros(len(healthy_data))
+		cosmic_vector[cosmic_mutations] = 1
+
+
+	#Assembling samples
+	edge_index = torch.LongTensor(np.array(ppi.edges()).transpose())
+	edge_index = add_remaining_self_loops(edge_index)[0]
+	edge_index = to_undirected(edge_index)
+	number_of_nodes = ppi.number_of_nodes()
+
+	#Pre-compute distances from each node to each of the nodes mutated and save in a dictionary
+	
+	G = nx.Graph() 
+	G.add_edges_from(tuple(zip(edge_index[0,:].tolist(), edge_index[1,:].tolist())))
+	# dict_node_mutation_spl = dict()
+	# for node_index in range(G.number_of_nodes()):
+	# 	for mutation in cosmic_mutations:
+	# 		spl = nx.shortest_path_length(G, node_index, mutation)
+	# 		dict_node_mutation_spl[(node_index, mutation)] = spl
+
+
+
+
+	#remove incoming edges to perturbed nodes (mutated nodes)
+	# mask = [e not in cosmic_mutations for e in edge_index[1,:]]
+	# edge_index_mutilated = edge_index[:, mask]
+	# edge_index_mutilated = add_remaining_self_loops(edge_index_mutilated)[0]
+	
+	dict_forward_sample_and_mutations = dict() #saves the mutation vector used in forward
+	forward_data_list = []
+	if healthy_data is not None: #Only process forward list if we have healthy data
+		#FORWARD DATA - healthy_data, cosmic_vector, obs_data
+			#each Data object will be a pairing of a random healthy_data column, the cosmic mutations, and a obs_data column
+			#will have as many as obs_data columns
+		i = 0
+		order = np.array(range(healthy_data.shape[1]))
+		np.random.shuffle(order)
+		
+		for sample_id in obs_data.columns:
+			#sample a random healthy GE vector
+			i = i % healthy_data.shape[1]
+			sample_index = order[i]
+			healthy_sample = healthy_data[healthy_data.columns[i]].values
+			healthy = torch.Tensor(healthy_sample)
+			#mutation
+			#randomize mutations. First select the percentage of mutations to include, then select the mutations
+		
+			perc_to_include = np.random.choice([0.25, 0.50, 0.75, 1], 1).item()
+			if int_metadata['cell_mfc_name'][0].split('.')[0] == 'PC3' or int_metadata['cell_mfc_name'][0].split('.')[0] == 'VCAP':
+				perc_to_include = 1
+			
+			cosmic_mutations_i = np.random.choice(cosmic_mutations, int(len(cosmic_mutations)* perc_to_include))
+			cosmic_vector = np.zeros(len(healthy_data))
+			cosmic_vector[cosmic_mutations_i] = 1
+			mutations = torch.Tensor(cosmic_vector)
+			#diseased
+			diseased = torch.Tensor(obs_data[sample_id])
+			#additional features
+			mutation_gene_indices = torch.where(mutations)[0].tolist()
+			# additional_features = []
+			# for node_index in range(len(healthy)):
+			# 	spls = [dict_node_mutation_spl[(node_index, e)] for e in mutation_gene_indices]
+			# 	additional_features.append(torch.Tensor([np.min(spls), np.max(spls), np.mean(spls)]))
+			data = Data(healthy = healthy, mutations=mutations, diseased=diseased, gene_symbols = gene_info['gene_symbol'].tolist())
+			data.num_nodes = number_of_nodes
+			forward_data_list.append(data)
+			#Save 
+			i +=1
+			dict_forward_sample_and_mutations[sample_id] = mutations
+
+		print('finished data forward')
+
+
+
+
+
+
+
+	#BACKWARD DATA - obs_data, int_data
+
+
+	#dict sample id: perturbed gene ordered index
+	
+	dict_sample_id_perturbed_gene_ordered_index = dict()
+	for sample_id, gene_symbols in zip(int_metadata['sample_id'], int_metadata['target_names']):
+		dict_sample_id_perturbed_gene_ordered_index[sample_id] = []
+		for gene in gene_symbols.replace("[\'", "").replace("\']", "").replace(' ','').replace('\'','').split(','):
+			if gene in gene_symbol_to_index:
+				 dict_sample_id_perturbed_gene_ordered_index[sample_id].append(gene_index_to_ordered_index[gene_symbol_to_index[gene]])
+		if len(dict_sample_id_perturbed_gene_ordered_index[sample_id])==0:
+			print(sample_id)
+
+	
+	#Fill dictionary with remaining spls
+	# perturbations = list(dict_sample_id_perturbed_gene_ordered_index.values())
+	# perturbations = [item for sublist in perturbations for item in sublist]
+	# perturbations = list(set(perturbations))
+	# dict_node_mutation_spl = dict()
+	# for node_index in range(G.number_of_nodes()):
+	# 	for mutation in perturbations:
+	# 		if (node_index, mutation) in dict_node_mutation_spl:
+	# 			continue
+	# 		else:
+	# 			spl = nx.shortest_path_length(G, node_index, mutation)
+	# 			dict_node_mutation_spl[(node_index, mutation)] = spl
+
+
+		#these are helpers to sample from obs_data
+	order = np.array(range(obs_data.shape[1]))
+	np.random.shuffle(order)
+	i = 0
+	#shuffle obs data columns
+	backward_data_list = []
+	unique_names_pert = set()
+	for sample_id in int_data.columns:
+		binary_indicator_perturbation = np.zeros(len(int_data))
+		binary_indicator_perturbation[dict_sample_id_perturbed_gene_ordered_index[sample_id]] = 1
+		#Get a random pre-intervention sample
+		i = i % obs_data.shape[1]
+		sample_index = order[i]
+		obs_sample_id = obs_data.columns[i]
+		obs_sample = obs_data[obs_data.columns[i]].values
+		#concat initial node features and perturbation indicator
+		diseased = torch.Tensor(obs_sample)
+		intervention = torch.Tensor(binary_indicator_perturbation)
+		if healthy_data is not None:
+			mutations = dict_forward_sample_and_mutations[obs_sample_id]
+		else:
+			mutations = torch.Tensor(np.zeros(len(diseased)))
+		# torch.Tensor(np.stack([obs_sample, binary_indicator_perturbation], 1))
+		#post-intervention
+		treated = torch.Tensor(int_data[sample_id])
+		#remove incoming edges to perturbed node
+		# perturbed_node = dict_sample_id_perturbed_gene_ordered_index[sample_id]
+		# edge_index_mutilated = edge_index[:, edge_index[1,:] != perturbed_node]
+		#additional features
+		# additional_features = []
+		# for node_index in range(len(diseased)):
+		# 	spls = [dict_node_mutation_spl[(node_index, e)] for e in dict_sample_id_perturbed_gene_ordered_index[sample_id]]
+		# 	additional_features.append(torch.Tensor([np.min(spls), np.max(spls), np.mean(spls)]))
+		drug_name = int_metadata[int_metadata['sample_id'] == sample_id]['cmap_name'].item()
+		unique_names_pert.add(drug_name)
+		data = Data(perturbagen_name = drug_name, diseased = diseased, intervention=intervention, treated = treated, gene_symbols = gene_info['gene_symbol'].tolist(), mutations = mutations)
+		data.num_nodes = number_of_nodes
+		backward_data_list.append(data)
+		i +=1
+
+	log_handle.write('Samples forward:\t{}\n'.format(len(forward_data_list)))
+	log_handle.write('Samples backward:\t{}\n'.format(len(backward_data_list)))
+	log_handle.write('Unique perturbagens:\t{}\n'.format(len(unique_names_pert)))
+	return forward_data_list, backward_data_list, edge_index
+
+
+
+
+def save_data(forward_data_list, backward_data_list, edge_index, cell_line, log_handle):
+	log_handle.write('Saving data {} ...\n\n\n'.format(cell_line))
+	torch.save(forward_data_list, osp.join(outdir, 'data_forward_{}.pt'.format(cell_line)))
+	torch.save(backward_data_list, osp.join(outdir, 'data_backward_{}.pt'.format(cell_line)))
+	torch.save(edge_index, osp.join(outdir, 'edge_index_{}.pt'.format(cell_line)))
+	return
+
+
+binarization = 'real_lognorm'
+outdir = '../../processed/torch_data/chemical/{}'.format(binarization)
+os.makedirs(outdir, exist_ok=True)
+
+
+
+
+def main():  
+	#cell-line wise
+	log_handle = open(osp.join(outdir, 'log_export_data.txt'), 'w')
+	data_root_dir = '../../processed/lincs/chemical/nofilter_dose_timepoint/{}'.format(binarization)
+	data_root_dir_healthy = '../../processed/lincs/chemical/{}'.format(binarization)
+	# , ('RWPE1', 'ctl_vector')
+
+	#Cell lines with healthy counterparts
+	for cell_line, healthy in zip(['A549', 'MCF7', 'PC3', 'VCAP', 'MDAMB231', 'BT20'], [('NL20', 'ctl_vehicle'), ('MCF10A', 'ctl_vehicle'), ('RWPE1', 'ctl_vector'), ('RWPE1', 'ctl_vector'), ('MCF10A', 'ctl_vehicle'), ('MCF10A', 'ctl_vehicle')]):
+		log_handle.write('----------------\n\nCELL LINE:{}\n----------------\n'.format(cell_line))
+		#PPI
+		ppi = load_ppi('../../processed/ppi/ppi_all_genes_edgelist.txt', log_handle)
+		#gene info
+		gene_info, dict_entrez_symbol, dict_symbol_entrez = load_gene_metadata('../../processed/lincs/chemical/nofilter_dose_timepoint/gene_info.txt', log_handle)
+		#FORWARD DATA
+			#healthy GE data
+		healthy_data, healthy_metadata = load_healthy_data(data_root_dir_healthy, healthy, log_handle)
+			#COSMIC
+		cosmic_data = load_cosmic('../../processed/cosmic/CosmicCLP_MutantExport_only_verified_and_curated.csv', log_handle)
+		cosmic_mutations = map_cosmic_to_lincs(cosmic_data, cell_line, gene_info, dict_symbol_entrez, log_handle)
+		#BACKWARD DATA
+			#LINCS
+		obs_metadata, obs_data, int_metadata, int_data = load_data(cell_line, data_root_dir, log_handle)
+		healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, gene_info = filter_data(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+
+
+		forward_data_list, backward_data_list, edge_index = assemble_data_list(healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+		# save_data(forward_data_list, backward_data_list, edge_index, cell_line, log_handle)
+
+
+	#Cell lines without healthy counterparts
+	for cell_line, healthy in zip(['HA1E', 'HT29', 'A375', 'HELA'], [None, None, None, None]):
+		log_handle.write('----------------\n\nCELL LINE:{}\n----------------\n'.format(cell_line))
+		#PPI
+		ppi = load_ppi('../../processed/ppi/ppi_all_genes_edgelist.txt', log_handle)
+		#gene info
+		gene_info, dict_entrez_symbol, dict_symbol_entrez = load_gene_metadata('../../processed/lincs/chemical/nofilter_dose_timepoint/gene_info.txt', log_handle)
+		#FORWARD DATA
+			#healthy GE data
+		# healthy_data, healthy_metadata = load_healthy_data(data_root_dir_healthy, healthy, log_handle)
+			#COSMIC
+		cosmic_data = load_cosmic('../../processed/cosmic/CosmicCLP_MutantExport_only_verified_and_curated.csv', log_handle)
+		cosmic_mutations = map_cosmic_to_lincs(cosmic_data, cell_line, gene_info, dict_symbol_entrez, log_handle)
+		#BACKWARD DATA
+			#LINCS
+		obs_metadata, obs_data, int_metadata, int_data = load_data(cell_line, data_root_dir, log_handle)
+		healthy_data, healthy_metadata, cosmic_mutations, obs_metadata, obs_data, int_metadata, int_data, gene_info = filter_data(None, None, None, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+
+
+		forward_data_list, backward_data_list, edge_index = assemble_data_list(None, None, None, obs_metadata, obs_data, int_metadata, int_data, ppi, gene_info, log_handle)
+		# save_data(forward_data_list, backward_data_list, edge_index, cell_line, log_handle)
+
+	log_handle.close()
+
+
+
+
+
+
+
+if __name__ == "__main__":
+    main()

PDGrapher/data/scripts/splits/create_standard_splits.py

@@ -0,0 +1,97 @@
+##Create splits to be used for training across all models
+
+import os
+import os.path as osp
+from sklearn.model_selection import KFold, train_test_split
+import torch
+from sklearn.metrics import jaccard_score
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+
+def create_splits_n_fold(dataset, splits_type, nfolds, outdir):
+	kf = KFold(nfolds, shuffle=True, random_state=42)
+	os.makedirs(outdir, exist_ok = True)
+
+	#datasets forward and backward
+	dataset_forward = dataset[0]; dataset_backward = dataset[1]
+	splits = {}
+
+
+	if splits_type =='random':
+		i = 1
+		if len(dataset_forward)> 0:
+			for train_test_index_forward, train_test_index_backward in zip(kf.split(dataset_forward), kf.split(dataset_backward)):
+				#Forward
+				train_index_forward = train_test_index_forward[0]; test_index_forward = train_test_index_forward[1]
+				train_index_backward = train_test_index_backward[0]; test_index_backward = train_test_index_backward[1]
+				#Backward
+				train_index_forward, val_index_forward = train_test_split(train_index_forward, test_size=0.2, random_state=42)
+				train_index_backward, val_index_backward = train_test_split(train_index_backward, test_size=0.2, random_state=42)
+
+				assert len(dataset_backward) == len(train_index_backward) +  len(val_index_backward) +  len(test_index_backward), 'Splitted datasets should have the same number of samples as full dataset'
+				assert  len(set(test_index_forward).intersection(train_index_forward)) ==0, "Overlap between train and test indices should be zero"
+				assert  len(set(test_index_forward).intersection(val_index_forward)) ==0, "Overlap between val and test indices should be zero"
+				assert  len(set(train_index_forward).intersection(val_index_forward)) ==0, "Overlap between train and val indices should be zero"
+				assert  len(set(test_index_backward).intersection(train_index_backward)) ==0, "Overlap between train and test indices should be zero"
+				assert  len(set(test_index_backward).intersection(val_index_backward)) ==0, "Overlap between val and test indices should be zero"
+				assert  len(set(train_index_backward).intersection(val_index_backward)) ==0, "Overlap between train and val indices should be zero"
+
+				splits[i] = {'train_index_forward': train_index_forward, 
+								'val_index_forward': val_index_forward,
+								'test_index_forward': test_index_forward,
+								'train_index_backward': train_index_backward,
+								'val_index_backward': val_index_backward,
+								'test_index_backward': test_index_backward}
+				i += 1
+		else:
+			for train_test_index_backward in kf.split(dataset_backward):
+				#Backward
+				train_index_backward = train_test_index_backward[0]; test_index_backward = train_test_index_backward[1]
+				train_index_backward, val_index_backward = train_test_split(train_index_backward, test_size=0.2, random_state=42)
+
+				assert len(dataset_backward) == len(train_index_backward) +  len(val_index_backward) +  len(test_index_backward), 'Splitted datasets should have the same number of samples as full dataset'
+				assert  len(set(test_index_backward).intersection(train_index_backward)) ==0, "Overlap between train and test indices should be zero"
+				assert  len(set(test_index_backward).intersection(val_index_backward)) ==0, "Overlap between val and test indices should be zero"
+				assert  len(set(train_index_backward).intersection(val_index_backward)) ==0, "Overlap between train and val indices should be zero"
+
+				splits[i] = {'train_index_forward': None, 
+								'val_index_forward': None,
+								'test_index_forward': None,
+								'train_index_backward': train_index_backward,
+								'val_index_backward': val_index_backward,
+								'test_index_backward': test_index_backward}
+				i += 1
+		
+	torch.save(splits, osp.join(outdir,'splits.pt'))
+	return
+
+
+
+
+###Generate splits
+os.makedirs('../../processed/splits/', exist_ok=True)
+nfolds = 5
+for dataset_type in ['genetic', 'chemical']:
+	for dataset_name in ['A375', 'A549', 'MCF7', 'PC3', 'HT29', 'ES2', 'BICR6', 'YAPC', 'AGS', 'U251MG', 'VCAP', 'MDAMB231', 'BT20', 'HA1E', 'HELA']:
+		for splits_type in ['random']:
+			splits_setting = '{}fold'.format(nfolds)
+			outdir = '../../processed/splits/{}/{}/{}/{}'.format(dataset_type, dataset_name, splits_type, splits_setting)
+
+			if dataset_type == 'genetic':
+				base_path = "../../processed/torch_data/real_lognorm"
+			elif dataset_type =='chemical':
+				base_path = "../../processed/torch_data/chemical/real_lognorm"
+
+			try:
+				path = osp.join(base_path, 'data_forward_{}.pt'.format(dataset_name))
+				dataset_forward = torch.load(path)
+				path = osp.join(base_path, 'data_backward_{}.pt'.format(dataset_name))
+				dataset_backward = torch.load(path)
+				dataset = [dataset_forward, dataset_backward]
+
+				create_splits_n_fold(dataset, splits_type, nfolds, outdir)
+			except:
+				continue