Bug fixes in stats and added gene dict

.gitattributes

@@ -14,11 +14,10 @@
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text 
 *.tar.* filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
@@ -26,4 +25,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zstandard filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-model.safetensors filter=lfs diff=lfs merge=lfs -text
+geneformer/gene_name_id_dict.pkl filter=lfs diff=lfs merge=lfs -text

.pre-commit-config.yaml

@@ -1,26 +0,0 @@
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-yaml
-    -   id: check-added-large-files
-    -   id: check-merge-conflict
-    -   id: mixed-line-ending
-    -   id: check-docstring-first
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-    -   id: isort
-        args: ["--profile", "black"]
--   repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.1.4
-    hooks:
-    # Run the Ruff linter.
-    -   id: ruff
-    # Run the Ruff formatter.
-    -   id: ruff-format

.readthedocs.yaml

@@ -1,19 +0,0 @@
-# Read the Docs configuration file
-
-# Required
-version: 2
-
-# Set the OS, Python version and other tools you might need
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.10"
-
-# Build documentation in the "docs/" directory with Sphinx
-sphinx:
-  configuration: docs/source/conf.py
-
-# Python requirements required build your documentation
-python:
-   install:
-   - requirements: docs/requirements.txt

Geneformer-V1-10M/config.json

@@ -1,23 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.02,
-  "gradient_checkpointing": false,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.02,
-  "hidden_size": 256,
-  "initializer_range": 0.02,
-  "intermediate_size": 512,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 2048,
-  "model_type": "bert",
-  "num_attention_heads": 4,
-  "num_hidden_layers": 6,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "transformers_version": "4.6.0",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 25426
-}

Geneformer-V1-10M/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a5e33a757431643b3697de7ef6127950cdc49e06e58d4266b3a3ab191b683f14
-size 41183536

Geneformer-V1-10M/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f0ec3459454205174c9d2e4d6c6930f6b0fbf3364fc03a6f4d99c4d3add2012b
-size 2607

Geneformer-V2-104M/config.json

@@ -1,24 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.44.2",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 20275
-}

Geneformer-V2-104M/generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.44.2"
-}

Geneformer-V2-104M/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fff5cba29ddd8792991fa77b4872246fbe548a178cebda3775cdc72b67780e7f
-size 417571156

Geneformer-V2-104M/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0d8ddd9e4f35b5fe23a3adaae03aa4480705ca82eed546a488f970adb3752d9d
-size 5496

Geneformer-V2-104M_CLcancer/config.json

@@ -1,25 +0,0 @@
-{
-  "_name_or_path": "/gladstone/theodoris/lab/ctheodoris/gf-104m/models/241127_143148_geneformer_94M_L12_emb768_SL4096_E3_B18_LR0.0002_LScosine_WR0.007_Oadamw_DS13/models",
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
-  "initializer_range": 0.02,
-  "intermediate_size": 3072,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
-  "model_type": "bert",
-  "num_attention_heads": 12,
-  "num_hidden_layers": 12,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.37.1",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 20275
-}

Geneformer-V2-104M_CLcancer/generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.37.1"
-}

Geneformer-V2-104M_CLcancer/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:827738139bfed4bafa9d1f3df7c6146da2e3b85f7225076adc32c6eda0ba4357
-size 417571156

Geneformer-V2-104M_CLcancer/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8cf8ce52b498253adc6df53197a99821fa145c19b8ae5eeb8d15be76b8b7ddb3
-size 4984

Geneformer-V2-316M/config.json

@@ -1,24 +0,0 @@
-{
-  "architectures": [
-    "BertForMaskedLM"
-  ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "hidden_act": "relu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 1152,
-  "initializer_range": 0.02,
-  "intermediate_size": 4608,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
-  "model_type": "bert",
-  "num_attention_heads": 18,
-  "num_hidden_layers": 18,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.44.2",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 20275
-}

Geneformer-V2-316M/generation_config.json

@@ -1,5 +0,0 @@
-{
-  "_from_model_config": true,
-  "pad_token_id": 0,
-  "transformers_version": "4.44.2"
-}

Geneformer-V2-316M/model.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:965ceccea81953d362081ef3843560a0e4fef88d396c28017881f1e94b1246f3
-size 1265455076

Geneformer-V2-316M/training_args.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e45150f9a4ca34cb4e91ce79f65f3d99d9d66df9f66a37517a352d291008e0b8
-size 5432

MANIFEST.in

@@ -1,9 +1,2 @@
-include geneformer/gene_median_dictionary_gc104M.pkl
-include geneformer/gene_name_id_dict_gc104M.pkl
-include geneformer/ensembl_mapping_dict_gc104M.pkl
-include geneformer/token_dictionary_gc104M.pkl
-
-include geneformer/gene_dictionaries_30m/gene_median_dictionary_gc30M.pkl
-include geneformer/gene_dictionaries_30m/gene_name_id_dict_gc30M.pkl
-include geneformer/gene_dictionaries_30m/ensembl_mapping_dict_gc30M.pkl
-include geneformer/gene_dictionaries_30m/token_dictionary_gc30M.pkl
+include geneformer/gene_median_dictionary.pkl
+include geneformer/token_dictionary.pkl

README.md

@@ -1,91 +1,30 @@
 ---
 datasets: ctheodoris/Genecorpus-30M
-license: apache-2.0
-tags:
-- single-cell
-- genomics
 ---
 # Geneformer
-Geneformer is a foundational transformer model pretrained on a large-scale corpus of single cell transcriptomes to enable context-aware predictions in settings with limited data in network biology.
+Geneformer is a foundation transformer model pretrained on a large-scale corpus of ~30 million single cell transcriptomes to enable context-aware predictions in settings with limited data in network biology. 
 
-- See [our manuscript](https://rdcu.be/ddrx0) for details of the original model trained on ~30 million transcriptomes in June 2021 and the initial report of our in silico perturbation and cell and gene classification strategies.
-- See [our manuscript](https://www.biorxiv.org/content/10.1101/2024.08.16.608180v1.full.pdf) for details of the expanded model, now trained on ~104 million transcriptomes, and our continual learning, multitask learning, and quantization strategies.
-- See [geneformer.readthedocs.io](https://geneformer.readthedocs.io) for documentation.
+See [our manuscript](https://www.nature.com/articles/s41586-023-06139-9) for details.
 
 # Model Description
-Geneformer is a foundational transformer model pretrained on a large-scale corpus of single cell transcriptomes representing a broad range of human tissues. Geneformer V1 was originally pretrained in June 2021 on [Genecorpus-30M](https://huggingface.co/datasets/ctheodoris/Genecorpus-30M), a corpus comprised of ~30 million human single cell transcriptomes. We excluded cells with high mutational burdens (e.g. malignant cells and immortalized cell lines) that could lead to substantial network rewiring without companion genome sequencing to facilitate interpretation. The current updated Geneformer V2 is pretrained on ~104 million human single cell transcriptomes (non-cancer). The cancer continual learning V2 variant was continually pretrained on ~14 million cancer transcriptomes to yield a cancer domain-tuned model.
+Geneformer is a foundation transformer model pretrained on [Genecorpus-30M](https://huggingface.co/datasets/ctheodoris/Genecorpus-30M), a pretraining corpus comprised of ~30 million single cell transcriptomes from a broad range of human tissues. Each single cell’s transcriptome is presented to the model as a rank value encoding where genes are ranked by their expression in that cell normalized by their expression across the entire Genecorpus-30M. The rank value encoding provides a nonparametric representation of that cell’s transcriptome and takes advantage of the many observations of each gene’s expression across Genecorpus-30M to prioritize genes that distinguish cell state. Specifically, this method will deprioritize ubiquitously highly-expressed housekeeping genes by normalizing them to a lower rank. Conversely, genes such as transcription factors that may be lowly expressed when they are expressed but highly distinguish cell state will move to a higher rank within the encoding. Furthermore, this rank-based approach may be more robust against technical artifacts that may systematically bias the absolute transcript counts value while the overall relative ranking of genes within each cell remains more stable. 
 
-Each single cell’s transcriptome is presented to the model as a rank value encoding where genes are ranked by their expression in that cell scaled by their expression across the entire Genecorpus (~30M for V1, ~104M for V2). The rank value encoding provides a nonparametric representation of that cell’s transcriptome and takes advantage of the many observations of each gene’s expression across the pretraining corpus to prioritize genes that distinguish cell state. Specifically, this method will deprioritize ubiquitously highly-expressed housekeeping genes by scaling them to a lower rank. Conversely, genes such as transcription factors that may be lowly expressed when they are expressed but highly distinguish cell state will move to a higher rank within the encoding. Furthermore, this rank-based approach may be more robust against technical artifacts that may systematically bias the absolute transcript counts value while the overall relative ranking of genes within each cell remains more stable.
+The rank value encoding of each single cell’s transcriptome then proceeds through six transformer encoder units. Pretraining was accomplished using a masked learning objective where 15% of the genes within each transcriptome were masked and the model was trained to predict which gene should be within each masked position in that specific cell state using the context of the remaining unmasked genes. A major strength of this approach is that it is entirely self-supervised and can be accomplished on completely unlabeled data, which allows the inclusion of large amounts of training data without being restricted to samples with accompanying labels.
 
-The rank value encoding of each single cell’s transcriptome then proceeds through N layers of transformer encoder units, where N varies dependent on the model size. Pretraining was accomplished using a masked learning objective where 15% of the genes within each transcriptome were masked and the model was trained to predict which gene should be within each masked position in that specific cell state using the context of the remaining unmasked genes. A major strength of this approach is that it is entirely self-supervised and can be accomplished on completely unlabeled data, which allows the inclusion of large amounts of training data without being restricted to samples with accompanying labels.
+We detail applications and results in [our manuscript](https://www.nature.com/articles/s41586-023-06139-9). 
 
-We detail applications and results in [our manuscript](https://rdcu.be/ddrx0).
-
-During pretraining, Geneformer gained a fundamental understanding of network dynamics, encoding network hierarchy in the model’s attention weights in a completely self-supervised manner. With both zero-shot learning and fine-tuning with limited task-specific data, Geneformer consistently boosted predictive accuracy in a diverse panel of downstream tasks relevant to chromatin and network dynamics. In silico perturbation with zero-shot learning identified a novel transcription factor in cardiomyocytes that we experimentally validated to be critical to their ability to generate contractile force. In silico treatment with limited patient data revealed candidate therapeutic targets for cardiomyopathy that we experimentally validated to significantly improve the ability of cardiomyocytes to generate contractile force in an induced pluripotent stem cell (iPSC) model of the disease. Overall, Geneformer represents a foundational AI model pretrained on a large-scale corpus human single cell transcriptomes to gain a fundamental understanding of gene network dynamics that can now be democratized to a vast array of downstream tasks to accelerate discovery of key network regulators and candidate therapeutic targets.
-
-The repository includes the following pretrained models:
-
-- Geneformer-V1-10M: original model trained June 2021 on ~30M human single cell transcriptomes, 10M parameters, input size 2048, vocabulary ~25K protein-coding or non-coding RNA genes
-- Geneformer-V2-104M and Geneformer-V2-316M: updated model trained Dec 2024 on ~104M human single cell transcriptomes, 104M or 316M parameters, input size 4096, vocabulary ~20K protein-coding genes
-
-The current default model in the main directory of the repository is Geneformer-V2-316M.
-
-The repository also contains fined tuned models in the fine_tuned_models directory and the cancer-tuned model following continual learning on ~14 million cancer cells, Geneformer-V2-104M_CLcancer.
+During pretraining, Geneformer gained a fundamental understanding of network dynamics, encoding network hierarchy in the model’s attention weights in a completely self-supervised manner. Fine-tuning Geneformer towards a diverse panel of downstream tasks relevant to chromatin and network dynamics using limited task-specific data demonstrated that Geneformer consistently boosted predictive accuracy. Applied to disease modeling with limited patient data, Geneformer identified candidate therapeutic targets. Overall, Geneformer represents an invaluable pretrained model from which fine-tuning towards a broad range of downstream applications can be pursued to accelerate discovery of key network regulators and candidate therapeutic targets.
 
 # Application
 The pretrained Geneformer model can be used directly for zero-shot learning, for example for in silico perturbation analysis, or by fine-tuning towards the relevant downstream task, such as gene or cell state classification.
 
-Example applications demonstrated in [our manuscript](https://rdcu.be/ddrx0) include:
-
-*Fine-tuning*:
-- transcription factor dosage sensitivity
-- chromatin dynamics (bivalently marked promoters)
-- transcription factor regulatory range
-- gene network centrality
-- transcription factor targets
-- cell type annotation
-- batch integration
-- cell state classification across differentiation
-- disease classification
-- in silico perturbation to determine disease-driving genes
-- in silico treatment to determine candidate therapeutic targets
-
-*Zero-shot learning*:
-- batch integration
-- gene context specificity
-- in silico reprogramming
-- in silico differentiation
-- in silico perturbation to determine impact on cell state
-- in silico perturbation to determine transcription factor targets
-- in silico perturbation to determine transcription factor cooperativity
-
 # Installation
-In addition to the pretrained model, contained herein are functions for tokenizing and collating data specific to single cell transcriptomics, pretraining the model, fine-tuning the model, extracting and plotting cell embeddings, and performing in silico pertrubation with either the pretrained or fine-tuned models. To install (~20s):
+In addition to the pretrained model, contained herein are functions for tokenizing and collating data specific to single cell transcriptomics, pretraining the model, and performing in silico pertrubation with either the pretrained or fine-tuned models. To install:
 
 ```bash
-# Make sure you have git-lfs installed (https://git-lfs.com)
-git lfs install
 git clone https://huggingface.co/ctheodoris/Geneformer
 cd Geneformer
 pip install .
 ```
 
-For usage, see [examples](https://huggingface.co/ctheodoris/Geneformer/tree/main/examples) for:
-- tokenizing transcriptomes
-- pretraining
-- hyperparameter tuning
-- fine-tuning
-- extracting and plotting cell embeddings
-- in silico perturbation
-
-Please also see [here](https://tinyurl.com/geneformertutorial) for a quickstart tutorial for predicting candidate therapeutic targets with Geneformer.
-
-Complete documentation is available at https://geneformer.readthedocs.io/en/latest/.
-
-Please note that the fine-tuning examples are meant to be generally applicable and the input datasets and labels will vary dependent on the downstream task. Example input files for a few of the downstream tasks demonstrated in the manuscript are located within the [example_input_files directory](https://huggingface.co/datasets/ctheodoris/Genecorpus-30M/tree/main/example_input_files) in the dataset repository, but these only represent a few example fine-tuning applications.
-
-Please note that GPU resources are required for efficient usage of Geneformer. Additionally, we strongly recommend tuning hyperparameters for each downstream fine-tuning application as this can significantly boost predictive potential in the downstream task (e.g. max learning rate, learning schedule, number of layers to freeze, etc.). Importantly, as usual for deep learning models, there are no uniformly applicable default hyperparameters for Geneformer.
-
-# Citations
-- C V Theodoris#, L Xiao, A Chopra, M D Chaffin, Z R Al Sayed, M C Hill, H Mantineo, E Brydon, Z Zeng, X S Liu, P T Ellinor#. Transfer learning enables predictions in network biology. _**Nature**_, 31 May 2023. (#co-corresponding authors)
-- H Chen*, M S Venkatesh*, J Gomez Ortega, S V Mahesh, T Nandi, R Madduri, K Pelka†, C V Theodoris†#. Quantized multi-task learning for context-specific representations of gene network dynamics. _**bioRxiv**_, 19 Aug 2024. (*co-first authors, †co-senior authors, #corresponding author)
+For usage, see [examples](https://huggingface.co/ctheodoris/Geneformer/tree/main/examples) for pretraining and fine-tuning. Please note that GPU resources are required for efficient usage of Geneformer.

benchmarking/castle_cell_type_annotation.r

@@ -0,0 +1,80 @@
+# Usage: Rscript castle_cell_type_annotation.r organ
+
+# parse ordered arguments
+args <- commandArgs(trailingOnly=TRUE)
+organ <- args[1]
+
+suppressPackageStartupMessages(library(scater))
+suppressPackageStartupMessages(library(xgboost))
+suppressPackageStartupMessages(library(igraph))
+BREAKS=c(-1, 0, 1, 6, Inf)
+nFeatures = 100
+
+print(paste("Training ", organ, sep=""))
+
+# import training and test data
+rootdir="/path/to/data/"
+train_counts <- t(as.matrix(read.csv(file = paste(rootdir, organ, "_filtered_data_train.csv", sep=""), row.names = 1)))
+test_counts <- t(as.matrix(read.csv(file = paste(rootdir, organ, "_filtered_data_test.csv", sep=""), row.names = 1)))
+train_celltype <- as.matrix(read.csv(file = paste(rootdir, organ, "_filtered_celltype_train.csv", sep="")))
+test_celltype <- as.matrix(read.csv(file = paste(rootdir, organ, "_filtered_celltype_test.csv", sep="")))
+
+# select features
+sourceCellTypes = as.factor(train_celltype[,"Cell_type"])
+ds = rbind(train_counts,test_counts)
+ds[is.na(ds)] <- 0
+isSource = c(rep(TRUE,nrow(train_counts)), rep(FALSE,nrow(test_counts)))
+topFeaturesAvg = colnames(ds[isSource,])[order(apply(ds[isSource,], 2, mean), decreasing = T)]
+topFeaturesMi = names(sort(apply(ds[isSource,],2,function(x) { compare(cut(x,breaks=BREAKS),sourceCellTypes,method = "nmi") }), decreasing = T))
+selectedFeatures = union(head(topFeaturesAvg, nFeatures) , head(topFeaturesMi, nFeatures) )
+tmp = cor(ds[isSource,selectedFeatures], method = "pearson")
+tmp[!lower.tri(tmp)] = 0
+selectedFeatures = selectedFeatures[apply(tmp,2,function(x) any(x < 0.9))]
+remove(tmp)
+
+# bin expression values and expand features by bins
+dsBins = apply(ds[, selectedFeatures], 2, cut, breaks= BREAKS)
+nUniq = apply(dsBins, 2, function(x) { length(unique(x)) })
+ds = model.matrix(~ . , as.data.frame(dsBins[,nUniq>1]))
+remove(dsBins, nUniq)
+
+# train model
+train = runif(nrow(ds[isSource,]))<0.8
+# slightly different setup for multiclass and binary classification
+if (length(unique(sourceCellTypes)) > 2) {
+  xg=xgboost(data=ds[isSource,][train, ] ,
+       label=as.numeric(sourceCellTypes[train])-1,
+       objective="multi:softmax", num_class=length(unique(sourceCellTypes)),
+       eta=0.7 , nthread=5, nround=20, verbose=0,
+       gamma=0.001, max_depth=5, min_child_weight=10)
+} else {
+  xg=xgboost(data=ds[isSource,][train, ] ,
+       label=as.numeric(sourceCellTypes[train])-1,
+       eta=0.7 , nthread=5, nround=20, verbose=0,
+       gamma=0.001, max_depth=5, min_child_weight=10)
+}
+
+# validate model
+predictedClasses = predict(xg, ds[!isSource, ])
+testCellTypes = as.factor(test_celltype[,"Cell_type"])
+trueClasses <- as.numeric(testCellTypes)-1
+
+cm <- as.matrix(table(Actual = trueClasses, Predicted = predictedClasses))
+n <- sum(cm)
+nc = nrow(cm) # number of classes
+diag = diag(cm) # number of correctly classified instances per class
+rowsums = apply(cm, 1, sum) # number of instances per class
+colsums = apply(cm, 2, sum) # number of predictions per class
+p = rowsums / n # distribution of instances over the actual classes
+q = colsums / n # distribution of instances over the predicted classes
+accuracy = sum(diag) / n
+precision = diag / colsums
+recall = diag / rowsums
+f1 = 2 * precision * recall / (precision + recall)
+macroF1 = mean(f1)
+
+print(paste(organ, " accuracy: ", accuracy, sep=""))
+print(paste(organ, " macroF1: ", macroF1, sep=""))
+
+results_df = data.frame(Accuracy=c(accuracy),macroF1=c(macroF1))
+write.csv(results_df,paste(rootdir, organ, "_castle_results_test.csv", sep=""), row.names = FALSE)

benchmarking/prepare_datasplits_for_cell_type_annotation.ipynb

@@ -0,0 +1,288 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "25107132",
+   "metadata": {},
+   "source": [
+    "### Preparing train and test data splits for cell type annotation application"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "83d8d249-affe-45dd-915e-992b4b35b31a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import pandas as pd\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "from tqdm.notebook import tqdm\n",
+    "from collections import Counter\n",
+    "import pickle"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "e3e6a2bf-44c8-4164-9ecd-1686230ea8be",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['pancreas',\n",
+       " 'liver',\n",
+       " 'blood',\n",
+       " 'lung',\n",
+       " 'spleen',\n",
+       " 'placenta',\n",
+       " 'colorectum',\n",
+       " 'kidney',\n",
+       " 'brain']"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rootdir = \"/path/to/data/\"\n",
+    "\n",
+    "# collect panel of tissues to test\n",
+    "dir_list = []\n",
+    "for dir_i in os.listdir(rootdir):\n",
+    "    if (\"results\" not in dir_i) & (os.path.isdir(os.path.join(rootdir, dir_i))):\n",
+    "        dir_list += [dir_i]\n",
+    "dir_list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "0b205eec-a518-472a-ab90-dd63ef9803cd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>filter_pass</th>\n",
+       "      <th>original_cell_id</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>C_1</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_2</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0</td>\n",
+       "      <td>C_3</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_4</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0</td>\n",
+       "      <td>C_5</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9590</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_9591</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9591</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_9592</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9592</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_9593</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9593</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_9594</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9594</th>\n",
+       "      <td>1</td>\n",
+       "      <td>C_9595</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>9595 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      filter_pass original_cell_id\n",
+       "0               0              C_1\n",
+       "1               1              C_2\n",
+       "2               0              C_3\n",
+       "3               1              C_4\n",
+       "4               0              C_5\n",
+       "...           ...              ...\n",
+       "9590            1           C_9591\n",
+       "9591            1           C_9592\n",
+       "9592            1           C_9593\n",
+       "9593            1           C_9594\n",
+       "9594            1           C_9595\n",
+       "\n",
+       "[9595 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# dictionary of cell barcodes that passed QC filtering applied by Geneformer \n",
+    "# to ensure same cells were used for comparison\n",
+    "with open(f\"{rootdir}deepsort_filter_dict.pickle\", \"rb\") as fp:\n",
+    "    filter_dict = pickle.load(fp)\n",
+    "\n",
+    "# for example:\n",
+    "filter_dict[\"human_Placenta9595_data\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "207e3571-0236-4493-83b3-a89b67b16cb2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for dir_name in tqdm(dir_list):\n",
+    "\n",
+    "    df = pd.DataFrame()\n",
+    "    ct_df = pd.DataFrame(columns=[\"Cell\",\"Cell_type\"])\n",
+    "    \n",
+    "    subrootdir = f\"{rootdir}{dir_name}/\"\n",
+    "    for subdir, dirs, files in os.walk(subrootdir):\n",
+    "        for i in range(len(files)):\n",
+    "            file = files[i]\n",
+    "            if file.endswith(\"_data.csv\"):\n",
+    "                file_prefix = file.replace(\"_data.csv\",\"\")\n",
+    "                sample_prefix = file.replace(\".csv\",\"\")\n",
+    "                filter_df = filter_dict[sample_prefix]\n",
+    "                sample_to_analyze = list(filter_df[filter_df[\"filter_pass\"]==1][\"original_cell_id\"])\n",
+    "                \n",
+    "                # collect data for each tissue\n",
+    "                df_i = pd.read_csv(f\"{subrootdir}{file}\", index_col=0)\n",
+    "                df_i = df_i[sample_to_analyze]\n",
+    "                df_i.columns = [f\"{i}_{cell_id}\" for cell_id in df_i.columns]\n",
+    "                df = pd.concat([df,df_i],axis=1)\n",
+    "                \n",
+    "                # collect cell type metadata\n",
+    "                ct_df_i = pd.read_csv(f\"{subrootdir}{file_prefix}_celltype.csv\", index_col=0)\n",
+    "                ct_df_i.columns = [\"Cell\",\"Cell_type\"]\n",
+    "                ct_df_i[\"Cell\"] = [f\"{i}_{cell_id}\" for cell_id in ct_df_i[\"Cell\"]]\n",
+    "                ct_df = pd.concat([ct_df,ct_df_i],axis=0)\n",
+    "        \n",
+    "    # per published scDeepsort method, filter data for cell types >0.5% of data\n",
+    "    ct_counts = Counter(ct_df[\"Cell_type\"])\n",
+    "    total_count = sum(ct_counts.values())\n",
+    "    nonrare_cell_types = [cell_type for cell_type,count in ct_counts.items() if count>(total_count*0.005)]\n",
+    "    nonrare_cells = list(ct_df[ct_df[\"Cell_type\"].isin(nonrare_cell_types)][\"Cell\"])\n",
+    "    df = df[df.columns.intersection(nonrare_cells)]\n",
+    "\n",
+    "    # split into 80/20 train/test data\n",
+    "    train, test = train_test_split(df.T, test_size=0.2)\n",
+    "    train = train.T\n",
+    "    test = test.T  \n",
+    "    \n",
+    "    # save filtered train/test data\n",
+    "    train.to_csv(f\"{subrootdir}{dir_name}_filtered_data_train.csv\")\n",
+    "    test.to_csv(f\"{subrootdir}{dir_name}_filtered_data_test.csv\")\n",
+    "\n",
+    "    # split metadata into train/test data\n",
+    "    ct_df_train = ct_df[ct_df[\"Cell\"].isin(list(train.columns))]\n",
+    "    ct_df_test = ct_df[ct_df[\"Cell\"].isin(list(test.columns))]\n",
+    "    train_order_dict = dict(zip(train.columns,[i for i in range(len(train.columns))]))\n",
+    "    test_order_dict = dict(zip(test.columns,[i for i in range(len(test.columns))]))\n",
+    "    ct_df_train[\"order\"] = [train_order_dict[cell_id] for cell_id in ct_df_train[\"Cell\"]]\n",
+    "    ct_df_test[\"order\"] = [test_order_dict[cell_id] for cell_id in ct_df_test[\"Cell\"]]\n",
+    "    ct_df_train = ct_df_train.sort_values(\"order\")\n",
+    "    ct_df_test = ct_df_test.sort_values(\"order\")\n",
+    "    ct_df_train = ct_df_train.drop(\"order\",axis=1)\n",
+    "    ct_df_test = ct_df_test.drop(\"order\",axis=1)\n",
+    "    assert list(ct_df_train[\"Cell\"]) == list(train.columns)\n",
+    "    assert list(ct_df_test[\"Cell\"]) == list(test.columns)\n",
+    "    train_labels = list(Counter(ct_df_train[\"Cell_type\"]).keys())\n",
+    "    test_labels = list(Counter(ct_df_test[\"Cell_type\"]).keys())\n",
+    "    assert set(train_labels) == set(test_labels)\n",
+    "    \n",
+    "    # save train/test cell type annotations\n",
+    "    ct_df_train.to_csv(f\"{subrootdir}{dir_name}_filtered_celltype_train.csv\")\n",
+    "    ct_df_test.to_csv(f\"{subrootdir}{dir_name}_filtered_celltype_test.csv\")\n",
+    "                "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.8.6 64-bit ('3.8.6')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.6"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "eba1599a1f7e611c14c87ccff6793920aa63510b01fc0e229d6dd014149b8829"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

benchmarking/scDeepsort_train_predict.ipynb

@@ -0,0 +1,166 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "83d8d249-affe-45dd-915e-992b4b35b31a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "import deepsort\n",
+    "from sklearn.metrics import accuracy_score, f1_score\n",
+    "from tqdm.notebook import tqdm\n",
+    "import pickle"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "25de46ec-8a41-484d-8e14-d2b19768fc2c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def compute_metrics(labels, preds):\n",
+    "\n",
+    "    # calculate accuracy and macro f1 using sklearn's function\n",
+    "    acc = accuracy_score(labels, preds)\n",
+    "    macro_f1 = f1_score(labels, preds, average='macro')\n",
+    "    return {\n",
+    "      'accuracy': acc,\n",
+    "      'macro_f1': macro_f1\n",
+    "    }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a4029b2b-afca-4300-82a2-082fec59f191",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['pancreas',\n",
+       " 'liver',\n",
+       " 'blood',\n",
+       " 'lung',\n",
+       " 'spleen',\n",
+       " 'placenta',\n",
+       " 'colorectum',\n",
+       " 'kidney',\n",
+       " 'brain']"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "rootdir = \"/path/to/data/\"\n",
+    "\n",
+    "dir_list = []\n",
+    "for dir_i in os.listdir(rootdir):\n",
+    "    if (\"results\" not in dir_i) & (os.path.isdir(os.path.join(rootdir, dir_i))):\n",
+    "        dir_list += [dir_i]\n",
+    "dir_list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ddcdc5cd-871e-4fd2-8457-18d3049fa76c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "output_dir = \"results_EDefault_filtered\"\n",
+    "n_epochs = \"Default\"  # scDeepsort default epochs = 300\n",
+    "\n",
+    "results_dict = dict()\n",
+    "for dir_name in tqdm(dir_list):\n",
+    "    print(f\"TRAINING: {dir_name}\")\n",
+    "    subrootdir = f\"{rootdir}{dir_name}/\"\n",
+    "    train_files = [(f\"{subrootdir}{dir_name}_filtered_data_train.csv\",f\"{subrootdir}{dir_name}_filtered_celltype_train.csv\")]\n",
+    "    test_file = f\"{subrootdir}{dir_name}_filtered_data_test.csv\"\n",
+    "    label_file = f\"{subrootdir}{dir_name}_filtered_celltype_test.csv\"\n",
+    "    \n",
+    "    # define the model\n",
+    "    model = deepsort.DeepSortClassifier(species='human',\n",
+    "                               tissue=dir_name,\n",
+    "                               gpu_id=0,\n",
+    "                               random_seed=1,\n",
+    "                               validation_fraction=0)  # use all training data (already held out 20% in test data file)\n",
+    "\n",
+    "    # fit the model\n",
+    "    model.fit(train_files, save_path=f\"{subrootdir}{output_dir}\")\n",
+    "    \n",
+    "    # use the saved model to predict cell types in test data\n",
+    "    model.predict(input_file=test_file,\n",
+    "                   model_path=f\"{subrootdir}{output_dir}\",\n",
+    "                   save_path=f\"{subrootdir}{output_dir}\",\n",
+    "                   unsure_rate=0,\n",
+    "                   file_type='csv')\n",
+    "    labels_df = pd.read_csv(label_file)\n",
+    "    preds_df = pd.read_csv(f\"{subrootdir}{output_dir}/human_{dir_name}_{dir_name}_filtered_data_test.csv\")\n",
+    "    label_cell_ids = labels_df[\"Cell\"]\n",
+    "    pred_cell_ids = preds_df[\"index\"]\n",
+    "    assert list(label_cell_ids) == list(pred_cell_ids)\n",
+    "    labels = list(labels_df[\"Cell_type\"])\n",
+    "    if isinstance(preds_df[\"cell_subtype\"][0],float):\n",
+    "        if np.isnan(preds_df[\"cell_subtype\"][0]):\n",
+    "            preds = list(preds_df[\"cell_type\"])\n",
+    "            results = compute_metrics(labels, preds)\n",
+    "    else:\n",
+    "        preds1 = list(preds_df[\"cell_type\"])\n",
+    "        preds2 = list(preds_df[\"cell_subtype\"])\n",
+    "        results1 = compute_metrics(labels, preds1)\n",
+    "        results2 = compute_metrics(labels, preds2)\n",
+    "        if results2[\"accuracy\"] > results1[\"accuracy\"]:\n",
+    "            results = results2\n",
+    "        else:\n",
+    "            results = results1\n",
+    "        \n",
+    "    print(f\"{dir_name}: {results}\")\n",
+    "    results_dict[dir_name] = results\n",
+    "    with open(f\"{subrootdir}deepsort_E{n_epochs}_filtered_pred_{dir_name}.pickle\", \"wb\") as output_file:\n",
+    "        pickle.dump(results, output_file)\n",
+    "\n",
+    "# save results\n",
+    "with open(f\"{rootdir}deepsort_E{n_epochs}_filtered_pred_dict.pickle\", \"wb\") as output_file:\n",
+    "    pickle.dump(results_dict, output_file)\n",
+    "    "
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.8.6 64-bit ('3.8.6')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.6"
+  },
+  "vscode": {
+   "interpreter": {
+    "hash": "eba1599a1f7e611c14c87ccff6793920aa63510b01fc0e229d6dd014149b8829"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

config.json

@@ -2,23 +2,22 @@
   "architectures": [
     "BertForMaskedLM"
   ],
-  "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
+  "attention_probs_dropout_prob": 0.02,
+  "gradient_checkpointing": false,
   "hidden_act": "relu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 1152,
+  "hidden_dropout_prob": 0.02,
+  "hidden_size": 256,
   "initializer_range": 0.02,
-  "intermediate_size": 4608,
+  "intermediate_size": 512,
   "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 4096,
+  "max_position_embeddings": 2048,
   "model_type": "bert",
-  "num_attention_heads": 18,
-  "num_hidden_layers": 18,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
   "pad_token_id": 0,
   "position_embedding_type": "absolute",
-  "torch_dtype": "float32",
-  "transformers_version": "4.44.2",
+  "transformers_version": "4.6.0",
   "type_vocab_size": 2,
   "use_cache": true,
-  "vocab_size": 20275
+  "vocab_size": 25426
 }

docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=source
-set BUILDDIR=build
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.https://www.sphinx-doc.org/
-	exit /b 1
-)
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd

docs/requirements.txt

@@ -1,3 +0,0 @@
-.
-sphinx_rtd_theme==2.0.0
-nbsphinx==0.9.3

docs/source/_static/css/custom.css

@@ -1,40 +0,0 @@
-/* top left logo */
-.wy-side-nav-search, .wy-nav-top {
-    background: linear-gradient(15deg, #13547a 0%, #80d0c7 100%);
-}
-
-
-/* unvisited link */
-.wy-nav-content a:link {
-  color: #067abd;
-}
-
-/* visited link */
-.wy-nav-content a:visited {
-  color: #4b827c;
-}
-
-/* mouse over link */
-.wy-nav-content a:hover {
-  color: #80d0c7;
-}
-
-/* selected link */
-.wy-nav-content a:active {
-  color: #4b827c;
-}
-
-/* class object */
-.sig.sig-object {
-    padding: 5px 5px 5px 5px;
-    background-color: #ececec;
-    border-style: solid;
-    border-color: black;
-    border-width: 1px 0;
-}
-
-/* parameter object */
-dt {
-    padding: 5px 5px 5px 5px;
-    background-color: #ececec;
-}

docs/source/about.rst

@@ -1,49 +0,0 @@
-About
-=====
-
-Model Description
------------------
-
-**Geneformer** is a context-aware, attention-based deep learning model pretrained on a large-scale corpus of single-cell transcriptomes to enable context-specific predictions in settings with limited data in network biology. During pretraining, Geneformer gained a fundamental understanding of network dynamics, encoding network hierarchy in the attention weights of the model in a completely self-supervised manner. With both zero-shot learning and fine-tuning with limited task-specific data, Geneformer consistently boosted predictive accuracy in a diverse panel of downstream tasks relevant to chromatin and network dynamics. In silico perturbation with zero-shot learning identified a novel transcription factor in cardiomyocytes that we experimentally validated to be critical to their ability to generate contractile force. In silico treatment with limited patient data revealed candidate therapeutic targets for cardiomyopathy that we experimentally validated to significantly improve the ability of cardiomyocytes to generate contractile force in an iPSC model of the disease. Overall, Geneformer represents a foundational deep learning model pretrained on a large-scale corpus of human single cell transcriptomes to gain a fundamental understanding of gene network dynamics that can now be democratized to a vast array of downstream tasks to accelerate discovery of key network regulators and candidate therapeutic targets.
-
-In `our manuscript <https://rdcu.be/ddrx0>`_, we report results for the original 6 layer Geneformer model pretrained on Genecorpus-30M. We additionally provide within the repository a 12 layer Geneformer model, scaled up with retained width:depth aspect ratio, also pretrained on Genecorpus-30M.
-
-Both the `6 <https://huggingface.co/ctheodoris/Geneformer/blob/main/gf-6L-30M-i2048/model.safetensors>`_ and `12 <https://huggingface.co/ctheodoris/Geneformer/blob/main/gf-12L-30M-i2048/pytorch_model.bin>`_ layer Geneformer models were pretrained in June 2021.
-
-Also see `our 2024 manuscript <https://www.biorxiv.org/content/10.1101/2024.08.16.608180v1.full.pdf>`_, for details of the `expanded model <https://huggingface.co/ctheodoris/Geneformer/blob/main/model.safetensors>`_ trained on ~95 million transcriptomes in April 2024 and our continual learning, multitask learning, and quantization strategies.
-
-Application
------------
-
-The pretrained Geneformer model can be used directly for zero-shot learning, for example for in silico perturbation analysis, or by fine-tuning towards the relevant downstream task, such as gene or cell state classification.
-
-Example applications demonstrated in `our manuscript <https://rdcu.be/ddrx0>`_ include:
-
-| *Fine-tuning*:
-| - transcription factor dosage sensitivity
-| - chromatin dynamics (bivalently marked promoters)
-| - transcription factor regulatory range
-| - gene network centrality
-| - transcription factor targets
-| - cell type annotation
-| - batch integration
-| - cell state classification across differentiation
-| - disease classification
-| - in silico perturbation to determine disease-driving genes
-| - in silico treatment to determine candidate therapeutic targets
-
-| *Zero-shot learning*:
-| - batch integration
-| - gene context specificity
-| - in silico reprogramming
-| - in silico differentiation
-| - in silico perturbation to determine impact on cell state
-| - in silico perturbation to determine transcription factor targets
-| - in silico perturbation to determine transcription factor cooperativity
-
-Citations
----------
-
-| C V Theodoris #, L Xiao, A Chopra, M D Chaffin, Z R Al Sayed, M C Hill, H Mantineo, E Brydon, Z Zeng, X S Liu, P T Ellinor #. `Transfer learning enables predictions in network biology. <https://rdcu.be/ddrx0>`_ *Nature*, 31 May 2023. (# co-corresponding authors)
-
-| H Chen \*, M S Venkatesh \*, J Gomez Ortega, S V Mahesh, T Nandi, R Madduri, K Pelka †, C V Theodoris † #. `Quantized multi-task learning for context-specific representations of gene network dynamics. <https://www.biorxiv.org/content/10.1101/2024.08.16.608180v1.full.pdf>`_ *bioRxiv*, 19 Aug 2024. (\* co-first authors, † co-senior authors, # corresponding author)

docs/source/api.rst

@@ -1,51 +0,0 @@
-API
-===
-
-Tokenizer
----------
-
-.. toctree::
-   :maxdepth: 1
-
-   geneformer.tokenizer
-
-Classifier
-----------
-
-.. toctree::
-   :maxdepth: 1
-
-   geneformer.classifier
-
-Multitask Classifier
---------------------
-
-.. toctree::
-   :maxdepth: 1
-
-   geneformer.mtl_classifier
-
-Embedding Extractor
--------------------
-
-.. toctree::
-   :maxdepth: 1
-
-   geneformer.emb_extractor
-
-In Silico Perturber
--------------------
-
-.. toctree::
-   :maxdepth: 1
-
-   geneformer.in_silico_perturber
-
-
-In Silico Perturber Stats
--------------------------
-
-.. toctree::
-   :maxdepth: 1
-
-   geneformer.in_silico_perturber_stats

docs/source/conf.py

@@ -1,80 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-import pathlib
-import re
-import sys
-
-from sphinx.ext import autodoc
-
-sys.path.insert(0, pathlib.Path(__file__).parents[2].resolve().as_posix())
-
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-
-project = "geneformer"
-copyright = "2024, Christina Theodoris"
-author = "Christina Theodoris"
-release = "0.1.0"
-repository_url = "https://huggingface.co/ctheodoris/Geneformer"
-
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-extensions = [
-    "sphinx.ext.autodoc",
-    "sphinx.ext.autosummary",
-    "nbsphinx",
-    "sphinx.ext.viewcode",
-    "sphinx.ext.doctest",
-]
-
-templates_path = ["_templates"]
-exclude_patterns = [
-    "**.ipynb_checkpoints",
-]
-autoclass_content = "both"
-
-
-class MockedClassDocumenter(autodoc.ClassDocumenter):
-    def add_line(self, line: str, source: str, *lineno: int) -> None:
-        if line == "   Bases: :py:class:`object`":
-            return
-        super().add_line(line, source, *lineno)
-
-
-autodoc.ClassDocumenter = MockedClassDocumenter
-add_module_names = False
-
-
-def process_signature(app, what, name, obj, options, signature, return_annotation):
-    # loop through each line in the docstring and replace path with
-    # the generic path text
-    signature = re.sub(r"PosixPath\(.*?\)", "FILEPATH", signature)
-    return (signature, None)
-
-
-def setup(app):
-    app.connect("autodoc-process-signature", process_signature)
-
-
-# -- Options for HTML output -------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-
-html_theme = "sphinx_rtd_theme"
-html_show_sphinx = False
-html_static_path = ["_static"]
-html_logo = "_static/gf_logo.png"
-html_theme_options = {
-    "collapse_navigation": False,
-    "sticky_navigation": True,
-    "navigation_depth": 3,
-    "logo_only": True,
-}
-html_css_files = [
-    "css/custom.css",
-]
-html_show_sourcelink = False

docs/source/geneformer.classifier.rst

@@ -1,10 +0,0 @@
-geneformer.classifier
-=====================
-
-.. automodule:: geneformer.classifier
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :exclude-members:
-        valid_option_dict,
-        validate_options

docs/source/geneformer.emb_extractor.rst

@@ -1,26 +0,0 @@
-geneformer.emb\_extractor
-=========================
-
-.. automodule:: geneformer.emb_extractor
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :exclude-members:
-       accumulate_tdigests,
-       gen_heatmap_class_colors,
-       gen_heatmap_class_dict,
-       get_embs,
-       label_cell_embs,
-       label_gene_embs,
-       make_colorbar,
-       plot_heatmap,
-       plot_umap,
-       summarize_gene_embs,
-       tdigest_mean,
-       tdigest_median,
-       test_emb,
-       update_tdigest_dict,
-       update_tdigest_dict_mean,
-       update_tdigest_dict_median,
-       valid_option_dict,
-       validate_options

docs/source/geneformer.in_silico_perturber.rst

@@ -1,8 +0,0 @@
-geneformer.in\_silico\_perturber
-=======================================
-
-.. automodule:: geneformer.in_silico_perturber
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :exclude-members:  valid_option_dict, validate_options, apply_additional_filters, isp_perturb_all, isp_perturb_set, , isp_perturb_all_special, isp_perturb_set_special, update_perturbation_dictionary

docs/source/geneformer.in_silico_perturber_stats.rst

@@ -1,25 +0,0 @@
-geneformer.in\_silico\_perturber\_stats
-==============================================
-
-.. automodule:: geneformer.in_silico_perturber_stats
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :exclude-members:
-        find,
-        get_fdr,
-        get_gene_list,
-        get_impact_component,
-        invert_dict,
-        isp_aggregate_gene_shifts,
-        isp_aggregate_grouped_perturb,
-        isp_stats_mixture_model,
-        isp_stats_to_goal_state,
-        isp_stats_vs_null,
-        n_detections,
-        read_dict,
-        read_dictionaries,
-        token_to_gene_name,
-        token_tuple_to_ensembl_ids,
-        valid_option_dict,
-        validate_options

docs/source/geneformer.mtl_classifier.rst

@@ -1,11 +0,0 @@
-geneformer.mtl\_classifier
-==========================
-
-.. automodule:: geneformer.mtl_classifier
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :exclude-members:
-        valid_option_dict,
-        validate_options,
-        validate_additional_options

docs/source/geneformer.tokenizer.rst

@@ -1,15 +0,0 @@
-geneformer.tokenizer
-====================
-
-.. automodule:: geneformer.tokenizer
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :exclude-members:
-        create_dataset,
-        tokenize_anndata,
-        tokenize_files,
-        tokenize_loom,
-        rank_genes,
-        tokenize_cell,
-        sum_ensembl_ids

docs/source/getstarted.rst

@@ -1,36 +0,0 @@
-Getting Started
-===============
-
-Installation
-------------
-
-Geneformer installation instructions.
-
-Make sure you have git-lfs installed (https://git-lfs.com).
-
-.. code-block:: bash
-
-    git lfs install
-    git clone https://huggingface.co/ctheodoris/Geneformer
-    cd Geneformer
-    pip install .
-
-
-Tutorials
----------
-
-| See `examples <https://huggingface.co/ctheodoris/Geneformer/tree/main/examples>`_ for:
-| - tokenizing transcriptomes
-| - pretraining
-| - hyperparameter tuning
-| - fine-tuning
-| - extracting and plotting cell embeddings
-| - in silico perturbation
-
-Please note that the fine-tuning examples are meant to be generally applicable and the input datasets and labels will vary dependent on the downstream task. Example input files for a few of the downstream tasks demonstrated in the manuscript are located within the `example_input_files directory <https://huggingface.co/datasets/ctheodoris/Genecorpus-30M/tree/main/example_input_files>`_ in the dataset repository, but these only represent a few example fine-tuning applications.
-
-
-Tips
-----
-
-Please note that GPU resources are required for efficient usage of Geneformer. Additionally, we strongly recommend tuning hyperparameters for each downstream fine-tuning application as this can significantly boost predictive potential in the downstream task (e.g. max learning rate, learning schedule, number of layers to freeze, etc.).

docs/source/index.rst

@@ -1,16 +0,0 @@
-Geneformer
-==========
-
-Geneformer is a foundation transformer model pretrained on a large-scale corpus of single cell transcriptomes to enable context-aware predictions in network biology.
-
-See `our manuscript <https://rdcu.be/ddrx0>`_ for details.
-
-Table of Contents
------------------
-
-.. toctree::
-   :maxdepth: 2
-
-   about
-   getstarted
-   api

examples/distributed_multitask_cell_classification.ipynb

@@ -1,149 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b3266a7b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import os\n",
-    "import torch\n",
-    "from geneformer import MTLClassifier"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3e12ac9f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Define paths\n",
-    "pretrained_path = \"/path/to/pretrained/Geneformer/model\" \n",
-    "# input data is tokenized rank value encodings generated by Geneformer tokenizer (see tokenizing_scRNAseq_data.ipynb)\n",
-    "train_path = \"/path/to/train/data.dataset\"\n",
-    "val_path = \"/path/to/val/data.dataset\"\n",
-    "test_path = \"/path/to/test/data.dataset\"\n",
-    "results_dir = \"/path/to/results/directory\"\n",
-    "model_save_path = \"/path/to/model/save/path\"\n",
-    "tensorboard_log_dir = \"/path/to/tensorboard/log/dir\"\n",
-    "\n",
-    "# Define tasks and hyperparameters\n",
-    "# task_columns should be a list of column names from your dataset\n",
-    "# Each column represents a specific classification task (e.g. cell type, disease state)\n",
-    "task_columns = [\"cell_type\", \"disease_state\"]  # Example task columns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c9bd7562",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Check GPU environment\n",
-    "num_gpus = torch.cuda.device_count()\n",
-    "use_distributed = num_gpus > 1\n",
-    "print(f\"Number of GPUs detected: {num_gpus}\")\n",
-    "print(f\"Using distributed training: {use_distributed}\")\n",
-    "\n",
-    "# Set environment variables for distributed training when multiple GPUs are available\n",
-    "if use_distributed:\n",
-    "    os.environ[\"MASTER_ADDR\"] = \"localhost\"  # hostname\n",
-    "    os.environ[\"MASTER_PORT\"] = \"12355\"      # Choose an available port\n",
-    "    print(\"Distributed environment variables set.\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "b6ff3618",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#Define Hyperparameters for Optimization\n",
-    "hyperparameters = {\n",
-    "    \"learning_rate\": {\"type\": \"float\", \"low\": 1e-5, \"high\": 1e-3, \"log\": True},\n",
-    "    \"warmup_ratio\": {\"type\": \"float\", \"low\": 0.005, \"high\": 0.01},\n",
-    "    \"weight_decay\": {\"type\": \"float\", \"low\": 0.01, \"high\": 0.1},\n",
-    "    \"dropout_rate\": {\"type\": \"float\", \"low\": 0.0, \"high\": 0.7},\n",
-    "    \"lr_scheduler_type\": {\"type\": \"categorical\", \"choices\": [\"cosine\"]},\n",
-    "    \"task_weights\": {\"type\": \"float\", \"low\": 0.1, \"high\": 2.0},\n",
-    "}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f665c5a7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "mc = MTLClassifier(\n",
-    "    task_columns=task_columns,  # Our defined classification tasks\n",
-    "    study_name=\"MTLClassifier_distributed\",\n",
-    "    pretrained_path=pretrained_path,\n",
-    "    train_path=train_path,\n",
-    "    val_path=val_path,\n",
-    "    test_path=test_path,\n",
-    "    model_save_path=model_save_path,\n",
-    "    results_dir=results_dir,\n",
-    "    tensorboard_log_dir=tensorboard_log_dir,\n",
-    "    hyperparameters=hyperparameters,\n",
-    "    # Distributed training parameters\n",
-    "    distributed_training=use_distributed,  # Enable distributed training if multiple GPUs available\n",
-    "    master_addr=\"localhost\" if use_distributed else None,\n",
-    "    master_port=\"12355\" if use_distributed else None,\n",
-    "    # Other training parameters\n",
-    "    n_trials=15,  # Number of trials for hyperparameter optimization\n",
-    "    epochs=1,     # Number of training epochs (1 suggested to prevent overfitting)\n",
-    "    batch_size=8, # Adjust based on available GPU memory\n",
-    "    gradient_accumulation_steps=4,  # Accumulate gradients over multiple steps\n",
-    "    gradient_clipping=True,         # Enable gradient clipping for stability\n",
-    "    max_grad_norm=1.0,              # Set maximum gradient norm\n",
-    "    seed=42\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f69f7b6a",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Run Hyperparameter Optimization with Distributed Training\n",
-    "if __name__ == \"__main__\":\n",
-    "    # This guard is required for distributed training to prevent\n",
-    "    # infinite subprocess spawning when using torch.multiprocessing\n",
-    "    mc.run_optuna_study()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3affd5dd",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Evaluate the Model on Test Data\n",
-    "if __name__ == \"__main__\":\n",
-    "    mc.load_and_evaluate_test_model()"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "bio",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "name": "python",
-   "version": "3.12.8"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

examples/example_input_files/bivalent_promoters/bivalent_gene_labels.txt

@@ -0,0 +1,107 @@
+ENSG00000005073
+ENSG00000007372
+ENSG00000007372
+ENSG00000043355
+ENSG00000068120
+ENSG00000075891
+ENSG00000078399
+ENSG00000105991
+ENSG00000105996
+ENSG00000105997
+ENSG00000106004
+ENSG00000106006
+ENSG00000106031
+ENSG00000106038
+ENSG00000107807
+ENSG00000107821
+ENSG00000107831
+ENSG00000107859
+ENSG00000107862
+ENSG00000108511
+ENSG00000108786
+ENSG00000108797
+ENSG00000110693
+ENSG00000110693
+ENSG00000113430
+ENSG00000115844
+ENSG00000117707
+ENSG00000117707
+ENSG00000119915
+ENSG00000120068
+ENSG00000120075
+ENSG00000120093
+ENSG00000120093
+ENSG00000120094
+ENSG00000122592
+ENSG00000125285
+ENSG00000125798
+ENSG00000125813
+ENSG00000125813
+ENSG00000125816
+ENSG00000125820
+ENSG00000128573
+ENSG00000128645
+ENSG00000128652
+ENSG00000128709
+ENSG00000128710
+ENSG00000128713
+ENSG00000128714
+ENSG00000129514
+ENSG00000131196
+ENSG00000131196
+ENSG00000136327
+ENSG00000136944
+ENSG00000138083
+ENSG00000139800
+ENSG00000143013
+ENSG00000143632
+ENSG00000144355
+ENSG00000148680
+ENSG00000148826
+ENSG00000151615
+ENSG00000152192
+ENSG00000152977
+ENSG00000159184
+ENSG00000159387
+ENSG00000163412
+ENSG00000163421
+ENSG00000163623
+ENSG00000164330
+ENSG00000164438
+ENSG00000164690
+ENSG00000164778
+ENSG00000165588
+ENSG00000165588
+ENSG00000165588
+ENSG00000166407
+ENSG00000166407
+ENSG00000168505
+ENSG00000168875
+ENSG00000169946
+ENSG00000170166
+ENSG00000170178
+ENSG00000170549
+ENSG00000170561
+ENSG00000170577
+ENSG00000170689
+ENSG00000173917
+ENSG00000174279
+ENSG00000174963
+ENSG00000174963
+ENSG00000175879
+ENSG00000176842
+ENSG00000177508
+ENSG00000178573
+ENSG00000182568
+ENSG00000182742
+ENSG00000185551
+ENSG00000185551
+ENSG00000187140
+ENSG00000196092
+ENSG00000197576
+ENSG00000198807
+ENSG00000253293
+ENSG00000256463
+ENSG00000260027
+ENSG00000276644
+ENSG00000285708

examples/example_input_files/bivalent_promoters/lys4_only_gene_labels.txt

@@ -0,0 +1,80 @@
+ENSG00000012048
+ENSG00000033627
+ENSG00000037042
+ENSG00000055950
+ENSG00000067596
+ENSG00000069248
+ENSG00000072682
+ENSG00000085274
+ENSG00000088035
+ENSG00000088930
+ENSG00000095539
+ENSG00000102471
+ENSG00000102967
+ENSG00000104313
+ENSG00000105146
+ENSG00000105379
+ENSG00000105982
+ENSG00000105983
+ENSG00000107816
+ENSG00000107819
+ENSG00000107829
+ENSG00000107833
+ENSG00000108784
+ENSG00000108799
+ENSG00000108828
+ENSG00000108830
+ENSG00000109911
+ENSG00000113522
+ENSG00000119487
+ENSG00000120049
+ENSG00000125347
+ENSG00000126581
+ENSG00000131374
+ENSG00000131437
+ENSG00000131462
+ENSG00000131467
+ENSG00000131469
+ENSG00000131470
+ENSG00000131475
+ENSG00000131477
+ENSG00000135272
+ENSG00000135776
+ENSG00000135801
+ENSG00000136158
+ENSG00000140262
+ENSG00000140450
+ENSG00000140563
+ENSG00000140829
+ENSG00000140830
+ENSG00000145494
+ENSG00000146909
+ENSG00000147905
+ENSG00000148688
+ENSG00000148840
+ENSG00000148950
+ENSG00000151332
+ENSG00000151338
+ENSG00000165637
+ENSG00000165644
+ENSG00000166135
+ENSG00000166136
+ENSG00000166167
+ENSG00000166169
+ENSG00000166189
+ENSG00000166197
+ENSG00000166377
+ENSG00000167081
+ENSG00000168118
+ENSG00000171421
+ENSG00000175832
+ENSG00000186480
+ENSG00000187098
+ENSG00000188554
+ENSG00000196628
+ENSG00000196628
+ENSG00000198728
+ENSG00000198728
+ENSG00000198863
+ENSG00000285283
+ENSG00000285708

examples/example_input_files/bivalent_promoters/no_methylation_gene_labels.txt

@@ -0,0 +1,42 @@
+ENSG00000068079
+ENSG00000068383
+ENSG00000075290
+ENSG00000104313
+ENSG00000105370
+ENSG00000105374
+ENSG00000105383
+ENSG00000106536
+ENSG00000113520
+ENSG00000113525
+ENSG00000118557
+ENSG00000125257
+ENSG00000128573
+ENSG00000131471
+ENSG00000131480
+ENSG00000131482
+ENSG00000134532
+ENSG00000136319
+ENSG00000138792
+ENSG00000140262
+ENSG00000140718
+ENSG00000147488
+ENSG00000147488
+ENSG00000148677
+ENSG00000151322
+ENSG00000151322
+ENSG00000156113
+ENSG00000164399
+ENSG00000164400
+ENSG00000167749
+ENSG00000167754
+ENSG00000167755
+ENSG00000169035
+ENSG00000170927
+ENSG00000182177
+ENSG00000186153
+ENSG00000187098
+ENSG00000204764
+ENSG00000213022
+ENSG00000213822
+ENSG00000261701
+ENSG00000285708

examples/example_input_files/dosage_sensitive_tfs/dosage_sens_tf_labels.csv

@@ -0,0 +1,369 @@
+dosage_sensitive,dosage_insensitive
+ENSG00000008197,ENSG00000010539
+ENSG00000008441,ENSG00000011590
+ENSG00000010818,ENSG00000063438
+ENSG00000011332,ENSG00000063587
+ENSG00000030419,ENSG00000064218
+ENSG00000062194,ENSG00000064489
+ENSG00000065970,ENSG00000067646
+ENSG00000067082,ENSG00000075407
+ENSG00000069667,ENSG00000079263
+ENSG00000072736,ENSG00000081386
+ENSG00000073282,ENSG00000083812
+ENSG00000073861,ENSG00000083814
+ENSG00000077092,ENSG00000083828
+ENSG00000083307,ENSG00000083838
+ENSG00000084676,ENSG00000083844
+ENSG00000085276,ENSG00000085644
+ENSG00000087510,ENSG00000089335
+ENSG00000087903,ENSG00000089775
+ENSG00000089225,ENSG00000102901
+ENSG00000091656,ENSG00000103199
+ENSG00000091831,ENSG00000105136
+ENSG00000095951,ENSG00000105610
+ENSG00000100644,ENSG00000105672
+ENSG00000100968,ENSG00000106410
+ENSG00000101076,ENSG00000106948
+ENSG00000101412,ENSG00000109705
+ENSG00000102870,ENSG00000115568
+ENSG00000102935,ENSG00000117010
+ENSG00000103449,ENSG00000118620
+ENSG00000105698,ENSG00000119574
+ENSG00000105866,ENSG00000120669
+ENSG00000106689,ENSG00000121406
+ENSG00000106852,ENSG00000121864
+ENSG00000111249,ENSG00000122085
+ENSG00000111783,ENSG00000124203
+ENSG00000112033,ENSG00000124232
+ENSG00000112246,ENSG00000124444
+ENSG00000112561,ENSG00000124613
+ENSG00000112837,ENSG00000125520
+ENSG00000115112,ENSG00000127081
+ENSG00000116809,ENSG00000127903
+ENSG00000116833,ENSG00000127989
+ENSG00000117000,ENSG00000129028
+ENSG00000118263,ENSG00000129071
+ENSG00000118922,ENSG00000129194
+ENSG00000119547,ENSG00000130544
+ENSG00000120798,ENSG00000130818
+ENSG00000121068,ENSG00000131848
+ENSG00000123358,ENSG00000132010
+ENSG00000123411,ENSG00000132846
+ENSG00000124496,ENSG00000133250
+ENSG00000124813,ENSG00000134874
+ENSG00000125398,ENSG00000135899
+ENSG00000125618,ENSG00000136866
+ENSG00000126368,ENSG00000137185
+ENSG00000127152,ENSG00000137504
+ENSG00000128573,ENSG00000138380
+ENSG00000129173,ENSG00000140993
+ENSG00000131759,ENSG00000141946
+ENSG00000132005,ENSG00000142556
+ENSG00000133794,ENSG00000143067
+ENSG00000134046,ENSG00000144026
+ENSG00000134317,ENSG00000144161
+ENSG00000134323,ENSG00000145908
+ENSG00000134852,ENSG00000146587
+ENSG00000135111,ENSG00000147183
+ENSG00000137203,ENSG00000147789
+ENSG00000137270,ENSG00000148300
+ENSG00000138795,ENSG00000149054
+ENSG00000139083,ENSG00000149922
+ENSG00000139793,ENSG00000151500
+ENSG00000140548,ENSG00000151650
+ENSG00000140968,ENSG00000151657
+ENSG00000142611,ENSG00000152439
+ENSG00000143033,ENSG00000152467
+ENSG00000143171,ENSG00000152475
+ENSG00000143190,ENSG00000153975
+ENSG00000143355,ENSG00000155592
+ENSG00000143365,ENSG00000156469
+ENSG00000143373,ENSG00000157429
+ENSG00000143437,ENSG00000159882
+ENSG00000144355,ENSG00000159885
+ENSG00000147862,ENSG00000159915
+ENSG00000148516,ENSG00000160224
+ENSG00000150907,ENSG00000160229
+ENSG00000151090,ENSG00000160352
+ENSG00000153234,ENSG00000160908
+ENSG00000158055,ENSG00000160961
+ENSG00000160007,ENSG00000161277
+ENSG00000160094,ENSG00000162086
+ENSG00000161405,ENSG00000163516
+ENSG00000162761,ENSG00000164011
+ENSG00000162924,ENSG00000164048
+ENSG00000164683,ENSG00000164296
+ENSG00000164684,ENSG00000164299
+ENSG00000167182,ENSG00000165066
+ENSG00000168610,ENSG00000165512
+ENSG00000168916,ENSG00000165643
+ENSG00000169554,ENSG00000165684
+ENSG00000169946,ENSG00000166529
+ENSG00000170370,ENSG00000166823
+ENSG00000172733,ENSG00000166860
+ENSG00000172819,ENSG00000167034
+ENSG00000177463,ENSG00000167384
+ENSG00000178177,ENSG00000167554
+ENSG00000179348,ENSG00000167625
+ENSG00000179361,ENSG00000167785
+ENSG00000179456,ENSG00000167800
+ENSG00000180357,ENSG00000167840
+ENSG00000185551,ENSG00000167962
+ENSG00000185591,ENSG00000167981
+ENSG00000187098,ENSG00000168152
+ENSG00000187605,ENSG00000168286
+ENSG00000189308,ENSG00000168769
+ENSG00000196092,ENSG00000169131
+ENSG00000196482,ENSG00000169136
+ENSG00000196628,ENSG00000169548
+ENSG00000197757,ENSG00000169951
+ENSG00000198815,ENSG00000169955
+ENSG00000198945,ENSG00000169989
+ENSG00000198963,ENSG00000170260
+ENSG00000204231,ENSG00000170608
+,ENSG00000170954
+,ENSG00000171291
+,ENSG00000171295
+,ENSG00000171425
+,ENSG00000171443
+,ENSG00000171466
+,ENSG00000171469
+,ENSG00000171574
+,ENSG00000171606
+,ENSG00000171827
+,ENSG00000171872
+,ENSG00000171970
+,ENSG00000172000
+,ENSG00000172888
+,ENSG00000173041
+,ENSG00000173258
+,ENSG00000173480
+,ENSG00000173673
+,ENSG00000173825
+,ENSG00000174255
+,ENSG00000174652
+,ENSG00000174796
+,ENSG00000175279
+,ENSG00000175325
+,ENSG00000175395
+,ENSG00000175691
+,ENSG00000176009
+,ENSG00000176024
+,ENSG00000176083
+,ENSG00000176222
+,ENSG00000176302
+,ENSG00000176472
+,ENSG00000176678
+,ENSG00000176679
+,ENSG00000177030
+,ENSG00000177494
+,ENSG00000177599
+,ENSG00000177683
+,ENSG00000177842
+,ENSG00000177873
+,ENSG00000177932
+,ENSG00000177946
+,ENSG00000178150
+,ENSG00000178229
+,ENSG00000178338
+,ENSG00000178386
+,ENSG00000178665
+,ENSG00000178917
+,ENSG00000178928
+,ENSG00000178935
+,ENSG00000179195
+,ENSG00000179772
+,ENSG00000179774
+,ENSG00000179886
+,ENSG00000179909
+,ENSG00000179922
+,ENSG00000179930
+,ENSG00000179943
+,ENSG00000179965
+,ENSG00000180257
+,ENSG00000180346
+,ENSG00000180532
+,ENSG00000180535
+,ENSG00000180938
+,ENSG00000181135
+,ENSG00000181444
+,ENSG00000181450
+,ENSG00000181638
+,ENSG00000181894
+,ENSG00000181896
+,ENSG00000182318
+,ENSG00000182983
+,ENSG00000182986
+,ENSG00000183340
+,ENSG00000183647
+,ENSG00000183734
+,ENSG00000183850
+,ENSG00000184221
+,ENSG00000184517
+,ENSG00000184635
+,ENSG00000184677
+,ENSG00000184895
+,ENSG00000185155
+,ENSG00000185252
+,ENSG00000185404
+,ENSG00000185730
+,ENSG00000186020
+,ENSG00000186026
+,ENSG00000186051
+,ENSG00000186103
+,ENSG00000186230
+,ENSG00000186300
+,ENSG00000186376
+,ENSG00000186446
+,ENSG00000186496
+,ENSG00000186777
+,ENSG00000186812
+,ENSG00000186814
+,ENSG00000187626
+,ENSG00000187801
+,ENSG00000187821
+,ENSG00000187855
+,ENSG00000187987
+,ENSG00000188033
+,ENSG00000188095
+,ENSG00000188171
+,ENSG00000188295
+,ENSG00000188321
+,ENSG00000188629
+,ENSG00000188785
+,ENSG00000188868
+,ENSG00000189164
+,ENSG00000189190
+,ENSG00000189298
+,ENSG00000189299
+,ENSG00000196152
+,ENSG00000196172
+,ENSG00000196214
+,ENSG00000196345
+,ENSG00000196357
+,ENSG00000196378
+,ENSG00000196381
+,ENSG00000196387
+,ENSG00000196391
+,ENSG00000196417
+,ENSG00000196418
+,ENSG00000196456
+,ENSG00000196460
+,ENSG00000196466
+,ENSG00000196605
+,ENSG00000196646
+,ENSG00000196652
+,ENSG00000196670
+,ENSG00000196693
+,ENSG00000196705
+,ENSG00000196812
+,ENSG00000196946
+,ENSG00000197008
+,ENSG00000197020
+,ENSG00000197037
+,ENSG00000197044
+,ENSG00000197054
+,ENSG00000197124
+,ENSG00000197134
+,ENSG00000197162
+,ENSG00000197213
+,ENSG00000197279
+,ENSG00000197343
+,ENSG00000197360
+,ENSG00000197363
+,ENSG00000197472
+,ENSG00000197779
+,ENSG00000197841
+,ENSG00000197857
+,ENSG00000197863
+,ENSG00000197928
+,ENSG00000197933
+,ENSG00000197951
+,ENSG00000198028
+,ENSG00000198039
+,ENSG00000198046
+,ENSG00000198185
+,ENSG00000198205
+,ENSG00000198300
+,ENSG00000198315
+,ENSG00000198342
+,ENSG00000198346
+,ENSG00000198429
+,ENSG00000198440
+,ENSG00000198464
+,ENSG00000198466
+,ENSG00000198482
+,ENSG00000198538
+,ENSG00000198546
+,ENSG00000198551
+,ENSG00000198556
+,ENSG00000198633
+,ENSG00000198939
+,ENSG00000203326
+,ENSG00000204514
+,ENSG00000204519
+,ENSG00000204532
+,ENSG00000204595
+,ENSG00000204604
+,ENSG00000204644
+,ENSG00000204946
+,ENSG00000213020
+,ENSG00000213799
+,ENSG00000213973
+,ENSG00000213988
+,ENSG00000214189
+,ENSG00000215271
+,ENSG00000215372
+,ENSG00000215612
+,ENSG00000220201
+,ENSG00000221923
+,ENSG00000223547
+,ENSG00000227124
+,ENSG00000229676
+,ENSG00000229809
+,ENSG00000230797
+,ENSG00000232040
+,ENSG00000234284
+,ENSG00000234444
+,ENSG00000235109
+,ENSG00000235608
+,ENSG00000236104
+,ENSG00000236609
+,ENSG00000237440
+,ENSG00000242852
+,ENSG00000243660
+,ENSG00000245680
+,ENSG00000248483
+,ENSG00000249459
+,ENSG00000249471
+,ENSG00000249709
+,ENSG00000250571
+,ENSG00000250709
+,ENSG00000251192
+,ENSG00000251247
+,ENSG00000251369
+,ENSG00000253831
+,ENSG00000254004
+,ENSG00000256087
+,ENSG00000256223
+,ENSG00000256229
+,ENSG00000256294
+,ENSG00000256463
+,ENSG00000256683
+,ENSG00000256771
+,ENSG00000257446
+,ENSG00000257591
+,ENSG00000258405
+,ENSG00000258873
+,ENSG00000263002
+,ENSG00000264668
+,ENSG00000265763
+,ENSG00000267041
+,ENSG00000267179
+,ENSG00000267281
+,ENSG00000267508
+,ENSG00000267680
+,ENSG00000269067
+,ENSG00000269343
+,ENSG00000269699
+,ENSG00000272602