Upload esm2-35m-sparse50 sparse model from PharmaCore

README.md

@@ -0,0 +1,32 @@
+---
+license: mit
+tags:
+  - pharmacore
+  - sparse
+  - drug-discovery
+  - apple-silicon
+base_model: facebook/esm2_t12_35M_UR50D
+---
+
+# esm2-35m-sparse50 (PharmaCore Sparse)
+
+50% magnitude-pruned version of [facebook/esm2_t12_35M_UR50D](https://huggingface.co/facebook/esm2_t12_35M_UR50D) 
+for efficient drug discovery on Apple Silicon.
+
+## Key Stats
+- **Sparsity:** 50%
+- **Quality Retention:** 97.3%
+- **Use Case:** Protein encoding in PharmaCore drug discovery pipeline
+
+## Usage
+
+```python
+from transformers import AutoModel, AutoTokenizer
+
+model = AutoModel.from_pretrained("stephenjun8192/esm2-35m-sparse50")
+tokenizer = AutoTokenizer.from_pretrained("facebook/esm2_t12_35M_UR50D")
+```
+
+## Part of PharmaCore
+
+[PharmaCore](https://github.com/stephenjun8192/PharmaCore) — Apple Silicon-native AI drug discovery platform.

config.json

@@ -0,0 +1,32 @@
+{
+  "add_cross_attention": false,
+  "architectures": [
+    "EsmModel"
+  ],
+  "attention_probs_dropout_prob": 0.0,
+  "classifier_dropout": null,
+  "dtype": "float32",
+  "emb_layer_norm_before": false,
+  "esmfold_config": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.0,
+  "hidden_size": 480,
+  "initializer_range": 0.02,
+  "intermediate_size": 1920,
+  "is_decoder": false,
+  "is_folding_model": false,
+  "layer_norm_eps": 1e-05,
+  "mask_token_id": 32,
+  "max_position_embeddings": 1026,
+  "model_type": "esm",
+  "num_attention_heads": 20,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "position_embedding_type": "rotary",
+  "tie_word_embeddings": true,
+  "token_dropout": true,
+  "transformers_version": "5.5.4",
+  "use_cache": true,
+  "vocab_list": null,
+  "vocab_size": 33
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b77052485a6517e12a856ec1514b1efcc7bd0bde997106b1e91fc94d02a205fe
+size 134025308

sparsification_results.json

@@ -0,0 +1,155 @@
+{
+  "model": "/tmp/esm2_35m",
+  "sparsity_target": 0.5,
+  "pruning_target": "all",
+  "pre_pruning": {
+    "total_params": 33500401,
+    "zero_params": 673,
+    "sparsity_pct": 0.0,
+    "top_layers": {
+      "embeddings.word_embeddings.weight": {
+        "total": 15840,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.attention.self.query.weight": {
+        "total": 230400,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.attention.self.key.weight": {
+        "total": 230400,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.attention.self.value.weight": {
+        "total": 230400,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.attention.output.dense.weight": {
+        "total": 230400,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.intermediate.dense.weight": {
+        "total": 921600,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.intermediate.dense.bias": {
+        "total": 1920,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.0.output.dense.weight": {
+        "total": 921600,
+        "zeros": 1,
+        "sparsity": 0.0
+      },
+      "encoder.layer.1.attention.self.query.weight": {
+        "total": 230400,
+        "zeros": 0,
+        "sparsity": 0.0
+      },
+      "encoder.layer.1.attention.self.key.weight": {
+        "total": 230400,
+        "zeros": 0,
+        "sparsity": 0.0
+      }
+    }
+  },
+  "post_pruning": {
+    "total_params": 33500401,
+    "zero_params": 16704659,
+    "sparsity_pct": 49.86,
+    "top_layers": {
+      "encoder.layer.0.attention.self.query.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.0.attention.self.key.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.0.attention.self.value.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.0.attention.output.dense.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.0.intermediate.dense.weight": {
+        "total": 921600,
+        "zeros": 460800,
+        "sparsity": 50.0
+      },
+      "encoder.layer.0.output.dense.weight": {
+        "total": 921600,
+        "zeros": 460800,
+        "sparsity": 50.0
+      },
+      "encoder.layer.1.attention.self.query.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.1.attention.self.key.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.1.attention.self.value.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      },
+      "encoder.layer.1.attention.output.dense.weight": {
+        "total": 230400,
+        "zeros": 115200,
+        "sparsity": 50.0
+      }
+    }
+  },
+  "quality": {
+    "per_protein_cosine_similarity": {
+      "insulin_human": 0.9178829789161682,
+      "insulin_mouse": 0.9191319942474365,
+      "egfr_fragment": 0.920665979385376,
+      "p53_fragment": 0.9336360096931458,
+      "hemoglobin_a": 0.9500190019607544,
+      "lysozyme": 0.9186660051345825,
+      "ubiquitin": 0.7709019780158997,
+      "cytochrome_c": 0.8732290267944336
+    },
+    "average_cosine_similarity": 0.900517,
+    "pairwise_rank_correlation": 0.558292,
+    "insulin_pair_similarity": 0.995582
+  },
+  "benchmark_original": {
+    "device": "mps",
+    "n_runs": 20,
+    "mean_ms": 8.46,
+    "std_ms": 0.17,
+    "min_ms": 8.22,
+    "max_ms": 8.78,
+    "throughput_seqs_per_sec": 118.2
+  },
+  "benchmark_sparse": {
+    "device": "mps",
+    "n_runs": 20,
+    "mean_ms": 5.97,
+    "std_ms": 0.2,
+    "min_ms": 5.69,
+    "max_ms": 6.42,
+    "throughput_seqs_per_sec": 167.6
+  },
+  "speedup": 1.417,
+  "device": "mps",
+  "torch_version": "2.11.0"
+}

tokenizer_config.json

@@ -0,0 +1,53 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<cls>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<eos>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32": {
+      "content": "<mask>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "backend": "custom",
+  "cls_token": "<cls>",
+  "eos_token": "<eos>",
+  "is_local": true,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "EsmTokenizer",
+  "unk_token": "<unk>"
+}

vocab.txt

@@ -0,0 +1,33 @@
+<cls>
+<pad>
+<eos>
+<unk>
+L
+A
+G
+V
+S
+E
+R
+T
+I
+D
+P
+K
+Q
+N
+F
+Y
+M
+H
+W
+C
+X
+B
+U
+Z
+O
+.
+-
+<null_1>
+<mask>