Create README.md

README.md

@@ -0,0 +1,46 @@
+Code for creating the tiny model:
+
+```python
+"""
+Create tiny random phi-4-mini-instruct model which preserves longrope
+"""
+
+import os
+import torch
+
+torch.set_default_dtype(torch.float32)
+
+from transformers import AutoTokenizer, AutoConfig, Phi3ForCausalLM, set_seed
+from optimum.intel import OVModelForCausalLM
+
+model_id = "microsoft/Phi-4-mini-instruct"
+output_dir = "phi-4-mini-tiny-random"
+ov_output_dir = output_dir + "-ov"
+
+set_seed(0)
+
+# === Step 1: Define tiny model config ===
+config = AutoConfig.from_pretrained(model_id)
+config.num_hidden_layers = 4
+config.num_attention_heads = 4
+config.num_key_value_heads = 2
+config.hidden_size = 64
+config.intermediate_size = 128
+config.initializer_range = 0.1
+
+
+if config.rope_scaling:
+    config.rope_scaling['short_factor'] = config.rope_scaling['short_factor'][::8]
+    config.rope_scaling['long_factor'] = config.rope_scaling['long_factor'][::8]
+
+# === Step 2: Create model from config ===
+model = Phi3ForCausalLM(config)
+
+# === Step 3: Load or create tokenizer ===
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+# === Step 4: Save model and tokenizer ===
+os.makedirs(output_dir, exist_ok=True)
+model.save_pretrained(output_dir)
+tokenizer.save_pretrained(output_dir)
+```