Upload folder using huggingface_hub

.gitattributes

@@ -33,5 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-blobs/6460cb80c6bcd981a75868e609bd77e1597421e9f2c37be887e513fee4ca65ec filter=lfs diff=lfs merge=lfs -text
-blobs/b51a86299f0279039b3d495ee70e4870ca7af3b9621ed93c1559ec659da79e60 filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,71 @@
+{
+  "architectures": [
+    "PEagleDraftModel"
+  ],
+  "auto_map": {
+    "": "config.PEagleSpeculatorConfig"
+  },
+  "down_sample_ratio": 0.7,
+  "down_sample_ratio_min": 0.2,
+  "draft_vocab_size": 151936,
+  "dtype": "bfloat16",
+  "eagle_aux_hidden_state_layer_ids": [
+    2,
+    18,
+    33
+  ],
+  "embed_requires_grad": true,
+  "mask_token_id": 151669,
+  "max_seq_len": 8192,
+  "norm_before_fc": false,
+  "norm_before_residual": false,
+  "num_depths": 4,
+  "speculators_config": {
+    "algorithm": "peagle",
+    "default_proposal_method": "greedy",
+    "proposal_methods": [
+      {
+        "accept_tolerance": 0.0,
+        "proposal_type": "greedy",
+        "speculative_tokens": 4,
+        "verifier_accept_k": 1
+      }
+    ],
+    "verifier": {
+      "architectures": [],
+      "name_or_path": "Qwen/Qwen3-8B"
+    }
+  },
+  "speculators_model_type": "peagle",
+  "speculators_version": "0.5.0.dev58",
+  "target_hidden_size": null,
+  "tie_word_embeddings": false,
+  "transformer_layer_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 12288,
+    "max_position_embeddings": 40960,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 4,
+    "num_key_value_heads": 8,
+    "pad_token_id": null,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    },
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "vocab_size": 151936
+  },
+  "transformers_version": "5.6.0"
+}

config.py

@@ -0,0 +1,82 @@
+from typing import Literal
+
+from pydantic import Field
+
+from speculators import SpeculatorModelConfig
+from speculators.models.eagle3.config import Eagle3SpeculatorConfig
+
+__all__ = [
+    "PEagleSpeculatorConfig",
+]
+
+
+@SpeculatorModelConfig.register("peagle")
+class PEagleSpeculatorConfig(Eagle3SpeculatorConfig):
+    """
+    Configuration for P-EAGLE (Parallel EAGLE) speculator.
+
+    P-EAGLE extends EAGLE-3 with parallel multi-token prediction using
+    Conditional Drop Token (COD) sampling for memory-efficient training.
+
+    :param para_depths: Number of parallel prediction groups (typically 8)
+    :param down_sample_ratio: Geometric decay ratio for COD sampling (r in [0,1])
+    :param down_sample_ratio_min: Minimum retention ratio floor
+    :param mask_token_id: Token ID used for masking
+    :param max_seq_len: Maximum sequence length for attention mask construction
+    """
+
+    speculators_model_type: Literal["peagle"] = "peagle"  # type: ignore[assignment]
+    architectures: list[str] = Field(
+        default_factory=lambda: ["PEagleSpeculator"],
+        description="Model architectures that can load these weights",
+    )
+
+    para_depths: int = Field(
+        default=8,
+        description="Number of parallel prediction groups (depths)",
+        ge=1,
+        le=16,
+    )
+
+    down_sample_ratio: float = Field(
+        default=0.7,
+        description="Geometric decay ratio for COD sampling (retention rate r)",
+        gt=0.0,
+        lt=1.0,
+    )
+
+    down_sample_ratio_min: float = Field(
+        default=0.1,
+        description="Minimum retention ratio floor to prevent over-sampling",
+        gt=0.0,
+        le=1.0,
+    )
+
+    mask_token_id: int | None = Field(
+        default=None,
+        description="Token ID used for padding unused positions in parallel groups",
+    )
+
+    max_seq_len: int = Field(
+        default=2048,
+        description="Maximum sequence length for attention mask construction",
+        ge=128,
+        le=8192,
+    )
+
+    # Override Eagle3 default: P-EAGLE requires trainable embeddings
+    # (matches p-eagle-train)
+    embed_requires_grad: bool = Field(
+        default=True,
+        description=(
+            "Whether embedding layer weights require gradients during "
+            "training (True for P-EAGLE)"
+        ),
+    )
+
+    prediction_loss_weight: float = Field(
+        default=1.0,
+        description="Weight for prediction loss (cross-entropy on logits). "
+        "P-eagle-train uses only prediction loss, no hidden state distillation.",
+        gt=0.0,
+    )

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6460cb80c6bcd981a75868e609bd77e1597421e9f2c37be887e513fee4ca65ec
+size 4183929280

optimizer_state_dict.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b51a86299f0279039b3d495ee70e4870ca7af3b9621ed93c1559ec659da79e60
+size 5878569834

scheduler_state_dict.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e65c261ecf76be2b856fa1d5b0f02d0e9bf164284775a68082ac9ebe1b6a7b7f
+size 1531

val_metrics.json

@@ -0,0 +1 @@
+{"loss_epoch": 2.2505897877767778, "full_acc_epoch": 0.4788787332964311, "position 0 acc_epoch": 0.6671951389062811, "position 1 acc_epoch": 0.4501549169114997, "position 2 acc_epoch": 0.31645943323750436, "position 3 acc_epoch": 0.22150448855365093}