Text Classification
Transformers
Safetensors
PyTorch
qwen2
text-generation
causal-lm
qwen
decision-making
system-one
reasoning
jev
jevbench
joint-expected-value
classification
qlora
agent
decision-model
fast-inference
text-embeddings-inference
Instructions to use devankit7873/QJev-1.5B with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use devankit7873/QJev-1.5B with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="devankit7873/QJev-1.5B")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("devankit7873/QJev-1.5B") model = AutoModelForCausalLM.from_pretrained("devankit7873/QJev-1.5B", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 7,488 Bytes
d9b5dc9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | import os
import re
from typing import Dict, List, Any, Union
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import AutoTokenizer, AutoModelForCausalLM
from safetensors.torch import load_file
class DecisionHead(nn.Module):
def __init__(self, hidden_size: int = 1536, intermediate_size: int = 512, max_choices: int = 10, dropout: float = 0.1):
super().__init__()
self.max_choices = max_choices
self.norm = nn.LayerNorm(hidden_size)
self.dense = nn.Linear(hidden_size, intermediate_size)
self.act = nn.GELU()
self.dropout = nn.Dropout(dropout)
self.out_proj = nn.Linear(intermediate_size, max_choices)
self.score_proj = nn.Linear(intermediate_size, 1)
def forward(self, x: torch.Tensor, num_choices: torch.Tensor = None) -> torch.Tensor:
h = self.norm(x)
h = self.act(self.dense(h))
h = self.dropout(h)
logits = self.out_proj(h)
if num_choices is not None:
batch_size = logits.size(0)
seq_range = torch.arange(self.max_choices, device=logits.device).unsqueeze(0).expand(batch_size, -1)
mask = seq_range >= num_choices.unsqueeze(1)
logits = logits.masked_fill(mask, -1e4)
return logits
class EndpointHandler:
"""
Hugging Face Inference Endpoints Custom Handler for QJev-1.5B.
Accepts:
{
"inputs": "Situation or context description",
"parameters": {
"choices": ["Choice 1", "Choice 2", ...]
}
}
OR:
{
"context": "...",
"question": "...",
"choices": [...]
}
Returns standard text-classification scores:
[
{"label": "Choice 1", "score": 0.8672},
{"label": "Choice 2", "score": 0.0703},
...
]
"""
def __init__(self, path: str = ""):
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
dtype = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float32
# 1. Load Tokenizer
self.tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
if self.tokenizer.pad_token is None:
self.tokenizer.pad_token = self.tokenizer.eos_token
# 2. Load Base Transformer Backbone
self.backbone = AutoModelForCausalLM.from_pretrained(
path,
torch_dtype=dtype,
device_map=self.device
)
self.backbone.eval()
# 3. Load MLP Decision Head
self.head = DecisionHead().to(device=self.device, dtype=dtype)
head_path = os.path.join(path, "decision_head.safetensors")
if not os.path.exists(head_path):
head_path = os.path.join(path, "decision_head.pt")
if os.path.exists(head_path):
if head_path.endswith(".safetensors"):
state = load_file(head_path)
else:
ckpt = torch.load(head_path, map_location="cpu")
state = ckpt.get("head_state_dict", ckpt)
self.head.load_state_dict(state)
self.head.eval()
def _extract_choices_from_text(self, text: str) -> List[str]:
# Try extracting patterns like "[1] ...", "1. ...", "Option A: ..."
lines = [line.strip() for line in text.split("\n") if line.strip()]
choices = []
for line in lines:
m = re.match(r"^(?:\[?\d+\]?[\.\)]?|[A-E][\.\)]|Option\s+[A-E]:?)\s*(.+)$", line, re.IGNORECASE)
if m:
choices.append(m.group(1).strip())
if len(choices) >= 2:
return choices[:10]
return ["Approve / Yes", "Reject / No"]
def _format_prompt(self, context: str, question: str, choices: List[str]) -> str:
choices_str = "\n".join([f"[{i+1}] {c}" for i, c in enumerate(choices)])
q_str = f"\n\nQuestion:\n{question.strip()}" if question else "\n\nQuestion:\nSelect the most appropriate option:"
return (
"<|im_start|>system\n"
"You are an expert decision judge. Evaluate the situation and determine the likelihood for each option.<|im_end|>\n"
f"<|im_start|>user\nContext:\n{context.strip()}{q_str}\n\nOptions:\n{choices_str}<|im_end|>\n"
"<|im_start|>assistant\nDecision:"
)
def _process_single(self, item: Dict[str, Any]) -> List[Dict[str, float]]:
# Parse inputs
context = ""
question = ""
choices = []
if isinstance(item, str):
context = item
choices = self._extract_choices_from_text(item)
elif isinstance(item, dict):
# Check standard Hugging Face inputs/parameters format
if "inputs" in item:
context = str(item.get("inputs", ""))
params = item.get("parameters", {}) or {}
if isinstance(params, dict):
choices = params.get("choices", [])
question = params.get("question", "")
# Check alternative keys
if not context and "context" in item:
context = str(item.get("context", ""))
if not question and "question" in item:
question = str(item.get("question", ""))
if not choices and "choices" in item:
choices = item.get("choices", [])
if not choices and "options" in item:
choices = item.get("options", [])
# Fallback if no choices provided
if not choices:
choices = self._extract_choices_from_text(context)
# Enforce max 10 choices (architecture limit)
choices = [str(c) for c in choices][:10]
if len(choices) < 2:
choices = ["Approve / Yes", "Reject / No"]
num_c = len(choices)
prompt = self._format_prompt(context, question, choices)
inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)
with torch.no_grad():
transformer = getattr(self.backbone, "model", self.backbone)
out = transformer(inputs["input_ids"], attention_mask=inputs["attention_mask"])
last_hidden = out.last_hidden_state[:, -1, :]
logits = self.head(last_hidden, num_choices=torch.tensor([num_c], device=self.device))
probs = F.softmax(logits[0, :num_c], dim=-1).cpu().tolist()
return [{"label": choice, "score": round(prob, 4)} for choice, prob in zip(choices, probs)]
def __call__(self, data: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]:
"""
Receives request payload, returns predictions.
"""
if isinstance(data, list):
return [self._process_single(item) for item in data]
elif isinstance(data, dict):
# If payload is {"inputs": ["str1", "str2"], ...}
if isinstance(data.get("inputs"), list):
results = []
for inp in data["inputs"]:
single_data = dict(data)
single_data["inputs"] = inp
results.append(self._process_single(single_data))
return results
return self._process_single(data)
elif isinstance(data, str):
return self._process_single(data)
else:
raise ValueError(f"Unsupported payload type: {type(data)}")
|