Instructions to use TheDeepDas/Harsh_GRPO with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use TheDeepDas/Harsh_GRPO with PEFT:

from peft import PeftModel
from transformers import AutoModelForCausalLM

base_model = AutoModelForCausalLM.from_pretrained("unsloth/Qwen2.5-Coder-3B-Instruct-bnb-4bit")
model = PeftModel.from_pretrained(base_model, "TheDeepDas/Harsh_GRPO")

Transformers

How to use TheDeepDas/Harsh_GRPO with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="TheDeepDas/Harsh_GRPO")
messages = [
    {"role": "user", "content": "Who are you?"},
]
pipe(messages)

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoModel
model = AutoModel.from_pretrained("TheDeepDas/Harsh_GRPO", device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use TheDeepDas/Harsh_GRPO with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "TheDeepDas/Harsh_GRPO"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "TheDeepDas/Harsh_GRPO",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/TheDeepDas/Harsh_GRPO

SGLang

How to use TheDeepDas/Harsh_GRPO with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "TheDeepDas/Harsh_GRPO" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "TheDeepDas/Harsh_GRPO",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "TheDeepDas/Harsh_GRPO" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "TheDeepDas/Harsh_GRPO",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Unsloth Desktop
Docker Model Runner
How to use TheDeepDas/Harsh_GRPO with Docker Model Runner:
```
docker model run hf.co/TheDeepDas/Harsh_GRPO
```

Harsh_GRPO / app.py

TheDeepDas

Upload folder using huggingface_hub

0af2a1b verified 18 days ago

Raw History Blame Contribute Delete

2.85 kB

from pathlib import Path

import gradio as gr
import spaces
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer


BASE_MODEL = "Qwen/Qwen2.5-Coder-3B-Instruct"
ADAPTER_DIR = Path(__file__).resolve().parent


def load_model():
    tokenizer = AutoTokenizer.from_pretrained(ADAPTER_DIR)
    use_cuda = torch.cuda.is_available()
    dtype = torch.float16 if use_cuda else torch.float32
    base_model = AutoModelForCausalLM.from_pretrained(
        BASE_MODEL,
        torch_dtype=dtype,
        low_cpu_mem_usage=True,
    )
    if use_cuda:
        # ZeroGPU requires the model to be placed on CUDA during app startup.
        base_model = base_model.to("cuda")
    else:
        base_model = base_model.to("cpu")
    model = PeftModel.from_pretrained(base_model, ADAPTER_DIR)
    model.eval()
    return model, tokenizer


model, tokenizer = load_model()


def gpu_duration(prompt: str, max_new_tokens: int, temperature: float) -> int:
    # Allow time for long requests while keeping typical calls in the fast queue.
    return min(120, max(30, 15 + int(max_new_tokens) // 4))


@spaces.GPU(duration=gpu_duration)
@torch.inference_mode()
def predict(prompt: str, max_new_tokens: int, temperature: float) -> str:
    if not prompt.strip():
        raise gr.Error("Enter a prompt before requesting an inference.")

    inputs = tokenizer.apply_chat_template(
        [{"role": "user", "content": prompt}],
        tokenize=True,
        add_generation_prompt=True,
        return_tensors="pt",
        return_dict=True,
    )
    inputs = inputs.to(model.get_input_embeddings().weight.device)
    generation_options = {
        "max_new_tokens": int(max_new_tokens),
        "eos_token_id": tokenizer.eos_token_id,
        "pad_token_id": tokenizer.pad_token_id or tokenizer.eos_token_id,
        "do_sample": temperature > 0,
    }
    if temperature > 0:
        generation_options["temperature"] = temperature

    output = model.generate(**inputs, **generation_options)
    generated_tokens = output[0, inputs["input_ids"].shape[1] :]
    return tokenizer.decode(generated_tokens, skip_special_tokens=True).strip()


demo = gr.Interface(
    fn=predict,
    inputs=[
        gr.Textbox(label="Prompt", lines=5, placeholder="Ask for help writing or explaining code..."),
        gr.Slider(minimum=16, maximum=512, value=128, step=16, label="Maximum new tokens"),
        gr.Slider(minimum=0, maximum=1.5, value=0.7, step=0.1, label="Temperature (0 = deterministic)"),
    ],
    outputs=gr.Textbox(label="Model response", lines=10),
    title="GRPO Qwen2.5 Coder",
    description="Qwen2.5-Coder-3B-Instruct fine-tuned with a GRPO LoRA adapter.",
    api_name="predict",
)

if __name__ == "__main__":
    demo.launch()

	from pathlib import Path

	import gradio as gr
	import spaces
	import torch
	from peft import PeftModel
	from transformers import AutoModelForCausalLM, AutoTokenizer


	BASE_MODEL = "Qwen/Qwen2.5-Coder-3B-Instruct"
	ADAPTER_DIR = Path(__file__).resolve().parent


	def load_model():
	tokenizer = AutoTokenizer.from_pretrained(ADAPTER_DIR)
	use_cuda = torch.cuda.is_available()
	dtype = torch.float16 if use_cuda else torch.float32
	base_model = AutoModelForCausalLM.from_pretrained(
	BASE_MODEL,
	torch_dtype=dtype,
	low_cpu_mem_usage=True,
	)
	if use_cuda:
	# ZeroGPU requires the model to be placed on CUDA during app startup.
	base_model = base_model.to("cuda")
	else:
	base_model = base_model.to("cpu")
	model = PeftModel.from_pretrained(base_model, ADAPTER_DIR)
	model.eval()
	return model, tokenizer


	model, tokenizer = load_model()


	def gpu_duration(prompt: str, max_new_tokens: int, temperature: float) -> int:
	# Allow time for long requests while keeping typical calls in the fast queue.
	return min(120, max(30, 15 + int(max_new_tokens) // 4))


	@spaces.GPU(duration=gpu_duration)
	@torch.inference_mode()
	def predict(prompt: str, max_new_tokens: int, temperature: float) -> str:
	if not prompt.strip():
	raise gr.Error("Enter a prompt before requesting an inference.")

	inputs = tokenizer.apply_chat_template(
	[{"role": "user", "content": prompt}],
	tokenize=True,
	add_generation_prompt=True,
	return_tensors="pt",
	return_dict=True,
	)
	inputs = inputs.to(model.get_input_embeddings().weight.device)
	generation_options = {
	"max_new_tokens": int(max_new_tokens),
	"eos_token_id": tokenizer.eos_token_id,
	"pad_token_id": tokenizer.pad_token_id or tokenizer.eos_token_id,
	"do_sample": temperature > 0,
	}
	if temperature > 0:
	generation_options["temperature"] = temperature

	output = model.generate(inputs, generation_options)
	generated_tokens = output[0, inputs["input_ids"].shape[1] :]
	return tokenizer.decode(generated_tokens, skip_special_tokens=True).strip()


	demo = gr.Interface(
	fn=predict,
	inputs=[
	gr.Textbox(label="Prompt", lines=5, placeholder="Ask for help writing or explaining code..."),
	gr.Slider(minimum=16, maximum=512, value=128, step=16, label="Maximum new tokens"),
	gr.Slider(minimum=0, maximum=1.5, value=0.7, step=0.1, label="Temperature (0 = deterministic)"),
	],
	outputs=gr.Textbox(label="Model response", lines=10),
	title="GRPO Qwen2.5 Coder",
	description="Qwen2.5-Coder-3B-Instruct fine-tuned with a GRPO LoRA adapter.",
	api_name="predict",
	)

	if __name__ == "__main__":
	demo.launch()