Spaces:
Sleeping
Sleeping
File size: 3,259 Bytes
0528593 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 | import gradio as gr
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM
import math
# Default public model
DEFAULT_MODEL = "HuggingFaceH4/zephyr-7b-beta"
def run_analysis(model_id, prompt_variations, temperature, max_new_tokens):
# Load model
tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32, device_map="auto")
results = []
for prompt in prompt_variations:
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
with torch.no_grad():
outputs = model.generate(
**inputs,
max_new_tokens=max_new_tokens,
temperature=temperature,
do_sample=True if temperature > 0 else False,
return_dict_in_generate=True,
output_scores=True
)
# Decode text
output_text = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
# Extract generated part
prompt_len = inputs["input_ids"].shape[1]
gen_ids = outputs.sequences[0][prompt_len:]
scores = outputs.scores
if len(scores) != len(gen_ids):
L = min(len(scores), len(gen_ids))
scores = scores[:L]
gen_ids = gen_ids[:L]
# Logprobs
logprobs = []
for step_logits, tid in zip(scores, gen_ids):
lp = torch.log_softmax(step_logits, dim=-1)[tid.item()].item()
logprobs.append(lp)
# Build table of top-5 alternatives
table = "Token | P(token) | Top alternatives\n"
table += "-"*50 + "\n"
for tid, lp, step_logits in zip(gen_ids, logprobs, scores):
alts = torch.topk(torch.log_softmax(step_logits, dim=-1), 5)
alt_tokens = [tokenizer.decode([i]) for i in alts.indices.tolist()]
alt_probs = [f"{math.exp(p)*100:.1f}%" for p in alts.values.tolist()]
alt_str = ", ".join([f"{t} ({p})" for t, p in zip(alt_tokens, alt_probs)])
table += f"{tokenizer.decode([tid])} | {math.exp(lp)*100:.1f}% | {alt_str}\n"
results.append(f"Prompt: {prompt}\n\nOutput:\n{output_text}\n\nToken Probabilities:\n{table}")
return "\n\n---\n\n".join(results)
# Gradio UI
with gr.Blocks() as demo:
gr.Markdown("# Prompt Variations & Token Analysis (Free, HF Models)")
model_id = gr.Textbox(value=DEFAULT_MODEL, label="Model ID", placeholder="Enter a model like HuggingFaceH4/zephyr-7b-beta")
prompts = gr.Textbox(lines=6, value="Explain dollar-cost averaging to a beginner in 6–8 sentences.\nIn 6–8 sentences, teach a newbie how dollar-cost averaging works.", label="Prompt Variations (one per line)")
temp = gr.Slider(0, 1, 0.2, label="Temperature")
max_tokens = gr.Slider(10, 500, 220, step=10, label="Max new tokens")
btn = gr.Button("Run Analysis")
output = gr.Textbox(lines=30, label="Results")
btn.click(
run_analysis,
inputs=[model_id, prompts, temp, max_tokens],
outputs=output
)
if __name__ == "__main__":
demo.launch()
|