| { |
| "kind": "documentation example checks, not benchmark", |
| "passed": true, |
| "outputs": { |
| "support-request.json": { |
| "model": "bev-bonsai-27b", |
| "backend": "prism_llama_cpp", |
| "backend_build": "b1-9a9394a", |
| "mode": "llama_full_vocab_categorical_scoring", |
| "strategy": "batch", |
| "requested_strategy": "auto", |
| "strategy_implementation": "bounded_concurrent_native_requests", |
| "parsed_json": { |
| "route": "billing" |
| }, |
| "fields": { |
| "route": { |
| "value": "billing", |
| "selected_value": "billing", |
| "selected_label": "A", |
| "selected_token_id": 32, |
| "probability": 0.9989061260210375, |
| "abstained": false, |
| "candidate_probability_mass": 0.9086667884355636, |
| "candidates": [ |
| { |
| "value": "billing", |
| "label": "A", |
| "token_id": 32, |
| "probability": 0.9989061260210375, |
| "logprob": -0.09687129408121109 |
| }, |
| { |
| "value": "technical", |
| "label": "B", |
| "token_id": 33, |
| "probability": 0.0008516050271616897, |
| "logprob": -7.1641645431518555 |
| }, |
| { |
| "value": "unknown", |
| "label": "C", |
| "token_id": 34, |
| "probability": 0.0002422689518008115, |
| "logprob": -8.421238899230957 |
| } |
| ], |
| "elapsed_ms": 474.855, |
| "queue_ms": 0.003, |
| "backend_requests": 1, |
| "score_retrieval": "selected_token_logprobs", |
| "requested_top_n": 3, |
| "prompt_tokens": 142, |
| "backend_timings": { |
| "cache_n": 0, |
| "prompt_n": 142, |
| "prompt_ms": 284.624, |
| "prompt_per_token_ms": 2.0043943661971833, |
| "prompt_per_second": 498.90381696554044, |
| "predicted_n": 1, |
| "predicted_ms": 0.001, |
| "predicted_per_token_ms": 0.0, |
| "predicted_per_second": 0.0 |
| }, |
| "usage": { |
| "prompt_tokens": 142, |
| "completion_tokens": 1, |
| "total_tokens": 143 |
| } |
| } |
| }, |
| "scores_are_calibrated": false, |
| "score_source": "pre_sampling_full_vocabulary_logprobs", |
| "selected_token_logprobs": true, |
| "score_temperature": 1.0, |
| "abstained_fields": [], |
| "elapsed_ms": 480.079, |
| "prompt_preparation_ms": 5.143, |
| "backend_requests": 1, |
| "scored_fields": 1, |
| "max_prompt_tokens": 142, |
| "max_model_len": 16384, |
| "backend_slots": 2, |
| "cache_block_tokens": null, |
| "usage": { |
| "prompt_tokens": 142, |
| "completion_tokens": 1, |
| "total_tokens": 143 |
| } |
| }, |
| "systemone-request.json": { |
| "model": "bev-bonsai-27b", |
| "answers": { |
| "queue": { |
| "type": "choice", |
| "choice": "billing", |
| "probabilities": { |
| "billing": 0.9982291183228973, |
| "technical": 0.0016640844610198414, |
| "general": 0.00010679721608279538 |
| }, |
| "confidence": 0.9982291183228973 |
| }, |
| "outage": { |
| "type": "noul", |
| "noul": 0.002787811663481365 |
| }, |
| "priority": { |
| "type": "score", |
| "score": 1.0002383132357986, |
| "probabilities": { |
| "0": 0.0007058337315065528, |
| "1": 0.9983500193011885, |
| "2": 0.0009441469673050024 |
| }, |
| "legend": { |
| "0": "General question", |
| "1": "Billing issue", |
| "2": "Outage" |
| }, |
| "confidence": 0.9983500193011885 |
| } |
| }, |
| "usage": { |
| "input_tokens": 547, |
| "output_tokens": 3, |
| "prompt_tokens": 547, |
| "completion_tokens": 3, |
| "total_tokens": 550 |
| }, |
| "usage_accounting": "logical_prompt_and_scoring_tokens_including_retries", |
| "scores_are_calibrated": false, |
| "confidence_definition": "maximum_candidate_relative_probability", |
| "evaluation_trace": { |
| "model": "bev-bonsai-27b", |
| "backend": "prism_llama_cpp", |
| "backend_build": "b1-9a9394a", |
| "mode": "llama_full_vocab_categorical_scoring", |
| "strategy": "batch", |
| "requested_strategy": "auto", |
| "strategy_implementation": "bounded_concurrent_native_requests", |
| "parsed_json": { |
| "queue": "billing", |
| "outage": false, |
| "priority": "1" |
| }, |
| "fields": { |
| "queue": { |
| "value": "billing", |
| "selected_value": "billing", |
| "selected_label": "A", |
| "selected_token_id": 32, |
| "probability": 0.9982291183228973, |
| "abstained": false, |
| "candidate_probability_mass": 0.8655803643819256, |
| "candidates": [ |
| { |
| "value": "billing", |
| "label": "A", |
| "token_id": 32, |
| "probability": 0.9982291183228973, |
| "logprob": -0.14612750709056854 |
| }, |
| { |
| "value": "technical", |
| "label": "B", |
| "token_id": 33, |
| "probability": 0.0016640844610198414, |
| "logprob": -6.542835235595703 |
| }, |
| { |
| "value": "general", |
| "label": "C", |
| "token_id": 34, |
| "probability": 0.00010679721608279538, |
| "logprob": -9.288933753967285 |
| } |
| ], |
| "elapsed_ms": 576.291, |
| "queue_ms": 0.002, |
| "backend_requests": 1, |
| "score_retrieval": "selected_token_logprobs", |
| "requested_top_n": 3, |
| "prompt_tokens": 190, |
| "backend_timings": { |
| "cache_n": 0, |
| "prompt_n": 190, |
| "prompt_ms": 572.964, |
| "prompt_per_token_ms": 3.0156000000000005, |
| "prompt_per_second": 331.60896670645974, |
| "predicted_n": 1, |
| "predicted_ms": 0.001, |
| "predicted_per_token_ms": 0.0, |
| "predicted_per_second": 0.0 |
| }, |
| "usage": { |
| "prompt_tokens": 190, |
| "completion_tokens": 1, |
| "total_tokens": 191 |
| } |
| }, |
| "outage": { |
| "value": false, |
| "selected_value": false, |
| "selected_label": "B", |
| "selected_token_id": 33, |
| "probability": 0.9972121883365187, |
| "abstained": false, |
| "candidate_probability_mass": 0.9862726128870609, |
| "candidates": [ |
| { |
| "value": true, |
| "label": "A", |
| "token_id": 32, |
| "probability": 0.002787811663481365, |
| "logprob": -5.896320819854736 |
| }, |
| { |
| "value": false, |
| "label": "B", |
| "token_id": 33, |
| "probability": 0.9972121883365187, |
| "logprob": -0.01661418378353119 |
| } |
| ], |
| "elapsed_ms": 578.692, |
| "queue_ms": 0.001, |
| "backend_requests": 1, |
| "score_retrieval": "selected_token_logprobs", |
| "requested_top_n": 2, |
| "prompt_tokens": 164, |
| "backend_timings": { |
| "cache_n": 0, |
| "prompt_n": 164, |
| "prompt_ms": 570.737, |
| "prompt_per_token_ms": 3.480103658536585, |
| "prompt_per_second": 287.34776262972264, |
| "predicted_n": 1, |
| "predicted_ms": 0.001, |
| "predicted_per_token_ms": 0.0, |
| "predicted_per_second": 0.0 |
| }, |
| "usage": { |
| "prompt_tokens": 164, |
| "completion_tokens": 1, |
| "total_tokens": 165 |
| } |
| }, |
| "priority": { |
| "value": "1", |
| "selected_value": "1", |
| "selected_label": "B", |
| "selected_token_id": 33, |
| "probability": 0.9983500193011885, |
| "abstained": false, |
| "candidate_probability_mass": 0.9648334149708054, |
| "candidates": [ |
| { |
| "value": "0", |
| "label": "A", |
| "token_id": 32, |
| "probability": 0.0007058337315065528, |
| "logprob": -7.291930675506592 |
| }, |
| { |
| "value": "1", |
| "label": "B", |
| "token_id": 33, |
| "probability": 0.9983500193011885, |
| "logprob": -0.037451162934303284 |
| }, |
| { |
| "value": "2", |
| "label": "C", |
| "token_id": 34, |
| "probability": 0.0009441469673050024, |
| "logprob": -7.001028537750244 |
| } |
| ], |
| "elapsed_ms": 977.959, |
| "queue_ms": 576.01, |
| "backend_requests": 1, |
| "score_retrieval": "selected_token_logprobs", |
| "requested_top_n": 3, |
| "prompt_tokens": 193, |
| "backend_timings": { |
| "cache_n": 0, |
| "prompt_n": 193, |
| "prompt_ms": 314.069, |
| "prompt_per_token_ms": 1.627300518134715, |
| "prompt_per_second": 614.5146448710315, |
| "predicted_n": 1, |
| "predicted_ms": 0.001, |
| "predicted_per_token_ms": 0.0, |
| "predicted_per_second": 0.0 |
| }, |
| "usage": { |
| "prompt_tokens": 193, |
| "completion_tokens": 1, |
| "total_tokens": 194 |
| } |
| } |
| }, |
| "scores_are_calibrated": false, |
| "score_source": "pre_sampling_full_vocabulary_logprobs", |
| "selected_token_logprobs": true, |
| "score_temperature": 1.0, |
| "abstained_fields": [], |
| "elapsed_ms": 983.191, |
| "prompt_preparation_ms": 4.88, |
| "backend_requests": 3, |
| "scored_fields": 3, |
| "max_prompt_tokens": 193, |
| "max_model_len": 16384, |
| "backend_slots": 2, |
| "cache_block_tokens": null, |
| "usage": { |
| "prompt_tokens": 547, |
| "completion_tokens": 3, |
| "total_tokens": 550 |
| } |
| } |
| } |
| } |
| } |
|
|
