File size: 21,047 Bytes
39fa862 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 | {"type":"snapshot","step":0,"timestamp":0.0,"payload":{"step":0,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":24,"num_cached_entries":0,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[],"waiting":[]}}
{"type":"request","step":1,"timestamp":0.5,"payload":{"request_id":"demo-aaaa1111","seq_id":1,"prompt":"Explain paged attention in two sentences. Then explain prefix caching.","prompt_len":20,"max_tokens":24}}
{"type":"step","step":1,"timestamp":0.55,"payload":{"duration_ms":280,"num_tokens":16,"num_seqs":1,"num_prefill_seqs":1,"num_decode_seqs":0,"deltas":[],"newly_admitted":[1],"finished":[],"preempted":[],"snapshot":{"step":1,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":22,"num_cached_entries":2,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[],"waiting":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"prefilling","prompt_len":20,"num_generated":0,"num_computed_tokens":16,"num_cached_prefix_tokens":0,"block_table":[0,1]}]}}}
{"type":"step","step":2,"timestamp":0.9,"payload":{"duration_ms":180,"num_tokens":4,"num_seqs":1,"num_prefill_seqs":1,"num_decode_seqs":0,"deltas":[],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":2,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":21,"num_cached_entries":2,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":0,"num_computed_tokens":20,"num_cached_prefix_tokens":0,"block_table":[0,1,2]}],"waiting":[]}}}
{"type":"step","step":3,"timestamp":1.22,"payload":{"duration_ms":95,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" Paged","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":3,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":21,"num_cached_entries":2,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":1,"num_computed_tokens":21,"num_cached_prefix_tokens":0,"block_table":[0,1,2]}],"waiting":[]}}}
{"type":"step","step":4,"timestamp":1.34,"payload":{"duration_ms":95,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" attention","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":4,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":21,"num_cached_entries":2,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":2,"num_computed_tokens":22,"num_cached_prefix_tokens":0,"block_table":[0,1,2]}],"waiting":[]}}}
{"type":"step","step":5,"timestamp":1.46,"payload":{"duration_ms":95,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" splits","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":5,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":21,"num_cached_entries":2,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":3,"num_computed_tokens":23,"num_cached_prefix_tokens":0,"block_table":[0,1,2]}],"waiting":[]}}}
{"type":"step","step":6,"timestamp":1.58,"payload":{"duration_ms":95,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" keys","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":6,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":21,"num_cached_entries":3,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":4,"num_computed_tokens":24,"num_cached_prefix_tokens":0,"block_table":[0,1,2]}],"waiting":[]}}}
{"type":"step","step":7,"timestamp":1.7,"payload":{"duration_ms":95,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" and","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":7,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":20,"num_cached_entries":3,"prefix_cache_hits":0,"prefix_cache_lookups":0,"ref_counts":[1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":5,"num_computed_tokens":25,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]}],"waiting":[]}}}
{"type":"request","step":8,"timestamp":2.3,"payload":{"request_id":"demo-bbbb2222","seq_id":2,"prompt":"Explain paged attention in two sentences. Then explain prefix caching.","prompt_len":20,"max_tokens":24}}
{"type":"step","step":9,"timestamp":2.35,"payload":{"duration_ms":140,"num_tokens":5,"num_seqs":2,"num_prefill_seqs":1,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" values","finished":false,"finish_reason":null}],"newly_admitted":[2],"finished":[],"preempted":[],"snapshot":{"step":9,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":19,"num_cached_entries":3,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":5,"num_computed_tokens":25,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]}],"waiting":[{"seq_id":2,"request_id":"demo-bbbb2222","status":"prefilling","prompt_len":20,"num_generated":0,"num_computed_tokens":16,"num_cached_prefix_tokens":16,"block_table":[0,1,4]}]}}}
{"type":"step","step":10,"timestamp":2.47,"payload":{"duration_ms":110,"num_tokens":5,"num_seqs":2,"num_prefill_seqs":1,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" into","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":10,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":19,"num_cached_entries":3,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":6,"num_computed_tokens":26,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":0,"num_computed_tokens":20,"num_cached_prefix_tokens":16,"block_table":[0,1,4]}],"waiting":[]}}}
{"type":"step","step":11,"timestamp":2.69,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":" small","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" Prefix","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":11,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":19,"num_cached_entries":3,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":7,"num_computed_tokens":27,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":1,"num_computed_tokens":21,"num_cached_prefix_tokens":16,"block_table":[0,1,4]}],"waiting":[]}}}
{"type":"step","step":12,"timestamp":2.79,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":",","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" caching","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":12,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":19,"num_cached_entries":3,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":8,"num_computed_tokens":28,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":2,"num_computed_tokens":22,"num_cached_prefix_tokens":16,"block_table":[0,1,4]}],"waiting":[]}}}
{"type":"step","step":13,"timestamp":2.89,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":" fixed","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" reuses","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":13,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":19,"num_cached_entries":3,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":9,"num_computed_tokens":29,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":3,"num_computed_tokens":23,"num_cached_prefix_tokens":16,"block_table":[0,1,4]}],"waiting":[]}}}
{"type":"step","step":14,"timestamp":2.99,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":"-size","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" those","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":14,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":19,"num_cached_entries":4,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":10,"num_computed_tokens":30,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":4,"num_computed_tokens":24,"num_cached_prefix_tokens":16,"block_table":[0,1,4]}],"waiting":[]}}}
{"type":"step","step":15,"timestamp":3.09,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":" blocks","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" blocks","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":15,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":18,"num_cached_entries":4,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":11,"num_computed_tokens":31,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":5,"num_computed_tokens":25,"num_cached_prefix_tokens":16,"block_table":[0,1,4,5]}],"waiting":[]}}}
{"type":"step","step":16,"timestamp":3.19,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":" stored","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" across","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":16,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":18,"num_cached_entries":4,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":12,"num_computed_tokens":32,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":6,"num_computed_tokens":26,"num_cached_prefix_tokens":16,"block_table":[0,1,4,5]}],"waiting":[]}}}
{"type":"step","step":17,"timestamp":3.29,"payload":{"duration_ms":105,"num_tokens":2,"num_seqs":2,"num_prefill_seqs":0,"num_decode_seqs":2,"deltas":[{"request_id":"demo-aaaa1111","new_text":" in","finished":false,"finish_reason":null},{"request_id":"demo-bbbb2222","new_text":" requests","finished":false,"finish_reason":null}],"newly_admitted":[],"finished":[],"preempted":[],"snapshot":{"step":17,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":18,"num_cached_entries":4,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[2,2,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,false,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":1,"request_id":"demo-aaaa1111","status":"running","prompt_len":20,"num_generated":13,"num_computed_tokens":33,"num_cached_prefix_tokens":0,"block_table":[0,1,2,3]},{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":7,"num_computed_tokens":27,"num_cached_prefix_tokens":16,"block_table":[0,1,4,5]}],"waiting":[]}}}
{"type":"step","step":18,"timestamp":3.39,"payload":{"duration_ms":85,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-aaaa1111","new_text":" the GPU.","finished":true,"finish_reason":"stop"}],"newly_admitted":[],"finished":["demo-aaaa1111"],"preempted":[],"snapshot":{"step":18,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":20,"num_cached_entries":5,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[{"seq_id":2,"request_id":"demo-bbbb2222","status":"running","prompt_len":20,"num_generated":8,"num_computed_tokens":28,"num_cached_prefix_tokens":16,"block_table":[0,1,4,5]}],"waiting":[]}}}
{"type":"step","step":19,"timestamp":3.49,"payload":{"duration_ms":80,"num_tokens":1,"num_seqs":1,"num_prefill_seqs":0,"num_decode_seqs":1,"deltas":[{"request_id":"demo-bbbb2222","new_text":" for the same prompts.","finished":true,"finish_reason":"stop"}],"newly_admitted":[],"finished":["demo-bbbb2222"],"preempted":[],"snapshot":{"step":19,"config":{"model":"Qwen/Qwen2.5-0.5B-Instruct","block_size":8,"num_blocks":24,"max_num_seqs":8,"max_num_batched_tokens":32,"prefix_caching":true},"block_pool":{"num_blocks":24,"block_size":8,"num_free_blocks":24,"num_cached_entries":6,"prefix_cache_hits":2,"prefix_cache_lookups":2,"ref_counts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"hashed":[true,true,true,true,true,true,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false,false]},"running":[],"waiting":[]}}}
|