Upload graph.pbtxt

graph.pbtxt

@@ -1,36 +1,36 @@
-input_stream: "HTTP_REQUEST_PAYLOAD:input"
-output_stream: "HTTP_RESPONSE_PAYLOAD:output"
-
-node: {
-  name: "LLMExecutor"
-  calculator: "HttpLLMCalculator"
-  input_stream: "LOOPBACK:loopback"
-  input_stream: "HTTP_REQUEST_PAYLOAD:input"
-  input_side_packet: "LLM_NODE_RESOURCES:llm"
-  output_stream: "LOOPBACK:loopback"
-  output_stream: "HTTP_RESPONSE_PAYLOAD:output"
-  input_stream_info: {
-    tag_index: 'LOOPBACK:0',
-    back_edge: true
-  }
-  node_options: {
-      [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {
-          models_path: "./",
-          plugin_config: '{"MAX_PROMPT_LEN": 8192}',
-          enable_prefix_caching: false,
-          cache_size: 10,
-          max_num_seqs: 256,
-          device: "NPU",
-      }
-  }
-  input_stream_handler {
-    input_stream_handler: "SyncSetInputStreamHandler",
-    options {
-      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
-        sync_set {
-          tag_index: "LOOPBACK:0"
-        }
-      }
-    }
-  }
+input_stream: "HTTP_REQUEST_PAYLOAD:input"
+output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+
+node: {
+  name: "LLMExecutor"
+  calculator: "HttpLLMCalculator"
+  input_stream: "LOOPBACK:loopback"
+  input_stream: "HTTP_REQUEST_PAYLOAD:input"
+  input_side_packet: "LLM_NODE_RESOURCES:llm"
+  output_stream: "LOOPBACK:loopback"
+  output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+  input_stream_info: {
+    tag_index: 'LOOPBACK:0',
+    back_edge: true
+  }
+  node_options: {
+      [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {
+          models_path: "./",
+          #plugin_config: '{"MAX_PROMPT_LEN": 2048}',
+          enable_prefix_caching: true,		  
+          cache_size: 10,
+          max_num_seqs: 256,
+          device: "GPU",
+      }
+  }
+  input_stream_handler {
+    input_stream_handler: "SyncSetInputStreamHandler",
+    options {
+      [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+        sync_set {
+          tag_index: "LOOPBACK:0"
+        }
+      }
+    }
+  }
 }