feat: integrate vision agent v3 for logs and code (#51)

what looks like for V2:



https://github.com/landing-ai/vision-agent/assets/132986242/95eb4605-0d37-402c-8401-b843bfd769ad


V3:


https://github.com/landing-ai/vision-agent/assets/132986242/4728641e-a9a7-49eb-81e7-2a5eff155224

app/api/vision-agent/route.ts

@@ -57,7 +57,7 @@ export const POST = withLogging(
 
     const fetchResponse = await fetch(
       `https://api.dev.landing.ai/v1/agent/chat?agent_class=vision_agent&visualize_output=true&self_reflection=${enableSelfReflection}`,
-      // `http://localhost:5001/v1/agent/chat?agent_class=vision_agent&visualize_output=true&self_reflection=${enableSelfReflection}`,
+      // `http://localhost:5001/v1/agent/chat?agent_class=vision_agent&self_reflection=${enableSelfReflection}`,
       {
         method: 'POST',
         headers: {
@@ -114,9 +114,70 @@ export const POST = withLogging(
       const stream = fetchResponse.body.pipeThrough(
         new TransformStream({
           transform: async (chunk, controller) => {
-            const message = decoder.decode(chunk);
-            logger.info(session, { message }, request, '__AGENT_RESPONSE');
-            controller.enqueue(encoder.encode(message));
+            const data = decoder.decode(chunk);
+            data.split('\n').forEach(line => {
+              if (!line.trim()) {
+                return;
+              }
+              try {
+                const json = JSON.parse(line);
+                let message = (json.log ?? '') + '\n';
+                if (json.task || json.plan || json.reflection) {
+                  const arr = json.plan
+                    ? json.plan
+                    : json.task
+                      ? [json.task]
+                      : [json.reflection];
+                  const keys = Object.keys(arr[0]);
+                  message += '\n';
+                  message += '| ' + keys.join(' | ') + ' |' + '\n';
+                  message +=
+                    new Array(keys.length + 1).fill('|').join(' :- ') + '\n';
+                  arr.forEach((obj: any) => {
+                    message +=
+                      '| ' +
+                      keys.map(key => obj[key]).join(' | ') +
+                      ' |' +
+                      '\n';
+                  });
+                  message += '\n';
+                }
+                if (json.tools) {
+                  message += '\n';
+                  message += '| ' + 'Descriptions' + ' |' + '\n';
+                  message += '| ' + ':-' + ' |' + '\n';
+                  json.tools.forEach((tool: string) => {
+                    message += '| ' + tool + ' |' + '\n';
+                  });
+                  message += '\n';
+                }
+                if (json.code) {
+                  message += `\`\`\`python\n${json.code}\n\`\`\`\n`;
+                }
+                if (json.result) {
+                  message += `\`\`\`\n${json.result}\n\`\`\`\n`;
+                }
+
+                logger.info(
+                  session,
+                  {
+                    message,
+                  },
+                  request,
+                  '__AGENT_RESPONSE',
+                );
+                controller.enqueue(encoder.encode(message));
+              } catch (e) {
+                console.log(data);
+                logger.error(
+                  session,
+                  { message: (e as Error).message, data },
+                  request,
+                );
+                controller.error(e);
+                controller.terminate();
+              }
+            });
           },
         }),
       );

components/chat/ChatMessage.tsx

@@ -48,119 +48,95 @@ export function ChatMessage({
         {message.role === 'user' ? <IconUser /> : <IconOpenAI />}
       </div>
       <div className="flex-1 px-1 ml-4 space-y-2 overflow-hidden">
-        {logs && message.role !== 'user' && (
-          <div className="bg-slate-100 dark:bg-slate-900 mb-4 p-4 max-h-[400px] overflow-auto">
-            <div className="text-xl font-bold">Thinking Process</div>
-            <MemoizedReactMarkdown
-              className="break-words text-sm"
-              remarkPlugins={[remarkGfm, remarkMath]}
-              components={{
-                p({ children }) {
+        {logs && (
+          <MemoizedReactMarkdown
+            className="break-words"
+            remarkPlugins={[remarkGfm, remarkMath]}
+            components={{
+              p({ children, ...props }) {
+                if (
+                  props.node.children.some(
+                    child =>
+                      child.type === 'element' && child.tagName === 'img',
+                  )
+                ) {
                   return (
-                    <p className="my-2 last:mb-0 whitespace-pre-line">
+                    <p className="flex flex-wrap gap-2 items-start">
                       {children}
                     </p>
                   );
-                },
-                code({ children, className, ...props }) {
-                  const match = /language-(\w+)/.exec(className || '');
-                  return (
-                    <CodeBlock
-                      key={Math.random()}
-                      language={(match && match[1]) || ''}
-                      value={String(children).replace(/\n$/, '')}
-                      {...props}
-                    />
-                  );
-                },
-              }}
-            >
-              {logs}
-            </MemoizedReactMarkdown>
-          </div>
-        )}
-        <MemoizedReactMarkdown
-          className="break-words"
-          remarkPlugins={[remarkGfm, remarkMath]}
-          components={{
-            p({ children, ...props }) {
-              if (
-                props.node.children.some(
-                  child => child.type === 'element' && child.tagName === 'img',
-                )
-              ) {
-                return (
-                  <p className="flex flex-wrap gap-2 items-start">{children}</p>
-                );
-              }
-              return (
-                <p className="my-2 last:mb-0 whitespace-pre-line">{children}</p>
-              );
-            },
-            img(props) {
-              if (props.src?.endsWith('.mp4')) {
+                }
                 return (
-                  <video src={props.src} controls width={500} height={500} />
+                  <p className="my-2 last:mb-0 whitespace-pre-line">
+                    {children}
+                  </p>
                 );
-              }
-              return (
-                <Tooltip>
-                  <TooltipTrigger asChild>
-                    <Img
-                      src={props.src ?? '/landing.png'}
-                      alt={props.alt ?? 'answer-image'}
-                      quality={100}
-                      className="cursor-zoom-in"
-                      sizes="(min-width: 66em) 25vw,
-                      (min-width: 44em) 40vw,
-                      100vw"
-                    />
-                  </TooltipTrigger>
-                  <TooltipContent>
-                    <Img
-                      className="m-2"
-                      src={props.src ?? '/landing.png'}
-                      alt={props.alt ?? 'answer-image'}
-                      quality={100}
-                      width={500}
-                    />
-                  </TooltipContent>
-                </Tooltip>
-              );
-            },
-            code({ node, inline, className, children, ...props }) {
-              if (children.length) {
-                if (children[0] == '▍') {
+              },
+              img(props) {
+                if (props.src?.endsWith('.mp4')) {
                   return (
-                    <span className="mt-1 cursor-default animate-pulse">▍</span>
+                    <video src={props.src} controls width={500} height={500} />
                   );
                 }
+                return (
+                  <Tooltip>
+                    <TooltipTrigger asChild>
+                      <Img
+                        src={props.src ?? '/landing.png'}
+                        alt={props.alt ?? 'answer-image'}
+                        quality={100}
+                        className="cursor-zoom-in"
+                        sizes="(min-width: 66em) 25vw,
+                      (min-width: 44em) 40vw,
+                      100vw"
+                      />
+                    </TooltipTrigger>
+                    <TooltipContent>
+                      <Img
+                        className="m-2"
+                        src={props.src ?? '/landing.png'}
+                        alt={props.alt ?? 'answer-image'}
+                        quality={100}
+                        width={500}
+                      />
+                    </TooltipContent>
+                  </Tooltip>
+                );
+              },
+              code({ node, inline, className, children, ...props }) {
+                // if (children.length) {
+                //   if (children[0] == '▍') {
+                //     return (
+                //       <span className="mt-1 cursor-default animate-pulse">▍</span>
+                //     );
+                //   }
 
-                children[0] = (children[0] as string).replace('`▍`', '▍');
-              }
+                //   children[0] = (children[0] as string).replace('`▍`', '▍');
+                // }
+
+                const match = /language-(\w+)/.exec(className || '');
+                // if (inline) {
+                //   return (
+                //     <code className={className} {...props}>
+                //       {children}
+                //     </code>
+                //   );
+                // }
 
-              const match = /language-(\w+)/.exec(className || '');
-              if (inline) {
                 return (
-                  <code className={className} {...props}>
-                    {children}
-                  </code>
+                  <CodeBlock
+                    key={Math.random()}
+                    language={(match && match[1]) || ''}
+                    value={String(children).replace(/\n$/, '')}
+                    {...props}
+                  />
                 );
-              }
-
-              return (
-                <CodeBlock
-                  key={Math.random()}
-                  language={(match && match[1]) || ''}
-                  value={String(children).replace(/\n$/, '')}
-                  {...props}
-                />
-              );
-            },
-          }}
-        >
-          {content}
-        </MemoizedReactMarkdown>
+              },
+            }}
+          >
+            {logs}
+          </MemoizedReactMarkdown>
+        )}
         {/* <ChatMessageActions message={message} /> */}
         {isLoading && <Loading />}
       </div>

lib/messageUtils.ts

@@ -42,11 +42,6 @@ export const getCleanedUpMessages = ({
   content,
   role,
 }: Pick<MessageBase, 'role' | 'content'>) => {
-  if (role === 'user') {
-    return {
-      content,
-    };
-  }
   if (content.split(CLEANED_SEPARATOR).length === 2) {
     return {
       logs: content.split(CLEANED_SEPARATOR)[0],
@@ -54,38 +49,39 @@ export const getCleanedUpMessages = ({
     };
   }
   const [logs = '', answer = ''] = content.split('<ANSWER>');
-  const cleanedLogs = [];
-  let left = 0;
-  let right = 0;
-  while (right < logs.length) {
-    if (Object.keys(PAIRS).includes(content[right])) {
-      cleanedLogs.push(content.substring(left, right));
-      left = right++;
-      while (
-        right < content.length &&
-        PAIRS[content[left]] !== content[right]
-      ) {
-        right++;
-      }
-      if (content[left] === MIDDLE_STARTER) {
-        // add the text alignment so it can be shown as a table
-        const separators = logs
-          .substring(left, right)
-          .split(MIDDLE_SEPARATOR).length;
-        if (separators > 0) {
-          cleanedLogs.push(
-            Array(separators + 1)
-              .fill('|')
-              .join(' :- '),
-          );
-        }
-      }
-      left = ++right;
-    } else {
-      right++;
-    }
-  }
-  cleanedLogs.push(content.substring(left, right));
+  // console.log(logs);
+  // const cleanedLogs = [];
+  // let left = 0;
+  // let right = 0;
+  // while (right < logs.length) {
+  //   if (Object.keys(PAIRS).includes(content[right])) {
+  //     cleanedLogs.push(content.substring(left, right));
+  //     left = right++;
+  //     while (
+  //       right < content.length &&
+  //       PAIRS[content[left]] !== content[right]
+  //     ) {
+  //       right++;
+  //     }
+  //     if (content[left] === MIDDLE_STARTER) {
+  //       // add the text alignment so it can be shown as a table
+  //       const separators = logs
+  //         .substring(left, right)
+  //         .split(MIDDLE_SEPARATOR).length;
+  //       if (separators > 0) {
+  //         cleanedLogs.push(
+  //           Array(separators + 1)
+  //             .fill('|')
+  //             .join(' :- '),
+  //         );
+  //       }
+  //     }
+  //     left = ++right;
+  //   } else {
+  //     right++;
+  //   }
+  // }
+  // cleanedLogs.push(content.substring(left, right));
   const [answerText, imagesStr = ''] = answer.split('<VIZ>');
   const [imagesArrayStr, ...rest] = imagesStr.split('</VIZ>');
   const images = imagesArrayStr
@@ -93,7 +89,7 @@ export const getCleanedUpMessages = ({
     .map(str => str.replace('<IMG>', ''))
     .slice(0, -1);
   return {
-    logs: cleanedLogs.join('').replace(/│/g, '|').split('|\n\n|').join('|\n|'),
+    logs: logs,
     content:
       answerText.replace('</</ANSWER>', '').replace('</ANSWER>', '') +
       '\n\n' +