Upload 5 files

InterpreterAtomicFlow.py

@@ -0,0 +1,130 @@
+import traceback
+from copy import deepcopy
+from typing import Dict, Any
+
+from .code_interpreters.create_code_interpreter import create_code_interpreter
+
+from aiflows.base_flows import AtomicFlow
+
+def truncate_output(data, max_output_chars=2000):
+    needs_truncation = False
+
+    message = f'Output truncated. Showing the last {max_output_chars} characters.\n\n'
+
+    # Remove previous truncation message if it exists
+    if data.startswith(message):
+        data = data[len(message):]
+        needs_truncation = True
+
+    # If data exceeds max length, truncate it and add message
+    if len(data) > max_output_chars or needs_truncation:
+        data = message + data[-max_output_chars:]
+
+    return data
+
+
+class InterpreterAtomicFlow(AtomicFlow):
+    """This flow is used to run the code passed from the caller.
+    *Expected Input*:
+    - `code`
+    - `language_of_code`
+    
+    *Expected Output*:
+    - `interpreter_output`: output of the code interpreter
+
+    **Full credits to open-interpreter (https://github.com/KillianLucas/open-interpreter)
+    for the usage of code interpreters (package `code_interpreters`) and the function truncate_output()**
+
+    I'm extracting the code interpreter part from open-interpreter because the litellm version of open-interpreter
+    is not compatible with that of the current version of aiflows(v.0.1.7).
+    """
+    def __init__(self,
+                 max_output=2000,
+                 **kwargs):
+        super().__init__(**kwargs)
+        self.max_output = max_output
+        self._code_interpreters = {}
+
+    @classmethod
+    def instantiate_from_config(cls, config):
+        flow_config = deepcopy(config)
+
+        kwargs = {"flow_config": flow_config}
+
+        # ~~~ Instantiate flow ~~~
+        return cls(**kwargs)
+
+
+    def set_up_flow_state(self):
+        """
+        class-specific flow state: language and code,
+        which describes the programming language and the code to run.
+        """
+        super().set_up_flow_state()
+        self.flow_state["language"] = None
+        self.flow_state["code"] = ""
+
+    def _state_update_add_language_and_code(self,
+                                            language: str,
+                                            code: str) -> None:
+        """
+        updates the language and code passed from _process_input_data
+        to the flow state
+        """
+        self.flow_state["language"] = language
+        self.flow_state["code"] = code
+
+    def _check_input(self, input_data: Dict[str, Any]):
+        """
+        Sanity check of input data
+        """
+         # ~~~ Sanity check of input_data ~~~
+        assert "language" in input_data, "attribute 'language' not in input data."
+        assert "code" in input_data, "attribute 'code' not in input data."
+
+
+    def _process_input_data(self, input_data: Dict[str, Any]):
+        """
+        Allocate interpreter if any, pass input data into flow state
+        """
+        # code in Jupyter notebook that starts with '!' is actually shell command.
+        if input_data["language"] == "python" and input_data["code"].startswith("!"):
+            input_data["language"] = "shell"
+            input_data["code"] = input_data["code"][1:]
+
+        # ~~~ Allocate interpreter ~~~
+        # interpreter existence is checked in create_code_interpreter()
+        # TODO: consider: should we put language not supported error into output?
+        language = input_data["language"]
+        if language not in self._code_interpreters:
+            self._code_interpreters[language] = create_code_interpreter(language)
+
+        # ~~~ Pass input data to flow state ~~~
+        self._state_update_add_language_and_code(
+            language=language,
+            code=input_data["code"]
+        )
+
+    def _call(self):
+        output = ""
+        try:
+            code_interpreter = self._code_interpreters[self.flow_state["language"]]
+            code = self.flow_state["code"]
+            for line in code_interpreter.run(code):
+                if "output" in line:
+                    output += "\n" + line["output"]
+                    # Truncate output
+                    output = truncate_output(output, self.max_output)
+                    output = output.strip()
+        except:
+            output = traceback.format_exc()
+            output = output.strip()
+        return output
+
+    def run(
+            self,
+            input_data: Dict[str, Any]):
+        self._check_input(input_data)
+        self._process_input_data(input_data)
+        response = self._call()
+        return {"interpreter_output": response}

InterpreterAtomicFlow.yaml

@@ -0,0 +1,2 @@
+name: "InterpreterAtomicFlow"
+description: "A flow that compiles and runs codes"

README.md

@@ -0,0 +1,363 @@
+# Table of Contents
+
+* [InterpreterAtomicFlow](#InterpreterAtomicFlow)
+  * [InterpreterAtomicFlow](#InterpreterAtomicFlow.InterpreterAtomicFlow)
+    * [set\_up\_flow\_state](#InterpreterAtomicFlow.InterpreterAtomicFlow.set_up_flow_state)
+* [\_\_init\_\_](#__init__)
+* [code\_interpreters](#code_interpreters)
+* [code\_interpreters.language\_map](#code_interpreters.language_map)
+* [code\_interpreters.create\_code\_interpreter](#code_interpreters.create_code_interpreter)
+* [code\_interpreters.languages.javascript](#code_interpreters.languages.javascript)
+  * [preprocess\_javascript](#code_interpreters.languages.javascript.preprocess_javascript)
+* [code\_interpreters.languages.shell](#code_interpreters.languages.shell)
+  * [preprocess\_shell](#code_interpreters.languages.shell.preprocess_shell)
+  * [add\_active\_line\_prints](#code_interpreters.languages.shell.add_active_line_prints)
+* [code\_interpreters.languages.python](#code_interpreters.languages.python)
+  * [preprocess\_python](#code_interpreters.languages.python.preprocess_python)
+  * [add\_active\_line\_prints](#code_interpreters.languages.python.add_active_line_prints)
+  * [AddLinePrints](#code_interpreters.languages.python.AddLinePrints)
+    * [insert\_print\_statement](#code_interpreters.languages.python.AddLinePrints.insert_print_statement)
+    * [process\_body](#code_interpreters.languages.python.AddLinePrints.process_body)
+    * [visit](#code_interpreters.languages.python.AddLinePrints.visit)
+* [code\_interpreters.languages.html](#code_interpreters.languages.html)
+* [code\_interpreters.languages.r](#code_interpreters.languages.r)
+  * [R](#code_interpreters.languages.r.R)
+    * [preprocess\_code](#code_interpreters.languages.r.R.preprocess_code)
+* [code\_interpreters.languages.powershell](#code_interpreters.languages.powershell)
+  * [preprocess\_powershell](#code_interpreters.languages.powershell.preprocess_powershell)
+  * [add\_active\_line\_prints](#code_interpreters.languages.powershell.add_active_line_prints)
+  * [wrap\_in\_try\_catch](#code_interpreters.languages.powershell.wrap_in_try_catch)
+* [code\_interpreters.languages](#code_interpreters.languages)
+* [code\_interpreters.languages.applescript](#code_interpreters.languages.applescript)
+  * [AppleScript](#code_interpreters.languages.applescript.AppleScript)
+    * [preprocess\_code](#code_interpreters.languages.applescript.AppleScript.preprocess_code)
+    * [add\_active\_line\_indicators](#code_interpreters.languages.applescript.AppleScript.add_active_line_indicators)
+    * [detect\_active\_line](#code_interpreters.languages.applescript.AppleScript.detect_active_line)
+    * [detect\_end\_of\_execution](#code_interpreters.languages.applescript.AppleScript.detect_end_of_execution)
+* [code\_interpreters.subprocess\_code\_interpreter](#code_interpreters.subprocess_code_interpreter)
+  * [SubprocessCodeInterpreter](#code_interpreters.subprocess_code_interpreter.SubprocessCodeInterpreter)
+    * [preprocess\_code](#code_interpreters.subprocess_code_interpreter.SubprocessCodeInterpreter.preprocess_code)
+* [code\_interpreters.base\_code\_interpreter](#code_interpreters.base_code_interpreter)
+  * [BaseCodeInterpreter](#code_interpreters.base_code_interpreter.BaseCodeInterpreter)
+
+<a id="InterpreterAtomicFlow"></a>
+
+# InterpreterAtomicFlow
+
+<a id="InterpreterAtomicFlow.InterpreterAtomicFlow"></a>
+
+## InterpreterAtomicFlow Objects
+
+```python
+class InterpreterAtomicFlow(AtomicFlow)
+```
+
+This flow is used to run the code passed from the caller.
+*Expected Input*:
+- `code`
+- `language_of_code`
+
+*Expected Output*:
+- `interpreter_output`: output of the code interpreter
+
+**Full credits to open-interpreter (https://github.com/KillianLucas/open-interpreter)
+for the usage of code interpreters (package `code_interpreters`) and the function truncate_output()**
+
+I'm extracting the code interpreter part from open-interpreter because the litellm version of open-interpreter
+is not compatible with that of the current version of aiflows(v.0.1.7).
+
+<a id="InterpreterAtomicFlow.InterpreterAtomicFlow.set_up_flow_state"></a>
+
+#### set\_up\_flow\_state
+
+```python
+def set_up_flow_state()
+```
+
+class-specific flow state: language and code,
+which describes the programming language and the code to run.
+
+<a id="__init__"></a>
+
+# \_\_init\_\_
+
+<a id="code_interpreters"></a>
+
+# code\_interpreters
+
+<a id="code_interpreters.language_map"></a>
+
+# code\_interpreters.language\_map
+
+<a id="code_interpreters.create_code_interpreter"></a>
+
+# code\_interpreters.create\_code\_interpreter
+
+<a id="code_interpreters.languages.javascript"></a>
+
+# code\_interpreters.languages.javascript
+
+<a id="code_interpreters.languages.javascript.preprocess_javascript"></a>
+
+#### preprocess\_javascript
+
+```python
+def preprocess_javascript(code)
+```
+
+Add active line markers
+Wrap in a try catch
+Add end of execution marker
+
+<a id="code_interpreters.languages.shell"></a>
+
+# code\_interpreters.languages.shell
+
+<a id="code_interpreters.languages.shell.preprocess_shell"></a>
+
+#### preprocess\_shell
+
+```python
+def preprocess_shell(code)
+```
+
+Add active line markers
+Wrap in a try except (trap in shell)
+Add end of execution marker
+
+<a id="code_interpreters.languages.shell.add_active_line_prints"></a>
+
+#### add\_active\_line\_prints
+
+```python
+def add_active_line_prints(code)
+```
+
+Add echo statements indicating line numbers to a shell string.
+
+<a id="code_interpreters.languages.python"></a>
+
+# code\_interpreters.languages.python
+
+<a id="code_interpreters.languages.python.preprocess_python"></a>
+
+#### preprocess\_python
+
+```python
+def preprocess_python(code)
+```
+
+Add active line markers
+Wrap in a try except
+Add end of execution marker
+
+<a id="code_interpreters.languages.python.add_active_line_prints"></a>
+
+#### add\_active\_line\_prints
+
+```python
+def add_active_line_prints(code)
+```
+
+Add print statements indicating line numbers to a python string.
+
+<a id="code_interpreters.languages.python.AddLinePrints"></a>
+
+## AddLinePrints Objects
+
+```python
+class AddLinePrints(ast.NodeTransformer)
+```
+
+Transformer to insert print statements indicating the line number
+before every executable line in the AST.
+
+<a id="code_interpreters.languages.python.AddLinePrints.insert_print_statement"></a>
+
+#### insert\_print\_statement
+
+```python
+def insert_print_statement(line_number)
+```
+
+Inserts a print statement for a given line number.
+
+<a id="code_interpreters.languages.python.AddLinePrints.process_body"></a>
+
+#### process\_body
+
+```python
+def process_body(body)
+```
+
+Processes a block of statements, adding print calls.
+
+<a id="code_interpreters.languages.python.AddLinePrints.visit"></a>
+
+#### visit
+
+```python
+def visit(node)
+```
+
+Overridden visit to transform nodes.
+
+<a id="code_interpreters.languages.html"></a>
+
+# code\_interpreters.languages.html
+
+<a id="code_interpreters.languages.r"></a>
+
+# code\_interpreters.languages.r
+
+<a id="code_interpreters.languages.r.R"></a>
+
+## R Objects
+
+```python
+class R(SubprocessCodeInterpreter)
+```
+
+<a id="code_interpreters.languages.r.R.preprocess_code"></a>
+
+#### preprocess\_code
+
+```python
+def preprocess_code(code)
+```
+
+Add active line markers
+Wrap in a tryCatch for better error handling in R
+Add end of execution marker
+
+<a id="code_interpreters.languages.powershell"></a>
+
+# code\_interpreters.languages.powershell
+
+<a id="code_interpreters.languages.powershell.preprocess_powershell"></a>
+
+#### preprocess\_powershell
+
+```python
+def preprocess_powershell(code)
+```
+
+Add active line markers
+Wrap in try-catch block
+Add end of execution marker
+
+<a id="code_interpreters.languages.powershell.add_active_line_prints"></a>
+
+#### add\_active\_line\_prints
+
+```python
+def add_active_line_prints(code)
+```
+
+Add Write-Output statements indicating line numbers to a PowerShell script.
+
+<a id="code_interpreters.languages.powershell.wrap_in_try_catch"></a>
+
+#### wrap\_in\_try\_catch
+
+```python
+def wrap_in_try_catch(code)
+```
+
+Wrap PowerShell code in a try-catch block to catch errors and display them.
+
+<a id="code_interpreters.languages"></a>
+
+# code\_interpreters.languages
+
+<a id="code_interpreters.languages.applescript"></a>
+
+# code\_interpreters.languages.applescript
+
+<a id="code_interpreters.languages.applescript.AppleScript"></a>
+
+## AppleScript Objects
+
+```python
+class AppleScript(SubprocessCodeInterpreter)
+```
+
+<a id="code_interpreters.languages.applescript.AppleScript.preprocess_code"></a>
+
+#### preprocess\_code
+
+```python
+def preprocess_code(code)
+```
+
+Inserts an end_of_execution marker and adds active line indicators.
+
+<a id="code_interpreters.languages.applescript.AppleScript.add_active_line_indicators"></a>
+
+#### add\_active\_line\_indicators
+
+```python
+def add_active_line_indicators(code)
+```
+
+Adds log commands to indicate the active line of execution in the AppleScript.
+
+<a id="code_interpreters.languages.applescript.AppleScript.detect_active_line"></a>
+
+#### detect\_active\_line
+
+```python
+def detect_active_line(line)
+```
+
+Detects active line indicator in the output.
+
+<a id="code_interpreters.languages.applescript.AppleScript.detect_end_of_execution"></a>
+
+#### detect\_end\_of\_execution
+
+```python
+def detect_end_of_execution(line)
+```
+
+Detects end of execution marker in the output.
+
+<a id="code_interpreters.subprocess_code_interpreter"></a>
+
+# code\_interpreters.subprocess\_code\_interpreter
+
+<a id="code_interpreters.subprocess_code_interpreter.SubprocessCodeInterpreter"></a>
+
+## SubprocessCodeInterpreter Objects
+
+```python
+class SubprocessCodeInterpreter(BaseCodeInterpreter)
+```
+
+<a id="code_interpreters.subprocess_code_interpreter.SubprocessCodeInterpreter.preprocess_code"></a>
+
+#### preprocess\_code
+
+```python
+def preprocess_code(code)
+```
+
+This needs to insert an end_of_execution marker of some kind,
+which can be detected by detect_end_of_execution.
+
+Optionally, add active line markers for detect_active_line.
+
+<a id="code_interpreters.base_code_interpreter"></a>
+
+# code\_interpreters.base\_code\_interpreter
+
+<a id="code_interpreters.base_code_interpreter.BaseCodeInterpreter"></a>
+
+## BaseCodeInterpreter Objects
+
+```python
+class BaseCodeInterpreter()
+```
+
+.run is a generator that yields a dict with attributes: active_line, output
+

__init__.py

@@ -0,0 +1,13 @@
+from .InterpreterAtomicFlow import InterpreterAtomicFlow
+from .code_interpreters import base_code_interpreter
+from .code_interpreters import languages
+from .code_interpreters.languages import python
+from .code_interpreters.languages import r
+from .code_interpreters.languages import applescript
+from .code_interpreters.languages import html
+from .code_interpreters.languages import javascript
+from .code_interpreters.languages import powershell
+from .code_interpreters.languages import shell
+from .code_interpreters import create_code_interpreter
+from .code_interpreters import language_map
+from .code_interpreters import subprocess_code_interpreter

pip_requirements.txt

@@ -0,0 +1,15 @@
+colorama==0.4.6
+pytest==7.3.1
+pytest-cov==4.1.0
+hydra-core==1.3.2
+hydra-colorlog==1.1.0
+wrapt-timeout-decorator==1.3.12.2
+diskcache==5.6.1
+openai==1.0.0
+huggingface_hub==0.19.4
+jsonlines==3.1.0
+jinja2==3.1.2
+mock==5.0.2
+rich==12.6.0
+litellm==1.0.0
+aiflows