vision module first version

.gitignore

@@ -0,0 +1,443 @@
+# Created by https://www.toptal.com/developers/gitignore/api/python,java,c++,pycharm,visualstudiocode,macos,linux,windows
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,java,c++,pycharm,visualstudiocode,macos,linux,windows
+
+### C++ ###
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+### Java ###
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+replay_pid*
+
+### Linux ###
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+### macOS ###
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+### macOS Patch ###
+# iCloud generated files
+*.icloud
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# Azure Toolkit for IntelliJ plugin
+# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
+.idea/**/azureSettings.xml
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+### Python Patch ###
+# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration
+poetry.toml
+
+# ruff
+.ruff_cache/
+
+# LSP config files
+pyrightconfig.json
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# End of https://www.toptal.com/developers/gitignore/api/python,java,c++,pycharm,visualstudiocode,macos,linux,windows
+
+.*
+flow_modules/

VisionAtomicFlow.py

@@ -0,0 +1,87 @@
+
+from typing import Dict, Any
+from flow_modules.aiflows.OpenAIChatFlowModule import OpenAIChatAtomicFlow
+from flows.utils.general_helpers import encode_image,encode_from_buffer
+import cv2
+
+
+class VisionAtomicFlow(OpenAIChatAtomicFlow):
+    
+    @staticmethod      
+    def get_image(image):
+        extension_dict = {
+            "jpg": "jpeg",
+            "jpeg": "jpeg",
+            "png": "png",
+            "webp": "webp",
+            "gif": "gif"
+        }
+        supported_image_types = ["local_path","url"]
+        assert image.get("type",None) in supported_image_types, f"Must define a valid image type for every image \n your type: {image.get('type',None)} \n supported types{supported_image_types} "
+        
+        processed_image = None
+        url = None
+        if image["type"] == "local_path":
+            processed_image = encode_image(image.get("image"))
+            image_extension_type = image.get("image").split(".")[-1]
+            url = f"data:image/{extension_dict[image_extension_type]};base64, {processed_image}"
+            
+        elif image["type"] == "url":
+            processed_image = image
+            url = image.get("image")
+            
+        return {"type": "image_url", "image_url": {"url": url}}
+    
+    @staticmethod
+    def get_video(video):
+        video_path = video["video_path"]
+        resize = video.get("resize",768)
+        frame_step_size = video.get("frame_step_size",10)
+        start_frame = video.get("start_frame",0)
+        end_frame = video.get("end_frame",None)
+        base64Frames = []
+        video = cv2.VideoCapture(video_path)
+        while video.isOpened():
+            success,frame = video.read()
+            if not success:
+                break
+            _,buffer = cv2.imencode(".jpg",frame)
+            base64Frames.append(encode_from_buffer(buffer))
+        video.release()
+        return map(lambda x: {"image": x, "resize": resize},base64Frames[start_frame:end_frame:frame_step_size])
+
+    @staticmethod
+    def get_user_message(prompt_template, input_data: Dict[str, Any]):
+        content = VisionAtomicFlow._get_message(prompt_template=prompt_template,input_data=input_data)
+        media_data = input_data["data"]
+        if "video" in media_data:
+            content = [ content[0], *VisionAtomicFlow.get_video(media_data["video"])]
+        if "images" in media_data:
+            images = [VisionAtomicFlow.get_image(image) for image in media_data["images"]]
+            content.extend(images)
+        return content
+    
+    @staticmethod
+    def _get_message(prompt_template, input_data: Dict[str, Any]):
+        template_kwargs = {}
+        for input_variable in prompt_template.input_variables:
+            template_kwargs[input_variable] = input_data[input_variable]
+        msg_content = prompt_template.format(**template_kwargs)
+        return [{"type": "text", "text": msg_content}]
+
+    def _process_input(self, input_data: Dict[str, Any]):
+        if self._is_conversation_initialized():
+            # Construct the message using the human message prompt template
+            user_message_content = self.get_user_message(self.human_message_prompt_template, input_data)
+
+        else:
+            # Initialize the conversation (add the system message, and potentially the demonstrations)
+            self._initialize_conversation(input_data)
+            if getattr(self, "init_human_message_prompt_template", None) is not None:
+                # Construct the message using the query message prompt template
+                user_message_content = self.get_user_message(self.init_human_message_prompt_template, input_data)
+            else:
+                user_message_content = self.get_user_message(self.human_message_prompt_template, input_data)
+
+        self._state_update_add_chat_message(role=self.flow_config["user_name"],
+                                            content=user_message_content)

VisionAtomicFlow.yaml

@@ -0,0 +1,49 @@
+# This is an abstract flow, therefore some required fields are not defined (and must be defined by the concrete flow)
+enable_cache: True
+
+n_api_retries: 6
+wait_time_between_retries: 20
+
+system_name: system
+user_name: user
+assistant_name: assistant
+
+backend:
+  _target_: flows.backends.llm_lite.LiteLLMBackend
+  api_infos: ???
+  model_name: "gpt-4-vision-preview"
+  n: 1
+  max_tokens: 2000
+  temperature: 0.3
+  top_p: 0.2
+  frequency_penalty: 0
+  presence_penalty: 0
+
+system_message_prompt_template:
+  _target_: flows.prompt_template.JinjaPrompt
+
+init_human_message_prompt_template:
+  _target_: flows.prompt_template.JinjaPrompt
+
+human_message_prompt_template:
+  _target_: flows.prompt_template.JinjaPrompt
+  template: "{{query}}"
+  input_variables:
+    - "query"
+input_interface_initialized:
+  - "query"
+  - "data"
+
+query_message_prompt_template:
+  _target_: flows.prompt_template.JinjaPrompt
+
+
+previous_messages:
+  first_k: null  # Note that the first message is the system prompt
+  last_k: null
+
+demonstrations: null
+demonstrations_response_template: null
+
+output_interface:
+  - "api_output"

__init__.py

@@ -0,0 +1,7 @@
+# ~~~ Specify the dependencies ~~
+dependencies = [
+    {"url": "aiflows/OpenAIChatFlowModule", "revision": "eeec09b71e967ce426553e2300c5689f6ea6a662"}
+]
+from flows import flow_verse
+flow_verse.sync_dependencies(dependencies)
+from .VisionAtomicFlow import VisionAtomicFlow

pip_requirements.txt

@@ -0,0 +1 @@
+opencv-python==4.8.1.78