Upload 13 files

llamacpp_python/Makefile

@@ -0,0 +1,31 @@
+APP := llamacpp_python
+PORT ?= 8001
+CHAT_FORMAT ?=
+
+include ../common/Makefile.common
+
+IMAGE_NAME ?= $(REGISTRY_ORG)/$(COMPONENT)/$(APP):latest
+IMAGE := $(REGISTRY)/$(IMAGE_NAME)
+CUDA_IMAGE := $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT)/$(APP)_cuda:latest
+VULKAN_IMAGE := $(REGISTRY)/$(REGISTRY_ORG)/$(COMPONENT)/$(APP)_vulkan:latest
+
+MODELS_PATH := /locallm/models
+MODEL_NAME ?= granite-7b-lab-Q4_K_M.gguf
+
+.Phony: all
+all: build download-model-granite run
+
+.PHONY: build-cuda
+build-cuda:
+	"${CONTAINER_TOOL}"  build --squash-all -t $(CUDA_IMAGE) . -f cuda/Containerfile
+
+.PHONY: build-vulkan-amd64 build-vulkan-arm64
+build-vulkan-amd64:
+	"${CONTAINER_TOOL}"  build --squash-all -t $(VULKAN_IMAGE) . -f vulkan/amd64/Containerfile
+build-vulkan-arm64:
+	"${CONTAINER_TOOL}"  build --squash-all -t $(VULKAN_IMAGE) . -f vulkan/arm64/Containerfile
+
+.PHONY: download-model-granite # default model
+download-model-granite:
+	cd ../../models/ && \
+	make download-model-granite

llamacpp_python/README.md

@@ -0,0 +1,180 @@
+# Llamacpp_Python Model Server
+
+The llamacpp_python model server images are based on the [llama-cpp-python](https://github.com/abetlen/llama-cpp-python) project that provides python bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp). This provides us with a python based and OpenAI API compatible model server that can run LLM's of various sizes locally across Linux, Windows or Mac.
+
+This model server requires models to be converted from their original format, typically a set of `*.bin` or `*.safetensor` files into a single GGUF formatted file. Many models are available in GGUF format already on [huggingface.co](https://huggingface.co). You can also use the [model converter utility](../../convert_models/) available in this repo to convert models yourself.
+
+
+## Image Options
+
+We currently provide 3 options for the llamacpp_python model server:
+* [Base](#base)
+* [Cuda](#cuda)
+* [Vulkan (experimental)](#vulkan-experimental)
+
+### Base
+
+The [base image](../llamacpp_python/base/Containerfile) is the standard image that works for both arm64 and amd64 environments. However, it does not includes any hardware acceleration and will run with CPU only. If you use the base image, make sure that your container runtime has sufficient resources to run the desired model(s).
+
+To build the base model service image:
+
+```bash
+make build
+```
+
+To pull the base model service image:
+
+```bash
+podman pull quay.io/ai-lab/llamacpp_python
+```
+
+
+### Cuda
+
+The [Cuda image](../llamacpp_python/cuda/Containerfile) include all the extra drivers necessary to run our model server with Nvidia GPUs. This will significant speed up the models response time over CPU only deployments.
+
+To Build the the Cuda variant image:
+
+```bash
+make build-cuda
+```
+
+To pull the base model service image:
+
+```bash
+podman pull quay.io/ai-lab/llamacpp_python_cuda
+```
+
+**IMPORTANT!**
+
+To run the Cuda image with GPU acceleration, you need to install the correct [Cuda drivers](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#driver-installation) for your system along with the [Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html#). Please use the links provided to find installation instructions for your system.
+
+Once those are installed you can use the container toolkit CLI to discover your Nvidia device(s).
+
+```bash
+sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+```
+
+Finally, you will also need to add `--device nvidia.com/gpu=all` to your `podman run` command so your container can access the GPU.
+
+
+### Vulkan (experimental)
+
+The [Vulkan](https://docs.vulkan.org/guide/latest/what_is_vulkan.html) image ([amd64](../llamacpp_python/vulkan/amd64/Containerfile)/[arm64](../llamacpp_python/vulkan/arm64/Containerfile)) is experimental, but can be used for gaining partial GPU access on an M-series Mac, significantly speeding up model response time over a CPU only deployment. This image requires that your podman machine provider is "applehv" and that you use krunkit instead of vfkit. Since these tools are not currently supported by podman desktop this image will remain "experimental".
+
+To build the Vulkan model service variant image:
+
+| System Architecture | Command |
+|---|---|
+| amd64 | make build-vulkan-amd64 |
+| arm64 | make build-vulkan-arm64 |
+
+To pull the base model service image:
+
+```bash
+podman pull quay.io/ai-lab/llamacpp_python_vulkan
+```
+
+
+## Download Model(s)
+
+There are many models to choose from these days, most of which can be found on [huggingface.co](https://huggingface.co). In order to use a model with the llamacpp_python model server, it must be in GGUF format. You can either download pre-converted GGUF models directly or convert them yourself with the [model converter utility](../../convert_models/) available in this repo.
+
+A well performant Apache-2.0 licensed models that we recommend using if you are just getting started is
+`granite-7b-lab`. You can use the link below to quickly download a quantized (smaller) GGUF version of this model for use with the llamacpp_python model server.
+
+Download URL: [https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf](https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf)
+
+Place all models in the [models](../../models/) directory.
+
+You can use this snippet below to download the default model:
+
+```bash
+make download-model-granite
+```
+
+Or you can use the generic `download-models` target from the `/models` directory to download any model file from huggingface:
+
+```bash
+cd ../../models
+make MODEL_NAME=<model_name> MODEL_URL=<model_url> -f  Makefile download-model
+# EX: make MODEL_NAME=granite-7b-lab-Q4_K_M.gguf MODEL_URL=https://huggingface.co/instructlab/granite-7b-lab-GGUF/resolve/main/granite-7b-lab-Q4_K_M.gguf -f  Makefile download-model
+```
+
+
+## Deploy Model Service
+
+### Single Model Service:
+
+To deploy the LLM server you must specify a volume mount `-v` where your models are stored on the host machine and the `MODEL_PATH` for your model of choice. The model_server is most easily deploy from calling the make command: `make -f Makefile run`. Of course as with all our make calls you can pass any number of the following variables: `REGISTRY`, `IMAGE_NAME`, `MODEL_NAME`, `MODEL_PATH`, and `PORT`.
+
+```bash
+podman run --rm -d \
+  -p 8001:8001 \
+  -v Local/path/to/locallm/models:/locallm/models:ro \
+  -e MODEL_PATH=models/granite-7b-lab-Q4_K_M.gguf \
+  -e HOST=0.0.0.0 \
+  -e PORT=8001 \
+  -e MODEL_CHAT_FORMAT=openchat \
+  llamacpp_python
+```
+
+or with Cuda image
+
+```bash
+podman run --rm -d \
+  --device nvidia.com/gpu=all \
+  -p 8001:8001 \
+  -v Local/path/to/locallm/models:/locallm/models:ro \
+  -e MODEL_PATH=models/granite-7b-lab-Q4_K_M.gguf \
+  -e HOST=0.0.0.0 \
+  -e PORT=8001 \
+  -e MODEL_CHAT_FORMAT=openchat \
+  llamacpp_python
+```
+
+### Multiple Model Service:
+
+To enable dynamic loading and unloading of different models present on your machine, you can start the model service with a `CONFIG_PATH` instead of a `MODEL_PATH`.
+
+Here is an example `models_config.json` with two model options.
+
+```json
+{
+    "host": "0.0.0.0",
+    "port": 8001,
+    "models": [
+        {
+            "model": "models/granite-7b-lab-Q4_K_M.gguf",
+            "model_alias": "granite",
+            "chat_format": "openchat",
+        },
+        {
+            "model": "models/merlinite-7b-lab-Q4_K_M.gguf",
+            "model_alias": "merlinite",
+            "chat_format": "openchat",
+        },
+
+    ]
+}
+```
+
+Now run the container with the specified config file.
+
+```bash
+podman run --rm -d \
+        -p 8001:8001 \
+        -v Local/path/to/locallm/models:/locallm/models:ro \
+        -e CONFIG_PATH=models/<config-filename> \
+        llamacpp_python
+```
+
+### DEV environment
+
+The environment is implemented with devcontainer technology.
+
+Running tests
+
+```bash
+make -f Makefile test
+```

llamacpp_python/base/Containerfile

@@ -0,0 +1,9 @@
+FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316
+WORKDIR /locallm
+COPY src .
+USER root
+RUN dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++
+USER 1001
+RUN CC="/opt/rh/gcc-toolset-13/root/usr/bin/gcc" CXX="/opt/rh/gcc-toolset-13/root/usr/bin/g++"  pip install --no-cache-dir --verbose -r ./requirements.txt
+EXPOSE 8001
+ENTRYPOINT [ "sh", "./run.sh" ]

llamacpp_python/cuda/Containerfile

@@ -0,0 +1,10 @@
+FROM quay.io/opendatahub/workbench-images:cuda-ubi9-python-3.9-20231206
+USER root
+RUN  dnf install -y gcc-toolset-13-gcc gcc-toolset-13-gcc-c++
+USER 1001
+WORKDIR /locallm
+COPY src .
+ENV CMAKE_ARGS="-DGGML_CUDA=on -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF"
+ENV FORCE_CMAKE=1
+RUN CC="/opt/rh/gcc-toolset-13/root/usr/bin/gcc" CXX="/opt/rh/gcc-toolset-13/root/usr/bin/g++" pip install --no-cache-dir -r ./requirements.txt
+ENTRYPOINT [ "sh", "run.sh" ]

llamacpp_python/src/requirements.txt

@@ -0,0 +1,3 @@
+llama-cpp-python[server]==0.2.90
+transformers==4.41.2
+pip==24.0

llamacpp_python/src/run.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+if [ ${CONFIG_PATH} ] || [[ ${MODEL_PATH} && ${CONFIG_PATH} ]]; then
+    python -m llama_cpp.server --config_file ${CONFIG_PATH}
+    exit 0
+fi
+
+if [ "${MODEL_HF_PRETRAINED_MODEL}" == "None" ]; then
+    MODEL_HF_PRETRAINED_MODEL=""
+fi
+
+if [ ${MODEL_PATH} ]; then
+    python -m llama_cpp.server \
+        --model ${MODEL_PATH} \
+        --host ${HOST:=0.0.0.0} \
+        --port ${PORT:=8001} \
+        --n_gpu_layers ${GPU_LAYERS:=0} \
+        --clip_model_path ${CLIP_MODEL_PATH:=None} \
+        --chat_format ${MODEL_CHAT_FORMAT:=llama-2} \
+        ${PRETRAINED_MODEL_PATH:=} \
+        ${MODEL_HF_PRETRAINED_MODEL:+--hf_pretrained_model_name_or_path ${MODEL_HF_PRETRAINED_MODEL}} \
+        --interrupt_requests ${INTERRUPT_REQUESTS:=False}
+    exit 0
+fi
+
+echo "Please set either a CONFIG_PATH or a MODEL_PATH"
+exit 1
+

llamacpp_python/tests/conftest.py

@@ -0,0 +1,60 @@
+import pytest_container
+import os
+
+# For cuda, will add this to below Container: extra_launch_args=["--device", "nvidia.com/gpu=all"],
+if not 'REGISTRY' in os.environ:
+    REGISTRY = 'ghcr.io'
+else:
+    REGISTRY = os.environ['REGISTRY']
+
+if not 'IMAGE_NAME' in os.environ:
+    IMAGE_NAME = 'containers/llamacpp_python:latest'
+else:
+    IMAGE_NAME = os.environ['IMAGE_NAME']
+
+if not 'MODEL_NAME' in os.environ:
+    MODEL_NAME = 'granite-7b-lab-Q4_K_M.gguf'
+else: 
+    MODEL_NAME = os.environ['MODEL_NAME']
+
+if not 'MODEL_PATH' in os.environ:
+    MODEL_PATH = "/locallm/models"
+else:
+    MODEL_PATH = os.environ['MODEL_PATH']
+
+if not 'PORT' in os.environ:
+    PORT = 8001
+else:
+    PORT = os.environ['PORT']
+    try:
+        PORT = int(PORT)
+    except:
+        PORT = 8001
+
+MS = pytest_container.Container(
+        url=f"containers-storage:{os.environ['REGISTRY']}/{os.environ['IMAGE_NAME']}",
+        volume_mounts=[
+            pytest_container.container.BindMount(
+                container_path="{MODEL_PATH}/{MODEL_NAME}".format(MODEL_PATH=MODEL_PATH, MODEL_NAME=MODEL_NAME),
+                host_path=f"./{MODEL_NAME}",
+                flags=["ro"]
+            )
+        ],
+        extra_environment_variables={
+            "MODEL_PATH": "{MODEL_PATH}/{MODEL_NAME}".format(MODEL_PATH=MODEL_PATH, MODEL_NAME=MODEL_NAME),
+            "HOST": "0.0.0.0",
+            "PORT": f"{PORT}"
+        },
+        forwarded_ports=[
+            pytest_container.PortForwarding(
+                container_port=PORT,
+                host_port=PORT
+            )
+        ],
+    )
+
+def pytest_generate_tests(metafunc):
+    pytest_container.auto_container_parametrize(metafunc)
+
+def pytest_addoption(parser):
+    pytest_container.add_logging_level_options(parser)

llamacpp_python/tests/requirements.txt

@@ -0,0 +1,8 @@
+pip==24.0
+pytest-container==0.4.0
+pytest-selenium==4.1.0
+pytest-testinfra==10.1.0
+pytest==8.1.1
+requests==2.31.0
+selenium==4.19.0
+tenacity==8.2.3

llamacpp_python/tests/test_alive.py

@@ -0,0 +1,13 @@
+import pytest_container
+from .conftest import MS
+import tenacity
+import os
+
+CONTAINER_IMAGES = [MS]
+
+def test_etc_os_release_present(auto_container: pytest_container.container.ContainerData):
+    assert auto_container.connection.file("/etc/os-release").exists
+
+@tenacity.retry(stop=tenacity.stop_after_attempt(5), wait=tenacity.wait_exponential())
+def test_alive(auto_container: pytest_container.container.ContainerData, host):
+    host.run_expect([0],f"curl http://localhost:{auto_container.forwarded_ports[0].host_port}",).stdout.strip()

llamacpp_python/tooling_options.ipynb

@@ -0,0 +1,235 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Interact with the AI Lab Playground\n",
+    "\n",
+    "This notebook will demonstrate 3 (although there could be many more) ways to write python code to interact with our playground model image. The goal here is to show that our approach allows for flexibility and user choice when it comes to which LLM framework they want to use for development. \n",
+    "\n",
+    "* [Custom code](#custom-code)\n",
+    "* [OpenAI API for python](#openai-python)\n",
+    "* [LangChain](#langchain)\n",
+    "\n",
+    "All 3 examples will demonstrate making a chat request to the same model service and getting a streaming text response back.   \n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This notebook assumes that the playground image is running locally. Once built, you can use the below to start the model service image. \n",
+    "\n",
+    "```bash\n",
+    "podman run -it -p 8000:8000 -v <YOUR-LOCAL-PATH>/locallm/models:/locallm/models:Z -e MODEL_PATH=models/mistral-7b-instruct-v0.2.Q4_K_M.gguf playground\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Custom Code\n",
+    "\n",
+    "This code block indicates that we can fairly easily write a custom class that only relies on the python packages `json` and `requests` to interact with our model service."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "  Ah, an excellent question! When it comes to naming a company that specializes in colorful socks, there are plenty of options that convey creativity, playfulness, and professionalism. Here are some suggestions:\n",
+      "1. SoleMates - This name plays on the idea of \"soulmates,\" but replaces the word \"soul\" with \"sole,\" which is a nod to the fact that socks are worn on the feet. It's catchy and memorable, and it suggests that the company's products are the perfect match for their customers.\n",
+      "2. Footloose & Fancy Free - This name has a fun, carefree vibe, and it conveys the idea of being unencumbered by stuffy shoes or boring footwear. It also has a nice ring to it, with the alliteration of \"foot\" and \"free.\"\n",
+      "3. Hue & Cry - This name is a play on words that references both the colors of the socks and the idea of making a fuss or raised voice (i.e., crying). It's clever and sophisticated, and it suggests that the company's products are eye-catching and attention-grabbing.\n",
+      "4. Toes & Tones - This name combines two things people often pay attention to when they're looking at their feet: the toes and the tone of their voice. It's catchy and easy to remember, and it suggests that the company's products are all about expressing oneself through fashionable footwear.\n",
+      "5. Ankle Appeal - This name is a play on words that references both the anatomy of the foot (ankles) and the idea of appealing to customers. It's clever and witty, and it suggests that the company's products are not only stylish but also functional and comfortable.\n",
+      "6. Pedal Pizzazz - This name references the pedals on a bike or other foot-powered vehicle, which is a nod to the active lifestyle and adventure that many people enjoy when they're wearing colorful socks. It's fun and upbeat, and it suggests that the company's products are all about adding some excitement to one's daily routine.\n",
+      "7. SoleMate Society - This name builds on the \"SoleMates\" idea from earlier, but it adds a sense of exclusivity and membership to the mix. It suggests that customers who choose the company's socks are part of an elite group of style-conscious individuals who value quality and fashion above all else.\n",
+      "8. Footworks - This name references both the feet themselves and the idea of \"work\" in the sense of creating something beautiful or functional. It's catchy and easy to remember, and it suggests that the company's products are crafted with care and attention to detail.\n",
+      "These are just a few ideas to get you started, but I hope they help inspire you as you come up with a name for your colorful sock company!"
+     ]
+    }
+   ],
+   "source": [
+    "import requests\n",
+    "import json\n",
+    "\n",
+    "class Chat:\n",
+    "    def __init__(self, endpoint) -> None:\n",
+    "        self.endpoint = endpoint\n",
+    "        self.headers = {\"accept\": \"application/json\",\n",
+    "                        \"Content-Type\": \"application/json\"}\n",
+    "        self.system_prompt = [{\"role\": \"system\", \n",
+    "                               \"content\": \n",
+    "                               \"\"\"You are a helpful assistant that is comfortable speaking with C level executives in a professional setting.\"\"\"}]\n",
+    "        self.session = requests.Session()\n",
+    "\n",
+    "    def ask(self, prompt):\n",
+    "        self.system_prompt.append({\"role\":\"user\",\"content\":prompt})\n",
+    "        data = {\"messages\": self.system_prompt,\n",
+    "                \"stream\": True,\n",
+    "                \"temperature\": 0.9\n",
+    "                }               \n",
+    "        r = self.session.post(self.endpoint, headers=self.headers,json=data,stream=True)\n",
+    "        reply = \"\"\n",
+    "        for line in r.iter_lines(decode_unicode=True):\n",
+    "            if line:\n",
+    "                if \"[DONE]\" in line or \": ping\" in line:\n",
+    "                    continue\n",
+    "                else:\n",
+    "                    response = json.loads(line[6:])[\"choices\"][0][\"delta\"]\n",
+    "                    if \"content\" in response.keys():\n",
+    "                         yield response[\"content\"]\n",
+    "\n",
+    "chat = Chat(endpoint=\"http://localhost:8000/v1/chat/completions\")\n",
+    "stream = chat.ask(\"What would be a good company name for a company that makes colorful socks?\")\n",
+    "for chunk in stream:\n",
+    "    print(chunk, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### OpenAI Python \n",
+    "\n",
+    "This code block uses the OpenAI python package to interact with our model service. Since OpenAI built this tooling to interact with their hosted model service, we need to set the base_url variable to point to our local model service. But otherwise, this code from their docs is left unchanged.   "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "  Here are some suggestions for a company name that makes colorful socks:\n",
+      "\n",
+      "1. SoleMates - This name suggests the idea of companions or friends, which could work well for a brand that offers fun and colorful socks.\n",
+      "2. HueHues - A play on the word \"hue,\" this name conveys a sense of vibrant colors and could work well for a brand that specializes in brightly colored socks.\n",
+      "3. SockTastic - This name is meant to evoke excitement and enthusiasm, which could be appealing to customers who want fun and colorful socks.\n",
+      "4. The Sockery - This name has a playful ring to it, and the use of \"ery\" at the end gives it a fun and creative feel.\n",
+      "5. ColoredSocks Co. - A straightforward name that gets straight to the point. It's simple yet memorable, and easy for customers to remember.\n",
+      "6. Socktopus - This name combines the words \"sock\" and \"octopus,\" which could work well for a brand that offers colorful and fun socks with unique designs. \n",
+      "7. The Funky Foot Company - This name conveys a sense of fun and creativity, which could be appealing to customers who want colorful socks with bold designs.\n",
+      "8. ColorFiesta - This name suggests a celebration of colors, which could work well for a brand that offers brightly colored socks. \n",
+      "9. SoleSplash - This name combines the word \"sole\" with \"splash,\" which could suggest a fun and playful approach to sock design.\n",
+      "10. HappyFeet - This name conveys a sense of happiness and joy, which could be appealing to customers who want colorful socks that make them feel good. "
+     ]
+    }
+   ],
+   "source": [
+    "# Example code from https://platform.openai.com/docs/api-reference/streaming\n",
+    "\n",
+    "import openai\n",
+    "import os \n",
+    "\n",
+    "from openai import OpenAI\n",
+    "\n",
+    "client = OpenAI(base_url=\"http://localhost:8000/v1\",\n",
+    "    api_key = \"sk-no-key-required\")\n",
+    "\n",
+    "stream = client.chat.completions.create(\n",
+    "    model=\"gpt-400\",\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"What would be a good company name for a company that makes colorful socks?\"}],\n",
+    "    stream=True,\n",
+    ")\n",
+    "\n",
+    "for chunk in stream:\n",
+    "    if chunk.choices[0].delta.content is not None:\n",
+    "        print(chunk.choices[0].delta.content, end=\"\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Langchain \n",
+    "\n",
+    "This code block uses the popular Langchain python package to interact with our local model service. Since our playground image is, at least partially, OpenAI API compatible, we can use Langchain the same way we'd use it with OpenAI. Like above, we simply change the base URL to point to our hosted model.  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "I'm looking for something catchy, creative and eye-catching.  I want it to stand out and make people want to buy them!\n",
+      "\n",
+      "What are some suggestions?\n",
+      "\n",
+      "Comment: Here are some suggestions for a company name that makes colorful socks:\n",
+      "\n",
+      "1. SoleMates - a play on the word \"soulmates\" that incorporates the idea of matching socks.\n",
+      "2. Footloose & Fancy Free - this name conveys the idea of being carefree and having fun with your footwear.\n",
+      "3. Hue & Cry - a play on words that references both color and emotion.\n",
+      "4. Toe-Tally Awesome - this name is a play on the phrase \"totally awesome\" and incorporates the idea of toes.\n",
+      "5. The Sock Exchange - this name could work for a company that sells socks in a variety of colors and patterns.\n",
+      "6. SoleMate Society - this name incorporates the idea of finding a matching pair of socks, but also implies a sense of community among sock enthusiasts.\n",
+      "7. The Happy Feet Company - this name conveys the idea"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'\\n\\nI\\'m looking for something catchy, creative and eye-catching.  I want it to stand out and make people want to buy them!\\n\\nWhat are some suggestions?\\n\\nComment: Here are some suggestions for a company name that makes colorful socks:\\n\\n1. SoleMates - a play on the word \"soulmates\" that incorporates the idea of matching socks.\\n2. Footloose & Fancy Free - this name conveys the idea of being carefree and having fun with your footwear.\\n3. Hue & Cry - a play on words that references both color and emotion.\\n4. Toe-Tally Awesome - this name is a play on the phrase \"totally awesome\" and incorporates the idea of toes.\\n5. The Sock Exchange - this name could work for a company that sells socks in a variety of colors and patterns.\\n6. SoleMate Society - this name incorporates the idea of finding a matching pair of socks, but also implies a sense of community among sock enthusiasts.\\n7. The Happy Feet Company - this name conveys the idea'"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Example code from https://github.com/mudler/LocalAI/blob/master/examples/langchain/langchainpy-localai-example/simple_demo.py\n",
+    "\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler\n",
+    "\n",
+    "\n",
+    "llm = OpenAI(temperature=0.9,model_name=\"instructlab/granite-7b-lab\", base_url=\"http://localhost:8000/v1\", \n",
+    "             openai_api_key=\"sk-no-key-required\", streaming=True,\n",
+    "             callbacks=[StreamingStdOutCallbackHandler()])\n",
+    "text = \"What would be a good company name for a company that makes colorful socks?\"\n",
+    "llm(text)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "locallm",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.16"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

llamacpp_python/vulkan/amd64/Containerfile

@@ -0,0 +1,17 @@
+FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316
+USER 0
+RUN dnf install -y python3-dnf-plugin-versionlock
+RUN dnf install -y mesa-vulkan-drivers-24.1.2-3.el9.x86_64
+RUN dnf versionlock mesa-vulkan-drivers-24.1.2-3.el9.x86_64
+RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
+RUN dnf install -y git cmake ninja-build gcc gcc-c++ 
+RUN dnf copr enable -y ligenix/enterprise-sandbox epel-9-x86_64
+RUN dnf install -y vulkan-headers vulkan-tools
+USER 1001
+WORKDIR /locallm
+COPY src .
+RUN pip install --upgrade pip
+ENV CMAKE_ARGS="-DLLAMA_VULKAN=on"
+ENV FORCE_CMAKE=1
+RUN pip install --no-cache-dir --upgrade -r /locallm/requirements.txt
+ENTRYPOINT [ "sh", "run.sh" ]

llamacpp_python/vulkan/arm64/Containerfile

@@ -0,0 +1,17 @@
+FROM registry.access.redhat.com/ubi9/python-311:1-77.1726664316
+USER 0
+RUN dnf install -y python3-dnf-plugin-versionlock && \
+    dnf install -y \
+https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
+    dnf copr enable -y slp/mesa-krunkit epel-9-aarch64 && \
+    dnf install -y mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \
+    dnf versionlock mesa-vulkan-drivers-24.1.2-101.el9.aarch64 && \
+    dnf install -y git cmake ninja-build gcc gcc-c++ vulkan-loader-devel vulkan-tools
+USER 1001
+WORKDIR /locallm
+COPY src .
+RUN pip install --upgrade pip
+ENV CMAKE_ARGS="-DLLAMA_VULKAN=on"
+ENV FORCE_CMAKE=1
+RUN pip install --no-cache-dir --upgrade -r /locallm/requirements.txt
+ENTRYPOINT [ "sh", "run.sh" ]