File size: 3,881 Bytes
789c7db |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
#! /bin/bash
################################################################################
# Shell script that starts a copy of Ollama with a base model plus all the
# available LoRA adapters in this repository.
#
# Converts `.safetensors` to `.gguf` for running in Ollama.
# Target application is GitHub Actions and CPU only systems.
# To keep the intermediate files, remove the `rm` command at the end.
#
# To run this script:
# 1. Install an appropriate build of Ollama for your machine and use port 55555
# See https://docs.ollama.com/linux#manual-install and set port with
# ```
# curl -fsSL https://ollama.com/download/ollama-linux-amd64.tgz \
# | sudo tar zx -C /usr
# OLLAMA_HOST="localhost:55555" ollama serve &
# ```
# See https://docs.ollama.com/faq#how-do-i-configure-ollama-server for other
# operating systems.
# 3. Download the intrinsics library by running:
# hf download ibm-granite/rag-intrinsics-lib --local-dir ./rag-intrinsics-lib
# 4. Edit the constants BASE_MODEL_NAME, BASE_MODEL_ORG as needed
# 5. Run this script from the root of your local copy of rag-intrinsics-lib.
################################################################################
echo "Setup"
OLLAMA_HOST="localhost:55555"
OLLAMA_DIR="_ollama"
mkdir -p $OLLAMA_DIR
echo ""
echo "Download base model"
MODEL_DIR="$OLLAMA_DIR/models"
mkdir -p $MODEL_DIR
OLLAMA_MODEL_NAME=granite3.3:8b # Quantized model on Ollama
BASE_MODEL_NAME=granite-3.3-8b-instruct
BASE_MODEL_NAME_UPPER=granite-3.3-8B-instruct # llama.cpp output
BASE_MODEL_ORG=ibm-granite
pip install huggingface_hub
hf download $BASE_MODEL_ORG/$BASE_MODEL_NAME --local-dir $MODEL_DIR/$BASE_MODEL_NAME
echo ""
echo "Clone llama.cpp and install dependencies"
LLAMA_CPP_FOLDER_NAME="$OLLAMA_DIR/llama.cpp"
git clone --single-branch --branch master https://github.com/ggml-org/llama.cpp.git $LLAMA_CPP_FOLDER_NAME
pip install -r $LLAMA_CPP_FOLDER_NAME/requirements/requirements-convert_hf_to_gguf.txt
pip install -r $LLAMA_CPP_FOLDER_NAME/requirements/requirements-convert_hf_to_gguf_update.txt
pip install -r $LLAMA_CPP_FOLDER_NAME/requirements/requirements-convert_lora_to_gguf.txt
echo ""
OUTTYPE="q8_0"
OUTTYPE_UPPER="Q8_0" # llama.cpp output
echo "Convert base model to GGUF: $OUTTYPE"
python $LLAMA_CPP_FOLDER_NAME/convert_hf_to_gguf.py $MODEL_DIR/$BASE_MODEL_NAME --outtype $OUTTYPE
MODEL_GGUF=$(realpath "$MODEL_DIR/$BASE_MODEL_NAME/$BASE_MODEL_NAME_UPPER-$OUTTYPE_UPPER.gguf")
echo ""
echo "Convert LoRA adapters to GGUF"
LORA_DIRS=$( find . -name "$BASE_MODEL_NAME" -path "*/lora/$BASE_MODEL_NAME*" | sort | cut -c 3- )
for LORA_DIR in $LORA_DIRS; do
LORA_GGUF="$LORA_DIR/$BASE_MODEL_NAME_UPPER-$OUTTYPE_UPPER-LoRA.gguf"
if [ ! -f "$LORA_GGUF" ]; then
python $LLAMA_CPP_FOLDER_NAME/convert_lora_to_gguf.py $LORA_DIR --base $MODEL_DIR/$BASE_MODEL_NAME --outtype $OUTTYPE
fi
done
echo ""
echo "Create Modelfiles and Ollama models"
MODELFILE_DIR="$OLLAMA_DIR/Modelfiles"
mkdir -p $MODELFILE_DIR
for LORA_DIR in $LORA_DIRS; do
LORA_GGUF=$(realpath "$LORA_DIR/$BASE_MODEL_NAME_UPPER-$OUTTYPE_UPPER-LoRA.gguf")
MODEL_FILE=$LORA_DIR
MODELFILE=${MODEL_FILE//\//_}
MODELFILE=${MODELFILE//../"Modelfile"}
MODELFILEPATH=$MODELFILE_DIR/$MODELFILE
LORA_NAME=$(echo "$LORA_DIR" | cut -d "/" -f 1)
echo ""
echo "Creating $LORA_NAME | $MODELFILEPATH"
# Use GGUF converted model
# printf "FROM $MODEL_GGUF\nADAPTER $LORA_GGUF\n" > $MODELFILEPATH
# printf "FROM $MODEL_GGUF\nADAPTER $LORA_GGUF\n"
# Use quantized model from Ollama
printf "FROM $OLLAMA_MODEL_NAME\nADAPTER $LORA_GGUF\n" > $MODELFILEPATH
printf "FROM $OLLAMA_MODEL_NAME\nADAPTER $LORA_GGUF\n"
echo ""
OLLAMA_HOST=$OLLAMA_HOST ollama create $LORA_NAME -f $MODELFILEPATH
done
echo ""
echo "Clean up"
echo "rm -rf $OLLAMA_DIR"
rm -rf $OLLAMA_DIR
|