| TMPDIR=$(mktemp -d /tmp/sima.XXXXXX) | |
| function cleanup () { | |
| rm -r ${TMPDIR} | |
| } | |
| trap cleanup EXIT | |
| cat >${TMPDIR}/config.py <<'EOF' | |
| def get_layer_configuration(model_properties, layer): | |
| if layer["part"] == "A_BF16_W_INT8": | |
| precision = "BF16" | |
| elif layer["is_group"]: | |
| precision = "A_BF16_W_INT8" | |
| else: | |
| precision = "A_BF16_W_INT4" | |
| return {"precision": precision} | |
| EOF | |
| llima-compile -c ${TMPDIR}/config.py --max_num_tokens 2048 --input_height 256 --input_width 256 $@ | |