Instructions to use HelloSun/flint_cpu_opt with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Notebooks
Google Colab
Kaggle
Local Apps Settings

How to use HelloSun/flint_cpu_opt with llama.cpp:

Install (macOS, Linux)

curl -LsSf https://llama.app/install.sh | sh
# Start a local OpenAI-compatible server with a web UI:
llama serve -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL
# Run inference directly in the terminal:
llama cli -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL

Install from WinGet (Windows)

winget install llama.cpp
# Start a local OpenAI-compatible server with a web UI:
llama serve -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL
# Run inference directly in the terminal:
llama cli -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL

Build from source code

git clone https://github.com/ggml-org/llama.cpp.git
cd llama.cpp
cmake -B build
cmake --build build -j --target llama-app
# Start a local OpenAI-compatible server with a web UI:
./build/bin/llama serve -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL
# Run inference directly in the terminal:
./build/bin/llama cli -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL

Use Docker

docker model run hf.co/HelloSun/flint_cpu_opt:UD-Q4_K_XL

Llama new
LM Studio
Jan

vLLM

How to use HelloSun/flint_cpu_opt with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "HelloSun/flint_cpu_opt"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/chat/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "HelloSun/flint_cpu_opt",
		"messages": [
			{
				"role": "user",
				"content": "What is the capital of France?"
			}
		]
	}'

Use Docker

docker model run hf.co/HelloSun/flint_cpu_opt:UD-Q4_K_XL

Ollama
How to use HelloSun/flint_cpu_opt with Ollama:
```
ollama run hf.co/HelloSun/flint_cpu_opt:UD-Q4_K_XL
```
Unsloth Desktop

How to use HelloSun/flint_cpu_opt with Pi:

Start the llama.cpp server

# Install llama.cpp:
brew install llama.cpp
# Start a local OpenAI-compatible server:
llama serve -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL

Configure the model in Pi

# Install Pi:
npm install -g @earendil-works/pi-coding-agent
# Add to ~/.pi/agent/models.json:
{
  "providers": {
    "llama-cpp": {
      "baseUrl": "http://localhost:8080/v1",
      "api": "openai-completions",
      "apiKey": "none",
      "models": [
        {
          "id": "HelloSun/flint_cpu_opt:UD-Q4_K_XL"
        }
      ]
    }
  }
}

Run Pi

# Start Pi in your project directory:
pi

Docker Model Runner
How to use HelloSun/flint_cpu_opt with Docker Model Runner:
```
docker model run hf.co/HelloSun/flint_cpu_opt:UD-Q4_K_XL
```

Lemonade

How to use HelloSun/flint_cpu_opt with Lemonade:

Pull the model

# Download Lemonade from https://lemonade-server.ai/
lemonade pull HelloSun/flint_cpu_opt:UD-Q4_K_XL

Run and chat with the model

lemonade run user.flint_cpu_opt-UD-Q4_K_XL

List all available models

lemonade list

Hermes Agent

How to use HelloSun/flint_cpu_opt with Hermes Agent:

Start the llama.cpp server

# Install llama.cpp:
brew install llama.cpp
# Start a local OpenAI-compatible server:
llama serve -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL

Configure Hermes

# Install Hermes:
curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash
hermes setup
# Point Hermes at the local server:
hermes config set model.provider custom
hermes config set model.base_url http://127.0.0.1:8080/v1
hermes config set model.default HelloSun/flint_cpu_opt:UD-Q4_K_XL

Run Hermes

hermes

Atomic Chat

OpenClaw

How to use HelloSun/flint_cpu_opt with OpenClaw:

Start the llama.cpp server

# Install llama.cpp:
brew install llama.cpp
# Start a local OpenAI-compatible server:
llama serve -hf HelloSun/flint_cpu_opt:UD-Q4_K_XL

Configure OpenClaw

# Install OpenClaw:
npm install -g openclaw@latest
# Register the local server and set it as the default model:
openclaw onboard --non-interactive --mode local \
  --auth-choice custom-api-key \
  --custom-base-url http://127.0.0.1:8080/v1 \
  --custom-model-id "HelloSun/flint_cpu_opt:UD-Q4_K_XL" \
  --custom-provider-id llama-cpp \
  --custom-compatibility openai \
  --custom-text-input \
  --accept-risk \
  --skip-health

Run OpenClaw

openclaw agent --local --agent main --message "Hello from Hugging Face"

flint_cpu_opt / benchmarks /nvfp4 /test_nvfp4.c

HelloSun

Add NVFP4 CPU kernels results, integrate model card, add handoff

45f1d0b verified 4 days ago

Raw History Blame Contribute Delete

2.34 kB

// Compare ggml_vec_dot_nvfp4_q8_0 (arch path) against ggml_vec_dot_nvfp4_q8_0_generic on random data.
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stddef.h>

typedef struct { uint8_t d[4]; uint8_t qs[32]; } block_nvfp4;   // 64 elems: 4 UE4M3 scales + 32 nibble bytes
typedef struct { _Float16 d; int8_t qs[32]; } block_q8_0;       // 32 elems

void ggml_vec_dot_nvfp4_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc);
void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc);

static uint32_t rng_state = 12345;
static uint32_t rnd(void) { rng_state = rng_state * 1664525u + 1013904223u; return rng_state >> 8; }

void ggml_cpu_init(void);
int main(void) {
    ggml_cpu_init();
    int fails = 0;
    for (int trial = 0; trial < 20; ++trial) {
        const int n = 64 * (1 + (trial * 37) % 200);
        const int nb = n / 64;
        block_nvfp4 * w = aligned_alloc(64, sizeof(block_nvfp4) * nb);
        block_q8_0  * a = aligned_alloc(64, sizeof(block_q8_0) * (n / 32));
        for (int i = 0; i < nb; ++i) {
            for (int s = 0; s < 4; ++s) w[i].d[s] = (uint8_t)(rnd() % 127);  // avoid NaN code 0x7F
            for (int j = 0; j < 32; ++j) w[i].qs[j] = (uint8_t)rnd();
        }
        for (int i = 0; i < n / 32; ++i) {
            float amax = 0.f; float xs[32];
            for (int j = 0; j < 32; ++j) { xs[j] = ((int)(rnd() % 20001) - 10000) / 1000.0f; if (fabsf(xs[j]) > amax) amax = fabsf(xs[j]); }
            float d = amax / 127.0f; float id = d ? 1.0f / d : 0.0f;
            a[i].d = (_Float16) d;
            for (int j = 0; j < 32; ++j) a[i].qs[j] = (int8_t) lrintf(xs[j] * id);
        }
        float r_gen = 0.f, r_arch = 0.f;
        ggml_vec_dot_nvfp4_q8_0_generic(n, &r_gen, 0, w, 0, a, 0, 1);
        ggml_vec_dot_nvfp4_q8_0(n, &r_arch, 0, w, 0, a, 0, 1);
        double rel = fabs((double) r_arch - r_gen) / (fabs((double) r_gen) + 1e-9);
        printf("n=%6d generic=% .6f arch=% .6f rel=%.3e %s\n", n, r_gen, r_arch, rel, rel < 1e-5 ? "OK" : "FAIL");
        if (rel >= 1e-5) fails++;
        free(w); free(a);
    }
    printf("%s (%d failures)\n", fails ? "FAILED" : "PASSED", fails);
    return fails != 0;
}

	// Compare ggml_vec_dot_nvfp4_q8_0 (arch path) against ggml_vec_dot_nvfp4_q8_0_generic on random data.
	#include <stdint.h>
	#include <stdio.h>
	#include <stdlib.h>
	#include <string.h>
	#include <math.h>
	#include <stddef.h>

	typedef struct { uint8_t d[4]; uint8_t qs[32]; } block_nvfp4; // 64 elems: 4 UE4M3 scales + 32 nibble bytes
	typedef struct { _Float16 d; int8_t qs[32]; } block_q8_0; // 32 elems

	void ggml_vec_dot_nvfp4_q8_0(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc);
	void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * s, size_t bs, const void * vx, size_t bx, const void * vy, size_t by, int nrc);

	static uint32_t rng_state = 12345;
	static uint32_t rnd(void) { rng_state = rng_state * 1664525u + 1013904223u; return rng_state >> 8; }

	void ggml_cpu_init(void);
	int main(void) {
	ggml_cpu_init();
	int fails = 0;
	for (int trial = 0; trial < 20; ++trial) {
	const int n = 64 * (1 + (trial * 37) % 200);
	const int nb = n / 64;
	block_nvfp4 * w = aligned_alloc(64, sizeof(block_nvfp4) * nb);
	block_q8_0 * a = aligned_alloc(64, sizeof(block_q8_0) * (n / 32));
	for (int i = 0; i < nb; ++i) {
	for (int s = 0; s < 4; ++s) w[i].d[s] = (uint8_t)(rnd() % 127); // avoid NaN code 0x7F
	for (int j = 0; j < 32; ++j) w[i].qs[j] = (uint8_t)rnd();
	}
	for (int i = 0; i < n / 32; ++i) {
	float amax = 0.f; float xs[32];
	for (int j = 0; j < 32; ++j) { xs[j] = ((int)(rnd() % 20001) - 10000) / 1000.0f; if (fabsf(xs[j]) > amax) amax = fabsf(xs[j]); }
	float d = amax / 127.0f; float id = d ? 1.0f / d : 0.0f;
	a[i].d = (_Float16) d;
	for (int j = 0; j < 32; ++j) a[i].qs[j] = (int8_t) lrintf(xs[j] * id);
	}
	float r_gen = 0.f, r_arch = 0.f;
	ggml_vec_dot_nvfp4_q8_0_generic(n, &r_gen, 0, w, 0, a, 0, 1);
	ggml_vec_dot_nvfp4_q8_0(n, &r_arch, 0, w, 0, a, 0, 1);
	double rel = fabs((double) r_arch - r_gen) / (fabs((double) r_gen) + 1e-9);
	printf("n=%6d generic=% .6f arch=% .6f rel=%.3e %s\n", n, r_gen, r_arch, rel, rel < 1e-5 ? "OK" : "FAIL");
	if (rel >= 1e-5) fails++;
	free(w); free(a);
	}
	printf("%s (%d failures)\n", fails ? "FAILED" : "PASSED", fails);
	return fails != 0;
	}