wingllm / internal /engine /supervisor.go
bennett-xdlabs's picture
Initial release: WingLLM source and binaries
9e2b962 verified
Raw History Blame Contribute Delete
3.82 kB
package engine
import (
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"time"
"github.com/Rbm3267/wingllm/internal/discovery"
)
type Config struct {
Port int
CtxSize int
DraftNMax int
Threads int
BackendPath string
UseGPUForDraft bool
}
type Supervisor struct {
Config Config
Pair *discovery.PairedModels
Cmd *exec.Cmd
Port int
}
func DefaultConfig() Config {
return Config{
Port: 8081, // llama-server listens on internal port, proxy sits on 8080
CtxSize: 2048,
DraftNMax: 3,
Threads: 6,
UseGPUForDraft: true,
}
}
// FindLlamaServer checks standard Ollama paths, Homebrew paths, and PATH for llama-server.
func FindLlamaServer() (string, error) {
if p, err := exec.LookPath("llama-server"); err == nil {
return p, nil
}
if runtime.GOOS == "windows" {
userProfile := os.Getenv("USERPROFILE")
ollamaServer := filepath.Join(userProfile, "AppData", "Local", "Programs", "Ollama", "lib", "ollama", "llama-server.exe")
if _, err := os.Stat(ollamaServer); err == nil {
return ollamaServer, nil
}
} else if runtime.GOOS == "darwin" {
// Standard macOS Homebrew paths
candidates := []string{
"/opt/homebrew/bin/llama-server",
"/usr/local/bin/llama-server",
}
for _, c := range candidates {
if _, err := os.Stat(c); err == nil {
return c, nil
}
}
}
return "", fmt.Errorf("llama-server executable not found. Install via 'brew install llama.cpp' (macOS) or verify Ollama installation")
}
func NewSupervisor(pair *discovery.PairedModels, cfg Config) *Supervisor {
return &Supervisor{
Config: cfg,
Pair: pair,
Port: cfg.Port,
}
}
// Start launches the heterogeneous speculative llama-server process.
func (s *Supervisor) Start() error {
binPath, err := FindLlamaServer()
if err != nil {
return err
}
args := []string{
"--port", fmt.Sprintf("%d", s.Port),
"-c", fmt.Sprintf("%d", s.Config.CtxSize),
"-t", fmt.Sprintf("%d", s.Config.Threads),
"--no-warmup",
"-m", s.Pair.Target.BlobPath,
"-ngl", "0", // Main target model stays 100% on CPU
"-md", s.Pair.Draft.BlobPath,
"--spec-draft-n-max", fmt.Sprintf("%d", s.Config.DraftNMax),
"--spec-draft-n-min", "1",
"-ctkd", "q8_0", // Quantized draft KV cache saves 50% draft memory
"-ctvd", "q8_0",
}
if s.Config.UseGPUForDraft {
args = append(args, "-ngld", "999") // Offload all draft layers to GPU VRAM
}
cmd := exec.Command(binPath, args...)
cmd.Dir = filepath.Dir(binPath) // Run in engine's own library directory for DLL resolution
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
// Ensure all GPU backend library paths (CUDA v12/v13, Vulkan) are present in PATH for DLL loading
ollamaLib := filepath.Dir(binPath)
cuda12 := filepath.Join(ollamaLib, "cuda_v12")
cuda13 := filepath.Join(ollamaLib, "cuda_v13")
vulkan := filepath.Join(ollamaLib, "vulkan")
cmd.Env = append(os.Environ(),
fmt.Sprintf("PATH=%s;%s;%s;%s;%s", cuda12, cuda13, vulkan, ollamaLib, os.Getenv("PATH")),
fmt.Sprintf("GGML_BACKEND_SEARCH_PATH=%s;%s", cuda12, vulkan),
)
if err := cmd.Start(); err != nil {
return fmt.Errorf("failed to start llama-server: %w", err)
}
s.Cmd = cmd
// Wait up to 30s for the server to be ready
ready := false
healthURL := fmt.Sprintf("http://127.0.0.1:%d/health", s.Port)
for i := 0; i < 30; i++ {
time.Sleep(1 * time.Second)
resp, err := http.Get(healthURL)
if err == nil && resp.StatusCode == http.StatusOK {
ready = true
resp.Body.Close()
break
}
}
if !ready {
s.Stop()
return fmt.Errorf("engine server did not become healthy within 30 seconds")
}
return nil
}
// Stop cleanly terminates the engine process.
func (s *Supervisor) Stop() error {
if s.Cmd != nil && s.Cmd.Process != nil {
return s.Cmd.Process.Kill()
}
return nil
}