Download internal/engine/supervisor.go from bennett-xdlabs/wingllm: direct link, hf CLI and curl.
- Browser
- Download file 3.82 kB
-
https://huggingface.co/bennett-xdlabs/wingllm/resolve/main/internal/engine/supervisor.go
- Command line
-
hf download hf://bennett-xdlabs/wingllm/internal/engine/supervisor.go
-
curl -L -o supervisor.go https://huggingface.co/bennett-xdlabs/wingllm/resolve/main/internal/engine/supervisor.go
3.82 kB
| package engine | |
| import ( | |
| "fmt" | |
| "net/http" | |
| "os" | |
| "os/exec" | |
| "path/filepath" | |
| "runtime" | |
| "time" | |
| "github.com/Rbm3267/wingllm/internal/discovery" | |
| ) | |
| type Config struct { | |
| Port int | |
| CtxSize int | |
| DraftNMax int | |
| Threads int | |
| BackendPath string | |
| UseGPUForDraft bool | |
| } | |
| type Supervisor struct { | |
| Config Config | |
| Pair *discovery.PairedModels | |
| Cmd *exec.Cmd | |
| Port int | |
| } | |
| func DefaultConfig() Config { | |
| return Config{ | |
| Port: 8081, // llama-server listens on internal port, proxy sits on 8080 | |
| CtxSize: 2048, | |
| DraftNMax: 3, | |
| Threads: 6, | |
| UseGPUForDraft: true, | |
| } | |
| } | |
| // FindLlamaServer checks standard Ollama paths, Homebrew paths, and PATH for llama-server. | |
| func FindLlamaServer() (string, error) { | |
| if p, err := exec.LookPath("llama-server"); err == nil { | |
| return p, nil | |
| } | |
| if runtime.GOOS == "windows" { | |
| userProfile := os.Getenv("USERPROFILE") | |
| ollamaServer := filepath.Join(userProfile, "AppData", "Local", "Programs", "Ollama", "lib", "ollama", "llama-server.exe") | |
| if _, err := os.Stat(ollamaServer); err == nil { | |
| return ollamaServer, nil | |
| } | |
| } else if runtime.GOOS == "darwin" { | |
| // Standard macOS Homebrew paths | |
| candidates := []string{ | |
| "/opt/homebrew/bin/llama-server", | |
| "/usr/local/bin/llama-server", | |
| } | |
| for _, c := range candidates { | |
| if _, err := os.Stat(c); err == nil { | |
| return c, nil | |
| } | |
| } | |
| } | |
| return "", fmt.Errorf("llama-server executable not found. Install via 'brew install llama.cpp' (macOS) or verify Ollama installation") | |
| } | |
| func NewSupervisor(pair *discovery.PairedModels, cfg Config) *Supervisor { | |
| return &Supervisor{ | |
| Config: cfg, | |
| Pair: pair, | |
| Port: cfg.Port, | |
| } | |
| } | |
| // Start launches the heterogeneous speculative llama-server process. | |
| func (s *Supervisor) Start() error { | |
| binPath, err := FindLlamaServer() | |
| if err != nil { | |
| return err | |
| } | |
| args := []string{ | |
| "--port", fmt.Sprintf("%d", s.Port), | |
| "-c", fmt.Sprintf("%d", s.Config.CtxSize), | |
| "-t", fmt.Sprintf("%d", s.Config.Threads), | |
| "--no-warmup", | |
| "-m", s.Pair.Target.BlobPath, | |
| "-ngl", "0", // Main target model stays 100% on CPU | |
| "-md", s.Pair.Draft.BlobPath, | |
| "--spec-draft-n-max", fmt.Sprintf("%d", s.Config.DraftNMax), | |
| "--spec-draft-n-min", "1", | |
| "-ctkd", "q8_0", // Quantized draft KV cache saves 50% draft memory | |
| "-ctvd", "q8_0", | |
| } | |
| if s.Config.UseGPUForDraft { | |
| args = append(args, "-ngld", "999") // Offload all draft layers to GPU VRAM | |
| } | |
| cmd := exec.Command(binPath, args...) | |
| cmd.Dir = filepath.Dir(binPath) // Run in engine's own library directory for DLL resolution | |
| cmd.Stdout = os.Stdout | |
| cmd.Stderr = os.Stderr | |
| // Ensure all GPU backend library paths (CUDA v12/v13, Vulkan) are present in PATH for DLL loading | |
| ollamaLib := filepath.Dir(binPath) | |
| cuda12 := filepath.Join(ollamaLib, "cuda_v12") | |
| cuda13 := filepath.Join(ollamaLib, "cuda_v13") | |
| vulkan := filepath.Join(ollamaLib, "vulkan") | |
| cmd.Env = append(os.Environ(), | |
| fmt.Sprintf("PATH=%s;%s;%s;%s;%s", cuda12, cuda13, vulkan, ollamaLib, os.Getenv("PATH")), | |
| fmt.Sprintf("GGML_BACKEND_SEARCH_PATH=%s;%s", cuda12, vulkan), | |
| ) | |
| if err := cmd.Start(); err != nil { | |
| return fmt.Errorf("failed to start llama-server: %w", err) | |
| } | |
| s.Cmd = cmd | |
| // Wait up to 30s for the server to be ready | |
| ready := false | |
| healthURL := fmt.Sprintf("http://127.0.0.1:%d/health", s.Port) | |
| for i := 0; i < 30; i++ { | |
| time.Sleep(1 * time.Second) | |
| resp, err := http.Get(healthURL) | |
| if err == nil && resp.StatusCode == http.StatusOK { | |
| ready = true | |
| resp.Body.Close() | |
| break | |
| } | |
| } | |
| if !ready { | |
| s.Stop() | |
| return fmt.Errorf("engine server did not become healthy within 30 seconds") | |
| } | |
| return nil | |
| } | |
| // Stop cleanly terminates the engine process. | |
| func (s *Supervisor) Stop() error { | |
| if s.Cmd != nil && s.Cmd.Process != nil { | |
| return s.Cmd.Process.Kill() | |
| } | |
| return nil | |
| } | |