Hugging Face's logo

Spaces:
YOGESHJIO5770
/
ai_api
Runtime error

App Files Community
ai_api
File size: 4,207 Bytes
cd8bd0a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
/**
 * OmniRoute — k6 Load / Performance Test (T-5)
 *
 * Tests the proxy endpoint under sustained load to measure:
 *   - Request throughput (RPS)
 *   - Response latency (p50, p95, p99)
 *   - Error rate
 *   - Concurrent connection handling
 *
 * Usage:
 *   k6 run tests/load/proxy-load.js
 *   k6 run tests/load/proxy-load.js --env BASE_URL=https://llms.omniroute.online
 *   k6 run tests/load/proxy-load.js --env VUS=50 --env DURATION=120s
 *
 * Prerequisites:
 *   - k6 installed: https://grafana.com/docs/k6/latest/set-up/install-k6/
 *   - OMNIROUTE_API_KEY env var or --env API_KEY=... set
 */

import http from "k6/http";
import { check, sleep } from "k6";
import { Rate, Trend } from "k6/metrics";

// ── Custom metrics ──
const errorRate = new Rate("errors");
const chatLatency = new Trend("chat_latency", true); // in ms
const healthLatency = new Trend("health_latency", true);

// ── Configuration ──
const BASE_URL = __ENV.BASE_URL || "http://localhost:3000";
const API_KEY = __ENV.API_KEY || __ENV.OMNIROUTE_API_KEY || "test-key";
const VUS = parseInt(__ENV.VUS || "10", 10);
const DURATION = __ENV.DURATION || "60s";

export const options = {
  scenarios: {
    // Ramp-up scenario for stress testing
    chat_stress: {
      executor: "ramping-vus",
      startVUs: 1,
      stages: [
        { duration: "10s", target: VUS }, // Ramp up
        { duration: DURATION, target: VUS }, // Sustained load
        { duration: "10s", target: 0 }, // Ramp down
      ],
      exec: "chatCompletions",
    },
    // Constant rate for health checks
    health_check: {
      executor: "constant-vus",
      vus: 2,
      duration: DURATION,
      exec: "healthCheck",
    },
  },
  thresholds: {
    http_req_duration: ["p(95)<5000"], // 95% of requests < 5s
    errors: ["rate<0.1"], // Error rate < 10%
    chat_latency: ["p(50)<3000", "p(95)<8000"],
    health_latency: ["p(95)<500"],
  },
};

// ── Headers ──
const headers = {
  "Content-Type": "application/json",
  Authorization: `Bearer ${API_KEY}`,
};

// ── Scenarios ──

/**
 * Chat Completions — main proxy endpoint
 * Sends a simple non-streaming chat request.
 */
export function chatCompletions() {
  const payload = JSON.stringify({
    model: "gpt-4o-mini",
    messages: [{ role: "user", content: "Say hello in one word." }],
    temperature: 0,
    max_tokens: 10,
    stream: false,
  });

  const res = http.post(`${BASE_URL}/v1/chat/completions`, payload, {
    headers,
    timeout: "15s",
  });

  chatLatency.add(res.timings.duration);

  const passed = check(res, {
    "status is 200": (r) => r.status === 200,
    "has choices": (r) => {
      try {
        const body = JSON.parse(r.body);
        return body.choices && body.choices.length > 0;
      } catch {
        return false;
      }
    },
    "response time < 10s": (r) => r.timings.duration < 10000,
  });

  errorRate.add(!passed);
  sleep(0.5);
}

/**
 * Health Check — lightweight endpoint to measure base latency.
 */
export function healthCheck() {
  const res = http.get(`${BASE_URL}/api/monitoring/health`, {
    headers: { Authorization: `Bearer ${API_KEY}` },
    timeout: "5s",
  });

  healthLatency.add(res.timings.duration);

  const passed = check(res, {
    "health status 200": (r) => r.status === 200,
    "response time < 1s": (r) => r.timings.duration < 1000,
  });

  errorRate.add(!passed);
  sleep(2);
}

/**
 * Summary handler — outputs a custom summary.
 */
export function handleSummary(data) {
  const summary = {
    timestamp: new Date().toISOString(),
    scenarios: Object.keys(options.scenarios),
    metrics: {
      http_reqs: data.metrics.http_reqs?.values?.count || 0,
      avg_duration_ms: Math.round(data.metrics.http_req_duration?.values?.avg || 0),
      p95_duration_ms: Math.round(data.metrics.http_req_duration?.values?.["p(95)"] || 0),
      p99_duration_ms: Math.round(data.metrics.http_req_duration?.values?.["p(99)"] || 0),
      error_rate: (data.metrics.errors?.values?.rate || 0).toFixed(4),
    },
  };

  return {
    stdout: `\n📊 Load Test Summary\n${JSON.stringify(summary, null, 2)}\n`,
    "tests/load/results.json": JSON.stringify(summary, null, 2),
  };
}