Codex commited on
Commit ·
7cd5cb8
1
Parent(s): 0cef4ab
Add async image task compatibility
Browse files- Dockerfile +9 -4
- README.md +6 -0
- codex-image-b64.patch +29 -272
- image_task_compat.go +370 -0
- image_task_compat_test.go +120 -0
- image_url_compat.go +284 -0
- image_url_compat_test.go +129 -0
Dockerfile
CHANGED
|
@@ -2,9 +2,10 @@ FROM alpine:3.22 AS source
|
|
| 2 |
|
| 3 |
RUN apk add --no-cache git
|
| 4 |
RUN git clone --depth 1 --branch v1.0.0-rc.22 https://github.com/QuantumNous/new-api.git /src
|
|
|
|
| 5 |
COPY codex-image-b64.patch /tmp/codex-image-b64.patch
|
| 6 |
-
RUN git -C /src apply --check /tmp/codex-image-b64.patch \
|
| 7 |
-
&& git -C /src apply /tmp/codex-image-b64.patch
|
| 8 |
|
| 9 |
FROM oven/bun:1 AS web-builder
|
| 10 |
|
|
@@ -24,7 +25,7 @@ RUN go mod download
|
|
| 24 |
COPY --from=source /src ./
|
| 25 |
COPY --from=web-builder /build/web/dist ./web/dist
|
| 26 |
RUN go test ./relay/channel/openai \
|
| 27 |
-
&& go build -ldflags "-s -w -X 'github.com/QuantumNous/new-api/common.Version=v1.0.0-rc.22-codex-image-
|
| 28 |
|
| 29 |
FROM debian:bookworm-slim
|
| 30 |
|
|
@@ -37,7 +38,11 @@ RUN apt-get update \
|
|
| 37 |
|
| 38 |
COPY --from=go-builder /build/new-api /new-api
|
| 39 |
COPY --from=source /src/LICENSE /src/NOTICE /src/THIRD-PARTY-LICENSES.md /licenses/
|
| 40 |
-
ENV PORT=7860
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
EXPOSE 7860
|
| 42 |
WORKDIR /data
|
| 43 |
USER root
|
|
|
|
| 2 |
|
| 3 |
RUN apk add --no-cache git
|
| 4 |
RUN git clone --depth 1 --branch v1.0.0-rc.22 https://github.com/QuantumNous/new-api.git /src
|
| 5 |
+
COPY image_url_compat.go image_url_compat_test.go image_task_compat.go image_task_compat_test.go /src/relay/channel/openai/
|
| 6 |
COPY codex-image-b64.patch /tmp/codex-image-b64.patch
|
| 7 |
+
RUN git -C /src apply --unidiff-zero --check /tmp/codex-image-b64.patch \
|
| 8 |
+
&& git -C /src apply --unidiff-zero /tmp/codex-image-b64.patch
|
| 9 |
|
| 10 |
FROM oven/bun:1 AS web-builder
|
| 11 |
|
|
|
|
| 25 |
COPY --from=source /src ./
|
| 26 |
COPY --from=web-builder /build/web/dist ./web/dist
|
| 27 |
RUN go test ./relay/channel/openai \
|
| 28 |
+
&& go build -ldflags "-s -w -X 'github.com/QuantumNous/new-api/common.Version=v1.0.0-rc.22-codex-image-compat'" -o new-api
|
| 29 |
|
| 30 |
FROM debian:bookworm-slim
|
| 31 |
|
|
|
|
| 38 |
|
| 39 |
COPY --from=go-builder /build/new-api /new-api
|
| 40 |
COPY --from=source /src/LICENSE /src/NOTICE /src/THIRD-PARTY-LICENSES.md /licenses/
|
| 41 |
+
ENV PORT=7860 \
|
| 42 |
+
VERSION=v1.0.0-rc.22-codex-image-compat \
|
| 43 |
+
CODEX_IMAGE_URL_TO_B64=true \
|
| 44 |
+
CODEX_IMAGE_TASK_POLL=true \
|
| 45 |
+
CODEX_IMAGE_TASK_POLL_TIMEOUT_SECONDS=120
|
| 46 |
EXPOSE 7860
|
| 47 |
WORKDIR /data
|
| 48 |
USER root
|
README.md
CHANGED
|
@@ -9,3 +9,9 @@ license: mit
|
|
| 9 |
---
|
| 10 |
|
| 11 |
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
| 12 |
+
|
| 13 |
+
This Space includes Codex image compatibility by default:
|
| 14 |
+
|
| 15 |
+
- asynchronous `202 image.task` responses are polled for up to 120 seconds;
|
| 16 |
+
- final image URLs are converted to `b64_json`;
|
| 17 |
+
- transient image download errors and upstream 5xx responses are retried up to three times without resubmitting the generation task.
|
codex-image-b64.patch
CHANGED
|
@@ -1,282 +1,42 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
-
|
| 7 |
-
.
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
diff --git a/.env.example b/.env.example
|
| 16 |
-
index d62e114..76ea82f 100644
|
| 17 |
-
--- a/.env.example
|
| 18 |
-
+++ b/.env.example
|
| 19 |
-
@@ -1,5 +1,7 @@
|
| 20 |
-
# 端口号
|
| 21 |
-
# PORT=3000
|
| 22 |
-
+# Convert URL-based OpenAI image responses to b64_json for strict clients such as Codex.
|
| 23 |
-
+# CODEX_IMAGE_URL_TO_B64=false
|
| 24 |
-
# 前端基础URL
|
| 25 |
-
# FRONTEND_BASE_URL=https://your-frontend-url.com
|
| 26 |
-
|
| 27 |
-
diff --git a/relay/channel/openai/image_url_compat.go b/relay/channel/openai/image_url_compat.go
|
| 28 |
-
new file mode 100644
|
| 29 |
-
index 0000000..e0f222c
|
| 30 |
-
--- /dev/null
|
| 31 |
-
+++ b/relay/channel/openai/image_url_compat.go
|
| 32 |
-
@@ -0,0 +1,162 @@
|
| 33 |
-
+package openai
|
| 34 |
-
+
|
| 35 |
-
+import (
|
| 36 |
-
+ "encoding/base64"
|
| 37 |
-
+ "encoding/json"
|
| 38 |
-
+ "fmt"
|
| 39 |
-
+ "io"
|
| 40 |
-
+ "net/http"
|
| 41 |
-
+ "strings"
|
| 42 |
-
+ "time"
|
| 43 |
-
+
|
| 44 |
-
+ "github.com/QuantumNous/new-api/common"
|
| 45 |
-
+)
|
| 46 |
-
+
|
| 47 |
-
+const maxImageURLCompatBytes int64 = 40 * 1024 * 1024
|
| 48 |
-
+
|
| 49 |
-
+type imageURLCompatDownloadFunc func(string, ...string) (*http.Response, error)
|
| 50 |
-
+
|
| 51 |
-
+type imageURLCompatStats struct {
|
| 52 |
-
+ Converted int
|
| 53 |
-
+ DownloadedBytes int64
|
| 54 |
-
+ DownloadTime time.Duration
|
| 55 |
-
+ EncodeTime time.Duration
|
| 56 |
-
+ TotalTime time.Duration
|
| 57 |
-
+}
|
| 58 |
-
+
|
| 59 |
-
+func (s imageURLCompatStats) ServerTiming() string {
|
| 60 |
-
+ return fmt.Sprintf(
|
| 61 |
-
+ "image-url-download;dur=%.3f, image-b64-encode;dur=%.3f, image-url-compat;dur=%.3f",
|
| 62 |
-
+ float64(s.DownloadTime.Microseconds())/1000,
|
| 63 |
-
+ float64(s.EncodeTime.Microseconds())/1000,
|
| 64 |
-
+ float64(s.TotalTime.Microseconds())/1000,
|
| 65 |
-
+ )
|
| 66 |
-
+}
|
| 67 |
-
+
|
| 68 |
-
+func (s imageURLCompatStats) LogMessage() string {
|
| 69 |
-
+ return fmt.Sprintf(
|
| 70 |
-
+ "image URL compatibility converted=%d bytes=%d download_ms=%.3f encode_ms=%.3f total_ms=%.3f",
|
| 71 |
-
+ s.Converted,
|
| 72 |
-
+ s.DownloadedBytes,
|
| 73 |
-
+ float64(s.DownloadTime.Microseconds())/1000,
|
| 74 |
-
+ float64(s.EncodeTime.Microseconds())/1000,
|
| 75 |
-
+ float64(s.TotalTime.Microseconds())/1000,
|
| 76 |
-
+ )
|
| 77 |
-
+}
|
| 78 |
-
+
|
| 79 |
-
+func convertOpenAIImageURLsToBase64(responseBody []byte, download imageURLCompatDownloadFunc) ([]byte, imageURLCompatStats, error) {
|
| 80 |
-
+ var stats imageURLCompatStats
|
| 81 |
-
+ if !common.GetEnvOrDefaultBool("CODEX_IMAGE_URL_TO_B64", false) {
|
| 82 |
-
+ return responseBody, stats, nil
|
| 83 |
-
+ }
|
| 84 |
-
+
|
| 85 |
-
+ startedAt := time.Now()
|
| 86 |
-
+ var payload map[string]json.RawMessage
|
| 87 |
-
+ if err := common.Unmarshal(responseBody, &payload); err != nil {
|
| 88 |
-
+ return nil, stats, fmt.Errorf("decode image compatibility response: %w", err)
|
| 89 |
-
+ }
|
| 90 |
-
+ dataJSON, ok := payload["data"]
|
| 91 |
-
+ if !ok {
|
| 92 |
-
+ return responseBody, stats, nil
|
| 93 |
-
+ }
|
| 94 |
-
+
|
| 95 |
-
+ var images []map[string]json.RawMessage
|
| 96 |
-
+ if err := common.Unmarshal(dataJSON, &images); err != nil {
|
| 97 |
-
+ return nil, stats, fmt.Errorf("decode image compatibility data: %w", err)
|
| 98 |
-
+ }
|
| 99 |
-
+
|
| 100 |
-
+ for _, image := range images {
|
| 101 |
-
+ if jsonString(image["b64_json"]) != "" {
|
| 102 |
-
+ continue
|
| 103 |
-
+ }
|
| 104 |
-
+
|
| 105 |
-
+ var imageURL string
|
| 106 |
-
+ var sourceField string
|
| 107 |
-
+ for _, field := range []string{"url", "result_url", "image_url"} {
|
| 108 |
-
+ if value := jsonString(image[field]); value != "" {
|
| 109 |
-
+ imageURL = value
|
| 110 |
-
+ sourceField = field
|
| 111 |
-
+ break
|
| 112 |
-
+ }
|
| 113 |
-
+ }
|
| 114 |
-
+ if imageURL == "" {
|
| 115 |
-
+ continue
|
| 116 |
-
+ }
|
| 117 |
-
+
|
| 118 |
-
+ downloadStartedAt := time.Now()
|
| 119 |
-
+ imageResponse, err := download(imageURL, "OpenAI image URL compatibility")
|
| 120 |
+ if err != nil {
|
| 121 |
-
+ return nil,
|
| 122 |
+ }
|
| 123 |
-
+
|
| 124 |
-
+ stats.DownloadTime += time.Since(downloadStartedAt)
|
| 125 |
-
+ if readErr != nil {
|
| 126 |
-
+ return nil, stats, readErr
|
| 127 |
-
+ }
|
| 128 |
-
+
|
| 129 |
-
+ encodeStartedAt := time.Now()
|
| 130 |
-
+ encodedJSON, err := common.Marshal(base64.StdEncoding.EncodeToString(imageBytes))
|
| 131 |
-
+ if err != nil {
|
| 132 |
-
+ return nil, stats, fmt.Errorf("encode image compatibility response: %w", err)
|
| 133 |
-
+ }
|
| 134 |
-
+ image["b64_json"] = encodedJSON
|
| 135 |
-
+ delete(image, sourceField)
|
| 136 |
-
+ stats.EncodeTime += time.Since(encodeStartedAt)
|
| 137 |
-
+ stats.Converted++
|
| 138 |
-
+ stats.DownloadedBytes += int64(len(imageBytes))
|
| 139 |
-
+ }
|
| 140 |
-
+
|
| 141 |
-
+ if stats.Converted == 0 {
|
| 142 |
-
+ return responseBody, stats, nil
|
| 143 |
+ }
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 144 |
+
|
| 145 |
-
+
|
| 146 |
-
+ dataJSON, err := common.Marshal(images)
|
| 147 |
+ if err != nil {
|
| 148 |
-
+ return nil,
|
| 149 |
+ }
|
| 150 |
-
+
|
| 151 |
-
+
|
| 152 |
-
+ stats.EncodeTime += time.Since(encodeStartedAt)
|
| 153 |
-
+ stats.TotalTime = time.Since(startedAt)
|
| 154 |
-
+ if err != nil {
|
| 155 |
-
+ return nil, stats, fmt.Errorf("encode image compatibility payload: %w", err)
|
| 156 |
-
+ }
|
| 157 |
-
+ return convertedBody, stats, nil
|
| 158 |
-
+}
|
| 159 |
-
+
|
| 160 |
-
+func jsonString(raw json.RawMessage) string {
|
| 161 |
-
+ if len(raw) == 0 {
|
| 162 |
-
+ return ""
|
| 163 |
-
+ }
|
| 164 |
-
+ var value string
|
| 165 |
-
+ if err := common.Unmarshal(raw, &value); err != nil {
|
| 166 |
-
+ return ""
|
| 167 |
-
+ }
|
| 168 |
-
+ return strings.TrimSpace(value)
|
| 169 |
-
+}
|
| 170 |
-
+
|
| 171 |
-
+func readImageURLCompatResponse(response *http.Response) ([]byte, error) {
|
| 172 |
-
+ if response == nil || response.Body == nil {
|
| 173 |
-
+ return nil, fmt.Errorf("image compatibility download returned an empty response")
|
| 174 |
-
+ }
|
| 175 |
-
+ defer response.Body.Close()
|
| 176 |
-
+ if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
| 177 |
-
+ return nil, fmt.Errorf("image compatibility download returned HTTP %d", response.StatusCode)
|
| 178 |
-
+ }
|
| 179 |
-
+ if response.ContentLength > maxImageURLCompatBytes {
|
| 180 |
-
+ return nil, fmt.Errorf("image compatibility download exceeds %d bytes", maxImageURLCompatBytes)
|
| 181 |
-
+ }
|
| 182 |
-
+
|
| 183 |
-
+ imageBytes, err := io.ReadAll(io.LimitReader(response.Body, maxImageURLCompatBytes+1))
|
| 184 |
-
+ if err != nil {
|
| 185 |
-
+ return nil, fmt.Errorf("read image compatibility download: %w", err)
|
| 186 |
-
+ }
|
| 187 |
-
+ if int64(len(imageBytes)) > maxImageURLCompatBytes {
|
| 188 |
-
+ return nil, fmt.Errorf("image compatibility download exceeds %d bytes", maxImageURLCompatBytes)
|
| 189 |
-
+ }
|
| 190 |
-
+ if len(imageBytes) == 0 {
|
| 191 |
-
+ return nil, fmt.Errorf("image compatibility download returned an empty body")
|
| 192 |
+ }
|
| 193 |
-
+ return
|
| 194 |
-
+}
|
| 195 |
-
diff --git a/relay/channel/openai/image_url_compat_test.go b/relay/channel/openai/image_url_compat_test.go
|
| 196 |
-
new file mode 100644
|
| 197 |
-
index 0000000..2722bba
|
| 198 |
-
--- /dev/null
|
| 199 |
-
+++ b/relay/channel/openai/image_url_compat_test.go
|
| 200 |
-
@@ -0,0 +1,70 @@
|
| 201 |
-
+package openai
|
| 202 |
-
+
|
| 203 |
-
+import (
|
| 204 |
-
+ "bytes"
|
| 205 |
-
+ "io"
|
| 206 |
-
+ "net/http"
|
| 207 |
-
+ "testing"
|
| 208 |
-
+
|
| 209 |
-
+ "github.com/QuantumNous/new-api/common"
|
| 210 |
-
+ "github.com/stretchr/testify/assert"
|
| 211 |
-
+ "github.com/stretchr/testify/require"
|
| 212 |
-
+)
|
| 213 |
-
+
|
| 214 |
-
+func TestConvertOpenAIImageURLsToBase64Disabled(t *testing.T) {
|
| 215 |
-
+ t.Setenv("CODEX_IMAGE_URL_TO_B64", "false")
|
| 216 |
-
+ body := []byte(`{"created":1,"data":[{"url":"https://example.com/image.png"}]}`)
|
| 217 |
-
+
|
| 218 |
-
+ converted, stats, err := convertOpenAIImageURLsToBase64(body, func(string, ...string) (*http.Response, error) {
|
| 219 |
-
+ t.Fatal("download must not run while compatibility is disabled")
|
| 220 |
-
+ return nil, nil
|
| 221 |
-
+ })
|
| 222 |
-
+
|
| 223 |
-
+ require.NoError(t, err)
|
| 224 |
-
+ assert.Equal(t, body, converted)
|
| 225 |
-
+ assert.Zero(t, stats.Converted)
|
| 226 |
-
+}
|
| 227 |
-
+
|
| 228 |
-
+func TestConvertOpenAIImageURLsToBase64PreservesResponseFields(t *testing.T) {
|
| 229 |
-
+ t.Setenv("CODEX_IMAGE_URL_TO_B64", "true")
|
| 230 |
-
+ image := []byte("test-image")
|
| 231 |
-
+ body := []byte(`{"created":1,"data":[{"url":"https://example.com/image.png","revised_prompt":"kept"}],"usage":{"total_tokens":7},"custom":{"kept":true}}`)
|
| 232 |
-
+
|
| 233 |
-
+ converted, stats, err := convertOpenAIImageURLsToBase64(body, func(rawURL string, reason ...string) (*http.Response, error) {
|
| 234 |
-
+ assert.Equal(t, "https://example.com/image.png", rawURL)
|
| 235 |
-
+ assert.Equal(t, []string{"OpenAI image URL compatibility"}, reason)
|
| 236 |
-
+ return &http.Response{
|
| 237 |
-
+ StatusCode: http.StatusOK,
|
| 238 |
-
+ ContentLength: int64(len(image)),
|
| 239 |
-
+ Body: io.NopCloser(bytes.NewReader(image)),
|
| 240 |
-
+ }, nil
|
| 241 |
-
+ })
|
| 242 |
-
+
|
| 243 |
-
+ require.NoError(t, err)
|
| 244 |
-
+ assert.Equal(t, 1, stats.Converted)
|
| 245 |
-
+ assert.Equal(t, int64(len(image)), stats.DownloadedBytes)
|
| 246 |
-
+
|
| 247 |
-
+ var payload map[string]any
|
| 248 |
-
+ require.NoError(t, common.Unmarshal(converted, &payload))
|
| 249 |
-
+ data := payload["data"].([]any)
|
| 250 |
-
+ item := data[0].(map[string]any)
|
| 251 |
-
+ assert.Equal(t, "dGVzdC1pbWFnZQ==", item["b64_json"])
|
| 252 |
-
+ assert.Equal(t, "kept", item["revised_prompt"])
|
| 253 |
-
+ assert.NotContains(t, item, "url")
|
| 254 |
-
+ assert.Equal(t, float64(7), payload["usage"].(map[string]any)["total_tokens"])
|
| 255 |
-
+ assert.Equal(t, true, payload["custom"].(map[string]any)["kept"])
|
| 256 |
-
+ assert.Contains(t, stats.ServerTiming(), "image-url-compat")
|
| 257 |
-
+}
|
| 258 |
-
+
|
| 259 |
-
+func TestReadImageURLCompatResponseRejectsOversize(t *testing.T) {
|
| 260 |
-
+ response := &http.Response{
|
| 261 |
-
+ StatusCode: http.StatusOK,
|
| 262 |
-
+ ContentLength: maxImageURLCompatBytes + 1,
|
| 263 |
-
+ Body: io.NopCloser(bytes.NewReader([]byte("unused"))),
|
| 264 |
-
+ }
|
| 265 |
-
+
|
| 266 |
-
+ _, err := readImageURLCompatResponse(response)
|
| 267 |
-
+
|
| 268 |
-
+ require.Error(t, err)
|
| 269 |
-
+ assert.Contains(t, err.Error(), "exceeds")
|
| 270 |
-
+}
|
| 271 |
diff --git a/relay/channel/openai/relay_image.go b/relay/channel/openai/relay_image.go
|
| 272 |
-
index
|
| 273 |
--- a/relay/channel/openai/relay_image.go
|
| 274 |
+++ b/relay/channel/openai/relay_image.go
|
| 275 |
-
@@ -
|
| 276 |
-
|
| 277 |
-
}
|
| 278 |
-
|
| 279 |
-
+ responseBody, compatStats, err := convertOpenAIImageURLsToBase64(responseBody, service.DoDownloadRequest)
|
| 280 |
+ if err != nil {
|
| 281 |
+ return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway)
|
| 282 |
+ }
|
|
@@ -286,6 +46,3 @@ index e0f09aa..a6cfad1 100644
|
|
| 286 |
+ logger.LogInfo(c, compatStats.LogMessage())
|
| 287 |
+ }
|
| 288 |
+
|
| 289 |
-
var usageResp dto.SimpleResponse
|
| 290 |
-
err = common.Unmarshal(responseBody, &usageResp)
|
| 291 |
-
if err != nil {
|
|
|
|
| 1 |
+
diff --git a/relay/channel/openai/adaptor.go b/relay/channel/openai/adaptor.go
|
| 2 |
+
index 4f1c4286..c05ef093 100644
|
| 3 |
+
--- a/relay/channel/openai/adaptor.go
|
| 4 |
+
+++ b/relay/channel/openai/adaptor.go
|
| 5 |
+
@@ -624,3 +624 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request
|
| 6 |
+
- if info.RelayMode == relayconstant.RelayModeAudioTranscription ||
|
| 7 |
+
- info.RelayMode == relayconstant.RelayModeAudioTranslation ||
|
| 8 |
+
- (info.RelayMode == relayconstant.RelayModeImagesEdits && !isJSONRequest(c)) {
|
| 9 |
+
+ if info.RelayMode == relayconstant.RelayModeAudioTranscription || info.RelayMode == relayconstant.RelayModeAudioTranslation {
|
| 10 |
+
@@ -628 +626,9 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request
|
| 11 |
+
- } else if info.RelayMode == relayconstant.RelayModeRealtime {
|
| 12 |
+
+ }
|
| 13 |
+
+ if info.RelayMode == relayconstant.RelayModeImagesEdits && !isJSONRequest(c) {
|
| 14 |
+
+ response, err := channel.DoFormRequest(a, c, info, requestBody)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
+ if err != nil {
|
| 16 |
+
+ return nil, err
|
| 17 |
+ }
|
| 18 |
+
+ return resolveOpenAIImageTask(c, info, response)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
+ }
|
| 20 |
+
+ if info.RelayMode == relayconstant.RelayModeRealtime {
|
| 21 |
+
@@ -630,2 +635,0 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request
|
| 22 |
+
- } else {
|
| 23 |
+
- return channel.DoApiRequest(a, c, info, requestBody)
|
| 24 |
+
@@ -632,0 +637,9 @@ func (a *Adaptor) DoRequest(c *gin.Context, info *relaycommon.RelayInfo, request
|
| 25 |
+
|
| 26 |
+
+ response, err := channel.DoApiRequest(a, c, info, requestBody)
|
|
|
|
| 27 |
+ if err != nil {
|
| 28 |
+
+ return nil, err
|
| 29 |
+ }
|
| 30 |
+
+ if info.RelayMode == relayconstant.RelayModeImagesGenerations || info.RelayMode == relayconstant.RelayModeImagesEdits {
|
| 31 |
+
+ return resolveOpenAIImageTask(c, info, response)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
+ }
|
| 33 |
+
+ return response, nil
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 34 |
diff --git a/relay/channel/openai/relay_image.go b/relay/channel/openai/relay_image.go
|
| 35 |
+
index 1e6be0dd..240e6eca 100644
|
| 36 |
--- a/relay/channel/openai/relay_image.go
|
| 37 |
+++ b/relay/channel/openai/relay_image.go
|
| 38 |
+
@@ -41,0 +42,10 @@ func OpenaiImageHandler(c *gin.Context, info *relaycommon.RelayInfo, resp *http.
|
| 39 |
+
+ responseBody, compatStats, err := convertOpenAIImageURLsToBase64(responseBody, downloadOpenAIImageURLCompat)
|
|
|
|
|
|
|
|
|
|
| 40 |
+ if err != nil {
|
| 41 |
+ return nil, types.NewOpenAIError(err, types.ErrorCodeBadResponseBody, http.StatusBadGateway)
|
| 42 |
+ }
|
|
|
|
| 46 |
+ logger.LogInfo(c, compatStats.LogMessage())
|
| 47 |
+ }
|
| 48 |
+
|
|
|
|
|
|
|
|
|
image_task_compat.go
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package openai
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"bytes"
|
| 5 |
+
"context"
|
| 6 |
+
"encoding/json"
|
| 7 |
+
"fmt"
|
| 8 |
+
"io"
|
| 9 |
+
"net/http"
|
| 10 |
+
"net/url"
|
| 11 |
+
"strconv"
|
| 12 |
+
"strings"
|
| 13 |
+
"time"
|
| 14 |
+
|
| 15 |
+
"github.com/QuantumNous/new-api/common"
|
| 16 |
+
"github.com/QuantumNous/new-api/logger"
|
| 17 |
+
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
| 18 |
+
"github.com/QuantumNous/new-api/service"
|
| 19 |
+
|
| 20 |
+
"github.com/gin-gonic/gin"
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
const (
|
| 24 |
+
maxImageTaskResponseBytes int64 = 64 * 1024 * 1024
|
| 25 |
+
defaultImageTaskPollTimeout = 120 * time.Second
|
| 26 |
+
defaultImageTaskPollDelay = 2 * time.Second
|
| 27 |
+
minImageTaskPollDelay = 250 * time.Millisecond
|
| 28 |
+
maxImageTaskPollDelay = 5 * time.Second
|
| 29 |
+
maxImageTaskPollErrors = 3
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
type openAIImageTaskEnvelope struct {
|
| 33 |
+
Object string `json:"object"`
|
| 34 |
+
Status string `json:"status"`
|
| 35 |
+
TaskID string `json:"task_id"`
|
| 36 |
+
ID string `json:"id"`
|
| 37 |
+
PollURL string `json:"poll_url"`
|
| 38 |
+
ResultURL string `json:"result_url"`
|
| 39 |
+
PollAfterMS int64 `json:"poll_after_ms"`
|
| 40 |
+
Error string `json:"error"`
|
| 41 |
+
ErrorCode string `json:"error_code"`
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
type imageTaskPollStats struct {
|
| 45 |
+
TaskID string
|
| 46 |
+
Attempts int
|
| 47 |
+
Duration time.Duration
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
type imageTaskWaitFunc func(context.Context, time.Duration) error
|
| 51 |
+
|
| 52 |
+
func resolveOpenAIImageTask(c *gin.Context, info *relaycommon.RelayInfo, response *http.Response) (*http.Response, error) {
|
| 53 |
+
if !common.GetEnvOrDefaultBool("CODEX_IMAGE_TASK_POLL", false) || response == nil || response.StatusCode != http.StatusAccepted {
|
| 54 |
+
return response, nil
|
| 55 |
+
}
|
| 56 |
+
if info == nil {
|
| 57 |
+
return nil, fmt.Errorf("image task relay info is missing")
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
client, err := service.GetHttpClientWithProxy(info.ChannelSetting.Proxy)
|
| 61 |
+
if err != nil {
|
| 62 |
+
return nil, fmt.Errorf("create image task poll client: %w", err)
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
timeoutSeconds := common.GetEnvOrDefault("CODEX_IMAGE_TASK_POLL_TIMEOUT_SECONDS", int(defaultImageTaskPollTimeout/time.Second))
|
| 66 |
+
if timeoutSeconds < 1 {
|
| 67 |
+
timeoutSeconds = int(defaultImageTaskPollTimeout / time.Second)
|
| 68 |
+
}
|
| 69 |
+
if timeoutSeconds > 600 {
|
| 70 |
+
timeoutSeconds = 600
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
startedAt := time.Now()
|
| 74 |
+
resolved, stats, err := pollOpenAIImageTask(
|
| 75 |
+
c.Request.Context(),
|
| 76 |
+
response,
|
| 77 |
+
client,
|
| 78 |
+
time.Duration(timeoutSeconds)*time.Second,
|
| 79 |
+
waitForOpenAIImageTask,
|
| 80 |
+
)
|
| 81 |
+
stats.Duration = time.Since(startedAt)
|
| 82 |
+
if err != nil {
|
| 83 |
+
return nil, err
|
| 84 |
+
}
|
| 85 |
+
if stats.Attempts > 0 {
|
| 86 |
+
logger.LogInfo(c, fmt.Sprintf(
|
| 87 |
+
"image task compatibility completed task_id=%s attempts=%d total_ms=%.3f",
|
| 88 |
+
maskImageTaskID(stats.TaskID),
|
| 89 |
+
stats.Attempts,
|
| 90 |
+
float64(stats.Duration.Microseconds())/1000,
|
| 91 |
+
))
|
| 92 |
+
resolved.Header.Set("X-New-API-Image-Task-Compat", "polled")
|
| 93 |
+
}
|
| 94 |
+
return resolved, nil
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
func pollOpenAIImageTask(
|
| 98 |
+
ctx context.Context,
|
| 99 |
+
initial *http.Response,
|
| 100 |
+
client *http.Client,
|
| 101 |
+
timeout time.Duration,
|
| 102 |
+
wait imageTaskWaitFunc,
|
| 103 |
+
) (*http.Response, imageTaskPollStats, error) {
|
| 104 |
+
var stats imageTaskPollStats
|
| 105 |
+
if initial == nil || initial.Body == nil {
|
| 106 |
+
return nil, stats, fmt.Errorf("image task response is empty")
|
| 107 |
+
}
|
| 108 |
+
if initial.StatusCode != http.StatusAccepted {
|
| 109 |
+
return initial, stats, nil
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
initialBody, err := readImageTaskResponseBody(initial)
|
| 113 |
+
if err != nil {
|
| 114 |
+
return nil, stats, err
|
| 115 |
+
}
|
| 116 |
+
initial.Body = io.NopCloser(bytes.NewReader(initialBody))
|
| 117 |
+
initial.ContentLength = int64(len(initialBody))
|
| 118 |
+
|
| 119 |
+
task, isTask, err := parseOpenAIImageTask(initialBody)
|
| 120 |
+
if err != nil {
|
| 121 |
+
return nil, stats, err
|
| 122 |
+
}
|
| 123 |
+
if !isTask {
|
| 124 |
+
return initial, stats, nil
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
stats.TaskID = firstNonEmpty(task.TaskID, task.ID)
|
| 128 |
+
baseURL, err := imageTaskBaseURL(initial)
|
| 129 |
+
if err != nil {
|
| 130 |
+
return nil, stats, err
|
| 131 |
+
}
|
| 132 |
+
pollClient := *client
|
| 133 |
+
pollClient.CheckRedirect = func(request *http.Request, via []*http.Request) error {
|
| 134 |
+
if len(via) >= 10 {
|
| 135 |
+
return fmt.Errorf("image task polling stopped after 10 redirects")
|
| 136 |
+
}
|
| 137 |
+
if request == nil || request.URL == nil ||
|
| 138 |
+
!strings.EqualFold(baseURL.Scheme, request.URL.Scheme) ||
|
| 139 |
+
!strings.EqualFold(baseURL.Host, request.URL.Host) {
|
| 140 |
+
return fmt.Errorf("image task redirect must use the original upstream origin")
|
| 141 |
+
}
|
| 142 |
+
return nil
|
| 143 |
+
}
|
| 144 |
+
client = &pollClient
|
| 145 |
+
pollURL, err := resolveImageTaskPollURL(baseURL, firstNonEmpty(task.PollURL, task.ResultURL))
|
| 146 |
+
if err != nil {
|
| 147 |
+
return nil, stats, err
|
| 148 |
+
}
|
| 149 |
+
|
| 150 |
+
deadline := time.Now().Add(timeout)
|
| 151 |
+
consecutiveErrors := 0
|
| 152 |
+
for {
|
| 153 |
+
if remaining := time.Until(deadline); remaining <= 0 {
|
| 154 |
+
return nil, stats, fmt.Errorf("image task polling timed out after %s (task_id=%s)", timeout, maskImageTaskID(stats.TaskID))
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
delay := imageTaskPollDelay(task.PollAfterMS)
|
| 158 |
+
if remaining := time.Until(deadline); delay > remaining {
|
| 159 |
+
delay = remaining
|
| 160 |
+
}
|
| 161 |
+
if err := wait(ctx, delay); err != nil {
|
| 162 |
+
return nil, stats, fmt.Errorf("image task polling stopped: %w", err)
|
| 163 |
+
}
|
| 164 |
+
|
| 165 |
+
request, err := http.NewRequestWithContext(ctx, http.MethodGet, pollURL.String(), nil)
|
| 166 |
+
if err != nil {
|
| 167 |
+
return nil, stats, fmt.Errorf("create image task poll request: %w", err)
|
| 168 |
+
}
|
| 169 |
+
copyImageTaskAuthHeaders(request.Header, initial.Request)
|
| 170 |
+
request.Header.Set("Accept", "application/json")
|
| 171 |
+
|
| 172 |
+
pollResponse, err := client.Do(request)
|
| 173 |
+
stats.Attempts++
|
| 174 |
+
if err != nil {
|
| 175 |
+
consecutiveErrors++
|
| 176 |
+
if consecutiveErrors >= maxImageTaskPollErrors {
|
| 177 |
+
return nil, stats, fmt.Errorf("poll image task after %d consecutive errors: %w", consecutiveErrors, err)
|
| 178 |
+
}
|
| 179 |
+
continue
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
pollBody, readErr := readImageTaskResponseBody(pollResponse)
|
| 183 |
+
if readErr != nil {
|
| 184 |
+
return nil, stats, readErr
|
| 185 |
+
}
|
| 186 |
+
if pollResponse.StatusCode >= http.StatusInternalServerError {
|
| 187 |
+
consecutiveErrors++
|
| 188 |
+
if consecutiveErrors >= maxImageTaskPollErrors {
|
| 189 |
+
return nil, stats, fmt.Errorf("poll image task returned HTTP %d after %d attempts", pollResponse.StatusCode, consecutiveErrors)
|
| 190 |
+
}
|
| 191 |
+
continue
|
| 192 |
+
}
|
| 193 |
+
consecutiveErrors = 0
|
| 194 |
+
if pollResponse.StatusCode < http.StatusOK || pollResponse.StatusCode >= http.StatusMultipleChoices {
|
| 195 |
+
return nil, stats, fmt.Errorf("poll image task returned HTTP %d", pollResponse.StatusCode)
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
if isFinalOpenAIImageResponse(pollBody) {
|
| 199 |
+
pollResponse.StatusCode = http.StatusOK
|
| 200 |
+
pollResponse.Status = fmt.Sprintf("%d %s", http.StatusOK, http.StatusText(http.StatusOK))
|
| 201 |
+
pollResponse.Body = io.NopCloser(bytes.NewReader(pollBody))
|
| 202 |
+
pollResponse.ContentLength = int64(len(pollBody))
|
| 203 |
+
pollResponse.Header.Del("Content-Encoding")
|
| 204 |
+
pollResponse.Header.Set("Content-Length", strconv.FormatInt(int64(len(pollBody)), 10))
|
| 205 |
+
return pollResponse, stats, nil
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
nextTask, nextIsTask, err := parseOpenAIImageTask(pollBody)
|
| 209 |
+
if err != nil {
|
| 210 |
+
return nil, stats, err
|
| 211 |
+
}
|
| 212 |
+
if !nextIsTask {
|
| 213 |
+
return nil, stats, fmt.Errorf("image task poll response is neither a task nor a final image response")
|
| 214 |
+
}
|
| 215 |
+
task = nextTask
|
| 216 |
+
if nextID := firstNonEmpty(task.TaskID, task.ID); nextID != "" {
|
| 217 |
+
stats.TaskID = nextID
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
switch strings.ToLower(strings.TrimSpace(task.Status)) {
|
| 221 |
+
case "failed", "cancelled", "canceled", "expired":
|
| 222 |
+
message := firstNonEmpty(task.Error, task.ErrorCode, task.Status)
|
| 223 |
+
return nil, stats, fmt.Errorf("image task %s: %s", maskImageTaskID(stats.TaskID), message)
|
| 224 |
+
case "completed", "succeeded", "success":
|
| 225 |
+
return nil, stats, fmt.Errorf("image task %s completed without image data", maskImageTaskID(stats.TaskID))
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
if nextRawURL := firstNonEmpty(task.PollURL, task.ResultURL); nextRawURL != "" {
|
| 229 |
+
pollURL, err = resolveImageTaskPollURL(baseURL, nextRawURL)
|
| 230 |
+
if err != nil {
|
| 231 |
+
return nil, stats, err
|
| 232 |
+
}
|
| 233 |
+
}
|
| 234 |
+
}
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
func parseOpenAIImageTask(body []byte) (openAIImageTaskEnvelope, bool, error) {
|
| 238 |
+
var task openAIImageTaskEnvelope
|
| 239 |
+
if err := common.Unmarshal(body, &task); err != nil {
|
| 240 |
+
return task, false, fmt.Errorf("decode image task response: %w", err)
|
| 241 |
+
}
|
| 242 |
+
if !strings.EqualFold(strings.TrimSpace(task.Object), "image.task") {
|
| 243 |
+
return task, false, nil
|
| 244 |
+
}
|
| 245 |
+
if firstNonEmpty(task.PollURL, task.ResultURL) == "" {
|
| 246 |
+
return task, false, fmt.Errorf("image task response is missing poll_url")
|
| 247 |
+
}
|
| 248 |
+
return task, true, nil
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
func isFinalOpenAIImageResponse(body []byte) bool {
|
| 252 |
+
var payload map[string]json.RawMessage
|
| 253 |
+
if err := common.Unmarshal(body, &payload); err != nil {
|
| 254 |
+
return false
|
| 255 |
+
}
|
| 256 |
+
if _, ok := payload["created"]; !ok {
|
| 257 |
+
return false
|
| 258 |
+
}
|
| 259 |
+
var images []json.RawMessage
|
| 260 |
+
if err := common.Unmarshal(payload["data"], &images); err != nil {
|
| 261 |
+
return false
|
| 262 |
+
}
|
| 263 |
+
return len(images) > 0
|
| 264 |
+
}
|
| 265 |
+
|
| 266 |
+
func imageTaskBaseURL(response *http.Response) (*url.URL, error) {
|
| 267 |
+
if response.Request == nil || response.Request.URL == nil {
|
| 268 |
+
return nil, fmt.Errorf("image task response is missing its upstream request URL")
|
| 269 |
+
}
|
| 270 |
+
base := response.Request.URL
|
| 271 |
+
if base.Scheme != "http" && base.Scheme != "https" {
|
| 272 |
+
return nil, fmt.Errorf("unsupported image task upstream scheme: %s", base.Scheme)
|
| 273 |
+
}
|
| 274 |
+
if base.User != nil || base.Hostname() == "" {
|
| 275 |
+
return nil, fmt.Errorf("invalid image task upstream URL")
|
| 276 |
+
}
|
| 277 |
+
return base, nil
|
| 278 |
+
}
|
| 279 |
+
|
| 280 |
+
func resolveImageTaskPollURL(baseURL *url.URL, rawURL string) (*url.URL, error) {
|
| 281 |
+
if baseURL == nil {
|
| 282 |
+
return nil, fmt.Errorf("image task upstream URL is missing")
|
| 283 |
+
}
|
| 284 |
+
reference, err := url.Parse(strings.TrimSpace(rawURL))
|
| 285 |
+
if err != nil {
|
| 286 |
+
return nil, fmt.Errorf("parse image task poll URL: %w", err)
|
| 287 |
+
}
|
| 288 |
+
resolved := baseURL.ResolveReference(reference)
|
| 289 |
+
if resolved.Scheme != "http" && resolved.Scheme != "https" {
|
| 290 |
+
return nil, fmt.Errorf("unsupported image task poll scheme: %s", resolved.Scheme)
|
| 291 |
+
}
|
| 292 |
+
if resolved.User != nil || resolved.Hostname() == "" {
|
| 293 |
+
return nil, fmt.Errorf("invalid image task poll URL")
|
| 294 |
+
}
|
| 295 |
+
if !strings.EqualFold(baseURL.Scheme, resolved.Scheme) || !strings.EqualFold(baseURL.Host, resolved.Host) {
|
| 296 |
+
return nil, fmt.Errorf("image task poll URL must use the original upstream origin")
|
| 297 |
+
}
|
| 298 |
+
return resolved, nil
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
func copyImageTaskAuthHeaders(destination http.Header, source *http.Request) {
|
| 302 |
+
if source == nil {
|
| 303 |
+
return
|
| 304 |
+
}
|
| 305 |
+
for _, name := range []string{"Authorization", "Api-Key", "X-Api-Key"} {
|
| 306 |
+
for _, value := range source.Header.Values(name) {
|
| 307 |
+
destination.Add(name, value)
|
| 308 |
+
}
|
| 309 |
+
}
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
func readImageTaskResponseBody(response *http.Response) ([]byte, error) {
|
| 313 |
+
if response == nil || response.Body == nil {
|
| 314 |
+
return nil, fmt.Errorf("image task response is empty")
|
| 315 |
+
}
|
| 316 |
+
defer response.Body.Close()
|
| 317 |
+
if response.ContentLength > maxImageTaskResponseBytes {
|
| 318 |
+
return nil, fmt.Errorf("image task response exceeds %d bytes", maxImageTaskResponseBytes)
|
| 319 |
+
}
|
| 320 |
+
body, err := io.ReadAll(io.LimitReader(response.Body, maxImageTaskResponseBytes+1))
|
| 321 |
+
if err != nil {
|
| 322 |
+
return nil, fmt.Errorf("read image task response: %w", err)
|
| 323 |
+
}
|
| 324 |
+
if int64(len(body)) > maxImageTaskResponseBytes {
|
| 325 |
+
return nil, fmt.Errorf("image task response exceeds %d bytes", maxImageTaskResponseBytes)
|
| 326 |
+
}
|
| 327 |
+
return body, nil
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
func imageTaskPollDelay(milliseconds int64) time.Duration {
|
| 331 |
+
if milliseconds <= 0 {
|
| 332 |
+
return defaultImageTaskPollDelay
|
| 333 |
+
}
|
| 334 |
+
delay := time.Duration(milliseconds) * time.Millisecond
|
| 335 |
+
if delay < minImageTaskPollDelay {
|
| 336 |
+
return minImageTaskPollDelay
|
| 337 |
+
}
|
| 338 |
+
if delay > maxImageTaskPollDelay {
|
| 339 |
+
return maxImageTaskPollDelay
|
| 340 |
+
}
|
| 341 |
+
return delay
|
| 342 |
+
}
|
| 343 |
+
|
| 344 |
+
func waitForOpenAIImageTask(ctx context.Context, delay time.Duration) error {
|
| 345 |
+
timer := time.NewTimer(delay)
|
| 346 |
+
defer timer.Stop()
|
| 347 |
+
select {
|
| 348 |
+
case <-ctx.Done():
|
| 349 |
+
return ctx.Err()
|
| 350 |
+
case <-timer.C:
|
| 351 |
+
return nil
|
| 352 |
+
}
|
| 353 |
+
}
|
| 354 |
+
|
| 355 |
+
func firstNonEmpty(values ...string) string {
|
| 356 |
+
for _, value := range values {
|
| 357 |
+
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
| 358 |
+
return trimmed
|
| 359 |
+
}
|
| 360 |
+
}
|
| 361 |
+
return ""
|
| 362 |
+
}
|
| 363 |
+
|
| 364 |
+
func maskImageTaskID(taskID string) string {
|
| 365 |
+
taskID = strings.TrimSpace(taskID)
|
| 366 |
+
if len(taskID) <= 12 {
|
| 367 |
+
return taskID
|
| 368 |
+
}
|
| 369 |
+
return taskID[:8] + "..." + taskID[len(taskID)-4:]
|
| 370 |
+
}
|
image_task_compat_test.go
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package openai
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"bytes"
|
| 5 |
+
"context"
|
| 6 |
+
"io"
|
| 7 |
+
"net/http"
|
| 8 |
+
"net/http/httptest"
|
| 9 |
+
"testing"
|
| 10 |
+
"time"
|
| 11 |
+
|
| 12 |
+
"github.com/QuantumNous/new-api/common"
|
| 13 |
+
"github.com/stretchr/testify/assert"
|
| 14 |
+
"github.com/stretchr/testify/require"
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
func TestPollOpenAIImageTaskReturnsFinalResponse(t *testing.T) {
|
| 18 |
+
attempts := 0
|
| 19 |
+
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
| 20 |
+
assert.Equal(t, "Bearer test-key", request.Header.Get("Authorization"))
|
| 21 |
+
attempts++
|
| 22 |
+
writer.Header().Set("Content-Type", "application/json")
|
| 23 |
+
if attempts == 1 {
|
| 24 |
+
writer.WriteHeader(http.StatusAccepted)
|
| 25 |
+
_, _ = writer.Write([]byte(`{"object":"image.task","status":"processing","task_id":"task-1","poll_url":"/v1/images/tasks/task-1","poll_after_ms":1}`))
|
| 26 |
+
return
|
| 27 |
+
}
|
| 28 |
+
_, _ = writer.Write([]byte(`{"created":123,"data":[{"url":"https://example.com/image.png"}],"usage":{"total_tokens":7}}`))
|
| 29 |
+
}))
|
| 30 |
+
defer server.Close()
|
| 31 |
+
|
| 32 |
+
request := httptest.NewRequest(http.MethodPost, server.URL+"/v1/images/edits", nil)
|
| 33 |
+
request.Header.Set("Authorization", "Bearer test-key")
|
| 34 |
+
initial := &http.Response{
|
| 35 |
+
StatusCode: http.StatusAccepted,
|
| 36 |
+
Header: make(http.Header),
|
| 37 |
+
Request: request,
|
| 38 |
+
Body: io.NopCloser(bytes.NewBufferString(
|
| 39 |
+
`{"object":"image.task","status":"queued","task_id":"task-1","poll_url":"/v1/images/tasks/task-1","poll_after_ms":1}`,
|
| 40 |
+
)),
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
resolved, stats, err := pollOpenAIImageTask(context.Background(), initial, server.Client(), time.Second, noImageTaskWait)
|
| 44 |
+
|
| 45 |
+
require.NoError(t, err)
|
| 46 |
+
require.NotNil(t, resolved)
|
| 47 |
+
assert.Equal(t, http.StatusOK, resolved.StatusCode)
|
| 48 |
+
assert.Equal(t, 2, stats.Attempts)
|
| 49 |
+
assert.Equal(t, "task-1", stats.TaskID)
|
| 50 |
+
resolvedBody, err := io.ReadAll(resolved.Body)
|
| 51 |
+
require.NoError(t, err)
|
| 52 |
+
assert.True(t, isFinalOpenAIImageResponse(resolvedBody))
|
| 53 |
+
|
| 54 |
+
var payload map[string]any
|
| 55 |
+
require.NoError(t, common.Unmarshal(resolvedBody, &payload))
|
| 56 |
+
assert.Equal(t, float64(123), payload["created"])
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
func TestPollOpenAIImageTaskRejectsCrossOriginPollURL(t *testing.T) {
|
| 60 |
+
request := httptest.NewRequest(http.MethodPost, "https://upstream.example/v1/images/edits", nil)
|
| 61 |
+
initial := &http.Response{
|
| 62 |
+
StatusCode: http.StatusAccepted,
|
| 63 |
+
Header: make(http.Header),
|
| 64 |
+
Request: request,
|
| 65 |
+
Body: io.NopCloser(bytes.NewBufferString(
|
| 66 |
+
`{"object":"image.task","status":"queued","task_id":"task-1","poll_url":"https://other.example/tasks/task-1"}`,
|
| 67 |
+
)),
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
_, _, err := pollOpenAIImageTask(context.Background(), initial, http.DefaultClient, time.Second, noImageTaskWait)
|
| 71 |
+
|
| 72 |
+
require.Error(t, err)
|
| 73 |
+
assert.Contains(t, err.Error(), "original upstream origin")
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
func TestPollOpenAIImageTaskReturnsFailure(t *testing.T) {
|
| 77 |
+
server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
|
| 78 |
+
writer.Header().Set("Content-Type", "application/json")
|
| 79 |
+
_, _ = writer.Write([]byte(`{"object":"image.task","status":"failed","task_id":"task-1","poll_url":"/v1/images/tasks/task-1","error":"generation failed"}`))
|
| 80 |
+
}))
|
| 81 |
+
defer server.Close()
|
| 82 |
+
|
| 83 |
+
request := httptest.NewRequest(http.MethodPost, server.URL+"/v1/images/edits", nil)
|
| 84 |
+
initial := &http.Response{
|
| 85 |
+
StatusCode: http.StatusAccepted,
|
| 86 |
+
Header: make(http.Header),
|
| 87 |
+
Request: request,
|
| 88 |
+
Body: io.NopCloser(bytes.NewBufferString(
|
| 89 |
+
`{"object":"image.task","status":"queued","task_id":"task-1","poll_url":"/v1/images/tasks/task-1"}`,
|
| 90 |
+
)),
|
| 91 |
+
}
|
| 92 |
+
|
| 93 |
+
_, _, err := pollOpenAIImageTask(context.Background(), initial, server.Client(), time.Second, noImageTaskWait)
|
| 94 |
+
|
| 95 |
+
require.Error(t, err)
|
| 96 |
+
assert.Contains(t, err.Error(), "generation failed")
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
func TestPollOpenAIImageTaskStopsWhenContextIsCancelled(t *testing.T) {
|
| 100 |
+
request := httptest.NewRequest(http.MethodPost, "https://upstream.example/v1/images/edits", nil)
|
| 101 |
+
initial := &http.Response{
|
| 102 |
+
StatusCode: http.StatusAccepted,
|
| 103 |
+
Header: make(http.Header),
|
| 104 |
+
Request: request,
|
| 105 |
+
Body: io.NopCloser(bytes.NewBufferString(
|
| 106 |
+
`{"object":"image.task","status":"queued","task_id":"task-1","poll_url":"/v1/images/tasks/task-1"}`,
|
| 107 |
+
)),
|
| 108 |
+
}
|
| 109 |
+
ctx, cancel := context.WithCancel(context.Background())
|
| 110 |
+
cancel()
|
| 111 |
+
|
| 112 |
+
_, _, err := pollOpenAIImageTask(ctx, initial, http.DefaultClient, time.Second, waitForOpenAIImageTask)
|
| 113 |
+
|
| 114 |
+
require.Error(t, err)
|
| 115 |
+
assert.ErrorIs(t, err, context.Canceled)
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
func noImageTaskWait(context.Context, time.Duration) error {
|
| 119 |
+
return nil
|
| 120 |
+
}
|
image_url_compat.go
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package openai
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"encoding/base64"
|
| 5 |
+
"encoding/json"
|
| 6 |
+
"fmt"
|
| 7 |
+
"io"
|
| 8 |
+
"net/http"
|
| 9 |
+
"net/url"
|
| 10 |
+
"strconv"
|
| 11 |
+
"strings"
|
| 12 |
+
"time"
|
| 13 |
+
|
| 14 |
+
"github.com/QuantumNous/new-api/common"
|
| 15 |
+
"github.com/QuantumNous/new-api/service"
|
| 16 |
+
"github.com/QuantumNous/new-api/setting/system_setting"
|
| 17 |
+
)
|
| 18 |
+
|
| 19 |
+
const maxImageURLCompatBytes int64 = 40 * 1024 * 1024
|
| 20 |
+
|
| 21 |
+
const maxImageURLCompatDownloadAttempts = 3
|
| 22 |
+
|
| 23 |
+
type imageURLCompatDownloadFunc func(string, ...string) (*http.Response, error)
|
| 24 |
+
|
| 25 |
+
type imageURLCompatStats struct {
|
| 26 |
+
Converted int
|
| 27 |
+
DownloadedBytes int64
|
| 28 |
+
DownloadTime time.Duration
|
| 29 |
+
EncodeTime time.Duration
|
| 30 |
+
TotalTime time.Duration
|
| 31 |
+
}
|
| 32 |
+
|
| 33 |
+
func downloadOpenAIImageURLCompat(rawURL string, reason ...string) (*http.Response, error) {
|
| 34 |
+
parsedURL, err := url.Parse(rawURL)
|
| 35 |
+
if err != nil {
|
| 36 |
+
return nil, fmt.Errorf("parse image compatibility URL: %w", err)
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
proxyURL, err := http.ProxyFromEnvironment(&http.Request{URL: parsedURL})
|
| 40 |
+
if err != nil {
|
| 41 |
+
return nil, fmt.Errorf("resolve image compatibility proxy: %w", err)
|
| 42 |
+
}
|
| 43 |
+
if proxyURL == nil {
|
| 44 |
+
return service.DoDownloadRequest(rawURL, reason...)
|
| 45 |
+
}
|
| 46 |
+
if err := validateImageURLCompatProxyTarget(parsedURL); err != nil {
|
| 47 |
+
return nil, fmt.Errorf("request reject: %w", err)
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
baseClient := service.GetHttpClient()
|
| 51 |
+
client := &http.Client{
|
| 52 |
+
Transport: baseClient.Transport,
|
| 53 |
+
Timeout: baseClient.Timeout,
|
| 54 |
+
Jar: baseClient.Jar,
|
| 55 |
+
CheckRedirect: func(request *http.Request, via []*http.Request) error {
|
| 56 |
+
if len(via) >= 10 {
|
| 57 |
+
return fmt.Errorf("stopped after 10 redirects")
|
| 58 |
+
}
|
| 59 |
+
redirectProxy, err := http.ProxyFromEnvironment(request)
|
| 60 |
+
if err != nil {
|
| 61 |
+
return fmt.Errorf("resolve redirect proxy: %w", err)
|
| 62 |
+
}
|
| 63 |
+
if redirectProxy == nil {
|
| 64 |
+
return fmt.Errorf("image compatibility redirect left the configured proxy")
|
| 65 |
+
}
|
| 66 |
+
return validateImageURLCompatProxyTarget(request.URL)
|
| 67 |
+
},
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
common.SysLog(fmt.Sprintf(
|
| 71 |
+
"downloading image compatibility URL through configured proxy: %s, reason: %s",
|
| 72 |
+
common.MaskSensitiveInfo(rawURL),
|
| 73 |
+
strings.Join(reason, ", "),
|
| 74 |
+
))
|
| 75 |
+
return retryImageURLCompatDownload(
|
| 76 |
+
func() (*http.Response, error) { return client.Get(rawURL) },
|
| 77 |
+
time.Sleep,
|
| 78 |
+
)
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
func retryImageURLCompatDownload(request func() (*http.Response, error), wait func(time.Duration)) (*http.Response, error) {
|
| 82 |
+
var lastErr error
|
| 83 |
+
for attempt := 1; attempt <= maxImageURLCompatDownloadAttempts; attempt++ {
|
| 84 |
+
response, err := request()
|
| 85 |
+
if err == nil && response != nil && response.StatusCode < http.StatusInternalServerError {
|
| 86 |
+
return response, nil
|
| 87 |
+
}
|
| 88 |
+
if response != nil && response.Body != nil {
|
| 89 |
+
response.Body.Close()
|
| 90 |
+
}
|
| 91 |
+
if err != nil {
|
| 92 |
+
lastErr = err
|
| 93 |
+
} else if response == nil {
|
| 94 |
+
lastErr = fmt.Errorf("image compatibility download returned an empty response")
|
| 95 |
+
} else {
|
| 96 |
+
lastErr = fmt.Errorf("image compatibility download returned HTTP %d", response.StatusCode)
|
| 97 |
+
}
|
| 98 |
+
if attempt < maxImageURLCompatDownloadAttempts {
|
| 99 |
+
wait(time.Duration(attempt) * 300 * time.Millisecond)
|
| 100 |
+
}
|
| 101 |
+
}
|
| 102 |
+
return nil, lastErr
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
// Clash Fake-IP addresses are local DNS placeholders. When an HTTP proxy is
|
| 106 |
+
// selected, validate the hostname and port without resolving that placeholder.
|
| 107 |
+
func validateImageURLCompatProxyTarget(parsedURL *url.URL) error {
|
| 108 |
+
if parsedURL == nil || parsedURL.Hostname() == "" {
|
| 109 |
+
return fmt.Errorf("invalid image compatibility URL")
|
| 110 |
+
}
|
| 111 |
+
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
| 112 |
+
return fmt.Errorf("unsupported image compatibility URL scheme: %s", parsedURL.Scheme)
|
| 113 |
+
}
|
| 114 |
+
if parsedURL.User != nil {
|
| 115 |
+
return fmt.Errorf("image compatibility URL must not contain credentials")
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
port := 80
|
| 119 |
+
if parsedURL.Scheme == "https" {
|
| 120 |
+
port = 443
|
| 121 |
+
}
|
| 122 |
+
if parsedURL.Port() != "" {
|
| 123 |
+
parsedPort, err := strconv.Atoi(parsedURL.Port())
|
| 124 |
+
if err != nil {
|
| 125 |
+
return fmt.Errorf("invalid image compatibility port: %w", err)
|
| 126 |
+
}
|
| 127 |
+
port = parsedPort
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
fetchSetting := system_setting.GetFetchSetting()
|
| 131 |
+
if fetchSetting == nil || !fetchSetting.EnableSSRFProtection {
|
| 132 |
+
return nil
|
| 133 |
+
}
|
| 134 |
+
protection, err := common.NewSSRFProtectionFromFetchSetting(
|
| 135 |
+
fetchSetting.AllowPrivateIp,
|
| 136 |
+
fetchSetting.DomainFilterMode,
|
| 137 |
+
fetchSetting.IpFilterMode,
|
| 138 |
+
fetchSetting.DomainList,
|
| 139 |
+
fetchSetting.IpList,
|
| 140 |
+
fetchSetting.AllowedPorts,
|
| 141 |
+
fetchSetting.ApplyIPFilterForDomain,
|
| 142 |
+
)
|
| 143 |
+
if err != nil {
|
| 144 |
+
return err
|
| 145 |
+
}
|
| 146 |
+
return protection.ValidateNetworkTarget(parsedURL.Hostname(), port)
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
func (s imageURLCompatStats) ServerTiming() string {
|
| 150 |
+
return fmt.Sprintf(
|
| 151 |
+
"image-url-download;dur=%.3f, image-b64-encode;dur=%.3f, image-url-compat;dur=%.3f",
|
| 152 |
+
float64(s.DownloadTime.Microseconds())/1000,
|
| 153 |
+
float64(s.EncodeTime.Microseconds())/1000,
|
| 154 |
+
float64(s.TotalTime.Microseconds())/1000,
|
| 155 |
+
)
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
func (s imageURLCompatStats) LogMessage() string {
|
| 159 |
+
return fmt.Sprintf(
|
| 160 |
+
"image URL compatibility converted=%d bytes=%d download_ms=%.3f encode_ms=%.3f total_ms=%.3f",
|
| 161 |
+
s.Converted,
|
| 162 |
+
s.DownloadedBytes,
|
| 163 |
+
float64(s.DownloadTime.Microseconds())/1000,
|
| 164 |
+
float64(s.EncodeTime.Microseconds())/1000,
|
| 165 |
+
float64(s.TotalTime.Microseconds())/1000,
|
| 166 |
+
)
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
func convertOpenAIImageURLsToBase64(responseBody []byte, download imageURLCompatDownloadFunc) ([]byte, imageURLCompatStats, error) {
|
| 170 |
+
var stats imageURLCompatStats
|
| 171 |
+
if !common.GetEnvOrDefaultBool("CODEX_IMAGE_URL_TO_B64", false) {
|
| 172 |
+
return responseBody, stats, nil
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
startedAt := time.Now()
|
| 176 |
+
var payload map[string]json.RawMessage
|
| 177 |
+
if err := common.Unmarshal(responseBody, &payload); err != nil {
|
| 178 |
+
return nil, stats, fmt.Errorf("decode image compatibility response: %w", err)
|
| 179 |
+
}
|
| 180 |
+
dataJSON, ok := payload["data"]
|
| 181 |
+
if !ok {
|
| 182 |
+
return responseBody, stats, nil
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
var images []map[string]json.RawMessage
|
| 186 |
+
if err := common.Unmarshal(dataJSON, &images); err != nil {
|
| 187 |
+
return nil, stats, fmt.Errorf("decode image compatibility data: %w", err)
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
for _, image := range images {
|
| 191 |
+
if jsonString(image["b64_json"]) != "" {
|
| 192 |
+
continue
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
var imageURL string
|
| 196 |
+
var sourceField string
|
| 197 |
+
for _, field := range []string{"url", "result_url", "image_url"} {
|
| 198 |
+
if value := jsonString(image[field]); value != "" {
|
| 199 |
+
imageURL = value
|
| 200 |
+
sourceField = field
|
| 201 |
+
break
|
| 202 |
+
}
|
| 203 |
+
}
|
| 204 |
+
if imageURL == "" {
|
| 205 |
+
continue
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
downloadStartedAt := time.Now()
|
| 209 |
+
imageResponse, err := download(imageURL, "OpenAI image URL compatibility")
|
| 210 |
+
if err != nil {
|
| 211 |
+
return nil, stats, fmt.Errorf("download image compatibility URL: %w", err)
|
| 212 |
+
}
|
| 213 |
+
imageBytes, readErr := readImageURLCompatResponse(imageResponse)
|
| 214 |
+
stats.DownloadTime += time.Since(downloadStartedAt)
|
| 215 |
+
if readErr != nil {
|
| 216 |
+
return nil, stats, readErr
|
| 217 |
+
}
|
| 218 |
+
|
| 219 |
+
encodeStartedAt := time.Now()
|
| 220 |
+
encodedJSON, err := common.Marshal(base64.StdEncoding.EncodeToString(imageBytes))
|
| 221 |
+
if err != nil {
|
| 222 |
+
return nil, stats, fmt.Errorf("encode image compatibility response: %w", err)
|
| 223 |
+
}
|
| 224 |
+
image["b64_json"] = encodedJSON
|
| 225 |
+
delete(image, sourceField)
|
| 226 |
+
stats.EncodeTime += time.Since(encodeStartedAt)
|
| 227 |
+
stats.Converted++
|
| 228 |
+
stats.DownloadedBytes += int64(len(imageBytes))
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
if stats.Converted == 0 {
|
| 232 |
+
return responseBody, stats, nil
|
| 233 |
+
}
|
| 234 |
+
|
| 235 |
+
encodeStartedAt := time.Now()
|
| 236 |
+
dataJSON, err := common.Marshal(images)
|
| 237 |
+
if err != nil {
|
| 238 |
+
return nil, stats, fmt.Errorf("encode image compatibility data: %w", err)
|
| 239 |
+
}
|
| 240 |
+
payload["data"] = dataJSON
|
| 241 |
+
convertedBody, err := common.Marshal(payload)
|
| 242 |
+
stats.EncodeTime += time.Since(encodeStartedAt)
|
| 243 |
+
stats.TotalTime = time.Since(startedAt)
|
| 244 |
+
if err != nil {
|
| 245 |
+
return nil, stats, fmt.Errorf("encode image compatibility payload: %w", err)
|
| 246 |
+
}
|
| 247 |
+
return convertedBody, stats, nil
|
| 248 |
+
}
|
| 249 |
+
|
| 250 |
+
func jsonString(raw json.RawMessage) string {
|
| 251 |
+
if len(raw) == 0 {
|
| 252 |
+
return ""
|
| 253 |
+
}
|
| 254 |
+
var value string
|
| 255 |
+
if err := common.Unmarshal(raw, &value); err != nil {
|
| 256 |
+
return ""
|
| 257 |
+
}
|
| 258 |
+
return strings.TrimSpace(value)
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
func readImageURLCompatResponse(response *http.Response) ([]byte, error) {
|
| 262 |
+
if response == nil || response.Body == nil {
|
| 263 |
+
return nil, fmt.Errorf("image compatibility download returned an empty response")
|
| 264 |
+
}
|
| 265 |
+
defer response.Body.Close()
|
| 266 |
+
if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices {
|
| 267 |
+
return nil, fmt.Errorf("image compatibility download returned HTTP %d", response.StatusCode)
|
| 268 |
+
}
|
| 269 |
+
if response.ContentLength > maxImageURLCompatBytes {
|
| 270 |
+
return nil, fmt.Errorf("image compatibility download exceeds %d bytes", maxImageURLCompatBytes)
|
| 271 |
+
}
|
| 272 |
+
|
| 273 |
+
imageBytes, err := io.ReadAll(io.LimitReader(response.Body, maxImageURLCompatBytes+1))
|
| 274 |
+
if err != nil {
|
| 275 |
+
return nil, fmt.Errorf("read image compatibility download: %w", err)
|
| 276 |
+
}
|
| 277 |
+
if int64(len(imageBytes)) > maxImageURLCompatBytes {
|
| 278 |
+
return nil, fmt.Errorf("image compatibility download exceeds %d bytes", maxImageURLCompatBytes)
|
| 279 |
+
}
|
| 280 |
+
if len(imageBytes) == 0 {
|
| 281 |
+
return nil, fmt.Errorf("image compatibility download returned an empty body")
|
| 282 |
+
}
|
| 283 |
+
return imageBytes, nil
|
| 284 |
+
}
|
image_url_compat_test.go
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package openai
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"bytes"
|
| 5 |
+
"errors"
|
| 6 |
+
"io"
|
| 7 |
+
"net/http"
|
| 8 |
+
"testing"
|
| 9 |
+
"time"
|
| 10 |
+
|
| 11 |
+
"github.com/QuantumNous/new-api/common"
|
| 12 |
+
"github.com/stretchr/testify/assert"
|
| 13 |
+
"github.com/stretchr/testify/require"
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
func TestConvertOpenAIImageURLsToBase64Disabled(t *testing.T) {
|
| 17 |
+
t.Setenv("CODEX_IMAGE_URL_TO_B64", "false")
|
| 18 |
+
body := []byte(`{"created":1,"data":[{"url":"https://example.com/image.png"}]}`)
|
| 19 |
+
|
| 20 |
+
converted, stats, err := convertOpenAIImageURLsToBase64(body, func(string, ...string) (*http.Response, error) {
|
| 21 |
+
t.Fatal("download must not run while compatibility is disabled")
|
| 22 |
+
return nil, nil
|
| 23 |
+
})
|
| 24 |
+
|
| 25 |
+
require.NoError(t, err)
|
| 26 |
+
assert.Equal(t, body, converted)
|
| 27 |
+
assert.Zero(t, stats.Converted)
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
func TestConvertOpenAIImageURLsToBase64PreservesResponseFields(t *testing.T) {
|
| 31 |
+
t.Setenv("CODEX_IMAGE_URL_TO_B64", "true")
|
| 32 |
+
image := []byte("test-image")
|
| 33 |
+
body := []byte(`{"created":1,"data":[{"url":"https://example.com/image.png","revised_prompt":"kept"}],"usage":{"total_tokens":7},"custom":{"kept":true}}`)
|
| 34 |
+
|
| 35 |
+
converted, stats, err := convertOpenAIImageURLsToBase64(body, func(rawURL string, reason ...string) (*http.Response, error) {
|
| 36 |
+
assert.Equal(t, "https://example.com/image.png", rawURL)
|
| 37 |
+
assert.Equal(t, []string{"OpenAI image URL compatibility"}, reason)
|
| 38 |
+
return &http.Response{
|
| 39 |
+
StatusCode: http.StatusOK,
|
| 40 |
+
ContentLength: int64(len(image)),
|
| 41 |
+
Body: io.NopCloser(bytes.NewReader(image)),
|
| 42 |
+
}, nil
|
| 43 |
+
})
|
| 44 |
+
|
| 45 |
+
require.NoError(t, err)
|
| 46 |
+
assert.Equal(t, 1, stats.Converted)
|
| 47 |
+
assert.Equal(t, int64(len(image)), stats.DownloadedBytes)
|
| 48 |
+
|
| 49 |
+
var payload map[string]any
|
| 50 |
+
require.NoError(t, common.Unmarshal(converted, &payload))
|
| 51 |
+
data := payload["data"].([]any)
|
| 52 |
+
item := data[0].(map[string]any)
|
| 53 |
+
assert.Equal(t, "dGVzdC1pbWFnZQ==", item["b64_json"])
|
| 54 |
+
assert.Equal(t, "kept", item["revised_prompt"])
|
| 55 |
+
assert.NotContains(t, item, "url")
|
| 56 |
+
assert.Equal(t, float64(7), payload["usage"].(map[string]any)["total_tokens"])
|
| 57 |
+
assert.Equal(t, true, payload["custom"].(map[string]any)["kept"])
|
| 58 |
+
assert.Contains(t, stats.ServerTiming(), "image-url-compat")
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
func TestReadImageURLCompatResponseRejectsOversize(t *testing.T) {
|
| 62 |
+
response := &http.Response{
|
| 63 |
+
StatusCode: http.StatusOK,
|
| 64 |
+
ContentLength: maxImageURLCompatBytes + 1,
|
| 65 |
+
Body: io.NopCloser(bytes.NewReader([]byte("unused"))),
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
_, err := readImageURLCompatResponse(response)
|
| 69 |
+
|
| 70 |
+
require.Error(t, err)
|
| 71 |
+
assert.Contains(t, err.Error(), "exceeds")
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
func TestDownloadOpenAIImageURLCompatRejectsPrivateLiteral(t *testing.T) {
|
| 75 |
+
_, err := downloadOpenAIImageURLCompat("https://127.0.0.1/image.png")
|
| 76 |
+
|
| 77 |
+
require.Error(t, err)
|
| 78 |
+
assert.Contains(t, err.Error(), "private IP address not allowed")
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
func TestRetryImageURLCompatDownloadRecoversFromTransientError(t *testing.T) {
|
| 82 |
+
attempts := 0
|
| 83 |
+
waits := 0
|
| 84 |
+
|
| 85 |
+
response, err := retryImageURLCompatDownload(func() (*http.Response, error) {
|
| 86 |
+
attempts++
|
| 87 |
+
if attempts == 1 {
|
| 88 |
+
return nil, errors.New("EOF")
|
| 89 |
+
}
|
| 90 |
+
return &http.Response{
|
| 91 |
+
StatusCode: http.StatusOK,
|
| 92 |
+
Body: io.NopCloser(bytes.NewBufferString("image")),
|
| 93 |
+
}, nil
|
| 94 |
+
}, func(time.Duration) {
|
| 95 |
+
waits++
|
| 96 |
+
})
|
| 97 |
+
|
| 98 |
+
require.NoError(t, err)
|
| 99 |
+
require.NotNil(t, response)
|
| 100 |
+
assert.Equal(t, 2, attempts)
|
| 101 |
+
assert.Equal(t, 1, waits)
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
func TestRetryImageURLCompatDownloadStopsAfterLimit(t *testing.T) {
|
| 105 |
+
attempts := 0
|
| 106 |
+
|
| 107 |
+
response, err := retryImageURLCompatDownload(func() (*http.Response, error) {
|
| 108 |
+
attempts++
|
| 109 |
+
return nil, errors.New("EOF")
|
| 110 |
+
}, func(time.Duration) {})
|
| 111 |
+
|
| 112 |
+
require.Error(t, err)
|
| 113 |
+
assert.Nil(t, response)
|
| 114 |
+
assert.Equal(t, maxImageURLCompatDownloadAttempts, attempts)
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
func TestRetryImageURLCompatDownloadRejectsEmptyResponse(t *testing.T) {
|
| 118 |
+
attempts := 0
|
| 119 |
+
|
| 120 |
+
response, err := retryImageURLCompatDownload(func() (*http.Response, error) {
|
| 121 |
+
attempts++
|
| 122 |
+
return nil, nil
|
| 123 |
+
}, func(time.Duration) {})
|
| 124 |
+
|
| 125 |
+
require.Error(t, err)
|
| 126 |
+
assert.Nil(t, response)
|
| 127 |
+
assert.Contains(t, err.Error(), "empty response")
|
| 128 |
+
assert.Equal(t, maxImageURLCompatDownloadAttempts, attempts)
|
| 129 |
+
}
|