fix audio convert

public/chatclient/media.js

@@ -112,6 +112,10 @@ function classifyFile(file) {
 function inferAudioFormat(file) {
   const mimeType = file?.type ?? file?.mimeType ?? "";
   const name = (file?.name ?? "").toLowerCase();
+  if (mimeType === "audio/mp4" || mimeType === "audio/x-m4a" || name.endsWith(".m4a")) {
+    return "m4a";
+  }
+
   if (mimeType === "audio/wav" || mimeType === "audio/x-wav" || name.endsWith(".wav")) {
     return "wav";
   }

public/chatclient/render.js

@@ -22,12 +22,18 @@ export function renderAttachments(card, container, summary, attachments, onRemov
 export function renderResponse(container, requestPayload, responseBody) {
   const assistant = responseBody?.choices?.[0]?.message ?? {};
   const text = extractAssistantText(assistant);
+  const audioTranscript = extractAssistantAudioTranscript(assistant);
   const images = extractAssistantImages(assistant);
   const audioUrl = assistant?.audio?.url || null;
+  const assistantBody = text || audioTranscript || "No assistant text returned.";
 
   container.innerHTML = "";
   container.appendChild(renderInfoCard("Request", describeRequest(requestPayload)));
-  container.appendChild(renderRichTextCard("Assistant", text || "No assistant text returned."));
+  container.appendChild(renderRichTextCard("Assistant", assistantBody));
+
+  if (audioTranscript && !sameText(text, audioTranscript)) {
+    container.appendChild(renderRichTextCard("Audio Transcript", audioTranscript));
+  }
 
   for (const imageUrl of images) {
     const imageCard = renderInfoCard("Image", imageUrl);
@@ -167,6 +173,18 @@ function extractAssistantImages(message) {
     .filter(Boolean);
 }
 
+function extractAssistantAudioTranscript(message) {
+  return typeof message?.audio?.transcript === "string" ? message.audio.transcript.trim() : "";
+}
+
+function sameText(left, right) {
+  return normalizeText(left) === normalizeText(right);
+}
+
+function normalizeText(value) {
+  return typeof value === "string" ? value.trim().replaceAll(/\s+/g, " ") : "";
+}
+
 function renderInfoCard(title, body) {
   const card = document.createElement("article");
   card.className = "response-block";

src/app.js

@@ -12,6 +12,20 @@ export function createApp({
   const app = express();
 
   app.disable("x-powered-by");
+  app.use((req, res, next) => {
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader("Access-Control-Allow-Methods", "GET,POST,PUT,PATCH,DELETE,OPTIONS");
+
+    const requestedHeaders = req.get("access-control-request-headers");
+    res.setHeader("Access-Control-Allow-Headers", requestedHeaders || "Content-Type, Authorization");
+
+    if (req.method === "OPTIONS") {
+      res.status(204).end();
+      return;
+    }
+
+    next();
+  });
   app.use(express.json({ limit: jsonLimit }));
   app.get("/", (_req, res) => {
     res.sendFile(path.join(publicDir, "index.html"));

src/services/audioConversionService.js

@@ -54,7 +54,8 @@ export function createAudioConversionService({ fetchImpl = fetch, maxAudioDownlo
         throw new HttpError(413, `Audio URL exceeded ${maxAudioDownloadMb}MB download limit.`);
       }
 
-      return transcodeAudioBuffer(audioBuffer, "mp3");
+      const inputFormat = inferAudioFormatFromUrl(url) || inferAudioFormatFromMimeType(response.headers.get("content-type")) || "unknown";
+      return transcodeAudioBuffer(audioBuffer, inputFormat);
     },
 
     async normalizeBase64Audio({ data, format }) {
@@ -67,8 +68,8 @@ export function createAudioConversionService({ fetchImpl = fetch, maxAudioDownlo
         throw new HttpError(413, `Audio input exceeded ${maxAudioDownloadMb}MB upload limit.`);
       }
 
-      if (!["mp3", "wav"].includes(format)) {
-        throw new HttpError(400, "Audio input format must be mp3 or wav.");
+      if (!["mp3", "wav", "m4a"].includes(format)) {
+        throw new HttpError(400, "Audio input format must be mp3, wav, or m4a.");
       }
 
       try {
@@ -83,19 +84,21 @@ export function createAudioConversionService({ fetchImpl = fetch, maxAudioDownlo
     }
   };
 
-  async function transcodeAudioBuffer(audioBuffer, outputFormat) {
+  async function transcodeAudioBuffer(audioBuffer, inputFormat) {
     const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "oapix-audio-"));
-    const inputPath = path.join(tempDir, "input-media");
-    const outputPath = path.join(tempDir, `output.${outputFormat}`);
+    const normalizedInputFormat = normalizeInputFormat(inputFormat);
+    const inputPath = path.join(tempDir, `input-media.${normalizedInputFormat === "unknown" ? "bin" : normalizedInputFormat}`);
+    const outputPath = path.join(tempDir, "output.mp3");
 
     try {
+      console.log(`convert audio: ${normalizedInputFormat}->mp3`);
       await fs.writeFile(inputPath, audioBuffer);
-      await runFfmpeg(inputPath, outputPath, ffmpegOutputArgs(outputFormat));
+      await runFfmpeg(inputPath, outputPath, ffmpegOutputArgs());
       const convertedBuffer = await fs.readFile(outputPath);
 
       return {
         data: convertedBuffer.toString("base64"),
-        format: outputFormat
+        format: "mp3"
       };
     } finally {
       await fs.rm(tempDir, { force: true, recursive: true });
@@ -103,10 +106,50 @@ export function createAudioConversionService({ fetchImpl = fetch, maxAudioDownlo
   }
 }
 
-function ffmpegOutputArgs(outputFormat) {
-  if (outputFormat === "wav") {
-    return ["-acodec", "pcm_s16le"];
+function ffmpegOutputArgs() {
+  return ["-acodec", "libmp3lame", "-q:a", "4"];
+}
+
+function inferAudioFormatFromUrl(url) {
+  try {
+    const pathname = new URL(url).pathname.toLowerCase();
+
+    if (pathname.endsWith(".m4a")) {
+      return "m4a";
+    }
+
+    if (pathname.endsWith(".wav")) {
+      return "wav";
+    }
+
+    if (pathname.endsWith(".mp3")) {
+      return "mp3";
+    }
+  } catch (_error) {
+    return "unknown";
   }
 
-  return ["-acodec", "libmp3lame"];
+  return "unknown";
+}
+
+function inferAudioFormatFromMimeType(mimeType) {
+  const value = String(mimeType || "").split(";")[0].trim().toLowerCase();
+
+  if (value === "audio/mp4" || value === "audio/x-m4a") {
+    return "m4a";
+  }
+
+  if (value === "audio/wav" || value === "audio/x-wav") {
+    return "wav";
+  }
+
+  if (value === "audio/mpeg" || value === "audio/mp3") {
+    return "mp3";
+  }
+
+  return "unknown";
+}
+
+function normalizeInputFormat(format) {
+  return ["m4a", "wav", "mp3"].includes(format) ? format : "unknown";
 }

src/services/requestNormalizationService.js

@@ -2,7 +2,11 @@ import { HttpError } from "../utils/httpError.js";
 import { imageMimeType } from "../utils/mediaTypes.js";
 import { isDataUrl, isLikelyBase64, parseDataUrl, stripDataUrl } from "../utils/dataUrl.js";
 
-const SUPPORTED_AUDIO_FORMATS = new Set(["mp3", "wav"]);
+const SUPPORTED_AUDIO_FORMATS = new Set(["mp3", "wav", "m4a"]);
+
+function isHttpUrl(value) {
+  return typeof value === "string" && (value.startsWith("http://") || value.startsWith("https://"));
+}
 
 function normalizeImageSource(source, mimeType) {
   if (typeof source !== "string" || source.length === 0) {
@@ -37,7 +41,7 @@ function normalizeImagePart(part) {
 function normalizeAudioBase64(audio) {
   const format = audio.format?.toLowerCase();
   if (!SUPPORTED_AUDIO_FORMATS.has(format)) {
-    throw new HttpError(400, "Audio input format must be mp3 or wav.");
+    throw new HttpError(400, "Audio input format must be mp3, wav, or m4a.");
   }
 
   if (typeof audio.data !== "string" || audio.data.length === 0) {
@@ -47,10 +51,28 @@ function normalizeAudioBase64(audio) {
   const parsed = parseDataUrl(audio.data);
   return {
     data: stripDataUrl(audio.data),
-    format: parsed?.mimeType === "audio/wav" ? "wav" : format
+    format: inferAudioFormat(parsed?.mimeType, format)
   };
 }
 
+function inferAudioFormat(mimeType, fallbackFormat) {
+  const normalizedMimeType = String(mimeType || "").split(";")[0].trim().toLowerCase();
+
+  if (normalizedMimeType === "audio/wav" || normalizedMimeType === "audio/x-wav") {
+    return "wav";
+  }
+
+  if (normalizedMimeType === "audio/mp4" || normalizedMimeType === "audio/x-m4a") {
+    return "m4a";
+  }
+
+  if (normalizedMimeType === "audio/mpeg" || normalizedMimeType === "audio/mp3") {
+    return "mp3";
+  }
+
+  return fallbackFormat;
+}
+
 export function createRequestNormalizationService({ audioConversionService }) {
   return {
     async normalize(body) {
@@ -76,9 +98,10 @@ export function createRequestNormalizationService({ audioConversionService }) {
 
           if (part.type === "input_audio") {
             const audio = part.input_audio ?? {};
+            const audioUrl = audio.url || (isHttpUrl(audio.data) ? audio.data : null);
 
-            if (audio.url) {
-              const converted = await audioConversionService.downloadAndConvertToMp3Base64(audio.url);
+            if (audioUrl) {
+              const converted = await audioConversionService.downloadAndConvertToMp3Base64(audioUrl);
               nextParts.push({
                 type: "input_audio",
                 input_audio: converted

src/utils/dataUrl.js

@@ -1,4 +1,3 @@
-const DATA_URL_PATTERN = /^data:([^;,]+);base64,(.+)$/s;
 const BASE64_PATTERN = /^[A-Za-z0-9+/=]+$/;
 
 export function isDataUrl(value) {
@@ -6,14 +5,28 @@ export function isDataUrl(value) {
 }
 
 export function parseDataUrl(value) {
-  const match = value.match(DATA_URL_PATTERN);
-  if (!match) {
+  if (!isDataUrl(value)) {
+    return null;
+  }
+
+  const commaIndex = value.indexOf(",");
+  if (commaIndex === -1) {
+    return null;
+  }
+
+  const metadata = value.slice(5, commaIndex);
+  const base64 = value.slice(commaIndex + 1);
+  const metadataParts = metadata.split(";");
+  const mimeType = metadataParts[0] || "";
+  const isBase64 = metadataParts.slice(1).some((part) => part.toLowerCase() === "base64");
+
+  if (!mimeType || !isBase64 || !base64) {
     return null;
   }
 
   return {
-    mimeType: match[1],
-    base64: match[2]
+    mimeType,
+    base64
   };
 }
 

test/app.test.js

@@ -0,0 +1,70 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createApp } from "../src/app.js";
+
+test("applies permissive CORS headers to API responses and preflight requests", async () => {
+  const publicDir = await mkdtemp(path.join(tmpdir(), "oapix-app-test-"));
+  await writeFile(path.join(publicDir, "index.html"), "<!doctype html><title>test</title>");
+
+  const app = createApp({
+    jsonLimit: "1mb",
+    publicDir,
+    chatController(_req, res) {
+      res.json({ ok: true });
+    },
+    mediaController(_req, res) {
+      res.json({ media: true });
+    }
+  });
+
+  const server = await new Promise((resolve) => {
+    const nextServer = app.listen(0, () => resolve(nextServer));
+  });
+
+  const address = server.address();
+  const baseUrl = `http://127.0.0.1:${address.port}`;
+
+  try {
+    const response = await fetch(`${baseUrl}/v1/chat/completions`, {
+      method: "POST",
+      headers: {
+        origin: "https://example.com",
+        "content-type": "application/json"
+      },
+      body: JSON.stringify({ model: "test", messages: [] })
+    });
+
+    assert.equal(response.status, 200);
+    assert.equal(response.headers.get("access-control-allow-origin"), "*");
+    assert.equal(response.headers.get("access-control-allow-methods"), "GET,POST,PUT,PATCH,DELETE,OPTIONS");
+
+    const preflight = await fetch(`${baseUrl}/v1/chat/completions`, {
+      method: "OPTIONS",
+      headers: {
+        origin: "https://example.com",
+        "access-control-request-method": "POST",
+        "access-control-request-headers": "content-type, authorization"
+      }
+    });
+
+    assert.equal(preflight.status, 204);
+    assert.equal(preflight.headers.get("access-control-allow-origin"), "*");
+    assert.equal(preflight.headers.get("access-control-allow-methods"), "GET,POST,PUT,PATCH,DELETE,OPTIONS");
+    assert.equal(preflight.headers.get("access-control-allow-headers"), "content-type, authorization");
+  } finally {
+    await new Promise((resolve, reject) => {
+      server.close((error) => {
+        if (error) {
+          reject(error);
+          return;
+        }
+
+        resolve();
+      });
+    });
+    await rm(publicDir, { recursive: true, force: true });
+  }
+});

test/requestNormalization.test.js

@@ -55,6 +55,48 @@ test("normalizes audio URLs to mp3 base64 and raw image base64 to data URLs", as
   assert.equal(responseContext.audioFormat, "wav");
 });
 
+test("downloads remote audio when the URL is provided in input_audio.data", async () => {
+  const service = createRequestNormalizationService({
+    audioConversionService: {
+      async downloadAndConvertToMp3Base64(url) {
+        assert.equal(url, "https://example.com/sample.mp4");
+        return {
+          data: "ZG93bmxvYWRlZC1hbmQtY29udmVydGVk",
+          format: "mp3"
+        };
+      },
+      async normalizeBase64Audio() {
+        throw new Error("unexpected base64 conversion");
+      }
+    }
+  });
+
+  const { normalizedBody } = await service.normalize({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "input_audio",
+            input_audio: {
+              data: "https://example.com/sample.mp4",
+              format: "m4a"
+            }
+          }
+        ]
+      }
+    ]
+  });
+
+  assert.deepEqual(normalizedBody.messages[0].content[0], {
+    type: "input_audio",
+    input_audio: {
+      data: "ZG93bmxvYWRlZC1hbmQtY29udmVydGVk",
+      format: "mp3"
+    }
+  });
+});
+
 test("normalizes uploaded wav audio before forwarding upstream", async () => {
   const service = createRequestNormalizationService({
     audioConversionService: {
@@ -100,3 +142,95 @@ test("normalizes uploaded wav audio before forwarding upstream", async () => {
     }
   });
 });
+
+test("preserves uploaded m4a format for conversion and forwards mp3 output", async () => {
+  const service = createRequestNormalizationService({
+    audioConversionService: {
+      async downloadAndConvertToMp3Base64() {
+        throw new Error("unexpected url conversion");
+      },
+      async normalizeBase64Audio(audio) {
+        assert.deepEqual(audio, {
+          data: "AAAAGGZ0eXBNNEEg",
+          format: "m4a"
+        });
+
+        return {
+          data: "Y29udmVydGVkLW1wMw==",
+          format: "mp3"
+        };
+      }
+    }
+  });
+
+  const { normalizedBody } = await service.normalize({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "input_audio",
+            input_audio: {
+              data: "AAAAGGZ0eXBNNEEg",
+              format: "m4a"
+            }
+          }
+        ]
+      }
+    ]
+  });
+
+  assert.deepEqual(normalizedBody.messages[0].content[0], {
+    type: "input_audio",
+    input_audio: {
+      data: "Y29udmVydGVkLW1wMw==",
+      format: "mp3"
+    }
+  });
+});
+
+test("parses m4a data urls with codec metadata before conversion", async () => {
+  const service = createRequestNormalizationService({
+    audioConversionService: {
+      async downloadAndConvertToMp3Base64() {
+        throw new Error("unexpected url conversion");
+      },
+      async normalizeBase64Audio(audio) {
+        assert.deepEqual(audio, {
+          data: "AAAAGGZ0eXBNNEEg",
+          format: "m4a"
+        });
+
+        return {
+          data: "Y29udmVydGVkLW1wMw==",
+          format: "mp3"
+        };
+      }
+    }
+  });
+
+  const { normalizedBody } = await service.normalize({
+    messages: [
+      {
+        role: "user",
+        content: [
+          {
+            type: "input_audio",
+            input_audio: {
+              data: "data:audio/mp4;codecs=mp4a.40.2;base64,AAAAGGZ0eXBNNEEg",
+              format: "m4a"
+            }
+          }
+        ]
+      }
+    ]
+  });
+
+  assert.deepEqual(normalizedBody.messages[0].content[0], {
+    type: "input_audio",
+    input_audio: {
+      data: "Y29udmVydGVkLW1wMw==",
+      format: "mp3"
+    }
+  });
+});