Add files using upload-large-folder tool

r1-a/response_generation/Kimi-Audio/.gitignore

@@ -0,0 +1,179 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python_goose script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python_goose-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.env_*
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+.idea/
+data
+logs
+*.zip
+conf
+.DS_Store
+.ruff_cache
+.log
+*.jsonl
+*.parquet
+*.progress
+# Vscode
+.vscode
+
+*.safetensors
+*.model
+*.pt
+*.pth
+test_audios/output

r1-a/response_generation/minicpm/MiniCPM-o/eval_mm/vlmevalkit/vlmeval/dataset/dude.py

@@ -0,0 +1,211 @@
+import math
+from typing import List
+
+from .utils.judge_util import build_judge
+from .image_base import ImageBaseDataset
+from .mmlongbench import concat_images, MMLongBench_auxeval, anls_compute
+from ..smp import *
+
+
+FAIL_MSG = 'Failed to obtain answer via API.'
+
+
+def DUDE_acc(result_file):
+    data = load(result_file)
+    overall_score = 0.0
+    score_list = list()
+    for i in range(len(data)):
+        item = data.iloc[i]
+        if isinstance(item['answer'], float) and math.isnan(item['answer']):
+            item['answer'] = 'Not answerable'
+
+        item['answer'] = item['answer'].lower()
+        item['pred'] = item['pred'].lower()
+        score = anls_compute(item['answer'], item['pred'])
+        score_list.append(score)
+        overall_score += score
+
+    data['score'] = score_list
+    dump(data, result_file)
+
+    res = dict()
+    res['category'], res['num'], res['avg_score'] = ['anls'], [len(data)], [overall_score / len(data)]
+    res = pd.DataFrame(res)
+    return res
+
+
+class DUDE(ImageBaseDataset):
+
+    TYPE = 'VQA'
+
+    DATASET_URL = {
+        'DUDE': 'https://opencompass.openxlab.space/utils/VLMEval/DUDE.tsv',
+        'DUDE_MINI': 'https://opencompass.openxlab.space/utils/VLMEval/DUDE_MINI.tsv',
+    }
+    DATASET_MD5 = {
+        'DUDE': '130d860d08206e1e407cd77150c10d88',
+        'DUDE_MINI': 'e0c0d998114f0cca7516d12039d2b538',
+    }
+
+    SUPPORTED_MODELS = {
+        'GPT4': (1, 1),
+        'GPT4V': (1, 1),
+        'GPT4V_HIGH': (1, 1),
+        'GPT4o': (1, 1),
+        'GPT4o_HIGH': (1, 1),
+        'GPT4o_MINI': (1, 1),
+        'XComposer2d5': (1, -1),
+        'XComposer2_4KHD': (1, -1),
+        'MiniCPM-Llama3-V-2_5': (1, 5),
+        'InternVL-Chat-V1-5': (5, 2),
+    }
+
+    def __init__(self, dataset, **kwargs):
+        self.model_list = list(self.SUPPORTED_MODELS.keys())
+        model_name = kwargs['model']
+        if not listinstr(self.model_list, model_name):
+            raise AssertionError("{} doesn't support the evaluation on DUDE.".format(model_name))
+        super(DUDE, self).__init__(dataset)
+
+        self.is_api = True if listinstr(['GPT4'], model_name) else False
+        self.max_pages = 120
+        concat_num, column_num = self.SUPPORTED_MODELS.get(model_name)
+        self.concat_num = concat_num
+        self.column_num = column_num
+
+    def prepare_tsv(self, url, file_md5=None):
+        data_root = LMUDataRoot()
+        os.makedirs(data_root, exist_ok=True)
+        file_name = url.split('/')[-1]
+        data_path = osp.join(data_root, file_name)
+        if osp.exists(data_path) and (file_md5 is None or md5(data_path) == file_md5):
+            pass
+        else:
+            warnings.warn('The dataset tsv is not downloaded')
+            download_file(url, data_path)
+        return load(data_path)
+
+    def dump_image(self, origin_line):
+        os.makedirs(self.img_root, exist_ok=True)
+        try:
+            import fitz
+        except Exception as e:
+            logging.critical(f'{type(e)}: {e}')
+            logging.critical('Please use `pip install pymupdf` to parse PDF files.')
+
+        line = origin_line.copy()
+        if not isinstance(line['image_path'], List):
+            line['image_path'] = [line['image_path']]
+        line['image_path'] = line['image_path'][:self.max_pages]
+        skip_pdf_parse = True
+        for im_name in line['image_path']:
+            path = osp.join(self.img_root, im_name)
+            if not read_ok(path):
+                skip_pdf_parse = False
+                break
+
+        # Just for being compatible with the zooped loop: zip(line['image'], line['image_path'])
+        if skip_pdf_parse:
+            line['image'] = line['image_path']
+        else:
+            pdf_data = base64.b64decode(line['image'])
+            pdf_file = io.BytesIO(pdf_data)
+            encoded_images = []
+            with fitz.open(stream=pdf_file, filetype='pdf') as doc:
+                doc = doc[:self.max_pages]
+                for page in doc:
+                    image = page.get_pixmap(dpi=144)
+                    image_file = io.BytesIO(image.tobytes(output='png'))
+                    image = Image.open(image_file)
+                    encoded_image = encode_image_to_base64(image)
+                    encoded_images.append(encoded_image)
+            line['image'] = encoded_images
+            print('process {}'.format(line['doc_id']))
+
+        if 'image' in line:
+            if isinstance(line['image'], list):
+                tgt_path = []
+                assert 'image_path' in line
+                for img, im_name in zip(line['image'], line['image_path']):
+                    path = osp.join(self.img_root, im_name)
+                    if not read_ok(path):
+                        decode_base64_to_image_file(img, path)
+                    tgt_path.append(path)
+            else:
+                tgt_path = osp.join(self.img_root, f"{line['index']}.jpg")
+                if not read_ok(tgt_path):
+                    decode_base64_to_image_file(line['image'], tgt_path)
+                tgt_path = [tgt_path]
+        else:
+            assert 'image_path' in line
+            tgt_path = toliststr(line['image_path'])
+
+        if self.concat_num > 0 and not self.is_api:
+            concatenated_images = concat_images(tgt_path, max_concat=self.concat_num, column_num=self.column_num)
+
+            old_tgt_path = tgt_path
+            assert isinstance(old_tgt_path, list)
+            if self.column_num != -1:
+                tgt_path = [
+                    '_'.join(old_tgt_path[0].split('_')[:-1]) + '_concat{}_{}.jpg'.format(self.concat_num, i)
+                    for i in range(len(concatenated_images))
+                ]
+            else:
+                tgt_path = ['_'.join(old_tgt_path[0].split('_')[:-1]) + '_concat_all.jpg']
+
+            for path, concatenated_image in zip(tgt_path, concatenated_images):
+                if not read_ok(path):
+                    decode_base64_to_image_file(encode_image_to_base64(concatenated_image), path)
+                    num_images, image_size = len(old_tgt_path), concatenated_image.size
+                    print('concat {} images to a new one with size {}. save at {}'.format(num_images, image_size, path))
+        return tgt_path
+
+    @classmethod
+    def evaluate(self, eval_file, **judge_kwargs):
+        logger = get_logger('Evaluation')
+        model = judge_kwargs['model']
+
+        suffix = eval_file.split('.')[-1]
+        storage = eval_file.replace(f'.{suffix}', f'_{model}.xlsx')
+        tmp_file = eval_file.replace(f'.{suffix}', f'_{model}.pkl')
+
+        if osp.exists(storage):
+            logger.warning(f'GPT scoring file {storage} already exists, will reuse it in DUDE_eval. ')
+        else:
+            data = load(eval_file)
+            model = build_judge(max_tokens=128, **judge_kwargs)
+            lt = len(data)
+            lines = [data.iloc[i] for i in range(lt)]
+            tups = [(model, line) for line in lines]
+            indices = [line['index'] for line in lines]
+
+            ans = {}
+            if osp.exists(tmp_file):
+                ans = load(tmp_file)
+            tups = [x for x, i in zip(tups, indices) if i not in ans]
+            indices = [i for i in indices if i not in ans]
+
+            if len(indices):
+                new_results = list()
+                for model, line in tqdm(tups):
+                    res = MMLongBench_auxeval(model, line)
+                    new_results.append(res)
+
+            log_map, res_map, pred_map = {}, {}, {}
+            all_inds = [line['index'] for line in lines]
+            for k, v in zip(all_inds, new_results):
+                log_map[k] = v['log']
+                res_map[k] = v['res']
+                pred_map[k] = v['pred']
+            data['res'] = [res_map[idx] for idx in data['index']]
+            data['log'] = [log_map[idx] for idx in data['index']]
+            data['pred'] = [pred_map[idx] for idx in data['index']]
+            dump(data, storage)
+
+        score = DUDE_acc(storage)
+        score_pth = storage.replace('.xlsx', '_score.csv')
+
+        dump(score, score_pth)
+        logger.info(f'DUDE successfully finished evaluating {eval_file}, results saved in {score_pth}')
+        logger.info('Score: ')
+        logger.info(score)

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/router/guard/stateGuard.js

@@ -0,0 +1 @@
+export function createStateGuard() {}

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/router/menu/index.js

@@ -0,0 +1,10 @@
+export const basicRoutes = [
+    {
+        path: '/',
+        component: () => import('@/views/home/index.vue')
+    },
+    {
+        path: '/:port',
+        component: () => import('@/views/home/index.vue')
+    }
+];

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/styles/base.css

@@ -0,0 +1,56 @@
+*,
+*::before,
+*::after {
+    box-sizing: border-box;
+    margin: 0;
+}
+
+::-webkit-scrollbar {
+    width: 6px;
+    height: 6px;
+}
+::-webkit-scrollbar-thumb {
+    background: #e0e4ee;
+    border-radius: 4px;
+}
+
+html,
+body {
+    width: 100%;
+    height: 100%;
+    font-family:
+        Inter,
+        -apple-system,
+        BlinkMacSystemFont,
+        Segoe UI,
+        SF Pro SC,
+        SF Pro Display,
+        SF Pro Icons,
+        PingFang SC,
+        Hiragino Sans GB,
+        Microsoft YaHei,
+        Helvetica Neue,
+        Helvetica,
+        Arial,
+        sans-serif !important;
+    background: #f3f3f3;
+    transition:
+        color 0.5s,
+        background-color 0.5s;
+    line-height: 1.3;
+    font-size: 14px;
+    font-weight: 400;
+    color: var(--el-text-color-regular);
+    text-rendering: optimizeLegibility;
+    -webkit-font-smoothing: antialiased;
+    -moz-osx-font-smoothing: grayscale;
+    margin: 0;
+    padding: 0;
+    overflow: hidden;
+}
+
+#app {
+    width: 100%;
+    height: 100%;
+    padding: 16px 4vw;
+}

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/styles/element/index.less

@@ -0,0 +1,79 @@
+@import url('./variable.less');
+
+.el-message {
+    box-shadow: 0px 4px 13px 2px rgba(75, 79, 88, 0.11);
+    border: none;
+    border-radius: 8px;
+    top: 60px !important;
+    &--success,
+    &--error {
+        .el-message__content {
+            color: rgb(10, 10, 10);
+            font-size: 14px;
+        }
+    }
+    .el-message-icon--error {
+        color: var(--el-color-danger);
+        font-size: 16px;
+    }
+    .el-message-icon--success {
+        color: var(--el-color-success);
+        font-size: 16px;
+    }
+}
+.el-message.time-warning,
+.el-message.system-error {
+    width: calc(100vw - 200px);
+    padding: 16px 12px;
+    border-radius: 12px;
+}
+.el-message.el-message--warning.time-warning {
+    border: 1px solid #f9ac2a;
+    background: #fef7ea;
+    .el-icon {
+        display: none;
+    }
+    .el-message__content {
+        color: #2f333e;
+        font-family: PingFang SC;
+        font-size: 14px;
+        font-style: normal;
+        font-weight: 400;
+        line-height: normal;
+        padding-left: 28px;
+        position: relative;
+    }
+    .el-message__content::before {
+        position: absolute;
+        content: '';
+        width: 20px;
+        height: 20px;
+        background: url('@/assets/svg/warning.svg') no-repeat;
+        left: 0;
+    }
+}
+.el-message.el-message--error.system-error {
+    border: 1px solid #e72b00;
+    background: #ffebe7;
+    .el-icon {
+        display: none;
+    }
+    .el-message__content {
+        color: #2f333e;
+        font-family: PingFang SC;
+        font-size: 14px;
+        font-style: normal;
+        font-weight: 400;
+        line-height: normal;
+        padding-left: 28px;
+        position: relative;
+    }
+    .el-message__content::before {
+        position: absolute;
+        content: '';
+        width: 20px;
+        height: 20px;
+        background: url('@/assets/svg/error.svg') no-repeat;
+        left: 0;
+    }
+}

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/styles/element/variable.less

@@ -0,0 +1,51 @@
+:root {
+    --el-component-size-large: 48px;
+    --el-component-size: 40px;
+    --el-color-primary: #7661ff;
+    --el-color-danger: #de0000;
+    --el-color-warning: #ff7d00;
+    --el-color-success: #00b42a;
+    --el-text-color-regular: #0a0a0a;
+}
+.el-button {
+    --el-button-height: var(--el-component-size);
+    height: var(--el-button-height);
+    &--large {
+        --el-button-height: var(--el-component-size-large);
+        height: var(--el-button-height);
+    }
+    &--primary {
+        --el-button-bg-color: #7661ff;
+        --el-button-text-color: var(--el-color-white);
+        --el-button-border-color: #7661ff;
+        --el-button-hover-bg-color: rgb(159, 144, 255);
+        --el-button-hover-text-color: var(--el-color-white);
+        --el-button-hover-border-color: rgb(159, 144, 255);
+        --el-button-active-bg-color: rgb(98, 82, 208);
+        --el-button-active-border-color: rgb(98, 82, 208);
+        --el-button-disabled-bg-color: #d4cdff;
+        --el-button-disabled-border-color: #d4cdff;
+    }
+}
+.el-checkbox {
+    --el-checkbox-border-radius: 4px;
+    --el-checkbox-input-border: 1px solid rgb(188, 188, 188);
+    --el-checkbox-input-border-color-hover: rgb(61, 92, 255);
+    --el-checkbox-checked-bg-color: rgb(61, 92, 255);
+    --el-checkbox-checked-input-border-color: rgb(61, 92, 255);
+}
+.el-dialog {
+    &__header {
+        padding-bottom: 20px;
+    }
+    &__title {
+        --el-text-color-primary: rgb(10, 10, 10);
+        --el-dialog-title-font-size: 18px;
+        --el-dialog-font-line-height: 20px;
+    }
+}
+.el-message {
+    --el-message-padding: 11px 20px;
+    --el-message-bg-color: rgb(255, 255, 255);
+    --el-message-text-color: #de0000;
+}

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/styles/main.css

@@ -0,0 +1,30 @@
+@import './base.css';
+@import './variable.css';
+
+.layout-root {
+    width: 100%;
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+}
+
+.layout-main {
+    flex: 1 1 0;
+    display: flex;
+    flex-direction: column;
+    padding: 0 var(--layout-main-padding);
+}
+
+.layout-footer {
+    width: 100%;
+    max-width: var(--layout-content-width);
+    height: fit-content;
+    display: flex;
+    flex-direction: column;
+    padding: 0 var(--layout-main-padding);
+    margin: auto;
+}
+
+:focus-visible {
+    outline: none;
+}

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/styles/variable.css

@@ -0,0 +1,7 @@
+:root {
+    --layout-sidebar-width: 56px;
+    --layout-sidebar-left-space: 8px;
+    --layout-main-padding: 8px;
+    --layout-main-minWidth: calc(var(--layout-content-width) + var(--layout-main-padding) * 2);
+    --layout-content-width: 780px;
+}

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/utils/index.js

@@ -0,0 +1,44 @@
+// 判断终端是pc还是移动端
+export const isMobile = () => {
+    let flag = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini|Linux/i.test(navigator.userAgent);
+    const platform = navigator.platform;
+    // iPad上的Safari
+    if (platform === 'MacIntel' && navigator.maxTouchPoints > 1) {
+        flag = true;
+    }
+    return flag;
+};
+// 单片语音长度(单位：ms)
+const voicePerLength = 200;
+
+// 图片计数，算出在哪一次发送语音时，同时发送图片。例如一片语音100ms，一秒钟发送一次语音，即发送的第10片语音时需要带一张图片
+export const maxCount = 1000 / voicePerLength;
+
+export const getChunkLength = sampleRate => {
+    return sampleRate * (voicePerLength / 1000);
+};
+
+export const isAvailablePort = port => {
+    return [
+        8000, 8001, 8002, 8003, 8004, 8010, 8011, 8012, 8013, 8014, 8020, 8021, 8022, 8023, 8024, 8025, 8026, 8027,
+        8028, 32449
+    ].includes(port);
+};
+
+// 文件转base64格式
+export const fileToBase64 = file => {
+    return new Promise((resolve, reject) => {
+        if (!file) {
+            reject('文件不能为空');
+        }
+        const reader = new FileReader();
+        reader.onload = e => {
+            const base64String = e.target.result;
+            resolve(base64String);
+        };
+        reader.onerror = () => {
+            reject('文件转码失败');
+        };
+        reader.readAsDataURL(file);
+    });
+};

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/utils/websocket.js

@@ -0,0 +1,91 @@
+class WebSocketClient {
+    constructor(url, maxReconnectAttempts = 5, reconnectInterval = 5000) {
+        this.url = url;
+        this.socket = null;
+        this.eventHandlers = {};
+        this.maxReconnectAttempts = maxReconnectAttempts;
+        this.reconnectInterval = reconnectInterval;
+        this.reconnectAttempts = 0;
+        this.reconnectTimer = null;
+    }
+
+    connect() {
+        this.reconnectAttempts = 0;
+        this.establishConnection();
+    }
+
+    establishConnection() {
+        this.socket = new WebSocket(this.url);
+
+        this.socket.onopen = () => {
+            console.log('WebSocket connection opened');
+            this.reconnectAttempts = 0; // Reset reconnect attempts on successful connection
+            this.emit('open');
+        };
+
+        this.socket.onclose = event => {
+            console.log('WebSocket connection closed', event);
+            this.emit('close', event);
+            // 1005为主动关闭websocket
+            if (event.code !== 1005) {
+                this.reconnect();
+            }
+        };
+
+        this.socket.onerror = error => {
+            console.error('WebSocket error', error);
+            this.emit('error', error);
+            // Optionally, you may want to trigger a reconnect on error as well
+            // this.reconnect();
+        };
+
+        this.socket.onmessage = message => {
+            // console.log('WebSocket message received', message.data);
+            this.emit('message', message.data);
+        };
+    }
+
+    send(data) {
+        if (this.socket && this.socket.readyState === WebSocket.OPEN) {
+            this.socket.send(data);
+        } else {
+            console.error('WebSocket is not open');
+        }
+    }
+
+    on(event, handler) {
+        // if (!this.eventHandlers[event]) {
+        this.eventHandlers[event] = [];
+        // }
+        this.eventHandlers[event].push(handler);
+        // console.log('Event handler added:', this.eventHandlers, event);
+    }
+
+    emit(event, ...args) {
+        if (this.eventHandlers[event]) {
+            this.eventHandlers[event].forEach(handler => handler(...args));
+        }
+    }
+
+    close() {
+        if (this.socket) {
+            this.socket.close();
+        }
+        clearTimeout(this.reconnectTimer);
+    }
+
+    reconnect() {
+        if (this.reconnectAttempts < this.maxReconnectAttempts) {
+            console.log(`Reconnecting attempt ${this.reconnectAttempts + 1}/${this.maxReconnectAttempts}...`);
+            this.reconnectTimer = setTimeout(() => {
+                this.reconnectAttempts++;
+                this.establishConnection();
+            }, this.reconnectInterval);
+        } else {
+            console.error('Max reconnect attempts reached. WebSocket will not attempt to reconnect.');
+            this.emit('max-reconnect-attempts');
+        }
+    }
+}
+
+export default WebSocketClient;

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/Chatbot.vue

@@ -0,0 +1,3 @@
+<template>
+    <div>Chatbot</div>
+</template>

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/VideoCall.vue

@@ -0,0 +1,971 @@
+<template>
+    <!-- <ExtraInfo webVersion="非websocket_0111" :modelVersion="modelVersion" /> -->
+    <div class="video-page">
+        <div class="video-page-header">
+            <div class="voice-container" v-if="!isCalling">
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+            </div>
+            <div class="voice-container" v-else>
+                <Voice
+                    :dataArray="dataArray"
+                    :isCalling="isCalling"
+                    :isPlaying="playing"
+                    :configList="videoConfigList"
+                    :boxStyle="{ height: '45px' }"
+                    :itemStyle="{ width: '3px', margin: '0 1px' }"
+                />
+            </div>
+            <!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
+        </div>
+        <div class="video-page-content">
+            <div class="video-page-content-video" v-loading="loading" element-loading-background="#f3f3f3">
+                <video ref="videoRef" autoplay playsinline muted />
+                <canvas ref="canvasRef" canvas-id="canvasId" style="display: none" />
+                <div class="switch-camera" v-if="isMobile()" @click="switchCamera">
+                    <SvgIcon name="switch-camera" class="icon" />
+                </div>
+            </div>
+            <div class="video-page-content-right">
+                <div class="output-content">
+                    <ModelOutput
+                        v-if="outputData.length > 0"
+                        :outputData="outputData"
+                        containerClass="output-content"
+                    />
+                </div>
+                <div class="skip-box">
+                    <!-- <DelayTips
+                        v-if="delayTimestamp > 200 || delayCount > 2"
+                        :delayTimestamp="delayTimestamp"
+                        :delayCount="delayCount"
+                    /> -->
+                    <LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
+                    <SkipBtn :disabled="skipDisabled" @click="skipVoice" />
+                </div>
+            </div>
+        </div>
+        <div class="video-page-btn">
+            <el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
+                {{ callDisabled ? t('notReadyBtn') : t('videoCallBtn') }}
+            </el-button>
+            <el-button v-show="isCalling" @click="stopRecording" type="danger">
+                <SvgIcon name="phone-icon" className="phone-icon" />
+                <span class="btn-text">{{ t('hangUpBtn') }}</span>
+                <CountDown v-model="isCalling" @timeUp="stopRecording" />
+            </el-button>
+        </div>
+        <IdeasList v-if="showIdeasList" :ideasList="videoIdeasList" />
+    </div>
+</template>
+<script setup>
+    import { sendMessage, stopMessage, uploadConfig } from '@/apis';
+    import { encodeWAV } from '@/hooks/useVoice';
+    import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
+    import { fetchEventSource } from '@microsoft/fetch-event-source';
+    import { MicVAD } from '@ricky0123/vad-web';
+    import { videoIdeasList, videoConfigList, showIdeasList } from '@/enums';
+    import { isMobile, maxCount, getChunkLength } from '@/utils';
+    import { mergeBase64ToBlob } from './merge';
+    import { useI18n } from 'vue-i18n';
+
+    const { t } = useI18n();
+    import WebSocketService from '@/utils/websocket';
+
+    let ctrl = new AbortController();
+    let socket = null;
+    const audioData = ref({
+        base64Str: '',
+        type: 'mp3'
+    }); // 自定义音色base64
+    const isCalling = defineModel();
+    const videoRef = ref();
+    const videoStream = ref(null);
+    const interval = ref();
+    const canvasRef = ref();
+    const videoImage = ref([]);
+    const videoLoaded = ref(false);
+    const taskQueue = ref([]);
+    const running = ref(false);
+    const outputData = ref([]);
+    const isFirstReturn = ref(true);
+    const audioPlayQueue = ref([]);
+    const base64List = ref([]);
+    const playing = ref(false);
+    const timbre = ref([1]);
+    const isReturnError = ref(false);
+
+    const textQueue = ref('');
+    const textAnimationInterval = ref();
+
+    const analyser = ref();
+    const dataArray = ref();
+    const animationFrameId = ref();
+    const skipDisabled = ref(true);
+    const stop = ref(false);
+    const isFrontCamera = ref(true);
+    const loading = ref(false);
+
+    const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
+
+    const isFirstPiece = ref(true);
+    const allVoice = ref([]);
+    const callDisabled = ref(true);
+
+    const feedbackStatus = ref('');
+    const curResponseId = ref('');
+    const delayTimestamp = ref(0); // 当前发送片延时
+    const delayCount = ref(0); // 当前���余多少ms未发送到接口
+
+    const modelVersion = ref('');
+
+    let mediaStream;
+    let audioRecorder;
+    let audioStream;
+    let intervalId;
+    let audioContext;
+    let audioChunks = [];
+    let count = 0;
+    let audioDOM;
+
+    onBeforeUnmount(() => {
+        stopRecording();
+    });
+    const vadStartTime = ref();
+    let myvad = null;
+    let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
+    const vadStart = async () => {
+        myvad = await MicVAD.new({
+            onSpeechStart: () => {
+                console.log('Speech start', +new Date());
+                // if (!skipDisabled.value) {
+                vadTimer && clearTimeout(vadTimer);
+                vadTimer = setTimeout(() => {
+                    // vadStartTime.value = +new Date();
+                    console.log('打断时间: ', +new Date());
+                    skipVoice();
+                }, 500);
+                // }
+            },
+            onSpeechEnd: audio => {
+                vadTimer && clearTimeout(vadTimer);
+                console.log('Speech end', +new Date());
+                // debugger;
+                // do something with `audio` (Float32Array of audio samples at sample rate 16000)...
+            },
+            baseAssetPath: '/'
+        });
+        myvad.start();
+    };
+    onMounted(async () => {
+        const { code, message } = await stopMessage();
+        if (code !== 0) {
+            ElMessage({
+                type: 'error',
+                message: message,
+                duration: 3000,
+                customClass: 'system-error'
+            });
+            return;
+        }
+        callDisabled.value = false;
+    });
+    const delay = ms => {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    };
+    const initRecording = async () => {
+        uploadUserConfig()
+            .then(async () => {
+                if (!audioDOM) {
+                    audioDOM = new Audio();
+                    audioDOM.playsinline = true;
+                    audioDOM.preload = 'auto';
+                }
+                // 每次call都需要生成新uid
+                setNewUserId();
+                buildConnect();
+                await delay(100);
+                // if (socket) {
+                //     socket.close();
+                // }
+                // socket = new WebSocketService(
+                //     `/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
+                // );
+                // socket.connect();
+
+                initVideoStream('environment');
+                if (localStorage.getItem('canStopByVoice') === 'true') {
+                    console.log('vad start');
+                    vadStart();
+                }
+            })
+            .catch(() => {});
+    };
+    // 切换摄像头
+    const switchCamera = () => {
+        if (!isCalling.value) {
+            return;
+        }
+        isFrontCamera.value = !isFrontCamera.value;
+        const facingMode = isFrontCamera.value ? 'environment' : 'user'; // 'user' 前置, 'environment' 后置
+        initVideoStream(facingMode);
+    };
+    const initVideoStream = async facingMode => {
+        if (mediaStream) {
+            mediaStream.getTracks().forEach(track => track.stop());
+            videoStream.value = null;
+        }
+        outputData.value = [];
+        isCalling.value = true;
+        loading.value = true;
+        if (!videoStream.value) {
+            try {
+                mediaStream = await window.navigator.mediaDevices.getUserMedia({
+                    video: { facingMode },
+                    audio: true
+                });
+                videoStream.value = mediaStream;
+                videoRef.value.srcObject = mediaStream;
+                loading.value = false;
+                console.log('打开后： ', +new Date());
+                // takePhotos();
+                audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
+                console.log('samplate: ', audioContext);
+                const audioSource = audioContext.createMediaStreamSource(mediaStream);
+                interval.value = setInterval(() => dealImage(), 50);
+                // 创建 ScriptProcessorNode 用于捕获音频数据
+                const processor = audioContext.createScriptProcessor(256, 1, 1);
+
+                processor.onaudioprocess = event => {
+                    if (!isCalling.value) return;
+                    if (isReturnError.value) {
+                        stopRecording();
+                        return;
+                    }
+                    const data = event.inputBuffer.getChannelData(0);
+                    audioChunks.push(new Float32Array(data));
+                    // 检查是否已经收集到1秒钟的数据
+                    const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
+                    const chunkLength = getChunkLength(audioContext.sampleRate);
+                    if (totalBufferLength >= chunkLength) {
+                        // 合并到一个完整的数据数组，并裁剪成1秒钟
+                        const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
+                        const oneSecondBuffer = mergedBuffer.slice(0, audioContext.sampleRate);
+
+                        // 保存并处理成WAV格式
+                        addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
+
+                        // 保留多余的数据备用
+                        audioChunks = [mergedBuffer.slice(audioContext.sampleRate)];
+                    }
+                };
+                analyser.value = audioContext.createAnalyser();
+                // 将音频节点连接到分析器
+                audioSource.connect(analyser.value);
+                // 分析器设置
+                analyser.value.fftSize = 256;
+                const bufferLength = analyser.value.frequencyBinCount;
+                dataArray.value = new Uint8Array(bufferLength);
+                // 开始绘制音波
+                drawBars();
+
+                audioSource.connect(processor);
+                processor.connect(audioContext.destination);
+            } catch {}
+        }
+    };
+    const drawText = async () => {
+        if (textQueue.value.length > 0) {
+            outputData.value[outputData.value.length - 1].text += textQueue.value[0];
+            textQueue.value = textQueue.value.slice(1);
+        } else {
+            cancelAnimationFrame(textAnimationInterval.value);
+        }
+        textAnimationInterval.value = requestAnimationFrame(drawText);
+    };
+    const getStopValue = () => {
+        return stop.value;
+    };
+    const getPlayingValue = () => {
+        return playing.value;
+    };
+    const getStopStatus = () => {
+        return localStorage.getItem('canStopByVoice') === 'true';
+    };
+    const saveAudioChunk = (buffer, timestamp) => {
+        return new Promise(resolve => {
+            if (!getStopStatus() && getPlayingValue()) {
+                resolve();
+                return;
+            }
+            const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
+            let reader = new FileReader();
+            reader.readAsDataURL(wavBlob);
+
+            reader.onloadend = async function () {
+                let base64data = reader.result.split(',')[1];
+                const imgBase64 = videoImage.value[videoImage.value.length - 1]?.src;
+                if (!(base64data && imgBase64)) {
+                    resolve();
+                    return;
+                }
+                const strBase64 = imgBase64.split(',')[1];
+                count++;
+                let obj = {
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'input_audio',
+                                    input_audio: {
+                                        data: base64data,
+                                        format: 'wav',
+                                        timestamp: String(timestamp)
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                };
+                obj.messages[0].content.unshift({
+                    type: 'image_data',
+                    image_data: {
+                        data: count === maxCount ? strBase64 : '',
+                        type: 2
+                    }
+                });
+                if (count === maxCount) {
+                    count = 0;
+                }
+                // socket.send(JSON.stringify(obj));
+                // socket.on('message', data => {
+                //     console.log('message: ', data);
+                //     delayTimestamp.value = +new Date() - timestamp;
+                //     delayCount.value = taskQueue.value.length;
+                //     resolve();
+                // });
+                // 将Base64音频数据发送到后端
+                try {
+                    await sendMessage(obj);
+                    delayTimestamp.value = +new Date() - timestamp;
+                    delayCount.value = taskQueue.value.length;
+                } catch (err) {}
+                resolve();
+            };
+        });
+    };
+    const mergeBuffers = (buffers, length) => {
+        const result = new Float32Array(length);
+        let offset = 0;
+        for (let buffer of buffers) {
+            result.set(buffer, offset);
+            offset += buffer.length;
+        }
+        return result;
+    };
+    const stopRecording = () => {
+        isCalling.value = false;
+        clearInterval(interval.value);
+        interval.value = null;
+        if (audioRecorder && audioRecorder.state !== 'inactive') {
+            audioRecorder.stop();
+        }
+        if (animationFrameId.value) {
+            cancelAnimationFrame(animationFrameId.value);
+        }
+        if (audioContext && audioContext.state !== 'closed') {
+            audioContext.close();
+        }
+        destroyVideoStream();
+        taskQueue.value = [];
+        audioPlayQueue.value = [];
+        base64List.value = [];
+        ctrl.abort();
+        ctrl = new AbortController();
+        isReturnError.value = false;
+        skipDisabled.value = true;
+        playing.value = false;
+        audioDOM?.pause();
+        stopMessage();
+        if (socket) {
+            socket.close();
+        }
+        if (
+            outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+            outputData.value[outputData.value.length - 1].audio === '' &&
+            allVoice.value.length > 0
+        ) {
+            outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+        }
+        myvad && myvad.destroy();
+    };
+    // 建立连接
+    const buildConnect = () => {
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [{ type: 'none' }]
+                }
+            ],
+            stream: true
+        };
+        isEnd.value = false;
+        ctrl.abort();
+        ctrl = new AbortController();
+        const url = `/api/v1/completions${window.location.search}`;
+
+        fetchEventSource(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                service: 'minicpmo-server',
+                uid: getNewUserId()
+            },
+            body: JSON.stringify(obj),
+            signal: ctrl.signal,
+            openWhenHidden: true,
+            async onopen(response) {
+                isFirstPiece.value = true;
+                isFirstReturn.value = true;
+                allVoice.value = [];
+                base64List.value = [];
+                console.log('onopen', response);
+                if (response.status !== 200) {
+                    ElMessage({
+                        type: 'error',
+                        message: 'At limit. Please try again soon.',
+                        duration: 3000,
+                        customClass: 'system-error'
+                    });
+                    isReturnError.value = true;
+                } else {
+                    isReturnError.value = false;
+                    drawText();
+                }
+            },
+            onmessage(msg) {
+                const data = JSON.parse(msg.data);
+                if (data.response_id) {
+                    curResponseId.value = data.response_id;
+                }
+                if (data.choices[0]?.text) {
+                    textQueue.value += data.choices[0].text.replace('<end>', '');
+                    console.warn('text return time -------------------------------', +new Date());
+                }
+                // 首次返回的是前端发给后端的音频片段，需要单独处理
+                if (isFirstReturn.value) {
+                    console.log('第一次');
+                    isFirstReturn.value = false;
+                    // 如果后端返回的音频为空，需要重连
+                    if (!data.choices[0].audio) {
+                        buildConnect();
+                        return;
+                    }
+                    outputData.value.push({
+                        type: 'USER',
+                        audio: `data:audio/wav;base64,${data.choices[0].audio}`
+                    });
+                    outputData.value.push({
+                        type: 'BOT',
+                        text: '',
+                        audio: ''
+                    });
+                    return;
+                }
+                if (data.choices[0]?.audio) {
+                    console.log('audio return time -------------------------------', +new Date());
+                    if (!getStopValue() && isCalling.value) {
+                        skipDisabled.value = false;
+                        base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                        addAudioQueue(() => truePlay(data.choices[0].audio));
+                    }
+                    allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                } else {
+                    // 发生异常了，直接重连
+                    buildConnect();
+                }
+                if (data.choices[0].text.includes('<end>')) {
+                    // isEnd.value = true;
+                    console.log('收到结束标记了:', +new Date());
+                    if (
+                        outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                        outputData.value[outputData.value.length - 1].audio === '' &&
+                        allVoice.value.length > 0
+                    ) {
+                        outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                    }
+                }
+            },
+            onclose() {
+                console.log('onclose', +new Date());
+                isEnd.value = true;
+                if (
+                    outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                    outputData.value[outputData.value.length - 1].audio === '' &&
+                    allVoice.value.length > 0
+                ) {
+                    outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                }
+                // sse关闭后，如果待播放的音频列表为空，说明模型出错了，此次连接没有返回音频，则直接重连
+                vadStartTime.value = +new Date();
+                if (audioPlayQueue.value.length === 0) {
+                    let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                    if (startIndex !== -1) {
+                        taskQueue.value = taskQueue.value.slice(startIndex);
+                    }
+                    buildConnect();
+                }
+            },
+            onerror(err) {
+                console.log('onerror', err);
+                ctrl.abort();
+                ctrl = new AbortController();
+                throw err;
+            }
+        });
+    };
+    // 返回的语音放到队列里，挨个播放
+    const addAudioQueue = async item => {
+        audioPlayQueue.value.push(item);
+        if (isFirstPiece.value) {
+            await delay(1500);
+            isFirstPiece.value = false;
+        }
+        if (audioPlayQueue.value.length > 0 && !playing.value) {
+            playing.value = true;
+            playAudio();
+        }
+    };
+    // 控制播放队列执行
+    const playAudio = () => {
+        console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
+
+        if (!isEnd.value && base64List.value.length >= 2) {
+            const remainLen = base64List.value.length;
+            const blob = mergeBase64ToBlob(base64List.value);
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('前期合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('前期合并后播放结束时间: ', +new Date());
+                base64List.value = base64List.value.slice(remainLen);
+                audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
+                playAudio();
+            };
+            return;
+        }
+        if (isEnd.value && base64List.value.length >= 2) {
+            const blob = mergeBase64ToBlob(base64List.value);
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('合并后播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                base64List.value = [];
+                audioPlayQueue.value = [];
+                playing.value = false;
+                skipDisabled.value = true;
+                if (isCalling.value && !isReturnError.value) {
+                    // skipDisabled.value = true;
+                    taskQueue.value = [];
+                    // 打断前记录一下打断时间或vad触发事件
+                    // vadStartTime.value = +new Date();
+                    // // 每次完成后只保留当前时刻往前推1s的语音
+                    // console.log(
+                    //     '截取前长度:',
+                    //     taskQueue.value.map(item => item.time)
+                    // );
+                    // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                    // if (startIndex !== -1) {
+                    //     taskQueue.value = taskQueue.value.slice(startIndex);
+                    //     console.log(
+                    //         '截取后长度:',
+                    //         taskQueue.value.map(item => item.time),
+                    //         vadStartTime.value
+                    //     );
+                    // }
+                    buildConnect();
+                }
+            };
+            return;
+        }
+        base64List.value.shift();
+        const _truePlay = audioPlayQueue.value.shift();
+        if (_truePlay) {
+            _truePlay().finally(() => {
+                playAudio();
+            });
+        } else {
+            playing.value = false;
+            if (isEnd.value) {
+                console.warn('play done................');
+                skipDisabled.value = true;
+            }
+            // 播放完成后且正在通话且接口未返回错误时开始下一次连接
+            if (isEnd.value && isCalling.value && !isReturnError.value) {
+                // skipDisabled.value = true;
+                taskQueue.value = [];
+                // 跳��之后，只保留当前时间点两秒内到之后的音频片段
+                // vadStartTime.value = +new Date();
+                // console.log(
+                //     '截取前长度:',
+                //     taskQueue.value.map(item => item.time)
+                // );
+                // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                // if (startIndex !== -1) {
+                //     taskQueue.value = taskQueue.value.slice(startIndex);
+                //     console.log(
+                //         '截取后长度:',
+                //         taskQueue.value.map(item => item.time),
+                //         vadStartTime.value
+                //     );
+                // }
+                buildConnect();
+            }
+        }
+    };
+    // 播放音频
+    const truePlay = voice => {
+        console.log('promise: ', +new Date());
+        return new Promise(resolve => {
+            audioDOM.src = 'data:audio/wav;base64,' + voice;
+            console.error('播放开始时间:', +new Date());
+            audioDOM
+                .play()
+                .then(() => {
+                    console.log('Audio played successfully');
+                })
+                .catch(error => {
+                    if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
+                        console.error('User interaction required or permission issue:', error);
+                        // ElMessage.warning('音频播放失败');
+                        console.error('播放失败时间');
+                        // alert('Please interact with the page (like clicking a button) to enable audio playback.');
+                    } else {
+                        console.error('Error playing audio:', error);
+                    }
+                });
+            // .finally(() => {
+            //     resolve();
+            // });
+            audioDOM.onerror = () => {
+                console.error('播放失败时间', +new Date());
+                resolve();
+            };
+            audioDOM.onended = () => {
+                console.error('播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                resolve();
+            };
+        });
+    };
+    // 当队列中任务数大于0时，开始处理队列中的任务
+    const addQueue = (time, item) => {
+        taskQueue.value.push({ func: item, time });
+        if (taskQueue.value.length > 0 && !running.value) {
+            running.value = true;
+            processQueue();
+        }
+    };
+    const processQueue = () => {
+        const item = taskQueue.value.shift();
+        if (item?.func) {
+            item.func()
+                .then(res => {
+                    console.log('已处理事件: ', res);
+                })
+                .finally(() => processQueue());
+        } else {
+            running.value = false;
+        }
+    };
+    const destroyVideoStream = () => {
+        videoStream.value?.getTracks().forEach(track => track.stop());
+        videoStream.value = null;
+        // 将srcObject设置为null以切断与MediaStream 对象的链接，以便将其释放
+        videoRef.value.srcObject = null;
+
+        videoImage.value = [];
+        videoLoaded.value = false;
+
+        clearInterval(intervalId);
+        clearInterval(interval.value);
+        interval.value = null;
+    };
+    const dealImage = () => {
+        if (!videoRef.value) {
+            return;
+        }
+        const canvas = canvasRef.value;
+        canvasRef.value.width = videoRef.value.videoWidth;
+        canvasRef.value.height = videoRef.value.videoHeight;
+        const context = canvas.getContext('2d');
+        context.drawImage(videoRef.value, 0, 0, canvasRef.value.width, canvasRef.value.height);
+        const imageDataUrl = canvas.toDataURL('image/webp', 0.8);
+
+        videoImage.value.push({ src: imageDataUrl });
+    };
+    const drawBars = () => {
+        // AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
+        analyser.value.getByteFrequencyData(dataArray.value);
+        animationFrameId.value = requestAnimationFrame(drawBars);
+    };
+    // 跳过当前片段
+    const skipVoice = async () => {
+        // 打断前记录一下打断时间或vad触发事件
+        vadStartTime.value = +new Date();
+        if (!skipDisabled.value) {
+            if (
+                outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                outputData.value[outputData.value.length - 1].audio === ''
+            ) {
+                outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+            }
+            base64List.value = [];
+            audioPlayQueue.value = [];
+            // 跳过之后，只保留当前时间点两秒内到之后的音频片段
+            console.log(
+                '截取前长度:',
+                taskQueue.value.map(item => item.time)
+            );
+            let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+            if (startIndex !== -1) {
+                taskQueue.value = taskQueue.value.slice(startIndex);
+                console.log(
+                    '截取后长度:',
+                    taskQueue.value.map(item => item.time),
+                    vadStartTime.value
+                );
+            }
+            stop.value = true;
+            audioDOM?.pause();
+            setTimeout(() => {
+                skipDisabled.value = true;
+            }, 300);
+            try {
+                playing.value = false;
+                await stopMessage();
+                stop.value = false;
+                // playing.value = false;
+                buildConnect();
+                // cancelAnimationFrame(animationFrameId.value);
+            } catch (err) {}
+        }
+    };
+    // 每次call先上传当前用户配置
+    const uploadUserConfig = async () => {
+        if (!localStorage.getItem('configData')) {
+            return new Promise(resolve => resolve());
+        }
+        const {
+            videoQuality,
+            useAudioPrompt,
+            voiceClonePrompt,
+            assistantPrompt,
+            vadThreshold,
+            audioFormat,
+            base64Str
+        } = JSON.parse(localStorage.getItem('configData'));
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'input_audio',
+                            input_audio: {
+                                data: base64Str,
+                                format: audioFormat
+                            }
+                        },
+                        {
+                            type: 'options',
+                            options: {
+                                hd_video: videoQuality,
+                                use_audio_prompt: useAudioPrompt,
+                                vad_threshold: vadThreshold,
+                                voice_clone_prompt: voiceClonePrompt,
+                                assistant_prompt: assistantPrompt
+                            }
+                        }
+                    ]
+                }
+            ]
+        };
+        const { code, message, data } = await uploadConfig(obj);
+        modelVersion.value = data?.choices?.content || '';
+        return new Promise((resolve, reject) => {
+            if (code !== 0) {
+                ElMessage({
+                    type: 'error',
+                    message: message,
+                    duration: 3000,
+                    customClass: 'system-error'
+                });
+                reject();
+            } else {
+                resolve();
+            }
+        });
+    };
+</script>
+<style lang="less" scoped>
+    .video-page {
+        flex: 1;
+        height: 100%;
+        display: flex;
+        flex-direction: column;
+        &-header {
+            width: 100%;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            padding: 0 16px 16px;
+            box-shadow: 0 0.5px 0 0 #e0e0e0;
+            margin-bottom: 16px;
+            .header-icon {
+                display: flex;
+                align-items: center;
+                img {
+                    width: 24px;
+                    height: 24px;
+                    margin-right: 8px;
+                }
+                span {
+                    color: rgba(23, 23, 23, 0.9);
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                    margin-right: 40px;
+                    flex-shrink: 0;
+                }
+            }
+            .voice-container {
+                display: flex;
+                .voice-icon {
+                    width: 191px;
+                    height: 45px;
+                }
+            }
+        }
+        &-content {
+            flex: 1;
+            margin-bottom: 16px;
+            display: flex;
+            height: 0;
+            &-video {
+                width: 50%;
+                height: 100%;
+                background: #f3f3f3;
+                flex-shrink: 0;
+                position: relative;
+                video {
+                    width: 100%;
+                    height: 100%;
+                    object-fit: contain;
+                }
+                .switch-camera {
+                    position: absolute;
+                    top: 10px;
+                    right: 10px;
+                    width: 36px;
+                    height: 36px;
+                    background: #ffffff;
+                    border-radius: 6px;
+                    display: flex;
+                    justify-content: center;
+                    align-items: center;
+                    font-size: 24px;
+                    z-index: 999;
+                    .icon {
+                        width: 20px;
+                        height: 20px;
+                    }
+                }
+            }
+            &-right {
+                margin-left: 16px;
+                flex: 1;
+                padding: 0 16px;
+                display: flex;
+                flex-direction: column;
+                .output-content {
+                    flex: 1;
+                    overflow: auto;
+                }
+                .skip-box {
+                    display: flex;
+                    align-items: center;
+                    justify-content: flex-end;
+                    margin-top: 16px;
+                }
+            }
+        }
+        &-btn {
+            text-align: center;
+            padding: 8px 0;
+            .el-button {
+                width: 284px;
+                height: 46px;
+                border-radius: 8px;
+            }
+            .el-button.el-button--success {
+                background: #647fff;
+                border-color: #647fff;
+                &:hover {
+                    opacity: 0.8;
+                }
+                span {
+                    color: #fff;
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                }
+            }
+            .el-button.el-button--success.is-disabled {
+                background: #f3f3f3;
+                border-color: #f3f3f3;
+                span {
+                    color: #d1d1d1;
+                }
+            }
+            .el-button.el-button--danger {
+                border-color: #dc3545;
+                background-color: #dc3545;
+                color: #ffffff;
+                font-family: PingFang SC;
+                font-size: 16px;
+                font-style: normal;
+                font-weight: 500;
+                line-height: normal;
+                .phone-icon {
+                    margin-right: 10px;
+                }
+                .btn-text {
+                    margin-right: 10px;
+                }
+                .btn-desc {
+                    margin-right: 16px;
+                }
+            }
+        }
+    }
+    .video-size {
+        position: absolute;
+        bottom: 10px;
+        right: 10px;
+        background: rgba(0, 0, 0, 0.5);
+        color: #fff;
+        padding: 4px 8px;
+        border-radius: 4px;
+        font-size: 12px;
+    }
+</style>

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/VideoCall_0105.vue

@@ -0,0 +1,955 @@
+<template>
+    <ExtraInfo webVersion="websocket_0107" :modelVersion="modelVersion" />
+    <div class="video-page">
+        <div class="video-page-header">
+            <div style="display: flex; align-items: center" class="header-icon">
+                <img src="@/assets/images/voice-icon.png" />
+                <span>Audio Choice</span>
+            </div>
+            <div class="voice-container" v-if="!isCalling">
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+            </div>
+            <div class="voice-container" v-else>
+                <Voice
+                    :dataArray="dataArray"
+                    :isCalling="isCalling"
+                    :isPlaying="playing"
+                    :configList="videoConfigList"
+                    :boxStyle="{ height: '45px' }"
+                    :itemStyle="{ width: '3px', margin: '0 1px' }"
+                />
+            </div>
+            <!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
+        </div>
+        <div class="video-page-content">
+            <div class="video-page-content-video" v-loading="loading" element-loading-background="#f3f3f3">
+                <video ref="videoRef" autoplay playsinline muted />
+                <canvas ref="canvasRef" canvas-id="canvasId" style="display: none" />
+                <div class="switch-camera" v-if="isMobile()" @click="switchCamera">
+                    <SvgIcon name="switch-camera" class="icon" />
+                </div>
+                <!-- <div class="video-size" v-if="width || height">{{ width }} x {{ height }}</div> -->
+            </div>
+            <div class="video-page-content-right">
+                <div class="output-content">
+                    <ModelOutput
+                        v-if="outputData.length > 0"
+                        :outputData="outputData"
+                        containerClass="output-content"
+                    />
+                </div>
+                <div class="skip-box">
+                    <DelayTips
+                        v-if="delayTimestamp > 200 || delayCount > 2"
+                        :delayTimestamp="delayTimestamp"
+                        :delayCount="delayCount"
+                    />
+                    <LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
+                    <SkipBtn :disabled="skipDisabled" @click="skipVoice" />
+                </div>
+            </div>
+        </div>
+        <div class="video-page-btn">
+            <el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
+                {{ callDisabled ? 'Not ready yet, please wait' : 'Call MiniCPM' }}
+            </el-button>
+            <el-button v-show="isCalling" @click="stopRecording" type="danger">
+                <SvgIcon name="phone-icon" className="phone-icon" />
+                <span class="btn-text">Hang Up</span>
+                <CountDown v-model="isCalling" @timeUp="stopRecording" />
+            </el-button>
+        </div>
+        <IdeasList v-if="showIdeasList" :ideasList="videoIdeasList" />
+    </div>
+</template>
+<script setup>
+    import { sendMessage, stopMessage, uploadConfig } from '@/apis';
+    import { encodeWAV } from '@/hooks/useVoice';
+    import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
+    import { fetchEventSource } from '@microsoft/fetch-event-source';
+    import { MicVAD } from '@ricky0123/vad-web';
+    import { videoIdeasList, videoConfigList, showIdeasList } from '@/enums';
+    import { isMobile, maxCount, getChunkLength } from '@/utils';
+    import { mergeBase64ToBlob } from './merge';
+    import WebSocketService from '@/utils/websocket';
+    let ctrl = new AbortController();
+    let socket = null;
+    const audioData = ref({
+        base64Str: '',
+        type: 'mp3'
+    }); // 自定义音色base64
+    const isCalling = defineModel();
+    const videoRef = ref();
+    const videoStream = ref(null);
+    const interval = ref();
+    const canvasRef = ref();
+    const videoImage = ref([]);
+    const videoLoaded = ref(false);
+    const taskQueue = ref([]);
+    const running = ref(false);
+    const outputData = ref([]);
+    const isFirstReturn = ref(true);
+    const audioPlayQueue = ref([]);
+    const base64List = ref([]);
+    const playing = ref(false);
+    const timbre = ref([1]);
+    const isReturnError = ref(false);
+
+    const textQueue = ref('');
+    const textAnimationInterval = ref();
+    const analyser = ref();
+    const dataArray = ref();
+    const animationFrameId = ref();
+    const skipDisabled = ref(true);
+    const stop = ref(false);
+    const isFrontCamera = ref(true);
+    const loading = ref(false);
+    const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
+    const isFirstPiece = ref(true);
+    const allVoice = ref([]);
+    const callDisabled = ref(true);
+    const feedbackStatus = ref('');
+    const curResponseId = ref('');
+    const delayTimestamp = ref(0); // 当前发送片延时
+    const delayCount = ref(0); // 当前剩余多少ms未发送到接口
+
+    const modelVersion = ref('');
+
+    let mediaStream;
+    let audioRecorder;
+    let audioStream;
+    let audioContext;
+    let audioChunks = [];
+    let count = 0;
+    let audioDOM;
+    onBeforeUnmount(() => {
+        stopRecording();
+    });
+    const vadStartTime = ref();
+    let myvad = null;
+    let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
+    const vadStart = async () => {
+        myvad = await MicVAD.new({
+            onSpeechStart: () => {
+                console.log('Speech start', +new Date());
+                if (!skipDisabled.value) {
+                    vadTimer && clearTimeout(vadTimer);
+                    vadTimer = setTimeout(() => {
+                        // vadStartTime.value = +new Date();
+                        console.log('打断时间: ', +new Date());
+                        skipVoice();
+                    }, 1000);
+                }
+            },
+            onSpeechEnd: audio => {
+                vadTimer && clearTimeout(vadTimer);
+                console.log('Speech end', +new Date());
+                // debugger;
+                // do something with `audio` (Float32Array of audio samples at sample rate 16000)...
+            }
+        });
+        myvad.start();
+    };
+    onMounted(async () => {
+        const { code, message } = await stopMessage();
+        if (code !== 0) {
+            ElMessage({
+                type: 'error',
+                message: message,
+                duration: 3000,
+                customClass: 'system-error'
+            });
+            return;
+        }
+        callDisabled.value = false;
+    });
+    const delay = ms => {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    };
+    const initRecording = async () => {
+        uploadUserConfig()
+            .then(async () => {
+                if (!audioDOM) {
+                    audioDOM = new Audio();
+                    audioDOM.playsinline = true;
+                    audioDOM.preload = 'auto';
+                }
+                // 每次call都需要生成新uid
+                setNewUserId();
+                buildConnect();
+                await delay(100);
+                initVideoStream('environment');
+                if (socket) {
+                    socket.close();
+                }
+                socket = new WebSocketService(
+                    `/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
+                );
+                socket.connect();
+                initVideoStream('environment');
+                if (localStorage.getItem('canStopByVoice') === 'true') {
+                    vadStart();
+                }
+            })
+            .catch(() => {});
+    };
+    // 切换摄像头
+    const switchCamera = () => {
+        if (!isCalling.value) {
+            return;
+        }
+        isFrontCamera.value = !isFrontCamera.value;
+        const facingMode = isFrontCamera.value ? 'environment' : 'user'; // 'user' 前置, 'environment' 后置
+        initVideoStream(facingMode);
+    };
+    const initVideoStream = async facingMode => {
+        if (mediaStream) {
+            mediaStream.getTracks().forEach(track => track.stop());
+            videoStream.value = null;
+        }
+        outputData.value = [];
+        isCalling.value = true;
+        loading.value = true;
+        if (!videoStream.value) {
+            try {
+                mediaStream = await window.navigator.mediaDevices.getUserMedia({
+                    video: { facingMode },
+                    audio: true
+                });
+                console.log('mediaStream', mediaStream);
+                videoStream.value = mediaStream;
+                videoRef.value.srcObject = mediaStream;
+                loading.value = false;
+                console.log('打开后： ', +new Date());
+                // takePhotos();
+                audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
+                console.log('samplate: ', audioContext);
+                const audioSource = audioContext.createMediaStreamSource(mediaStream);
+                interval.value = setInterval(() => dealImage(), 50);
+                // 创建 ScriptProcessorNode 用于捕获音频数据
+                const processor = audioContext.createScriptProcessor(256, 1, 1);
+                processor.onaudioprocess = event => {
+                    if (!isCalling.value) return;
+                    if (isReturnError.value) {
+                        stopRecording();
+                        return;
+                    }
+                    const data = event.inputBuffer.getChannelData(0);
+                    audioChunks.push(new Float32Array(data));
+                    // 检查是否已经收集到1秒钟的数据
+                    const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
+                    // const chunkLength = audioContext.sampleRate;
+                    const chunkLength = getChunkLength(audioContext.sampleRate);
+                    if (totalBufferLength >= chunkLength) {
+                        // 合并到一个完整的数据数组，并裁剪成1秒钟
+                        const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
+                        const oneSecondBuffer = mergedBuffer.slice(0, audioContext.sampleRate);
+
+                        // 保存并处理成WAV格式
+                        addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
+
+                        // 保留多余的数据备用
+                        audioChunks = [mergedBuffer.slice(audioContext.sampleRate)];
+                    }
+                };
+                analyser.value = audioContext.createAnalyser();
+                // 将音频节点连接到分析器
+                audioSource.connect(analyser.value);
+                // 分析器设置
+                analyser.value.fftSize = 256;
+                const bufferLength = analyser.value.frequencyBinCount;
+                dataArray.value = new Uint8Array(bufferLength);
+                // 开始绘制音波
+                drawBars();
+
+                audioSource.connect(processor);
+                processor.connect(audioContext.destination);
+            } catch {}
+        }
+    };
+    const drawText = async () => {
+        if (textQueue.value.length > 0) {
+            outputData.value[outputData.value.length - 1].text += textQueue.value[0];
+            textQueue.value = textQueue.value.slice(1);
+        } else {
+            cancelAnimationFrame(textAnimationInterval.value);
+        }
+        textAnimationInterval.value = requestAnimationFrame(drawText);
+    };
+    const getStopValue = () => {
+        return stop.value;
+    };
+    const getPlayingValue = () => {
+        return playing.value;
+    };
+    const getStopStatus = () => {
+        return localStorage.getItem('canStopByVoice') === 'true';
+    };
+    const saveAudioChunk = (buffer, timestamp) => {
+        return new Promise(resolve => {
+            if (!getStopStatus() && getPlayingValue()) {
+                resolve();
+                return;
+            }
+            const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
+            let reader = new FileReader();
+            reader.readAsDataURL(wavBlob);
+            reader.onloadend = async function () {
+                let base64data = reader.result.split(',')[1];
+                const imgBase64 = videoImage.value[videoImage.value.length - 1]?.src;
+                if (!(base64data && imgBase64)) {
+                    resolve();
+                    return;
+                }
+                const strBase64 = imgBase64.split(',')[1];
+                count++;
+                let obj = {
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'input_audio',
+                                    input_audio: {
+                                        data: base64data,
+                                        format: 'wav',
+                                        timestamp: String(timestamp)
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                };
+                obj.messages[0].content.unshift({
+                    type: 'image_data',
+                    image_data: {
+                        data: count === maxCount ? strBase64 : '',
+                        type: 2
+                    }
+                });
+                if (count === maxCount) {
+                    count = 0;
+                }
+                socket.send(JSON.stringify(obj));
+                socket.on('message', data => {
+                    console.log('message: ', data);
+                    delayTimestamp.value = +new Date() - timestamp;
+                    delayCount.value = taskQueue.value.length;
+                    resolve();
+                });
+                // 将Base64音频数据发送到后端
+                // try {
+                //     await sendMessage(obj);
+                //     delayTimestamp.value = +new Date() - timestamp;
+                //     delayCount.value = taskQueue.value.length;
+                // } catch (err) {}
+                // resolve();
+            };
+        });
+    };
+    const mergeBuffers = (buffers, length) => {
+        const result = new Float32Array(length);
+        let offset = 0;
+        for (let buffer of buffers) {
+            result.set(buffer, offset);
+            offset += buffer.length;
+        }
+        return result;
+    };
+    const stopRecording = () => {
+        isCalling.value = false;
+        clearInterval(interval.value);
+        interval.value = null;
+        if (audioRecorder && audioRecorder.state !== 'inactive') {
+            audioRecorder.stop();
+        }
+        if (animationFrameId.value) {
+            cancelAnimationFrame(animationFrameId.value);
+        }
+        if (audioContext && audioContext.state !== 'closed') {
+            audioContext.close();
+        }
+        destroyVideoStream();
+        taskQueue.value = [];
+        audioPlayQueue.value = [];
+        base64List.value = [];
+        ctrl.abort();
+        ctrl = new AbortController();
+        isReturnError.value = false;
+        skipDisabled.value = true;
+        playing.value = false;
+        audioDOM?.pause();
+        stopMessage();
+        if (socket) {
+            socket.close();
+        }
+        if (
+            outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+            outputData.value[outputData.value.length - 1].audio === '' &&
+            allVoice.value.length > 0
+        ) {
+            outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+        }
+        myvad && myvad.destroy();
+    };
+    // 建立连接
+    const buildConnect = () => {
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [{ type: 'none' }]
+                }
+            ],
+            stream: true
+        };
+        isEnd.value = false;
+        ctrl.abort();
+        ctrl = new AbortController();
+        const url = `/api/v1/completions${window.location.search}`;
+
+        fetchEventSource(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                service: 'minicpmo-server',
+                uid: getNewUserId()
+            },
+            body: JSON.stringify(obj),
+            signal: ctrl.signal,
+            openWhenHidden: true,
+            async onopen(response) {
+                isFirstPiece.value = true;
+                isFirstReturn.value = true;
+                allVoice.value = [];
+                base64List.value = [];
+                console.log('onopen', response);
+                if (response.status !== 200) {
+                    ElMessage({
+                        type: 'error',
+                        message: 'At limit. Please try again soon.',
+                        duration: 3000,
+                        customClass: 'system-error'
+                    });
+                    isReturnError.value = true;
+                } else {
+                    isReturnError.value = false;
+                    drawText();
+                }
+            },
+            onmessage(msg) {
+                const data = JSON.parse(msg.data);
+                if (data.response_id) {
+                    curResponseId.value = data.response_id;
+                }
+                if (data.choices[0]?.text) {
+                    textQueue.value += data.choices[0].text.replace('<end>', '');
+                    console.warn('text return time -------------------------------', +new Date());
+                }
+                // 首次返回的是前端发给后端的音频片段，需要单独处理
+                if (isFirstReturn.value) {
+                    console.log('第一次');
+                    isFirstReturn.value = false;
+                    // 如果后端返回的音频为空，需要重连
+                    if (!data.choices[0].audio) {
+                        buildConnect();
+                        return;
+                    }
+                    outputData.value.push({
+                        type: 'USER',
+                        audio: `data:audio/wav;base64,${data.choices[0].audio}`
+                    });
+                    outputData.value.push({
+                        type: 'BOT',
+                        text: '',
+                        audio: ''
+                    });
+                    return;
+                }
+                if (data.choices[0]?.audio) {
+                    console.log('audio return time -------------------------------', +new Date());
+                    if (!getStopValue() && isCalling.value) {
+                        skipDisabled.value = false;
+                        base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                        addAudioQueue(() => truePlay(data.choices[0].audio));
+                    }
+                    allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                } else {
+                    // 发生异常了，直接重连
+                    buildConnect();
+                }
+                if (data.choices[0].text.includes('<end>')) {
+                    console.log('收到结束标记了:', +new Date());
+                    if (
+                        outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                        outputData.value[outputData.value.length - 1].audio === '' &&
+                        allVoice.value.length > 0
+                    ) {
+                        outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                    }
+                }
+            },
+            onclose() {
+                console.log('onclose', +new Date());
+                isEnd.value = true;
+                outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                // sse关闭后，如果待播放的音频列表为空，说明模型出错了，此次连接没有返回音频，则直接重连
+                vadStartTime.value = +new Date();
+                if (audioPlayQueue.value.length === 0) {
+                    let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                    console.log('taskQueue111111111: ', taskQueue.value, startIndex);
+                    if (startIndex !== -1) {
+                        taskQueue.value = taskQueue.value.slice(startIndex);
+                        console.log('截取后长度:', taskQueue.value, vadStartTime.value);
+                    }
+                    buildConnect();
+                }
+            },
+            onerror(err) {
+                console.log('onerror', err);
+                ctrl.abort();
+                ctrl = new AbortController();
+                throw err;
+            }
+        });
+    };
+    // 返回的语音放到队列里，挨个播放
+    const addAudioQueue = async item => {
+        audioPlayQueue.value.push(item);
+        if (isFirstPiece.value) {
+            await delay(1500);
+            isFirstPiece.value = false;
+        }
+        if (audioPlayQueue.value.length > 0 && !playing.value) {
+            playing.value = true;
+            playAudio();
+        }
+    };
+    // 控制播放队列执行
+    const playAudio = () => {
+        console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
+
+        if (!isEnd.value && base64List.value.length >= 2) {
+            const remainLen = base64List.value.length;
+            const blob = mergeBase64ToBlob(base64List.value);
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('前期合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('前期合并后播放结束时间: ', +new Date());
+                base64List.value = base64List.value.slice(remainLen);
+                audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
+                playAudio();
+            };
+            return;
+        }
+        if (isEnd.value && base64List.value.length >= 2) {
+            const blob = mergeBase64ToBlob(base64List.value);
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('合并后播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                base64List.value = [];
+                audioPlayQueue.value = [];
+                playing.value = false;
+                skipDisabled.value = true;
+                if (isCalling.value && !isReturnError.value) {
+                    // skipDisabled.value = true;
+                    taskQueue.value = [];
+                    // 打断前记录一下打断时间或vad触发事件
+                    // vadStartTime.value = +new Date();
+                    // // 每次完成后只保留当前时刻往前推1s的语音
+                    // console.log(
+                    //     '截取前长度:',
+                    //     taskQueue.value.map(item => item.time)
+                    // );
+                    // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                    // if (startIndex !== -1) {
+                    //     taskQueue.value = taskQueue.value.slice(startIndex);
+                    //     console.log(
+                    //         '截取后长度:',
+                    //         taskQueue.value.map(item => item.time),
+                    //         vadStartTime.value
+                    //     );
+                    // }
+                    buildConnect();
+                }
+            };
+            return;
+        }
+        base64List.value.shift();
+        const _truePlay = audioPlayQueue.value.shift();
+        if (_truePlay) {
+            _truePlay().finally(() => {
+                playAudio();
+            });
+        } else {
+            playing.value = false;
+            if (isEnd.value) {
+                console.warn('play done................');
+                skipDisabled.value = true;
+            }
+            // 播放完成后且正在通话且接口未返回错误时开始下一次连接
+            if (isEnd.value && isCalling.value && !isReturnError.value) {
+                // skipDisabled.value = true;
+                taskQueue.value = [];
+                // // 跳过之后，只保留当前时间点两秒内到之后的音频片段
+                // vadStartTime.value = +new Date();
+                // console.log(
+                //     '截取前长度:',
+                //     taskQueue.value.map(item => item.time)
+                // );
+                // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                // if (startIndex !== -1) {
+                //     taskQueue.value = taskQueue.value.slice(startIndex);
+                //     console.log(
+                //         '截取后长度:',
+                //         taskQueue.value.map(item => item.time),
+                //         vadStartTime.value
+                //     );
+                // }
+                buildConnect();
+            }
+        }
+    };
+    // 播放音频
+    const truePlay = voice => {
+        console.log('promise: ', +new Date());
+        return new Promise(resolve => {
+            audioDOM.src = 'data:audio/wav;base64,' + voice;
+            console.error('播放开始时间:', +new Date());
+            audioDOM
+                .play()
+                .then(() => {
+                    console.log('Audio played successfully');
+                })
+                .catch(error => {
+                    if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
+                        console.error('User interaction required or permission issue:', error);
+                        // ElMessage.warning('音频播放失败');
+                        console.error('播放失败时间');
+                        // alert('Please interact with the page (like clicking a button) to enable audio playback.');
+                    } else {
+                        console.error('Error playing audio:', error);
+                    }
+                });
+            // .finally(() => {
+            //     resolve();
+            // });
+            audioDOM.onerror = () => {
+                console.error('播放失败时间', +new Date());
+                resolve();
+            };
+            audioDOM.onended = () => {
+                console.error('播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                resolve();
+            };
+        });
+    };
+    // 当队列中任务数大于0时，开始处理队列中的任务
+    const addQueue = (time, item) => {
+        taskQueue.value.push({ func: item, time });
+        if (taskQueue.value.length > 0 && !running.value) {
+            running.value = true;
+            processQueue();
+        }
+    };
+    const processQueue = () => {
+        const item = taskQueue.value.shift();
+        if (item?.func) {
+            item.func()
+                .then(res => {
+                    console.log('已处理事件: ', res);
+                })
+                .finally(() => processQueue());
+        } else {
+            running.value = false;
+        }
+    };
+    const destroyVideoStream = () => {
+        videoStream.value?.getTracks().forEach(track => track.stop());
+        videoStream.value = null;
+        // 将srcObject设置为null以切断与MediaStream 对象的链接，以便将其释放
+        videoRef.value.srcObject = null;
+
+        videoImage.value = [];
+        videoLoaded.value = false;
+
+        clearInterval(interval.value);
+        interval.value = null;
+    };
+    const dealImage = () => {
+        if (!videoRef.value) {
+            return;
+        }
+        const canvas = canvasRef.value;
+        canvasRef.value.width = videoRef.value.videoWidth;
+        canvasRef.value.height = videoRef.value.videoHeight;
+        const context = canvas.getContext('2d');
+        context.drawImage(videoRef.value, 0, 0, canvasRef.value.width, canvasRef.value.height);
+        const imageDataUrl = canvas.toDataURL('image/webp', 0.8);
+        videoImage.value.push({ src: imageDataUrl });
+    };
+    const drawBars = () => {
+        // AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
+        analyser.value.getByteFrequencyData(dataArray.value);
+        animationFrameId.value = requestAnimationFrame(drawBars);
+    };
+    // 跳过当前片段
+    const skipVoice = async () => {
+        // 打断前记录一下打断时间或vad触发事件
+        vadStartTime.value = +new Date();
+        if (!skipDisabled.value) {
+            if (
+                outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                outputData.value[outputData.value.length - 1].audio === ''
+            ) {
+                outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+            }
+            base64List.value = [];
+            audioPlayQueue.value = [];
+            // 跳过之后，只保留当前时间点两秒内到之后的音频片段
+            console.log(
+                '截取前长度:',
+                taskQueue.value.map(item => item.time)
+            );
+            let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+            if (startIndex !== -1) {
+                taskQueue.value = taskQueue.value.slice(startIndex);
+                console.log(
+                    '截取后长度:',
+                    taskQueue.value.map(item => item.time),
+                    vadStartTime.value
+                );
+            }
+            stop.value = true;
+            audioDOM?.pause();
+            setTimeout(() => {
+                skipDisabled.value = true;
+            }, 300);
+            try {
+                playing.value = false;
+                await stopMessage();
+                stop.value = false;
+                // playing.value = false;
+                buildConnect();
+                // cancelAnimationFrame(animationFrameId.value);
+            } catch (err) {}
+        }
+    };
+    // 每次call先上传当前用户配置
+    const uploadUserConfig = async () => {
+        if (!localStorage.getItem('configData')) {
+            return new Promise(resolve => resolve());
+        }
+        const {
+            videoQuality,
+            useAudioPrompt,
+            voiceClonePrompt,
+            assistantPrompt,
+            vadThreshold,
+            audioFormat,
+            base64Str
+        } = JSON.parse(localStorage.getItem('configData'));
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'input_audio',
+                            input_audio: {
+                                data: base64Str,
+                                format: audioFormat
+                            }
+                        },
+                        {
+                            type: 'options',
+                            options: {
+                                hd_video: videoQuality,
+                                use_audio_prompt: useAudioPrompt,
+                                vad_threshold: vadThreshold,
+                                voice_clone_prompt: voiceClonePrompt,
+                                assistant_prompt: assistantPrompt
+                            }
+                        }
+                    ]
+                }
+            ]
+        };
+        const { code, message, data } = await uploadConfig(obj);
+        modelVersion.value = data?.choices?.content || '';
+        return new Promise((resolve, reject) => {
+            if (code !== 0) {
+                ElMessage({
+                    type: 'error',
+                    message: message,
+                    duration: 3000,
+                    customClass: 'system-error'
+                });
+                reject();
+            } else {
+                resolve();
+            }
+        });
+    };
+</script>
+<style lang="less">
+    .video-page {
+        height: 100%;
+        display: flex;
+        flex-direction: column;
+        &-header {
+            display: flex;
+            align-items: center;
+            padding: 0 16px 16px;
+            box-shadow: 0 0.5px 0 0 #e0e0e0;
+            margin-bottom: 16px;
+            justify-content: space-between;
+            .header-icon {
+                display: flex;
+                align-items: center;
+                img {
+                    width: 24px;
+                    height: 24px;
+                    margin-right: 8px;
+                }
+                span {
+                    color: rgba(23, 23, 23, 0.9);
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                    margin-right: 40px;
+                    flex-shrink: 0;
+                }
+            }
+            .voice-container {
+                display: flex;
+                .voice-icon {
+                    width: 191px;
+                    height: 45px;
+                }
+            }
+        }
+        &-content {
+            flex: 1;
+            margin-bottom: 16px;
+            display: flex;
+            height: 0;
+            &-video {
+                width: 50%;
+                height: 100%;
+                background: #f3f3f3;
+                flex-shrink: 0;
+                position: relative;
+                video {
+                    width: 100%;
+                    height: 100%;
+                    object-fit: contain;
+                }
+                .switch-camera {
+                    position: absolute;
+                    top: 10px;
+                    right: 10px;
+                    width: 36px;
+                    height: 36px;
+                    background: #ffffff;
+                    border-radius: 6px;
+                    display: flex;
+                    justify-content: center;
+                    align-items: center;
+                    font-size: 24px;
+                    z-index: 999;
+                    .icon {
+                        width: 20px;
+                        height: 20px;
+                    }
+                }
+            }
+            &-right {
+                margin-left: 16px;
+                flex: 1;
+                padding: 0 16px;
+                display: flex;
+                flex-direction: column;
+                .output-content {
+                    flex: 1;
+                    overflow: auto;
+                }
+                .skip-box {
+                    display: flex;
+                    align-items: center;
+                    justify-content: flex-end;
+                    margin-top: 16px;
+                }
+            }
+        }
+        &-btn {
+            text-align: center;
+            padding: 8px 0;
+            .el-button {
+                width: 284px;
+                height: 46px;
+                border-radius: 8px;
+            }
+            .el-button.el-button--success {
+                background: #647fff;
+                border-color: #647fff;
+                &:hover {
+                    opacity: 0.8;
+                }
+                span {
+                    color: #fff;
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                }
+            }
+            .el-button.el-button--success.is-disabled {
+                background: #f3f3f3;
+                border-color: #f3f3f3;
+                span {
+                    color: #d1d1d1;
+                }
+            }
+            .el-button.el-button--danger {
+                border-color: #dc3545;
+                background-color: #dc3545;
+                color: #ffffff;
+                font-family: PingFang SC;
+                font-size: 16px;
+                font-style: normal;
+                font-weight: 500;
+                line-height: normal;
+                .phone-icon {
+                    margin-right: 10px;
+                }
+                .btn-text {
+                    margin-right: 10px;
+                }
+                .btn-desc {
+                    margin-right: 16px;
+                }
+            }
+        }
+    }
+    .video-size {
+        position: absolute;
+        bottom: 10px;
+        right: 10px;
+        background: rgba(0, 0, 0, 0.5);
+        color: #fff;
+        padding: 4px 8px;
+        border-radius: 4px;
+        font-size: 12px;
+    }
+</style>

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/VoiceCall.vue

@@ -0,0 +1,833 @@
+<template>
+    <!-- <ExtraInfo webVersion="非websocket_0112" :modelVersion="modelVersion" /> -->
+    <div class="voice-page">
+        <div class="voice-page-header">
+            <div class="voice-container" v-if="!isCalling">
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+            </div>
+            <div class="voice-container" v-else>
+                <Voice
+                    :dataArray="dataArray"
+                    :isCalling="isCalling"
+                    :isPlaying="playing"
+                    :configList="videoConfigList"
+                    :boxStyle="{ height: '45px' }"
+                    :itemStyle="{ width: '3px', margin: '0 1px' }"
+                />
+            </div>
+            <!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
+        </div>
+        <div class="voice-page-output">
+            <div class="output-content">
+                <ModelOutput v-if="outputData.length > 0" :outputData="outputData" containerClass="output-content" />
+            </div>
+            <div class="skip-box">
+                <!-- <DelayTips
+                    v-if="delayTimestamp > 200 || delayCount > 2"
+                    :delayTimestamp="delayTimestamp"
+                    :delayCount="delayCount"
+                /> -->
+                <LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
+                <SkipBtn :disabled="skipDisabled" @click="skipVoice" />
+            </div>
+        </div>
+        <div class="voice-page-btn">
+            <el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
+                {{ callDisabled ? t('notReadyBtn') : t('audioCallBtn') }}
+            </el-button>
+            <el-button v-show="isCalling" @click="stopRecording" type="danger">
+                <SvgIcon name="phone-icon" className="phone-icon" />
+                <span class="btn-text">{{ t('hangUpBtn') }}</span>
+                <CountDown v-model="isCalling" @timeUp="stopRecording" />
+            </el-button>
+        </div>
+        <IdeasList v-if="showIdeasList" :ideasList="voiceIdeasList" />
+    </div>
+</template>
+<script setup>
+    import { sendMessage, stopMessage, uploadConfig } from '@/apis';
+    import { encodeWAV } from '@/hooks/useVoice';
+    import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
+    import { fetchEventSource } from '@microsoft/fetch-event-source';
+    import { MicVAD } from '@ricky0123/vad-web';
+    import { videoConfigList, voiceConfigList, voiceIdeasList, showIdeasList } from '@/enums';
+    import { getChunkLength } from '@/utils';
+    import { mergeBase64ToBlob } from './merge';
+    import WebSocketService from '@/utils/websocket';
+    import { useI18n } from 'vue-i18n';
+
+    const { t } = useI18n();
+
+    let ctrl = new AbortController();
+    let socket = null;
+    const audioData = ref({
+        base64Str: '',
+        type: 'mp3'
+    }); // 自定义音色base64
+    const isCalling = defineModel();
+    const taskQueue = ref([]);
+    const running = ref(false);
+    const outputData = ref([]);
+    const textQueue = ref('');
+    const textAnimationInterval = ref();
+
+    const isFirstReturn = ref(true); // 首次返回的音频是前端发给后端的音频片段，需要单独处理
+
+    const audioPlayQueue = ref([]);
+    const base64List = ref([]);
+    const playing = ref(false);
+    const skipDisabled = ref(true);
+    const stop = ref(false);
+    const timbre = ref([1]);
+    const isReturnError = ref(false);
+    const allVoice = ref([]);
+    const callDisabled = ref(true);
+
+    const feedbackStatus = ref('');
+    const curResponseId = ref('');
+    const delayTimestamp = ref(0); // 当前发送片延时
+    const delayCount = ref(0); // 当前剩余多少ms未发送到接口
+
+    const modelVersion = ref('');
+
+    let audioDOM = new Audio();
+
+    const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
+    // 页面卸载时关闭录音
+    onBeforeUnmount(() => {
+        stopRecording();
+    });
+    const vadStartTime = ref();
+    let myvad = null;
+    let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
+    const vadStart = async () => {
+        myvad = await MicVAD.new({
+            onSpeechStart: () => {
+                console.log('Speech start detected');
+                // if (!skipDisabled.value) {
+                vadTimer && clearTimeout(vadTimer);
+                vadTimer = setTimeout(() => {
+                    console.log('打断时间: ', +new Date());
+                    skipVoice();
+                }, 500);
+                // }
+            },
+            onSpeechEnd: audio => {
+                vadTimer && clearTimeout(vadTimer);
+                // debugger;
+                // do something with `audio` (Float32Array of audio samples at sample rate 16000)...
+            },
+            baseAssetPath: '/'
+        });
+        console.log('vad: ', myvad);
+        myvad.start();
+    };
+    onMounted(async () => {
+        const { code, message } = await stopMessage();
+        if (code !== 0) {
+            ElMessage({
+                type: 'error',
+                message: message,
+                duration: 3000,
+                customClass: 'system-error'
+            });
+            return;
+        }
+        callDisabled.value = false;
+    });
+    const delay = ms => {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    };
+    const initRecording = async () => {
+        uploadUserConfig()
+            .then(async () => {
+                // 每次call都需要生成新uid
+                setNewUserId();
+
+                outputData.value = [];
+                buildConnect();
+                isCalling.value = true;
+                await delay(100);
+                // if (socket) {
+                //     socket.close();
+                // }
+                // socket = new WebSocketService(
+                //     `/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
+                // );
+                // socket.connect();
+                // 建立连接后稍等一会儿再传送数据
+                startRecording();
+                if (localStorage.getItem('canStopByVoice') === 'true') {
+                    vadStart();
+                }
+            })
+            .catch(() => {});
+    };
+    let audioContext;
+    const analyser = ref();
+    const dataArray = ref();
+    let mediaRecorder;
+    let audioChunks = [];
+    const animationFrameId = ref();
+
+    const isFirstPiece = ref(true);
+
+    const startRecording = async () => {
+        // 获取用户音频流
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+        // 创建 AudioContext 和 MediaStreamAudioSourceNode
+        audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
+        const source = audioContext.createMediaStreamSource(stream);
+
+        analyser.value = audioContext.createAnalyser();
+        // 将音频节点连接到分析器
+        source.connect(analyser.value);
+        // 分析器设置
+        analyser.value.fftSize = 256;
+        const bufferLength = analyser.value.frequencyBinCount;
+        dataArray.value = new Uint8Array(bufferLength);
+        // 开始绘制音波
+        drawBars();
+
+        // 创建 ScriptProcessorNode 用于捕获音频数据
+        const processor = audioContext.createScriptProcessor(256, 1, 1);
+
+        processor.onaudioprocess = event => {
+            if (!isCalling.value) return;
+            if (isReturnError.value) {
+                stopRecording();
+                return;
+            }
+            const data = event.inputBuffer.getChannelData(0);
+            audioChunks.push(new Float32Array(data));
+            // 检查是否已经收集到1秒钟的数据
+            const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
+            const chunkLength = getChunkLength(audioContext.sampleRate);
+            if (totalBufferLength >= chunkLength) {
+                // 合并到一个完整的数据数组，并裁剪成1秒钟
+                const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
+                const oneSecondBuffer = mergedBuffer.slice(0, chunkLength);
+                // 保存并处理成WAV格式
+                addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
+                // 保留多余的数据备用
+                audioChunks = [mergedBuffer.slice(chunkLength)];
+            }
+        };
+
+        source.connect(processor);
+        processor.connect(audioContext.destination);
+    };
+    const stopRecording = () => {
+        isCalling.value = false;
+        if (animationFrameId.value) {
+            cancelAnimationFrame(animationFrameId.value);
+        }
+        if (audioContext && audioContext.state !== 'closed') {
+            audioContext.close();
+        }
+        ctrl.abort();
+        ctrl = new AbortController();
+        taskQueue.value = [];
+        audioPlayQueue.value = [];
+        base64List.value = [];
+        isReturnError.value = false;
+        skipDisabled.value = true;
+        playing.value = false;
+        audioDOM.pause();
+        stopMessage();
+        if (socket) {
+            socket.close();
+        }
+        if (
+            outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+            outputData.value[outputData.value.length - 1].audio === '' &&
+            allVoice.value.length > 0
+        ) {
+            outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+        }
+        myvad && myvad.destroy();
+    };
+    const getStopValue = () => {
+        return stop.value;
+    };
+    const getPlayingValue = () => {
+        return playing.value;
+    };
+    const getStopStatus = () => {
+        return localStorage.getItem('canStopByVoice') === 'true';
+    };
+    const saveAudioChunk = (buffer, timestamp) => {
+        return new Promise(resolve => {
+            if (!getStopStatus() && getPlayingValue()) {
+                resolve();
+                return;
+            }
+            const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
+            let reader = new FileReader();
+            reader.readAsDataURL(wavBlob);
+
+            reader.onloadend = async function () {
+                let base64data = reader.result.split(',')[1];
+                if (!base64data) {
+                    resolve();
+                    return;
+                }
+                const obj = {
+                    uid: getNewUserId(),
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'input_audio',
+                                    input_audio: {
+                                        data: base64data,
+                                        format: 'wav',
+                                        timestamp: String(timestamp)
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                };
+                // socket.send(JSON.stringify(obj));
+                // socket.on('message', data => {
+                //     console.log('message: ', data);
+                //     delayTimestamp.value = +new Date() - timestamp;
+                //     delayCount.value = taskQueue.value.length;
+                //     resolve();
+                // });
+                // 将Base64音频数据发送到后端
+                try {
+                    await sendMessage(obj);
+                    delayTimestamp.value = +new Date() - timestamp;
+                    delayCount.value = taskQueue.value.length;
+                } catch (err) {}
+                resolve();
+            };
+        });
+    };
+    const mergeBuffers = (buffers, length) => {
+        const result = new Float32Array(length);
+        let offset = 0;
+        for (let buffer of buffers) {
+            result.set(buffer, offset);
+            offset += buffer.length;
+        }
+        return result;
+    };
+    // 建立连接
+    const buildConnect = async () => {
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [{ type: 'none' }]
+                }
+            ],
+            stream: true
+        };
+        isEnd.value = false;
+        ctrl.abort();
+        ctrl = new AbortController();
+        const url = `/api/v1/completions${window.location.search}`;
+        fetchEventSource(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                service: 'minicpmo-server',
+                uid: getNewUserId()
+            },
+            body: JSON.stringify(obj),
+            signal: ctrl.signal,
+            openWhenHidden: true,
+            async onopen(response) {
+                console.log('onopen', response);
+                isFirstPiece.value = true;
+                isFirstReturn.value = true;
+                allVoice.value = [];
+                base64List.value = [];
+                if (response.status !== 200) {
+                    ElMessage({
+                        type: 'error',
+                        message: 'At limit. Please try again soon.',
+                        duration: 3000,
+                        customClass: 'system-error'
+                    });
+                    isReturnError.value = true;
+                } else {
+                    isReturnError.value = false;
+                    // skipDisabled.value = false;
+                    drawText();
+                }
+            },
+            onmessage(msg) {
+                const data = JSON.parse(msg.data);
+                if (data.response_id) {
+                    curResponseId.value = data.response_id;
+                }
+                if (data.choices[0]?.text) {
+                    textQueue.value += data.choices[0].text.replace('<end>', '');
+                    console.warn('text return time -------------------------------', +new Date());
+                }
+                // 首次返回的是前端发给后端的音频片段，需要单独处理
+                if (isFirstReturn.value) {
+                    console.log('第一次');
+                    isFirstReturn.value = false;
+                    // 如果后端返回的音频为空，需要重连
+                    if (!data.choices[0].audio) {
+                        buildConnect();
+                        return;
+                    }
+                    outputData.value.push({
+                        type: 'USER',
+                        audio: `data:audio/wav;base64,${data.choices[0].audio}`
+                    });
+                    outputData.value.push({
+                        type: 'BOT',
+                        text: '',
+                        audio: ''
+                    });
+                    return;
+                }
+                if (data.choices[0]?.audio) {
+                    console.warn('audio return time -------------------------------', +new Date());
+                    if (!getStopValue() && isCalling.value) {
+                        skipDisabled.value = false;
+                        base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                        addAudioQueue(() => truePlay(data.choices[0].audio));
+                    }
+                    allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                } else {
+                    // 发生异常了，直接重连
+                    buildConnect();
+                }
+                if (data.choices[0].text.includes('<end>')) {
+                    // isEnd.value = true;
+                    console.log('收到结束标记了:', +new Date());
+                    if (
+                        outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                        outputData.value[outputData.value.length - 1].audio === '' &&
+                        allVoice.value.length > 0
+                    ) {
+                        outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                    }
+                }
+            },
+            onclose() {
+                console.log('onclose', +new Date());
+                isEnd.value = true;
+                if (
+                    outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                    outputData.value[outputData.value.length - 1].audio === '' &&
+                    allVoice.value.length > 0
+                ) {
+                    outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                }
+                vadStartTime.value = +new Date();
+                if (audioPlayQueue.value.length === 0) {
+                    let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 2000);
+                    console.log('taskQueue111111111: ', taskQueue.value, startIndex);
+                    if (startIndex !== -1) {
+                        taskQueue.value = taskQueue.value.slice(startIndex);
+                        console.log('截取后长度:', taskQueue.value, vadStartTime.value);
+                    }
+                    buildConnect();
+                }
+            },
+            onerror(err) {
+                console.log('onerror', err);
+                ctrl.abort();
+                ctrl = new AbortController();
+                throw err;
+            }
+        });
+    };
+    const drawText = async () => {
+        if (textQueue.value.length > 0) {
+            outputData.value[outputData.value.length - 1].text += textQueue.value[0];
+            textQueue.value = textQueue.value.slice(1);
+        } else {
+            cancelAnimationFrame(textAnimationInterval.value);
+        }
+        textAnimationInterval.value = requestAnimationFrame(drawText);
+    };
+    // 返回的语音放到队列里，挨个播放
+    const addAudioQueue = async item => {
+        audioPlayQueue.value.push(item);
+        if (isFirstPiece.value) {
+            await delay(500);
+            isFirstPiece.value = false;
+        }
+        if (audioPlayQueue.value.length > 0 && !playing.value) {
+            playing.value = true;
+            playAudio();
+        }
+    };
+    // 控制播放队列执行
+    const playAudio = () => {
+        console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
+
+        if (!isEnd.value && base64List.value.length >= 2) {
+            const remainLen = base64List.value.length;
+            const blob = mergeBase64ToBlob(base64List.value);
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('前期合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('前期合并后播放结束时间: ', +new Date());
+                base64List.value = base64List.value.slice(remainLen);
+                audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
+                playAudio();
+            };
+            return;
+        }
+        if (isEnd.value && base64List.value.length >= 2) {
+            const blob = mergeBase64ToBlob(base64List.value);
+            // let audio = new Audio();
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('最后合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('合并后播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                base64List.value = [];
+                audioPlayQueue.value = [];
+                playing.value = false;
+                skipDisabled.value = true;
+                if (isCalling.value && !isReturnError.value) {
+                    // skipDisabled.value = true;
+                    taskQueue.value = [];
+                    // 打断前记录一下打断时间或vad触发事件
+                    // vadStartTime.value = +new Date();
+                    // // 每次完成后只保留当前时刻往前推1s的语音
+                    // console.log('截取前长度:', JSON.parse(JSON.stringify(taskQueue.value.map(item => item.time))));
+                    // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 2000);
+                    // if (startIndex !== -1) {
+                    //     taskQueue.value = taskQueue.value.slice(startIndex);
+                    //     console.log(
+                    //         '截取后长度:',
+                    //         taskQueue.value.map(item => item.time),
+                    //         vadStartTime.value
+                    //     );
+                    // }
+                    buildConnect();
+                }
+            };
+            return;
+        }
+        base64List.value.shift();
+        const item = audioPlayQueue.value.shift();
+        if (item) {
+            item().finally(() => playAudio());
+        } else {
+            playing.value = false;
+            if (isEnd.value) {
+                console.warn('play done................');
+                skipDisabled.value = true;
+            }
+            // 播放完成后且正在通话且接口未返回错误时开始下一次连接
+            if (isEnd.value && isCalling.value && !isReturnError.value) {
+                // skipDisabled.value = true;
+                taskQueue.value = [];
+                // 打断前记录一下打断时间或vad触发事件
+                // vadStartTime.value = +new Date();
+                // // 每次完成后只保留当前时刻往前推1s的语音
+                // console.log(
+                //     '截取前长度:',
+                //     taskQueue.value.map(item => item.time)
+                // );
+                // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 2000);
+                // if (startIndex !== -1) {
+                //     taskQueue.value = taskQueue.value.slice(startIndex);
+                //     console.log(
+                //         '截取后长度:',
+                //         taskQueue.value.map(item => item.time),
+                //         vadStartTime.value
+                //     );
+                // }
+                buildConnect();
+            }
+        }
+    };
+
+    // 播放音频
+    const truePlay = async voice => {
+        return new Promise(resolve => {
+            audioDOM.src = 'data:audio/wav;base64,' + voice;
+            console.error('播放开始时间:', +new Date());
+            audioDOM
+                .play()
+                .then(() => {
+                    // console.error('播放结束时间: ', +new Date());
+                })
+                .catch(error => {
+                    resolve();
+                    if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
+                        console.error('User interaction required or permission issue:', error);
+                        ElMessage.warning('音频播放失败');
+                    } else {
+                        console.error('Error playing audio:', error);
+                    }
+                });
+
+            audioDOM.onended = () => {
+                console.error('播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                resolve();
+            };
+        });
+    };
+    // 当队列中任务数大于0时，开始处理队列中的任务
+    const addQueue = (time, item) => {
+        taskQueue.value.push({ func: item, time });
+        if (taskQueue.value.length > 0 && !running.value) {
+            running.value = true;
+            processQueue();
+        }
+    };
+    const processQueue = () => {
+        const item = taskQueue.value.shift();
+        if (item?.func) {
+            item.func().then(() => {
+                console.warn('shift!!!!!!!!!');
+                processQueue();
+            });
+        } else {
+            running.value = false;
+        }
+    };
+    const drawBars = () => {
+        // AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
+        analyser.value.getByteFrequencyData(dataArray.value);
+        animationFrameId.value = requestAnimationFrame(drawBars);
+    };
+    // 跳过当前片段
+    const skipVoice = async () => {
+        // 打断前记录一下打断时间或vad触发事件
+        vadStartTime.value = +new Date();
+        if (!skipDisabled.value) {
+            if (
+                outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                outputData.value[outputData.value.length - 1].audio === ''
+            ) {
+                outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+            }
+            base64List.value = [];
+            audioPlayQueue.value = [];
+            // 跳过之后，只保留当前时间点两秒内到之后的音频片段
+            console.log(
+                '截取前长度:',
+                taskQueue.value.map(item => item.time)
+            );
+            let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+            if (startIndex !== -1) {
+                taskQueue.value = taskQueue.value.slice(startIndex);
+                console.log(
+                    '截取后长度:',
+                    taskQueue.value.map(item => item.time),
+                    vadStartTime.value
+                );
+            }
+            stop.value = true;
+            audioDOM.pause();
+            setTimeout(() => {
+                skipDisabled.value = true;
+            }, 300);
+            try {
+                playing.value = false;
+                await stopMessage();
+                stop.value = false;
+                // playing.value = false;
+                buildConnect();
+                // cancelAnimationFrame(animationFrameId.value);
+            } catch (err) {}
+        }
+    };
+    // 每次call先上传当前用户配置
+    const uploadUserConfig = async () => {
+        if (!localStorage.getItem('configData')) {
+            return new Promise(resolve => resolve());
+        }
+        const {
+            videoQuality,
+            useAudioPrompt,
+            voiceClonePrompt,
+            assistantPrompt,
+            vadThreshold,
+            audioFormat,
+            base64Str
+        } = JSON.parse(localStorage.getItem('configData'));
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'input_audio',
+                            input_audio: {
+                                data: base64Str,
+                                format: audioFormat
+                            }
+                        },
+                        {
+                            type: 'options',
+                            options: {
+                                hd_video: videoQuality,
+                                use_audio_prompt: useAudioPrompt,
+                                vad_threshold: vadThreshold,
+                                voice_clone_prompt: voiceClonePrompt,
+                                assistant_prompt: assistantPrompt
+                            }
+                        }
+                    ]
+                }
+            ]
+        };
+        const { code, message, data } = await uploadConfig(obj);
+        modelVersion.value = data?.choices?.content || '';
+        return new Promise((resolve, reject) => {
+            if (code !== 0) {
+                ElMessage({
+                    type: 'error',
+                    message: message,
+                    duration: 3000,
+                    customClass: 'system-error'
+                });
+                reject();
+            } else {
+                resolve();
+            }
+        });
+    };
+</script>
+<style lang="less" scoped>
+    .voice-page {
+        flex: 1;
+        height: 100%;
+        display: flex;
+        flex-direction: column;
+        &-header {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            padding: 0 16px 16px;
+            box-shadow: 0 0.5px 0 0 #e0e0e0;
+            margin-bottom: 16px;
+            .header-icon {
+                display: flex;
+                align-items: center;
+                img {
+                    width: 24px;
+                    height: 24px;
+                    margin-right: 8px;
+                }
+                span {
+                    color: rgba(23, 23, 23, 0.9);
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                    margin-right: 40px;
+                    flex-shrink: 0;
+                }
+            }
+            .voice-container {
+                display: flex;
+                .voice-icon {
+                    width: 191px;
+                    height: 45px;
+                }
+            }
+        }
+        &-output {
+            flex: 1;
+            height: 0;
+            padding: 0 16px;
+            margin-bottom: 16px;
+            display: flex;
+            flex-direction: column;
+            .output-content {
+                flex: 1;
+                overflow: auto;
+            }
+            .skip-box {
+                display: flex;
+                align-items: center;
+                justify-content: flex-end;
+                margin-top: 16px;
+            }
+        }
+        &-btn {
+            text-align: center;
+            padding: 8px 0;
+            .el-button {
+                width: 284px;
+                height: 46px;
+                border-radius: 8px;
+            }
+            .el-button.el-button--success {
+                background: #647fff;
+                border-color: #647fff;
+                &:hover {
+                    opacity: 0.8;
+                }
+                span {
+                    color: #fff;
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                }
+            }
+            .el-button.el-button--success.is-disabled {
+                background: #f3f3f3;
+                border-color: #f3f3f3;
+                span {
+                    color: #d1d1d1;
+                }
+            }
+            .el-button.el-button--danger {
+                border-color: #dc3545;
+                background-color: #dc3545;
+                color: #ffffff;
+                font-family: PingFang SC;
+                font-size: 16px;
+                font-style: normal;
+                font-weight: 500;
+                line-height: normal;
+                .phone-icon {
+                    margin-right: 10px;
+                }
+                .btn-text {
+                    margin-right: 10px;
+                }
+                .btn-desc {
+                    margin-right: 16px;
+                }
+                .time {
+                    display: flex;
+                    align-items: center;
+                    .time-minute,
+                    .time-second {
+                        width: 26px;
+                        height: 26px;
+                        display: flex;
+                        justify-content: center;
+                        align-items: center;
+                        border-radius: 3.848px;
+                        background: rgba(47, 47, 47, 0.5);
+                    }
+                    .time-colon {
+                        margin: 0 3px;
+                    }
+                }
+            }
+        }
+    }
+</style>

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/VoiceCall_0105.vue

@@ -0,0 +1,829 @@
+<template>
+    <ExtraInfo webVersion="websocket_0107" :modelVersion="modelVersion" />
+    <div class="voice-page">
+        <div class="voice-page-header">
+            <div class="header-icon">
+                <img src="@/assets/images/voice-icon.png" />
+                <span>Audio Choice</span>
+            </div>
+            <div class="voice-container" v-if="!isCalling">
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+                <SvgIcon name="voice" class="voice-icon" />
+            </div>
+            <div class="voice-container" v-else>
+                <Voice
+                    :dataArray="dataArray"
+                    :isCalling="isCalling"
+                    :isPlaying="playing"
+                    :configList="videoConfigList"
+                    :boxStyle="{ height: '45px' }"
+                    :itemStyle="{ width: '3px', margin: '0 1px' }"
+                />
+            </div>
+            <!-- <SelectTimbre v-model:timbre="timbre" v-model:audioData="audioData" v-model:disabled="isCalling" /> -->
+        </div>
+        <div class="voice-page-output">
+            <div class="output-content">
+                <ModelOutput v-if="outputData.length > 0" :outputData="outputData" containerClass="output-content" />
+            </div>
+            <div class="skip-box">
+                <DelayTips
+                    v-if="delayTimestamp > 200 || delayCount > 2"
+                    :delayTimestamp="delayTimestamp"
+                    :delayCount="delayCount"
+                />
+                <LikeAndDislike v-model:feedbackStatus="feedbackStatus" v-model:curResponseId="curResponseId" />
+                <SkipBtn :disabled="skipDisabled" @click="skipVoice" />
+            </div>
+        </div>
+        <div class="voice-page-btn">
+            <el-button v-show="!isCalling" type="success" :disabled="callDisabled" @click="initRecording">
+                {{ callDisabled ? 'Not ready yet, please wait' : 'Call MiniCPM' }}
+            </el-button>
+            <el-button v-show="isCalling" @click="stopRecording" type="danger">
+                <SvgIcon name="phone-icon" className="phone-icon" />
+                <span class="btn-text">Hang Up</span>
+                <CountDown v-model="isCalling" @timeUp="stopRecording" />
+            </el-button>
+        </div>
+        <IdeasList v-if="showIdeasList" :ideasList="voiceIdeasList" />
+    </div>
+</template>
+<script setup>
+    import { sendMessage, stopMessage, uploadConfig } from '@/apis';
+    import { encodeWAV } from '@/hooks/useVoice';
+    import { getNewUserId, setNewUserId } from '@/hooks/useRandomId';
+    import { fetchEventSource } from '@microsoft/fetch-event-source';
+    import { MicVAD } from '@ricky0123/vad-web';
+    import { videoConfigList, voiceConfigList, voiceIdeasList, showIdeasList } from '@/enums';
+    import { getChunkLength } from '@/utils';
+    import { mergeBase64ToBlob } from './merge';
+    import WebSocketService from '@/utils/websocket';
+
+    let ctrl = new AbortController();
+    let socket = null;
+    const audioData = ref({
+        base64Str: '',
+        type: 'mp3'
+    }); // 自定义音色base64
+    const isCalling = defineModel();
+    const taskQueue = ref([]);
+    const running = ref(false);
+    const outputData = ref([]);
+    const textQueue = ref('');
+    const textAnimationInterval = ref();
+
+    const isFirstReturn = ref(true); // 首次返回的音频是前端发给后端的音频片段，需要单独处理
+
+    const audioPlayQueue = ref([]);
+    const base64List = ref([]);
+    const playing = ref(false);
+    const skipDisabled = ref(true);
+    const stop = ref(false);
+    const timbre = ref([1]);
+    const isReturnError = ref(false);
+    const allVoice = ref([]);
+    const callDisabled = ref(true);
+
+    const feedbackStatus = ref('');
+    const curResponseId = ref('');
+    const delayTimestamp = ref(0); // 当前发送片延时
+    const delayCount = ref(0); // 当前剩余多少ms未发送到接口
+
+    const modelVersion = ref('');
+
+    let audioDOM = new Audio();
+
+    const isEnd = ref(false); // sse接口关闭，认为模型已完成本次返回
+    // 页面卸载时关闭录音
+    onBeforeUnmount(() => {
+        stopRecording();
+    });
+    const vadStartTime = ref();
+    let myvad = null;
+    let vadTimer = null; // vad定时器，用于检测1s内人声是否停止，1s内停止，可认为是vad误触，直接忽略，1s内未停止，则认为是人声，已自动跳过当前对话
+    const vadStart = async () => {
+        myvad = await MicVAD.new({
+            onSpeechStart: () => {
+                console.log('Speech start detected');
+                if (!skipDisabled.value) {
+                    vadTimer && clearTimeout(vadTimer);
+                    vadTimer = setTimeout(() => {
+                        console.log('打断时间: ', +new Date());
+                        skipVoice();
+                    }, 500);
+                }
+            },
+            onSpeechEnd: audio => {
+                vadTimer && clearTimeout(vadTimer);
+                // debugger;
+                // do something with `audio` (Float32Array of audio samples at sample rate 16000)...
+            }
+        });
+        console.log('vad: ', myvad);
+        myvad.start();
+    };
+    onMounted(async () => {
+        const { code, message } = await stopMessage();
+        if (code !== 0) {
+            ElMessage({
+                type: 'error',
+                message: message,
+                duration: 3000,
+                customClass: 'system-error'
+            });
+            return;
+        }
+        callDisabled.value = false;
+    });
+    const delay = ms => {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    };
+    const initRecording = async () => {
+        uploadUserConfig()
+            .then(async () => {
+                // 每次call都需要生成新uid
+                setNewUserId();
+
+                outputData.value = [];
+                buildConnect();
+                isCalling.value = true;
+                await delay(100);
+                if (socket) {
+                    socket.close();
+                }
+                socket = new WebSocketService(
+                    `/ws/stream${window.location.search}&uid=${getNewUserId()}&service=minicpmo-server`
+                );
+                socket.connect();
+                // 建立连接后稍等一会儿再传送数据
+                startRecording();
+                if (localStorage.getItem('canStopByVoice') === 'true') {
+                    vadStart();
+                }
+            })
+            .catch(() => {});
+    };
+    let audioContext;
+    const analyser = ref();
+    const dataArray = ref();
+    let mediaRecorder;
+    let audioChunks = [];
+    const animationFrameId = ref();
+
+    const isFirstPiece = ref(true);
+
+    const startRecording = async () => {
+        // 获取用户音频流
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+        // 创建 AudioContext 和 MediaStreamAudioSourceNode
+        audioContext = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
+        const source = audioContext.createMediaStreamSource(stream);
+
+        analyser.value = audioContext.createAnalyser();
+        // 将音频节点连接到分析器
+        source.connect(analyser.value);
+        // 分析器设置
+        analyser.value.fftSize = 256;
+        const bufferLength = analyser.value.frequencyBinCount;
+        dataArray.value = new Uint8Array(bufferLength);
+        // 开始绘制音波
+        drawBars();
+
+        // 创建 ScriptProcessorNode 用于捕获音频数据
+        const processor = audioContext.createScriptProcessor(256, 1, 1);
+
+        processor.onaudioprocess = event => {
+            if (!isCalling.value) return;
+            if (isReturnError.value) {
+                stopRecording();
+                return;
+            }
+            const data = event.inputBuffer.getChannelData(0);
+            audioChunks.push(new Float32Array(data));
+            // 检查是否已经收集到1秒钟的数据
+            const totalBufferLength = audioChunks.reduce((total, curr) => total + curr.length, 0);
+            const chunkLength = getChunkLength(audioContext.sampleRate);
+            if (totalBufferLength >= chunkLength) {
+                // 合并到一个完整的数据数组，并裁剪成1秒钟
+                const mergedBuffer = mergeBuffers(audioChunks, totalBufferLength);
+                const oneSecondBuffer = mergedBuffer.slice(0, chunkLength);
+                // 保存并处理成WAV格式
+                addQueue(+new Date(), () => saveAudioChunk(oneSecondBuffer, +new Date()));
+                // 保留多余的数据备用
+                audioChunks = [mergedBuffer.slice(chunkLength)];
+            }
+        };
+
+        source.connect(processor);
+        processor.connect(audioContext.destination);
+    };
+    const stopRecording = () => {
+        isCalling.value = false;
+        if (animationFrameId.value) {
+            cancelAnimationFrame(animationFrameId.value);
+        }
+        if (audioContext && audioContext.state !== 'closed') {
+            audioContext.close();
+        }
+        ctrl.abort();
+        ctrl = new AbortController();
+        taskQueue.value = [];
+        audioPlayQueue.value = [];
+        base64List.value = [];
+        isReturnError.value = false;
+        skipDisabled.value = true;
+        playing.value = false;
+        audioDOM.pause();
+        stopMessage();
+        if (socket) {
+            socket.close();
+        }
+        if (
+            outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+            outputData.value[outputData.value.length - 1].audio === '' &&
+            allVoice.value.length > 0
+        ) {
+            outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+        }
+        myvad && myvad.destroy();
+    };
+    const getStopValue = () => {
+        return stop.value;
+    };
+    const getPlayingValue = () => {
+        return playing.value;
+    };
+    const getStopStatus = () => {
+        return localStorage.getItem('canStopByVoice') === 'true';
+    };
+    const saveAudioChunk = (buffer, timestamp) => {
+        return new Promise(resolve => {
+            if (!getStopStatus() && getPlayingValue()) {
+                resolve();
+                return;
+            }
+            const wavBlob = encodeWAV(buffer, audioContext.sampleRate);
+            let reader = new FileReader();
+            reader.readAsDataURL(wavBlob);
+
+            reader.onloadend = async function () {
+                let base64data = reader.result.split(',')[1];
+                if (!base64data) {
+                    resolve();
+                    return;
+                }
+                const obj = {
+                    uid: getNewUserId(),
+                    messages: [
+                        {
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'input_audio',
+                                    input_audio: {
+                                        data: base64data,
+                                        format: 'wav',
+                                        timestamp: String(timestamp)
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                };
+                socket.send(JSON.stringify(obj));
+                socket.on('message', data => {
+                    console.log('message: ', data);
+                    delayTimestamp.value = +new Date() - timestamp;
+                    delayCount.value = taskQueue.value.length;
+                    resolve();
+                });
+                // 将Base64音频数据发送到后端
+                // try {
+                //     await sendMessage(obj);
+                //     delayTimestamp.value = +new Date() - timestamp;
+                //     delayCount.value = taskQueue.value.length;
+                // } catch (err) {}
+                // resolve();
+            };
+        });
+    };
+    const mergeBuffers = (buffers, length) => {
+        const result = new Float32Array(length);
+        let offset = 0;
+        for (let buffer of buffers) {
+            result.set(buffer, offset);
+            offset += buffer.length;
+        }
+        return result;
+    };
+    // 建立连接
+    const buildConnect = async () => {
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [{ type: 'none' }]
+                }
+            ],
+            stream: true
+        };
+        isEnd.value = false;
+        ctrl.abort();
+        ctrl = new AbortController();
+        const url = `/api/v1/completions${window.location.search}`;
+        fetchEventSource(url, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                service: 'minicpmo-server',
+                uid: getNewUserId()
+            },
+            body: JSON.stringify(obj),
+            signal: ctrl.signal,
+            openWhenHidden: true,
+            async onopen(response) {
+                console.log('onopen', response);
+                isFirstPiece.value = true;
+                isFirstReturn.value = true;
+                allVoice.value = [];
+                base64List.value = [];
+                if (response.status !== 200) {
+                    ElMessage({
+                        type: 'error',
+                        message: 'At limit. Please try again soon.',
+                        duration: 3000,
+                        customClass: 'system-error'
+                    });
+                    isReturnError.value = true;
+                } else {
+                    isReturnError.value = false;
+                    // skipDisabled.value = false;
+                    drawText();
+                }
+            },
+            onmessage(msg) {
+                const data = JSON.parse(msg.data);
+                if (data.response_id) {
+                    curResponseId.value = data.response_id;
+                }
+                if (data.choices[0]?.text) {
+                    textQueue.value += data.choices[0].text.replace('<end>', '');
+                    console.warn('text return time -------------------------------', +new Date());
+                }
+                // 首次返回的是前端发给后端的音频片段，需要单独处理
+                if (isFirstReturn.value) {
+                    console.log('第一次');
+                    isFirstReturn.value = false;
+                    // 如果后端返回的音频为空，需要重连
+                    if (!data.choices[0].audio) {
+                        buildConnect();
+                        return;
+                    }
+                    outputData.value.push({
+                        type: 'USER',
+                        audio: `data:audio/wav;base64,${data.choices[0].audio}`
+                    });
+                    outputData.value.push({
+                        type: 'BOT',
+                        text: '',
+                        audio: ''
+                    });
+                    return;
+                }
+                if (data.choices[0]?.audio) {
+                    console.warn('audio return time -------------------------------', +new Date());
+                    if (!getStopValue() && isCalling.value) {
+                        skipDisabled.value = false;
+                        base64List.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                        addAudioQueue(() => truePlay(data.choices[0].audio));
+                    }
+                    allVoice.value.push(`data:audio/wav;base64,${data.choices[0].audio}`);
+                } else {
+                    // 发生异常了，直接重连
+                    buildConnect();
+                }
+                if (data.choices[0].text.includes('<end>')) {
+                    console.log('收到结束标记了:', +new Date());
+                    if (
+                        outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                        outputData.value[outputData.value.length - 1].audio === '' &&
+                        allVoice.value.length > 0
+                    ) {
+                        outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                    }
+                }
+            },
+            onclose() {
+                console.log('onclose', +new Date());
+                isEnd.value = true;
+                outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+                vadStartTime.value = +new Date();
+                if (audioPlayQueue.value.length === 0) {
+                    let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                    console.log('taskQueue111111111: ', taskQueue.value, startIndex);
+                    if (startIndex !== -1) {
+                        taskQueue.value = taskQueue.value.slice(startIndex);
+                        console.log('截取后长度:', taskQueue.value, vadStartTime.value);
+                    }
+                    buildConnect();
+                }
+            },
+            onerror(err) {
+                console.log('onerror', err);
+                ctrl.abort();
+                ctrl = new AbortController();
+                throw err;
+            }
+        });
+    };
+    const drawText = async () => {
+        if (textQueue.value.length > 0) {
+            outputData.value[outputData.value.length - 1].text += textQueue.value[0];
+            textQueue.value = textQueue.value.slice(1);
+        } else {
+            cancelAnimationFrame(textAnimationInterval.value);
+        }
+        textAnimationInterval.value = requestAnimationFrame(drawText);
+    };
+    // 返回的语音放到队列里，挨个播放
+    const addAudioQueue = async item => {
+        audioPlayQueue.value.push(item);
+        if (isFirstPiece.value) {
+            await delay(500);
+            isFirstPiece.value = false;
+        }
+        if (audioPlayQueue.value.length > 0 && !playing.value) {
+            playing.value = true;
+            playAudio();
+        }
+    };
+    // 控制播放队列执行
+    const playAudio = () => {
+        console.log('剩余播放列表:', audioPlayQueue.value, +new Date());
+
+        if (!isEnd.value && base64List.value.length >= 2) {
+            const remainLen = base64List.value.length;
+            const blob = mergeBase64ToBlob(base64List.value);
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('前期合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('前期合并后播放结束时间: ', +new Date());
+                base64List.value = base64List.value.slice(remainLen);
+                audioPlayQueue.value = audioPlayQueue.value.slice(remainLen);
+                playAudio();
+            };
+            return;
+        }
+        if (isEnd.value && base64List.value.length >= 2) {
+            const blob = mergeBase64ToBlob(base64List.value);
+            // let audio = new Audio();
+            audioDOM.src = blob;
+            audioDOM.play();
+            console.error('最后合并后播放开始时间: ', +new Date());
+            audioDOM.onended = () => {
+                console.error('合并后播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                base64List.value = [];
+                audioPlayQueue.value = [];
+                playing.value = false;
+                skipDisabled.value = true;
+                if (isCalling.value && !isReturnError.value) {
+                    // skipDisabled.value = true;
+                    taskQueue.value = [];
+                    // 打断前记录一下打断时间或vad触发事件
+                    // vadStartTime.value = +new Date();
+                    // // 每次完成后只保留当前时刻往前推1s的语音
+                    // console.log(
+                    //     '截取前长度:',
+                    //     taskQueue.value.map(item => item.time)
+                    // );
+                    // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                    // if (startIndex !== -1) {
+                    //     taskQueue.value = taskQueue.value.slice(startIndex);
+                    //     console.log(
+                    //         '截取后长度:',
+                    //         taskQueue.value.map(item => item.time),
+                    //         vadStartTime.value
+                    //     );
+                    // }
+                    buildConnect();
+                }
+            };
+            return;
+        }
+        base64List.value.shift();
+        const item = audioPlayQueue.value.shift();
+        if (item) {
+            item().finally(() => playAudio());
+        } else {
+            playing.value = false;
+            if (isEnd.value) {
+                console.warn('play done................');
+                skipDisabled.value = true;
+            }
+            // 播放完成后且正在通话且接口未返回错误时开始下一次连接
+            if (isEnd.value && isCalling.value && !isReturnError.value) {
+                // skipDisabled.value = true;
+                taskQueue.value = [];
+                // 打断前记录一下打断时间或vad触发事件
+                // vadStartTime.value = +new Date();
+                // // 每次完成后只保留当前时刻往前推1s的语音
+                // console.log(
+                //     '截取前长度:',
+                //     taskQueue.value.map(item => item.time)
+                // );
+                // let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+                // if (startIndex !== -1) {
+                //     taskQueue.value = taskQueue.value.slice(startIndex);
+                //     console.log(
+                //         '截取后长度:',
+                //         taskQueue.value.map(item => item.time),
+                //         vadStartTime.value
+                //     );
+                // }
+                buildConnect();
+            }
+        }
+    };
+
+    // 播放音频
+    const truePlay = async voice => {
+        return new Promise(resolve => {
+            audioDOM.src = 'data:audio/wav;base64,' + voice;
+            console.error('播放开始时间:', +new Date());
+            audioDOM
+                .play()
+                .then(() => {
+                    // console.error('播放结束时间: ', +new Date());
+                })
+                .catch(error => {
+                    resolve();
+                    if (error.name === 'NotAllowedError' || error.name === 'SecurityError') {
+                        console.error('User interaction required or permission issue:', error);
+                        ElMessage.warning('音频播放失败');
+                    } else {
+                        console.error('Error playing audio:', error);
+                    }
+                });
+
+            audioDOM.onended = () => {
+                console.error('播放结束时间: ', +new Date());
+                // URL.revokeObjectURL(url);
+                resolve();
+            };
+        });
+    };
+    // 当队列中任务数大于0时，开始处理队列中的任务
+    const addQueue = (time, item) => {
+        taskQueue.value.push({ func: item, time });
+        if (taskQueue.value.length > 0 && !running.value) {
+            running.value = true;
+            processQueue();
+        }
+    };
+    const processQueue = () => {
+        const item = taskQueue.value.shift();
+        if (item?.func) {
+            item.func().then(() => {
+                console.warn('shift!!!!!!!!!');
+                processQueue();
+            });
+        } else {
+            running.value = false;
+        }
+    };
+    const drawBars = () => {
+        // AnalyserNode接口的 getByteFrequencyData() 方法将当前频率数据复制到传入的 Uint8Array（无符号字节数组）中。
+        analyser.value.getByteFrequencyData(dataArray.value);
+        animationFrameId.value = requestAnimationFrame(drawBars);
+    };
+    // 跳过当前片段
+    const skipVoice = async () => {
+        // 打断前记录一下打断时间或vad触发事件
+        vadStartTime.value = +new Date();
+        if (!skipDisabled.value) {
+            if (
+                outputData.value[outputData.value.length - 1]?.type === 'BOT' &&
+                outputData.value[outputData.value.length - 1].audio === ''
+            ) {
+                outputData.value[outputData.value.length - 1].audio = mergeBase64ToBlob(allVoice.value);
+            }
+            base64List.value = [];
+            audioPlayQueue.value = [];
+            // 跳过之后，只保留当前时间点两秒内到之后的音频片段
+            console.log(
+                '截取前长度:',
+                taskQueue.value.map(item => item.time)
+            );
+            let startIndex = taskQueue.value.findIndex(item => item.time >= vadStartTime.value - 1000);
+            if (startIndex !== -1) {
+                taskQueue.value = taskQueue.value.slice(startIndex);
+                console.log(
+                    '截取后长度:',
+                    taskQueue.value.map(item => item.time),
+                    vadStartTime.value
+                );
+            }
+            stop.value = true;
+            audioDOM.pause();
+            setTimeout(() => {
+                skipDisabled.value = true;
+            }, 300);
+            try {
+                playing.value = false;
+                await stopMessage();
+                stop.value = false;
+                // playing.value = false;
+                buildConnect();
+                // cancelAnimationFrame(animationFrameId.value);
+            } catch (err) {}
+        }
+    };
+    // 每次call先上传当前用户配置
+    const uploadUserConfig = async () => {
+        if (!localStorage.getItem('configData')) {
+            return new Promise(resolve => resolve());
+        }
+        const {
+            videoQuality,
+            useAudioPrompt,
+            voiceClonePrompt,
+            assistantPrompt,
+            vadThreshold,
+            audioFormat,
+            base64Str
+        } = JSON.parse(localStorage.getItem('configData'));
+        const obj = {
+            messages: [
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'input_audio',
+                            input_audio: {
+                                data: base64Str,
+                                format: audioFormat
+                            }
+                        },
+                        {
+                            type: 'options',
+                            options: {
+                                hd_video: videoQuality,
+                                use_audio_prompt: useAudioPrompt,
+                                vad_threshold: vadThreshold,
+                                voice_clone_prompt: voiceClonePrompt,
+                                assistant_prompt: assistantPrompt
+                            }
+                        }
+                    ]
+                }
+            ]
+        };
+        const { code, message, data } = await uploadConfig(obj);
+        modelVersion.value = data?.choices?.content || '';
+        return new Promise((resolve, reject) => {
+            if (code !== 0) {
+                ElMessage({
+                    type: 'error',
+                    message: message,
+                    duration: 3000,
+                    customClass: 'system-error'
+                });
+                reject();
+            } else {
+                resolve();
+            }
+        });
+    };
+</script>
+<style lang="less">
+    .voice-page {
+        flex: 1;
+        height: 100%;
+        display: flex;
+        flex-direction: column;
+        &-header {
+            display: flex;
+            align-items: center;
+            padding: 0 16px 16px;
+            box-shadow: 0 0.5px 0 0 #e0e0e0;
+            margin-bottom: 16px;
+            justify-content: space-between;
+            .header-icon {
+                display: flex;
+                align-items: center;
+                img {
+                    width: 24px;
+                    height: 24px;
+                    margin-right: 8px;
+                }
+                span {
+                    color: rgba(23, 23, 23, 0.9);
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                    margin-right: 40px;
+                    flex-shrink: 0;
+                }
+            }
+            .voice-container {
+                display: flex;
+                .voice-icon {
+                    width: 191px;
+                    height: 45px;
+                }
+            }
+        }
+        &-output {
+            flex: 1;
+            height: 0;
+            padding: 0 16px;
+            margin-bottom: 16px;
+            display: flex;
+            flex-direction: column;
+            .output-content {
+                flex: 1;
+                overflow: auto;
+            }
+            .skip-box {
+                display: flex;
+                align-items: center;
+                justify-content: flex-end;
+                margin-top: 16px;
+            }
+        }
+        &-btn {
+            text-align: center;
+            padding: 8px 0;
+            .el-button {
+                width: 284px;
+                height: 46px;
+                border-radius: 8px;
+            }
+            .el-button.el-button--success {
+                background: #647fff;
+                border-color: #647fff;
+                &:hover {
+                    opacity: 0.8;
+                }
+                span {
+                    color: #fff;
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 500;
+                    line-height: normal;
+                }
+            }
+            .el-button.el-button--success.is-disabled {
+                background: #f3f3f3;
+                border-color: #f3f3f3;
+                span {
+                    color: #d1d1d1;
+                }
+            }
+            .el-button.el-button--danger {
+                border-color: #dc3545;
+                background-color: #dc3545;
+                color: #ffffff;
+                font-family: PingFang SC;
+                font-size: 16px;
+                font-style: normal;
+                font-weight: 500;
+                line-height: normal;
+                .phone-icon {
+                    margin-right: 10px;
+                }
+                .btn-text {
+                    margin-right: 10px;
+                }
+                .btn-desc {
+                    margin-right: 16px;
+                }
+                .time {
+                    display: flex;
+                    align-items: center;
+                    .time-minute,
+                    .time-second {
+                        width: 26px;
+                        height: 26px;
+                        display: flex;
+                        justify-content: center;
+                        align-items: center;
+                        border-radius: 3.848px;
+                        background: rgba(47, 47, 47, 0.5);
+                    }
+                    .time-colon {
+                        margin: 0 3px;
+                    }
+                }
+            }
+        }
+    }
+</style>

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/audioBufferToMp3Base64.js

@@ -0,0 +1,36 @@
+import lame from '@breezystack/lamejs';
+
+export const audioBufferToMp3Base64 = audioBuffer => {
+    const mp3Encoder = new lame.Mp3Encoder(1, 16000, 128);
+    const sampleBlockSize = 1152;
+    const mp3Data = [];
+
+    for (let i = 0; i < audioBuffer.length; i += sampleBlockSize) {
+        const sampleChunk = audioBuffer.subarray(i, i + sampleBlockSize);
+        const mp3buf = mp3Encoder.encodeBuffer(sampleChunk);
+        if (mp3buf.length > 0) {
+            mp3Data.push(new Int8Array(mp3buf));
+        }
+    }
+
+    const mp3buf = mp3Encoder.flush();
+    if (mp3buf.length > 0) {
+        mp3Data.push(new Int8Array(mp3buf));
+    }
+
+    const mp3Blob = new Blob(mp3Data, { type: 'audio/mp3' });
+    const url = URL.createObjectURL(mp3Blob);
+    let dom = document.querySelector('#voice-box');
+    let audio = document.createElement('audio');
+    audio.controls = true;
+    audio.src = url;
+    dom.appendChild(audio);
+    return new Promise(resolve => {
+        const reader = new FileReader();
+        reader.onloadend = () => {
+            const base64String = reader.result.split(',')[1];
+            resolve(base64String);
+        };
+        reader.readAsDataURL(mp3Blob);
+    });
+};

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/merge.js

@@ -0,0 +1,132 @@
+// Convert Base64 to ArrayBuffer
+const base64ToArrayBuffer = base64 => {
+    const binaryString = atob(base64.split(',')[1]); // Remove data URI scheme if present
+    const len = binaryString.length;
+    const bytes = new Uint8Array(len);
+    for (let i = 0; i < len; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes.buffer;
+};
+
+// Parse WAV header and get audio data section
+const parseWav = buffer => {
+    const view = new DataView(buffer);
+    const format = view.getUint16(20, true);
+    const channels = view.getUint16(22, true);
+    const sampleRate = view.getUint32(24, true);
+    const bitsPerSample = view.getUint16(34, true);
+    const dataOffset = 44;
+    const dataSize = view.getUint32(40, true);
+    const audioData = new Uint8Array(buffer, dataOffset, dataSize);
+
+    return {
+        format,
+        channels,
+        sampleRate,
+        bitsPerSample,
+        audioData
+    };
+};
+
+// Create WAV header for combined audio data
+const createWavHeader = (audioDataSize, sampleRate, channels, bitsPerSample) => {
+    const arrayBuffer = new ArrayBuffer(44);
+    const view = new DataView(arrayBuffer);
+
+    const writeString = (view, offset, string) => {
+        for (let i = 0; i < string.length; i++) {
+            view.setUint8(offset + i, string.charCodeAt(i));
+        }
+    };
+
+    writeString(view, 0, 'RIFF'); // ChunkID
+    view.setUint32(4, 36 + audioDataSize, true); // ChunkSize
+    writeString(view, 8, 'WAVE'); // Format
+    writeString(view, 12, 'fmt '); // Subchunk1ID
+    view.setUint32(16, 16, true); // Subchunk1Size (PCM)
+    view.setUint16(20, 1, true); // AudioFormat (PCM)
+    view.setUint16(22, channels, true); // NumChannels
+    view.setUint32(24, sampleRate, true); // SampleRate
+    view.setUint32(28, (sampleRate * channels * bitsPerSample) / 8, true); // ByteRate
+    view.setUint16(32, (channels * bitsPerSample) / 8, true); // BlockAlign
+    view.setUint16(34, bitsPerSample, true); // BitsPerSample
+    writeString(view, 36, 'data'); // Subchunk2ID
+    view.setUint32(40, audioDataSize, true); // Subchunk2Size
+
+    return arrayBuffer;
+};
+
+// Merge multiple Base64 audio files and return a Blob
+const mergeAudioFiles = base64AudioArray => {
+    let sampleRate, channels, bitsPerSample;
+    let combinedAudioData = new Uint8Array();
+
+    for (let i = 0; i < base64AudioArray.length; i++) {
+        const arrayBuffer = base64ToArrayBuffer(base64AudioArray[i]);
+        const wav = parseWav(arrayBuffer);
+
+        // Initialize properties based on the first audio file
+        if (i === 0) {
+            sampleRate = wav.sampleRate;
+            channels = wav.channels;
+            bitsPerSample = wav.bitsPerSample;
+        }
+
+        // Ensure all files have the same format
+        if (wav.sampleRate !== sampleRate || wav.channels !== channels || wav.bitsPerSample !== bitsPerSample) {
+            throw new Error('All audio files must have the same format.');
+        }
+
+        // Combine audio data
+        const newCombinedData = new Uint8Array(combinedAudioData.byteLength + wav.audioData.byteLength);
+        newCombinedData.set(combinedAudioData, 0);
+        newCombinedData.set(wav.audioData, combinedAudioData.byteLength);
+        combinedAudioData = newCombinedData;
+    }
+
+    const combinedAudioDataSize = combinedAudioData.byteLength;
+    const wavHeader = createWavHeader(combinedAudioDataSize, sampleRate, channels, bitsPerSample);
+    const combinedWavBuffer = new Uint8Array(wavHeader.byteLength + combinedAudioData.byteLength);
+    combinedWavBuffer.set(new Uint8Array(wavHeader), 0);
+    combinedWavBuffer.set(combinedAudioData, wavHeader.byteLength);
+
+    // Create a Blob from the combined audio data
+    const combinedBlob = new Blob([combinedWavBuffer], { type: 'audio/wav' });
+    return combinedBlob;
+};
+export const mergeBase64ToBlob = base64List => {
+    const combinedBlob = mergeAudioFiles(base64List);
+    const audioUrl = URL.createObjectURL(combinedBlob);
+    return audioUrl;
+};
+
+// 假设 base64Strings 是一个包含多个 Base64 编码 WAV 文件的数组
+// 注意：这些 Base64 字符串不应该包含 URI 前缀 (例如 "audio/wav;base64,")
+/**
+ *
+ * @param {Array} base64Strings
+ * @returns
+ */
+// 解码 Base64 字符串并合并二进制数据
+export const mergeBase64WavFiles = base64Strings => {
+    const binaryDataArray = base64Strings.map(base64 => {
+        return Uint8Array.from(atob(base64), c => c.charCodeAt(0));
+    });
+
+    const totalLength = binaryDataArray.reduce((sum, arr) => sum + arr.length, 0);
+
+    const mergedArray = new Uint8Array(totalLength);
+    let offset = 0;
+
+    binaryDataArray.forEach(arr => {
+        mergedArray.set(arr, offset);
+        offset += arr.length;
+    });
+
+    // 重新编码为 Base64 字符串
+    const binaryString = String.fromCharCode(...mergedArray);
+    const mergedBase64 = btoa(binaryString);
+
+    return mergedBase64;
+};

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/components/mergeMp3Base64.js

@@ -0,0 +1,29 @@
+const base64ToArrayBuffer = base64 => {
+    let binaryString = atob(base64);
+    let len = binaryString.length;
+    let bytes = new Uint8Array(len);
+    for (let i = 0; i < len; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    return bytes.buffer;
+};
+
+const concatenateArrayBuffers = buffers => {
+    let totalLength = buffers.reduce((acc, value) => acc + value.byteLength, 0);
+    let result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (let buffer of buffers) {
+        result.set(new Uint8Array(buffer), offset);
+        offset += buffer.byteLength;
+    }
+    return result.buffer;
+};
+
+export const mergeMp3Base64ToBlob = base64Strings => {
+    let arrayBuffers = base64Strings.map(base64ToArrayBuffer);
+    let combinedArrayBuffer = concatenateArrayBuffers(arrayBuffers);
+    const blob = new Blob([combinedArrayBuffer], { type: 'audio/mp3' });
+    const url = URL.createObjectURL(blob);
+    console.log('url', url);
+    return url;
+};

r1-a/response_generation/minicpm/MiniCPM-o/web_demos/minicpm-o_2.6/web_server/src/views/home/index.vue

@@ -0,0 +1,262 @@
+<template>
+    <div class="home-page">
+        <div class="home-page-header">
+            <div class="home-page-header-logo">
+                <!-- <img src="@/assets/images/logo.png" /> -->
+                <SvgIcon name="miniCPM2.6" class="logo-icon" />
+            </div>
+            <div class="home-page-header-menu">
+                <div
+                    class="home-page-header-menu-item"
+                    v-for="(item, index) in tabList"
+                    :key="item.type"
+                    :class="`home-page-header-menu-item ${activeTab === item.type ? 'active-tab' : ''} ${item.disabled ? 'disabled-tab' : ''}`"
+                    @click="handleClickTab(item.type, index)"
+                >
+                    {{ getMenuTab(item.type) }}
+                </div>
+            </div>
+
+            <div class="home-page-header-switch">
+                <div class="change-language">
+                    <div
+                        :class="`change-language-item ${language === 'en' ? 'active-language' : ''}`"
+                        @click="handleChangeLanguage('en')"
+                    >
+                        English
+                    </div>
+                    <div
+                        :class="`change-language-item ${language === 'zh' ? 'active-language' : ''}`"
+                        @click="handleChangeLanguage('zh')"
+                    >
+                        中文
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div :class="`home-page-content ${activeTab === 'chatbot' && 'no-padding'}`">
+            <VoiceCallWs v-if="isWebSocket && activeTab === 'voice'" v-model="isCalling" />
+            <VoiceCall v-else-if="!isWebSocket && activeTab === 'voice'" v-model="isCalling" />
+            <VideoCallWs v-else-if="isWebSocket && activeTab === 'video'" v-model="isCalling" />
+            <VideoCall v-else-if="!isWebSocket && activeTab === 'video'" v-model="isCalling" />
+            <!-- TODO: https is required to support chatbot in iframe -->
+            <iframe
+                src="http://127.0.0.1:8000/"
+                frameborder="0"
+                width="100%"
+                height="100%"
+                v-else
+            />
+            <div class="config-box" v-if="activeTab !== 'chatbot'">
+                <ModelConfig v-model:isCalling="isCalling" v-model:type="activeTab" />
+            </div>
+        </div>
+    </div>
+</template>
+
+<script setup>
+    import VoiceCall from './components/VoiceCall.vue';
+    import VoiceCallWs from './components/VoiceCall_0105.vue';
+    import VideoCall from './components/VideoCall.vue';
+    import VideoCallWs from './components/VideoCall_0105.vue';
+    import { useI18n } from 'vue-i18n';
+    import { useRoute, useRouter } from 'vue-router';
+
+    const route = useRoute();
+    const router = useRouter();
+    const typeObj = {
+        0: 'video',
+        1: 'voice',
+        2: 'chatbot'
+    };
+    const defaultType = typeObj[route.query.type] || 'voice';
+
+    const { t, locale } = useI18n();
+    const activeTab = ref(defaultType);
+    const language = ref(localStorage.getItem('language') || 'zh');
+    const isWebSocket = false;
+    const tabList = ref([
+        {
+            type: 'video',
+            text: 'Realtime Video Call'
+        },
+        {
+            type: 'voice',
+            text: 'Realtime Voice Call'
+        },
+        {
+            type: 'chatbot',
+            text: 'Chatbot'
+            // disabled: true
+        }
+    ]);
+    const isCalling = ref(false);
+    const handleChangeLanguage = val => {
+        console.log('val: ', val);
+        language.value = val;
+        locale.value = val;
+        localStorage.setItem('language', val);
+    };
+    const getMenuTab = val => {
+        let text = '';
+        switch (val) {
+            case 'video':
+                text = t('menuTabVideo');
+                break;
+            case 'voice':
+                text = t('menuTabAudio');
+                break;
+            case 'chatbot':
+                text = t('menuTabChatbot');
+                break;
+            default:
+                break;
+        }
+        return text;
+    };
+    const handleClickTab = (val, index) => {
+        activeTab.value = val;
+        const port = route.query.port;
+        const type = index;
+        router.push({
+            path: '/',
+            query: {
+                port,
+                type
+            }
+        });
+    };
+</script>
+
+<style lang="less" scoped>
+    .home-page {
+        width: 100%;
+        height: 100%;
+        display: flex;
+        flex-direction: column;
+        &-header {
+            display: flex;
+            align-items: center;
+            &-logo {
+                width: 174px;
+                height: 46px;
+                display: flex;
+                align-items: center;
+                justify-content: center;
+                border-radius: 12px;
+                background: #ffffff;
+                flex-shrink: 0;
+                padding: 0 24px;
+                .logo-icon {
+                    width: 100%;
+                    height: 100%;
+                }
+            }
+            &-menu {
+                display: flex;
+                align-items: center;
+                margin-left: 16px;
+                &-item {
+                    width: 260px;
+                    height: 46px;
+                    display: flex;
+                    align-items: center;
+                    justify-content: center;
+                    background: #ffffff;
+                    color: #252525;
+                    font-family: PingFang SC;
+                    font-size: 16px;
+                    font-style: normal;
+                    font-weight: 400;
+                    line-height: normal;
+                    border: 1px solid #dde1eb;
+                    cursor: pointer;
+                    user-select: none;
+                }
+                &-item + &-item {
+                    border-left: none;
+                }
+                &-item:first-of-type {
+                    border-radius: 12px 0 0 12px;
+                }
+                &-item:last-of-type {
+                    border-radius: 0 12px 12px 0;
+                }
+                .active-tab {
+                    color: #ffffff;
+                    background: linear-gradient(90deg, #789efe 0.02%, #647fff 75.28%);
+                    font-weight: 500;
+                }
+                .disabled-tab {
+                    cursor: not-allowed;
+                    border-color: #dde1eb;
+                    color: #d1d1d1;
+                }
+            }
+            &-switch {
+                flex: 1;
+                display: flex;
+                align-items: center;
+                justify-content: flex-end;
+                .change-language {
+                    display: flex;
+                    align-items: center;
+                    &-item {
+                        width: 80px;
+                        height: 32px;
+                        display: flex;
+                        justify-content: center;
+                        align-items: center;
+                        border: 1px solid #dde1eb;
+                        background: #ffffff;
+                        color: #252525;
+                        font-family: PingFang SC;
+                        font-size: 14px;
+                        font-weight: 400;
+                        line-height: normal;
+                        cursor: pointer;
+                        user-select: none;
+                    }
+                    &-item:first-of-type {
+                        border-right: none;
+                        border-radius: 12px 0 0 12px;
+                    }
+                    &-item:last-of-type {
+                        border-radius: 0 12px 12px 0;
+                    }
+                    &-item.active-language {
+                        color: #ffffff;
+                        background: linear-gradient(90deg, #789efe 0.02%, #647fff 75.28%);
+                    }
+                }
+            }
+        }
+        &-content {
+            flex: 1;
+            height: 0;
+            border-radius: 12px;
+            margin-top: 16px;
+            background: #ffffff;
+            padding: 18px;
+            display: flex;
+            .config-box {
+                width: 322px;
+                margin-left: 16px;
+                // border-left: 1px solid black;
+                box-shadow: -0.5px 0 0 0 #e0e0e0;
+                overflow: auto;
+            }
+        }
+        .no-padding {
+            padding: 0;
+            overflow: hidden;
+            background: #ffffff;
+        }
+    }
+</style>
+<style lang="less">
+    .el-popover.el-popper.config-popover {
+        padding: 18px;
+        border-radius: 12px;
+    }
+</style>