first upload

Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,12 +1,12 @@
 ---
 title: Hokkien
-emoji: 🔥
-colorFrom: gray
-colorTo: indigo
+emoji: 👀
+colorFrom: green
+colorTo: pink
 sdk: docker
 pinned: false
 license: apache-2.0
-short_description: ' 台語語音辨識示範，使用 Wav2Vec2 模型將錄音轉成羅馬拼音；使用 whisper 模型將錄音轉成中文。'
+short_description: 台語語音辨識示範，使用 Wav2Vec2 模型將錄音轉成羅馬拼音；使用 whisper 模型將錄音轉成中文。
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,15 @@
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+
+from demo.app.api import router as api_router  # 注意這裡改成 app.api
+
+app = FastAPI()
+
+app.include_router(api_router)
+
+app.mount("/static", StaticFiles(directory="demo/static"), name="static")
+
+@app.get("/")
+async def index():
+    return FileResponse("demo/static/index.html")

demo/app/__init__.py

@@ -0,0 +1 @@
+# nothing

demo/app/api.py

@@ -0,0 +1,27 @@
+from fastapi import APIRouter, UploadFile
+from fastapi.responses import JSONResponse
+import os
+import tempfile
+import torch
+
+from demo.app.preprocess import preprocess_audio
+from demo.app.model_wav2vec2 import infer as wav2vec2_infer
+
+router = APIRouter()
+
+@router.post("/transcribe")
+async def transcribe(file: UploadFile):
+    suffix = os.path.splitext(file.filename)[1]
+    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+        tmp.write(await file.read())
+        tmp_path = tmp.name
+
+    waveform, sr = preprocess_audio(tmp_path)
+    os.remove(tmp_path)
+
+    wav2vec2_text, wav2vec2_conf = wav2vec2_infer(waveform.squeeze(0).cpu().numpy(), sr)
+
+    return JSONResponse({
+        "transcription": wav2vec2_text,
+        "confidence": round(wav2vec2_conf, 4)
+    })

demo/app/model_wav2vec2.py

@@ -0,0 +1,31 @@
+import torch
+from torch.nn.functional import softmax
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+model = Wav2Vec2ForCTC.from_pretrained("demo/my-wav2vec2").to(device).eval()
+processor = Wav2Vec2Processor.from_pretrained("demo/my-wav2vec2")
+
+def ctc_decode(token_ids, blank_id=0):
+    prev_token = None
+    decoded_ids = []
+    for token in token_ids:
+        if token != prev_token and token != blank_id:
+            decoded_ids.append(token)
+        prev_token = token
+    tokens = processor.tokenizer.convert_ids_to_tokens(decoded_ids)
+    return "".join(tokens).replace(processor.tokenizer.pad_token, "").strip()
+
+def infer(waveform, sample_rate):
+    inputs = processor(waveform, sampling_rate=sample_rate, return_tensors="pt").to(device)
+    with torch.no_grad():
+        logits = model(**inputs).logits
+    predicted_ids = torch.argmax(logits, dim=-1)[0].tolist()
+    transcription = ctc_decode(predicted_ids)
+
+    probs = softmax(logits, dim=-1)
+    confidences, _ = torch.max(probs, dim=-1)
+    confidence = confidences.mean().item()
+
+    return transcription, confidence

demo/app/preprocess.py

@@ -0,0 +1,28 @@
+import torch
+import soundfile as sf
+import os
+import torchaudio
+
+target_sample_rate = 16000
+
+def preprocess_audio(file_path):
+    suffix = os.path.splitext(file_path)[1]
+    if suffix == ".pt":
+        waveform = torch.load(file_path)
+        sr = target_sample_rate
+    else:
+        waveform, sr = sf.read(file_path)
+        waveform = torch.tensor(waveform).float()
+        if waveform.dim() == 1:
+            waveform = waveform.unsqueeze(0)
+        else:
+            waveform = waveform.permute(1, 0)
+
+    if waveform.shape[0] > 1:
+        waveform = waveform.mean(dim=0, keepdim=True)
+
+    if sr != target_sample_rate:
+        resampler = torchaudio.transforms.Resample(sr, target_sample_rate)
+        waveform = resampler(waveform)
+
+    return waveform, target_sample_rate

demo/my-wav2vec2/config.json

@@ -0,0 +1,117 @@
+{
+  "activation_dropout": 0.0,
+  "adapter_attn_dim": null,
+  "adapter_kernel_size": 3,
+  "adapter_stride": 2,
+  "add_adapter": false,
+  "apply_spec_augment": true,
+  "architectures": [
+    "Wav2Vec2ForCTC"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": 1,
+  "classifier_proj_size": 256,
+  "codevector_dim": 256,
+  "contrastive_logits_temperature": 0.1,
+  "conv_bias": false,
+  "conv_dim": [
+    512,
+    512,
+    512,
+    512,
+    512,
+    512,
+    512
+  ],
+  "conv_kernel": [
+    10,
+    3,
+    3,
+    3,
+    3,
+    2,
+    2
+  ],
+  "conv_stride": [
+    5,
+    2,
+    2,
+    2,
+    2,
+    2,
+    2
+  ],
+  "ctc_loss_reduction": "sum",
+  "ctc_zero_infinity": false,
+  "diversity_loss_weight": 0.1,
+  "do_stable_layer_norm": false,
+  "eos_token_id": 2,
+  "feat_extract_activation": "gelu",
+  "feat_extract_norm": "group",
+  "feat_proj_dropout": 0.1,
+  "feat_quantizer_dropout": 0.0,
+  "final_dropout": 0.0,
+  "freeze_feat_extract_train": true,
+  "hidden_act": "gelu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.0,
+  "mask_channel_length": 10,
+  "mask_channel_min_space": 1,
+  "mask_channel_other": 0.0,
+  "mask_channel_prob": 0.0,
+  "mask_channel_selection": "static",
+  "mask_feature_length": 10,
+  "mask_feature_min_masks": 0,
+  "mask_feature_prob": 0.0,
+  "mask_time_length": 10,
+  "mask_time_min_masks": 2,
+  "mask_time_min_space": 1,
+  "mask_time_other": 0.0,
+  "mask_time_prob": 0.05,
+  "mask_time_selection": "static",
+  "model_type": "wav2vec2",
+  "no_mask_channel_overlap": false,
+  "no_mask_time_overlap": false,
+  "num_adapter_layers": 3,
+  "num_attention_heads": 12,
+  "num_codevector_groups": 2,
+  "num_codevectors_per_group": 320,
+  "num_conv_pos_embedding_groups": 16,
+  "num_conv_pos_embeddings": 128,
+  "num_feat_extract_layers": 7,
+  "num_hidden_layers": 12,
+  "num_negatives": 100,
+  "output_hidden_size": 768,
+  "pad_token_id": 0,
+  "proj_codevector_dim": 256,
+  "tdnn_dilation": [
+    1,
+    2,
+    3,
+    1,
+    1
+  ],
+  "tdnn_dim": [
+    512,
+    512,
+    512,
+    512,
+    1500
+  ],
+  "tdnn_kernel": [
+    5,
+    3,
+    3,
+    1,
+    1
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.53.3",
+  "use_weighted_layer_sum": false,
+  "vocab_size": 99,
+  "xvector_output_dim": 512
+}

demo/my-wav2vec2/preprocessor_config.json

@@ -0,0 +1,10 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "Wav2Vec2Processor",
+  "return_attention_mask": false,
+  "sampling_rate": 16000
+}

demo/my-wav2vec2/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

demo/my-wav2vec2/tokenizer.json

@@ -0,0 +1,453 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 98,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Split",
+    "pattern": {
+      "String": ""
+    },
+    "behavior": "Isolated",
+    "invert": false
+  },
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "Unigram",
+    "unk_id": null,
+    "vocab": [
+      [
+        "<unk>",
+        0.0
+      ],
+      [
+        "<s>",
+        0.0
+      ],
+      [
+        "</s>",
+        0.0
+      ],
+      [
+        " ",
+        -2.0491303237853096
+      ],
+      [
+        "-",
+        -2.438041258277112
+      ],
+      [
+        "n",
+        -2.501524477266583
+      ],
+      [
+        "t",
+        -2.638661273777304
+      ],
+      [
+        "h",
+        -2.692965388541229
+      ],
+      [
+        "i",
+        -2.694754512457008
+      ],
+      [
+        "s",
+        -2.8923410887869334
+      ],
+      [
+        "k",
+        -3.1970415940597174
+      ],
+      [
+        "u",
+        -3.2651700304177123
+      ],
+      [
+        "g",
+        -3.4248376939813774
+      ],
+      [
+        "a",
+        -3.4963570859315585
+      ],
+      [
+        "á",
+        -3.9240834819779824
+      ],
+      [
+        "o",
+        -3.975795370165889
+      ],
+      [
+        "l",
+        -4.007994364937257
+      ],
+      [
+        "̍",
+        -4.170577777562061
+      ],
+      [
+        ".",
+        -4.189869429609686
+      ],
+      [
+        "â",
+        -4.25267258530512
+      ],
+      [
+        "ā",
+        -4.298943240036779
+      ],
+      [
+        "p",
+        -4.3723955360015605
+      ],
+      [
+        "à",
+        -4.461337584827133
+      ],
+      [
+        "b",
+        -4.466076323196452
+      ],
+      [
+        ",",
+        -4.499891555448983
+      ],
+      [
+        "í",
+        -4.578458027626528
+      ],
+      [
+        "m",
+        -4.5786016851469995
+      ],
+      [
+        "e",
+        -4.749594791761277
+      ],
+      [
+        "ó",
+        -4.759358486978165
+      ],
+      [
+        "ī",
+        -4.845970910394813
+      ],
+      [
+        "ì",
+        -4.851051839639437
+      ],
+      [
+        "ê",
+        -4.877819521265044
+      ],
+      [
+        "ē",
+        -4.927070017931555
+      ],
+      [
+        "ō",
+        -5.013998122557426
+      ],
+      [
+        "î",
+        -5.099728488704239
+      ],
+      [
+        "ū",
+        -5.347765015858414
+      ],
+      [
+        "ô",
+        -5.365594702921749
+      ],
+      [
+        "T",
+        -5.418076380746381
+      ],
+      [
+        "ò",
+        -5.444021231995655
+      ],
+      [
+        "I",
+        -5.644843649981011
+      ],
+      [
+        "ú",
+        -5.656596146044344
+      ],
+      [
+        "̄",
+        -5.816993865454794
+      ],
+      [
+        "è",
+        -5.825455837126889
+      ],
+      [
+        "L",
+        -5.997384198304952
+      ],
+      [
+        "j",
+        -6.023851907952154
+      ],
+      [
+        "é",
+        -6.0893458766159565
+      ],
+      [
+        "û",
+        -6.36828010754205
+      ],
+      [
+        "ù",
+        -6.395308778217399
+      ],
+      [
+        "K",
+        -6.453533494168924
+      ],
+      [
+        "G",
+        -6.49075152735316
+      ],
+      [
+        "?",
+        -6.808536997604716
+      ],
+      [
+        "H",
+        -6.961241358131142
+      ],
+      [
+        "S",
+        -7.013969214753503
+      ],
+      [
+        "“",
+        -7.060997180654681
+      ],
+      [
+        "”",
+        -7.060997180654681
+      ],
+      [
+        "!",
+        -7.1527881697284315
+      ],
+      [
+        "̂",
+        -7.164162696539445
+      ],
+      [
+        "ǹ",
+        -7.416567092235995
+      ],
+      [
+        "B",
+        -7.561557318255249
+      ],
+      [
+        "ń",
+        -7.616965035855269
+      ],
+      [
+        "P",
+        -7.724493393771001
+      ],
+      [
+        "A",
+        -7.790034021304488
+      ],
+      [
+        "M",
+        -8.091475968442492
+      ],
+      [
+        "N",
+        -8.58636336389603
+      ],
+      [
+        "Ū",
+        -8.677334622535016
+      ],
+      [
+        "J",
+        -8.836547433198561
+      ],
+      [
+        "U",
+        -9.428393546992902
+      ],
+      [
+        "À",
+        -9.976578745300651
+      ],
+      [
+        "O",
+        -10.076652901770805
+      ],
+      [
+        "Í",
+        -10.112367187485088
+      ],
+      [
+        ";",
+        -10.187865762983664
+      ],
+      [
+        "E",
+        -10.227865762983663
+      ],
+      [
+        "Ī",
+        -10.45608428359349
+      ],
+      [
+        "Â",
+        -10.623094947508182
+      ],
+      [
+        "Ô",
+        -10.685594947508188
+      ],
+      [
+        "Ē",
+        -10.685594947508188
+      ],
+      [
+        "Ā",
+        -10.752261614174865
+      ],
+      [
+        "Á",
+        -10.752261614174865
+      ],
+      [
+        "─",
+        -10.98394659586001
+      ],
+      [
+        "Î",
+        -11.920490607456603
+      ],
+      [
+        "ḿ",
+        -11.920490607456603
+      ],
+      [
+        "‘",
+        -12.170490607456603
+      ],
+      [
+        "…",
+        -12.170490607456603
+      ],
+      [
+        "’",
+        -12.170490607456603
+      ],
+      [
+        "Ì",
+        -12.503823940789935
+      ],
+      [
+        "0",
+        -12.503823940789935
+      ],
+      [
+        "Ê",
+        -12.503823940789935
+      ],
+      [
+        "2",
+        -13.003823940789935
+      ],
+      [
+        "1",
+        -13.003823940789935
+      ],
+      [
+        "Ó",
+        -13.003823940789935
+      ],
+      [
+        "9",
+        -13.003823940789935
+      ],
+      [
+        "Ǹ",
+        -13.003823940789935
+      ],
+      [
+        "̋",
+        -13.003823940789935
+      ],
+      [
+        "Ō",
+        -14.003823940789935
+      ],
+      [
+        "Ú",
+        -14.003823940789935
+      ],
+      [
+        "3",
+        -14.003823940789935
+      ],
+      [
+        " ",
+        -14.003823940789935
+      ],
+      [
+        "4",
+        -14.003823940789935
+      ]
+    ],
+    "byte_fallback": false
+  }
+}

demo/my-wav2vec2/tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "98": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "processor_class": "Wav2Vec2Processor",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}

demo/static/app.js

@@ -0,0 +1,136 @@
+// 上傳檔案並辨識
+async function uploadAudio() {
+    const fileInput = document.getElementById('audio');
+    if (!fileInput.files.length) {
+        alert('請先選擇音檔！');
+        return;
+    }
+    const formData = new FormData();
+    formData.append("file", fileInput.files[0]);
+
+    document.getElementById('result').innerText = '辨識中...';
+
+    const response = await fetch("/transcribe", {
+        method: "POST",
+        body: formData
+    });
+
+    const data = await response.json();
+    document.getElementById('result').innerText =
+        `辨識結果：${data.transcription}\n信心分數：${data.confidence}`;
+}
+
+// 錄音相關
+let recorder;
+let audioChunks = [];
+
+document.getElementById("record").onclick = async () => {
+    try {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+        let mimeType = "";
+        if (MediaRecorder.isTypeSupported("audio/mp4")) {
+            mimeType = "audio/mp4";
+        } else if (MediaRecorder.isTypeSupported("audio/webm")) {
+            mimeType = "audio/webm";
+        } else {
+            mimeType = "";
+        }
+
+        recorder = new MediaRecorder(stream, mimeType ? { mimeType } : {});
+        audioChunks = [];
+
+        recorder.ondataavailable = e => audioChunks.push(e.data);
+
+        recorder.onstart = () => {
+            document.getElementById("status").innerText = "錄音中...";
+            document.getElementById("record").disabled = true;
+            document.getElementById("stop").disabled = false;
+            document.getElementById("result").innerText = "";
+        };
+
+        recorder.onstop = async () => {
+            const webmBlob = new Blob(audioChunks, { type: "audio/webm" });
+            const arrayBuffer = await webmBlob.arrayBuffer();
+            const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+            const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+
+            // 轉 WAV
+            const wavBuffer = audioBufferToWav(audioBuffer);
+            const wavBlob = new Blob([wavBuffer], { type: "audio/wav" });
+
+            const formData = new FormData();
+            formData.append("file", wavBlob, "record.wav");
+
+            document.getElementById("status").innerText = "辨識中...";
+
+            const response = await fetch("/transcribe", {
+                method: "POST",
+                body: formData
+            });
+
+            const result = await response.json();
+            document.getElementById("status").innerText = "辨識完成";
+            document.getElementById("result").innerText =
+                `辨識結果：${result.transcription}\n信心分數：${result.confidence}`;
+        };
+
+        recorder.start();
+    } catch (e) {
+        alert("無法存取麥克風，請確認權限。");
+    }
+};
+
+document.getElementById("stop").onclick = () => {
+    recorder.stop();
+    document.getElementById("stop").disabled = true;
+    document.getElementById("record").disabled = false;
+    document.getElementById("status").innerText = "停止錄音，等待辨識...";
+};
+
+// Web Audio API -> WAV 轉換
+function audioBufferToWav(buffer) {
+    const numOfChan = buffer.numberOfChannels;
+    const length = buffer.length * numOfChan * 2 + 44;
+    const bufferArray = new ArrayBuffer(length);
+    const view = new DataView(bufferArray);
+    const channels = [];
+
+    function writeString(view, offset, string) {
+        for (let i = 0; i < string.length; i++) {
+            view.setUint8(offset + i, string.charCodeAt(i));
+        }
+    }
+
+    writeString(view, 0, 'RIFF');
+    view.setUint32(4, length - 8, true);
+    writeString(view, 8, 'WAVE');
+    writeString(view, 12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true);
+    view.setUint16(22, numOfChan, true);
+    view.setUint32(24, buffer.sampleRate, true);
+    view.setUint32(28, buffer.sampleRate * 2 * numOfChan, true);
+    view.setUint16(32, numOfChan * 2, true);
+    view.setUint16(34, 16, true);
+    writeString(view, 36, 'data');
+    view.setUint32(40, length - 44, true);
+
+    for (let i = 0; i < numOfChan; i++) {
+        channels.push(buffer.getChannelData(i));
+    }
+
+    let interleaved = new Float32Array(buffer.length * numOfChan);
+    for (let i = 0; i < buffer.length; i++) {
+        for (let ch = 0; ch < numOfChan; ch++) {
+            interleaved[i * numOfChan + ch] = channels[ch][i];
+        }
+    }
+
+    let index = 44;
+    for (let i = 0; i < interleaved.length; i++, index += 2) {
+        let s = Math.max(-1, Math.min(1, interleaved[i]));
+        view.setInt16(index, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+    }
+    return bufferArray;
+}

demo/static/index.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="zh">
+<head>
+    <meta charset="UTF-8">
+    <title>Wav2Vec2 台語羅馬拼音辨識 Demo</title>
+    <link rel="stylesheet" href="/static/style.css">
+</head>
+<body>
+    <h1>Wav2Vec2 台語羅馬拼音辨識 Demo</h1>
+    <p>請使用手機或電腦錄音，然後上傳音檔，等待辨識結果。</p>
+
+    <!-- 檔案上傳 -->
+    <input type="file" id="audio" accept="audio/*">
+    <button onclick="uploadAudio()">上傳並辨識</button>
+
+    <hr>
+
+    <!-- 錄音功能 -->
+    <button id="record">開始錄音</button>
+    <button id="stop" disabled>停止錄音</button>
+    <p id="status"></p>
+
+    <div id="result"></div>
+    <script src="/static/app.js"></script>
+</html>

demo/static/style.css

@@ -0,0 +1,17 @@
+body {
+    font-family: sans-serif;
+    margin: 40px;
+    text-align: center;
+}
+input[type="file"] {
+    margin: 20px 0;
+}
+button {
+    padding: 10px 20px;
+    margin: 5px;
+}
+#result {
+    margin-top: 20px;
+    font-size: 1.2em;
+    white-space: pre-line;
+}

requirements.txt

@@ -0,0 +1,7 @@
+fastapi
+uvicorn[standard]
+torch
+transformers
+torchaudio
+soundfile
+python-multipart