diff --git a/Containerfile b/Containerfile
index 87c865e..3f8efb8 100644
--- a/Containerfile
+++ b/Containerfile
@@ -38,6 +38,19 @@ RUN python -m venv --system-site-packages /opt/venvs/xtts && \
/opt/venvs/xtts/bin/pip install --upgrade pip && \
/opt/venvs/xtts/bin/pip install -r /build/envs/xtts.txt
+# vibevoice는 pip 패키지가 아니라 GitHub 소스를 직접 빌드해야 함(커스텀 모델링 코드,
+# transformers Auto* 레지스트리에 없음). 특정 커밋에 고정해 업스트림 변경에 흔들리지 않게 함.
+RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* && \
+ git clone https://github.com/microsoft/VibeVoice.git /opt/vibevoice_src && \
+ cd /opt/vibevoice_src && git checkout 303b2833e01cff4578ec278bbfe536da54bd19fe
+
+# vibevoice venv — base image의 torch/CUDA 상속. fastapi/uvicorn/python-multipart는
+# vibevoice의 pyproject.toml(직접 의존성 또는 gradio 경유 간접 의존성)으로 이미 호환되는
+# 버전이 설치되므로 별도 requirements 파일 불필요.
+RUN python -m venv --system-site-packages /opt/venvs/vibevoice && \
+ /opt/venvs/vibevoice/bin/pip install --upgrade pip && \
+ /opt/venvs/vibevoice/bin/pip install -e /opt/vibevoice_src
+
COPY app/ /app/
RUN chmod +x /app/start.sh
diff --git a/app/asr/config.py b/app/asr/config.py
index 7c66d06..beb5edd 100644
--- a/app/asr/config.py
+++ b/app/asr/config.py
@@ -19,6 +19,7 @@ DEFAULT_LANGUAGE = env_str("DEFAULT_LANGUAGE", "ko")
FASTER_WHISPER_URL = env_str("FASTER_WHISPER_URL", "http://127.0.0.1:8001")
QWEN3_URL = env_str("QWEN3_URL", "http://127.0.0.1:8004")
+VIBEVOICE_URL = env_str("VIBEVOICE_URL", "http://127.0.0.1:8006")
PYANNOTE_HF_TOKEN = env_str("PYANNOTE_HF_TOKEN", "")
diff --git a/app/asr/router.py b/app/asr/router.py
index 4117295..1165784 100644
--- a/app/asr/router.py
+++ b/app/asr/router.py
@@ -18,6 +18,7 @@ from asr.config import (
DEFAULT_MODEL,
FASTER_WHISPER_URL,
QWEN3_URL,
+ VIBEVOICE_URL,
RESULT_DIR,
UPLOAD_DIR,
ensure_runtime_dirs,
@@ -29,6 +30,7 @@ ws_router = APIRouter()
BACKENDS = {
"faster-whisper": FASTER_WHISPER_URL,
"qwen3": QWEN3_URL,
+ "vibevoice": VIBEVOICE_URL,
}
REALTIME_SAMPLE_RATE = 16000
@@ -50,6 +52,9 @@ def config() -> Dict[str, Any]:
"qwen3": {
"models": ["Qwen/Qwen3-ASR-0.6B-hf", "Qwen/Qwen3-ASR-1.7B-hf"],
},
+ "vibevoice": {
+ "models": ["microsoft/VibeVoice-ASR"],
+ },
},
}
@@ -168,6 +173,10 @@ async def transcribe(
"language": effective_language,
"task": task,
}
+ elif backend == "vibevoice":
+ data = {
+ "model": model,
+ }
else:
# faster-whisper
data = {
diff --git a/app/asr/workers/vibevoice_worker.py b/app/asr/workers/vibevoice_worker.py
new file mode 100644
index 0000000..5a1707e
--- /dev/null
+++ b/app/asr/workers/vibevoice_worker.py
@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import argparse
+import gc
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import uvicorn
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.responses import JSONResponse
+
+import sys
+sys.path.insert(0, "/app")
+from asr.config import DEVICE, MODEL_CACHE, ensure_runtime_dirs
+
+import os
+# 컨테이너는 983 유저로 실행되는데 HOME(/app)이 root 소유라 numba/matplotlib/torch가
+# 각자 ~/.cache, ~/.config 아래에 쓰려다 실패한다. HOME을 쓰기 가능한 곳으로 돌린다.
+os.environ["HOME"] = "/tmp" # 컨테이너 기본 HOME=/app은 983 유저가 쓰기 불가 (setdefault로는 덮어쓰기 안 됨)
+os.environ.setdefault("NUMBA_CACHE_DIR", "/tmp/numba_cache")
+
+app = FastAPI(title="ASR VibeVoice Worker")
+
+_MODEL_CACHE: Dict[str, Any] = {}
+
+
+def _log(msg: str) -> None:
+ print(f"[vibevoice] {msg}", flush=True)
+
+
+def _load_model(model_id: str) -> Any:
+ if model_id in _MODEL_CACHE:
+ return _MODEL_CACHE[model_id]
+
+ import torch
+ from vibevoice.modular.modeling_vibevoice_asr import VibeVoiceASRForConditionalGeneration
+ from vibevoice.processor.vibevoice_asr_processor import VibeVoiceASRProcessor
+
+ _log(f"loading model {model_id}")
+ processor = VibeVoiceASRProcessor.from_pretrained(
+ model_id,
+ language_model_pretrained_name="Qwen/Qwen2.5-1.5B",
+ cache_dir=str(MODEL_CACHE),
+ )
+ model = VibeVoiceASRForConditionalGeneration.from_pretrained(
+ model_id,
+ dtype=torch.bfloat16 if DEVICE == "cuda" else torch.float32,
+ attn_implementation="sdpa",
+ trust_remote_code=True,
+ cache_dir=str(MODEL_CACHE),
+ )
+ model.to(DEVICE)
+ model.eval()
+
+ _MODEL_CACHE[model_id] = (model, processor)
+ _log(f"model {model_id} loaded")
+ return _MODEL_CACHE[model_id]
+
+
+@app.on_event("startup")
+def startup() -> None:
+ ensure_runtime_dirs()
+
+
+@app.get("/health")
+def health() -> Dict[str, Any]:
+ return {"status": "ok", "device": DEVICE, "loaded_models": list(_MODEL_CACHE.keys())}
+
+
+@app.post("/transcribe")
+async def transcribe(
+ file: UploadFile = File(...),
+ model: str = Form("microsoft/VibeVoice-ASR"),
+ max_new_tokens: int = Form(4096),
+) -> JSONResponse:
+ ensure_runtime_dirs()
+ suffix = Path(file.filename or "audio.bin").suffix or ".wav"
+
+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+ tmp.write(await file.read())
+ tmp_path = Path(tmp.name)
+
+ try:
+ import torch
+
+ vv_model, processor = _load_model(model)
+
+ inputs = processor(
+ audio=[str(tmp_path)],
+ sampling_rate=None,
+ return_tensors="pt",
+ padding=True,
+ add_generation_prompt=True,
+ )
+ inputs = {k: v.to(DEVICE) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
+
+ gen_config = {
+ "max_new_tokens": max_new_tokens,
+ "pad_token_id": processor.pad_id,
+ "eos_token_id": processor.tokenizer.eos_token_id,
+ "do_sample": False,
+ }
+
+ _log(f"transcribing model={model}")
+ with torch.no_grad():
+ output_ids = vv_model.generate(**inputs, **gen_config)
+
+ input_length = inputs["input_ids"].shape[1]
+ generated_ids = output_ids[0, input_length:]
+ raw_text = processor.decode(generated_ids, skip_special_tokens=True)
+
+ try:
+ raw_segments = processor.post_process_transcription(raw_text)
+ except Exception as e:
+ _log(f"post_process_transcription failed: {e}")
+ raw_segments = []
+
+ segments: List[Dict[str, Any]] = []
+ full_text_parts: List[str] = []
+ for i, seg in enumerate(raw_segments):
+ text = (seg.get("text") or "").strip()
+ speaker_id = seg.get("speaker_id")
+ start = seg.get("start_time")
+ end = seg.get("end_time")
+ segments.append({
+ "id": i,
+ "start": round(float(start), 3) if start is not None else None,
+ "end": round(float(end), 3) if end is not None else None,
+ "text": text,
+ "speaker": f"SPEAKER_{speaker_id:02d}" if speaker_id is not None else "UNKNOWN",
+ })
+ if text:
+ full_text_parts.append(text)
+
+ duration = segments[-1]["end"] if segments and segments[-1]["end"] is not None else None
+ full_text = " ".join(full_text_parts)
+
+ return JSONResponse({
+ "backend": "vibevoice",
+ "model": model,
+ "language": None,
+ "duration": duration,
+ "text": full_text,
+ "segments": segments,
+ "diarized": True,
+ })
+
+ except HTTPException:
+ raise
+ except Exception as e:
+ _log(f"error: {type(e).__name__}: {e}")
+ raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}")
+ finally:
+ tmp_path.unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--host", default="0.0.0.0")
+ parser.add_argument("--port", type=int, default=8006)
+ args = parser.parse_args()
+ _log(f"starting host={args.host} port={args.port} device={DEVICE}")
+ uvicorn.run(app, host=args.host, port=args.port)
diff --git a/app/supervisord.conf b/app/supervisord.conf
index ba164fe..7c05282 100644
--- a/app/supervisord.conf
+++ b/app/supervisord.conf
@@ -26,6 +26,17 @@ stderr_logfile=/dev/fd/2
stderr_logfile_maxbytes=0
priority=20
+[program:vibevoice]
+command=/opt/venvs/vibevoice/bin/python /app/asr/workers/vibevoice_worker.py --host 0.0.0.0 --port 8006
+directory=/app
+autostart=true
+autorestart=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0
+priority=25
+
[program:xtts]
command=/opt/venvs/xtts/bin/python /app/tts/workers/xtts_worker.py --host 0.0.0.0 --port 8005
directory=/app
diff --git a/app/ui/assets/js/asr.js b/app/ui/assets/js/asr.js
index e7c2a64..1b6dd63 100644
--- a/app/ui/assets/js/asr.js
+++ b/app/ui/assets/js/asr.js
@@ -14,6 +14,7 @@ async function loadConfig() {
backends: {
'faster-whisper': { models: ['tiny','base','small','medium','large-v3','large-v2','turbo'] },
'qwen3': { models: ['Qwen/Qwen3-ASR-0.6B-hf','Qwen/Qwen3-ASR-1.7B-hf'] },
+ 'vibevoice': { models: ['microsoft/VibeVoice-ASR'] },
},
};
}
@@ -52,8 +53,8 @@ function populateModels(backend) {
}
function onBackendChange() {
- const isQwen = backendSel.value === 'qwen3';
- fwOptions.style.display = isQwen ? 'none' : '';
+ const isFasterWhisper = backendSel.value === 'faster-whisper';
+ fwOptions.style.display = isFasterWhisper ? '' : 'none';
populateModels(backendSel.value);
}
diff --git a/app/ui/index.html b/app/ui/index.html
index f8a78d6..651c8c2 100644
--- a/app/ui/index.html
+++ b/app/ui/index.html
@@ -47,6 +47,7 @@