Self-restart worker process when model loading fails

Discovered vibevoice sitting at 3.2GB resident GPU memory with
loaded_models: [] and no idle-unload ever firing for it again. Root cause:
a load attempt had OOM'd partway through (competing with an unrelated
ollama process on the same GPU), so _MODEL_CACHE was never populated —
the idle-unload loop only clears that cache, so it had nothing to act on,
even though the partially-constructed model had already left memory
allocated. gc.collect()+empty_cache() don't reliably reclaim memory from
an interrupted from_pretrained() call.

Confirmed a plain process restart does fully reclaim it, so each worker
now treats any load failure as fatal: log it and os._exit(1), letting
supervisord's autorestart=true respawn a clean process immediately.
Verified with a bogus model id — worker exits, respawns, and passes the
smoke test right after.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Containerfile

@@ -38,6 +38,19 @@ RUN python -m venv --system-site-packages /opt/venvs/xtts && \
     /opt/venvs/xtts/bin/pip install --upgrade pip && \
     /opt/venvs/xtts/bin/pip install -r /build/envs/xtts.txt
 
+# vibevoice는 pip 패키지가 아니라 GitHub 소스를 직접 빌드해야 함(커스텀 모델링 코드,
+# transformers Auto* 레지스트리에 없음). 특정 커밋에 고정해 업스트림 변경에 흔들리지 않게 함.
+RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* && \
+    git clone https://github.com/microsoft/VibeVoice.git /opt/vibevoice_src && \
+    cd /opt/vibevoice_src && git checkout 303b2833e01cff4578ec278bbfe536da54bd19fe
+
+# vibevoice venv — base image의 torch/CUDA 상속. fastapi/uvicorn/python-multipart는
+# vibevoice의 pyproject.toml(직접 의존성 또는 gradio 경유 간접 의존성)으로 이미 호환되는
+# 버전이 설치되므로 별도 requirements 파일 불필요.
+RUN python -m venv --system-site-packages /opt/venvs/vibevoice && \
+    /opt/venvs/vibevoice/bin/pip install --upgrade pip && \
+    /opt/venvs/vibevoice/bin/pip install -e /opt/vibevoice_src
+
 COPY app/ /app/
 
 RUN chmod +x /app/start.sh

app/asr/config.py

@@ -19,6 +19,7 @@ DEFAULT_LANGUAGE = env_str("DEFAULT_LANGUAGE", "ko")
 
 FASTER_WHISPER_URL = env_str("FASTER_WHISPER_URL", "http://127.0.0.1:8001")
 QWEN3_URL = env_str("QWEN3_URL", "http://127.0.0.1:8004")
+VIBEVOICE_URL = env_str("VIBEVOICE_URL", "http://127.0.0.1:8006")
 
 PYANNOTE_HF_TOKEN = env_str("PYANNOTE_HF_TOKEN", "")
 

app/asr/router.py

@@ -18,6 +18,7 @@ from asr.config import (
     DEFAULT_MODEL,
     FASTER_WHISPER_URL,
     QWEN3_URL,
+    VIBEVOICE_URL,
     RESULT_DIR,
     UPLOAD_DIR,
     ensure_runtime_dirs,
@@ -29,6 +30,7 @@ ws_router = APIRouter()
 BACKENDS = {
     "faster-whisper": FASTER_WHISPER_URL,
     "qwen3": QWEN3_URL,
+    "vibevoice": VIBEVOICE_URL,
 }
 
 REALTIME_SAMPLE_RATE = 16000
@@ -48,7 +50,10 @@ def config() -> Dict[str, Any]:
                 "models": ["tiny", "base", "small", "medium", "large-v3", "large-v2", "turbo"],
             },
             "qwen3": {
-                "models": ["Qwen/Qwen3-ASR-2B", "Qwen/Qwen3-ASR-8B"],
+                "models": ["Qwen/Qwen3-ASR-0.6B-hf", "Qwen/Qwen3-ASR-1.7B-hf"],
+            },
+            "vibevoice": {
+                "models": ["microsoft/VibeVoice-ASR"],
             },
         },
     }
@@ -168,6 +173,10 @@ async def transcribe(
                 "language": effective_language,
                 "task": task,
             }
+        elif backend == "vibevoice":
+            data = {
+                "model": model,
+            }
         else:
             # faster-whisper
             data = {

app/asr/workers/faster_whisper_worker.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
 import argparse
+import asyncio
+import gc
 import tempfile
+import time
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
 
@@ -32,10 +35,55 @@ app = FastAPI(title="ASR Faster-Whisper Worker")
 _MODEL_CACHE: Dict[str, WhisperModel] = {}
 _DIARIZATION_PIPELINE: Any = None
 
+# Idle-unload / model-switch eviction: only one whisper model stays resident
+# at a time (switching sizes frees the old one), and everything (including
+# the diarization pipeline) is dropped after IDLE_UNLOAD_SECONDS of no
+# requests, so this backend doesn't permanently hog GPU memory shared with
+# the other ASR workers.
+IDLE_UNLOAD_SECONDS = 120
+_last_used: float = 0.0
+_active_requests: int = 0
+
+
+def _free_gpu() -> None:
+    gc.collect()
+    try:
+        import torch
+        if DEVICE == "cuda" and torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+
+
+def _unload_whisper_models() -> None:
+    if not _MODEL_CACHE:
+        return
+    print(f"[faster_whisper] unloading {list(_MODEL_CACHE.keys())}", flush=True)
+    _MODEL_CACHE.clear()
+    _free_gpu()
+
+
+def _unload_diarization() -> None:
+    global _DIARIZATION_PIPELINE
+    if _DIARIZATION_PIPELINE is None:
+        return
+    print("[faster_whisper] unloading diarization pipeline", flush=True)
+    _DIARIZATION_PIPELINE = None
+    _free_gpu()
+
+
+async def _idle_unload_loop() -> None:
+    while True:
+        await asyncio.sleep(30)
+        if _active_requests == 0 and _last_used and (time.monotonic() - _last_used) >= IDLE_UNLOAD_SECONDS:
+            _unload_whisper_models()
+            _unload_diarization()
+
 
 @app.on_event("startup")
-def startup() -> None:
+async def startup() -> None:
     ensure_runtime_dirs()
+    asyncio.create_task(_idle_unload_loop())
 
 
 @app.get("/health")
@@ -75,6 +123,8 @@ async def transcribe(
         tmp.write(await file.read())
         tmp_path = Path(tmp.name)
 
+    global _active_requests, _last_used
+    _active_requests += 1
     try:
         resolved_model = resolve_custom_model_path(custom_model_path) or model
         result = _transcribe(
@@ -107,16 +157,33 @@ async def transcribe(
         raise HTTPException(status_code=500, detail=f"Worker failure: {exc}") from exc
     finally:
         tmp_path.unlink(missing_ok=True)
+        _free_gpu()
+        _active_requests -= 1
+        _last_used = time.monotonic()
 
 
 def _load_model(model_name: str) -> WhisperModel:
     if model_name not in _MODEL_CACHE:
+        if _MODEL_CACHE:
+            # Only one whisper model resident at a time — switching sizes frees the old one.
+            _unload_whisper_models()
+        try:
             _MODEL_CACHE[model_name] = WhisperModel(
                 model_name,
                 device=DEVICE,
                 compute_type=COMPUTE_TYPE,
                 download_root=str(MODEL_CACHE),
             )
+        except Exception as e:
+            # A load interrupted partway (e.g. OOM) can leave CUDA memory
+            # fragmented/leaked in ways gc.collect()+empty_cache() don't
+            # reliably reclaim, and since _MODEL_CACHE never got populated
+            # the idle-unload loop has nothing to clean up either.
+            # Restarting the whole process is the only guaranteed way to get
+            # that memory back — supervisord's autorestart=true respawns it
+            # immediately.
+            print(f"[faster_whisper] model load failed, restarting process to reclaim GPU memory: {type(e).__name__}: {e}", flush=True)
+            os._exit(1)
     return _MODEL_CACHE[model_name]
 
 
@@ -209,7 +276,7 @@ def _get_diarization_pipeline():
             raise RuntimeError("화자 분리를 사용하려면 PYANNOTE_HF_TOKEN 환경변수를 설정하세요.")
         _DIARIZATION_PIPELINE = Pipeline.from_pretrained(
             "pyannote/speaker-diarization-3.1",
-            use_auth_token=PYANNOTE_HF_TOKEN,
+            token=PYANNOTE_HF_TOKEN,
         )
         if DEVICE == "cuda":
             import torch
@@ -234,7 +301,9 @@ def _apply_diarization(
         if max_speakers is not None:
             kwargs["max_speakers"] = max_speakers
 
-    diarization = pipeline(str(audio_path), **kwargs)
+    # pyannote.audio 4.x: pipeline() returns a DiarizeOutput dataclass instead of
+    # an Annotation directly; the itertracks-capable Annotation is .speaker_diarization.
+    diarization = pipeline(str(audio_path), **kwargs).speaker_diarization
     segments = result.get("segments", [])
     for seg in segments:
         seg_start = float(seg.get("start", 0))

app/asr/workers/qwen3_worker.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
 import argparse
+import asyncio
 import gc
+import subprocess
 import tempfile
+import time
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional, Tuple
 
 import uvicorn
 from fastapi import FastAPI, File, Form, HTTPException, UploadFile
@@ -22,7 +25,19 @@ os.environ.setdefault("NUMBA_CACHE_DIR", "/tmp/numba_cache")
 
 app = FastAPI(title="ASR Qwen3 Worker")
 
-_PIPE_CACHE: Dict[str, Any] = {}
+# model_id -> (model, processor). Qwen3-ASR is a chat-style audio LM, not a
+# Whisper/CTC model, so it cannot go through the generic HF ASR `pipeline()`
+# (no return_timestamps, no task/language generate_kwargs). Load it directly
+# and drive it through Qwen3ASRProcessor.apply_transcription_request().
+_MODEL_CACHE: Dict[str, Tuple[Any, Any]] = {}
+
+# Idle-unload / model-switch eviction: only one model stays resident at a
+# time, and the whole cache is dropped after IDLE_UNLOAD_SECONDS of no
+# requests, so this backend doesn't permanently hog GPU memory shared with
+# the other ASR workers.
+IDLE_UNLOAD_SECONDS = 120
+_last_used: float = 0.0
+_active_requests: int = 0
 
 LANG_MAP = {
     "ko": "korean", "en": "english", "ja": "japanese", "zh": "chinese",
@@ -51,65 +66,90 @@ def _free_gpu(*objs: Any) -> None:
         pass
 
 
-def _load_pipe(model_id: str) -> Any:
-    if model_id in _PIPE_CACHE:
-        return _PIPE_CACHE[model_id]
+def _to_wav(src_path: Path) -> Path:
+    """librosa/soundfile (used by Qwen3ASRFeatureExtractor) can't decode m4a/mp3/etc,
+    so normalize everything to 16kHz mono WAV via ffmpeg before handing it to the
+    processor, the same way faster-whisper does internally."""
+    wav_path = src_path.with_suffix(".conv.wav")
+    result = subprocess.run(
+        ["ffmpeg", "-y", "-i", str(src_path), "-ar", "16000", "-ac", "1", "-f", "wav", str(wav_path)],
+        capture_output=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"ffmpeg conversion failed: {result.stderr.decode(errors='replace')[-500:]}")
+    return wav_path
+
+
+def _unload_models() -> None:
+    if not _MODEL_CACHE:
+        return
+    _log(f"unloading {list(_MODEL_CACHE.keys())}")
+    _MODEL_CACHE.clear()
+    _free_gpu()
+
+
+async def _idle_unload_loop() -> None:
+    while True:
+        await asyncio.sleep(30)
+        if _active_requests == 0 and _last_used and (time.monotonic() - _last_used) >= IDLE_UNLOAD_SECONDS:
+            _unload_models()
+
+
+def _load_model(model_id: str) -> Tuple[Any, Any]:
+    if model_id in _MODEL_CACHE:
+        return _MODEL_CACHE[model_id]
+
+    if _MODEL_CACHE:
+        # Only one model resident at a time — switching models frees the old one.
+        _unload_models()
 
     import torch
-    from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+    from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
 
     _log(f"loading model {model_id}")
     dtype = torch.float16 if DEVICE == "cuda" else torch.float32
 
     try:
+        processor = AutoProcessor.from_pretrained(model_id, cache_dir=str(MODEL_CACHE))
         model = AutoModelForSpeechSeq2Seq.from_pretrained(
             model_id,
-            torch_dtype=dtype,
+            dtype=dtype,
             low_cpu_mem_usage=True,
             cache_dir=str(MODEL_CACHE),
         )
         model.to(DEVICE)
-        processor = AutoProcessor.from_pretrained(model_id, cache_dir=str(MODEL_CACHE))
-        pipe = pipeline(
-            "automatic-speech-recognition",
-            model=model,
-            tokenizer=processor.tokenizer,
-            feature_extractor=processor.feature_extractor,
-            torch_dtype=dtype,
-            device=DEVICE,
-        )
-    except Exception:
-        # Fallback: let pipeline handle loading (uses device_map)
-        _log("direct load failed, trying pipeline auto-load")
-        pipe = pipeline(
-            "automatic-speech-recognition",
-            model=model_id,
-            torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32,
-            device_map="auto" if DEVICE == "cuda" else None,
-            model_kwargs={"cache_dir": str(MODEL_CACHE)},
-        )
+        model.eval()
+    except Exception as e:
+        # A load interrupted partway (e.g. OOM) can leave CUDA memory
+        # fragmented/leaked in ways gc.collect()+empty_cache() don't reliably
+        # reclaim, and since _MODEL_CACHE never got populated the idle-unload
+        # loop has nothing to clean up either. Restarting the whole process
+        # is the only guaranteed way to get that memory back — supervisord's
+        # autorestart=true respawns it immediately.
+        _log(f"model load failed, restarting process to reclaim GPU memory: {type(e).__name__}: {e}")
+        os._exit(1)
 
-    _PIPE_CACHE[model_id] = pipe
+    _MODEL_CACHE[model_id] = (model, processor)
     _log(f"model {model_id} loaded")
-    return pipe
+    return _MODEL_CACHE[model_id]
 
 
 @app.on_event("startup")
-def startup() -> None:
+async def startup() -> None:
     ensure_runtime_dirs()
+    asyncio.create_task(_idle_unload_loop())
 
 
 @app.get("/health")
 def health() -> Dict[str, Any]:
-    return {"status": "ok", "device": DEVICE, "loaded_models": list(_PIPE_CACHE.keys())}
+    return {"status": "ok", "device": DEVICE, "loaded_models": list(_MODEL_CACHE.keys())}
 
 
 @app.post("/transcribe")
 async def transcribe(
     file: UploadFile = File(...),
-    model: str = Form("Qwen/Qwen3-ASR-2B"),
+    model: str = Form("Qwen/Qwen3-ASR-1.7B-hf"),
     language: Optional[str] = Form("ko"),
-    task: str = Form("transcribe"),
 ) -> JSONResponse:
     ensure_runtime_dirs()
     suffix = Path(file.filename or "audio.bin").suffix or ".wav"
@@ -118,49 +158,38 @@ async def transcribe(
         tmp.write(await file.read())
         tmp_path = Path(tmp.name)
 
+    global _active_requests, _last_used
+    _active_requests += 1
+    wav_path: Optional[Path] = None
     try:
-        pipe = _load_pipe(model)
+        import torch
 
-        lang_code = (language or "ko").strip().lower()
-        lang_name = LANG_MAP.get(lang_code, lang_code)
+        model_obj, processor = _load_model(model)
 
-        generate_kwargs: Dict[str, Any] = {"task": task}
-        if lang_code:
-            generate_kwargs["language"] = lang_name
+        lang_code = (language or "").strip().lower()
+        lang_name = LANG_MAP.get(lang_code, lang_code) if lang_code else None
 
-        _log(f"transcribing language={lang_code} model={model}")
-        raw = pipe(
-            str(tmp_path),
-            return_timestamps=True,
-            generate_kwargs=generate_kwargs,
-        )
+        _log(f"transcribing language={lang_code or 'auto'} model={model}")
 
-        # raw may be {"text": "...", "chunks": [...]} or {"text": "..."}
-        full_text: str = raw.get("text", "").strip()
-        chunks: List[Dict[str, Any]] = raw.get("chunks", [])
+        wav_path = _to_wav(tmp_path)
+        inputs = processor.apply_transcription_request(audio=str(wav_path), language=lang_name)
+        inputs = inputs.to(model_obj.device, dtype=model_obj.dtype)
+        prompt_len = inputs["input_ids"].shape[1]
 
-        segments = []
-        for i, chunk in enumerate(chunks):
-            ts = chunk.get("timestamp") or (None, None)
-            start = float(ts[0]) if ts[0] is not None else None
-            end = float(ts[1]) if ts[1] is not None else None
-            segments.append({
-                "id": i,
-                "start": round(start, 3) if start is not None else None,
-                "end": round(end, 3) if end is not None else None,
-                "text": chunk.get("text", "").strip(),
-            })
+        with torch.no_grad():
+            generated = model_obj.generate(**inputs)
 
-        # Estimate duration from last segment end
-        duration = None
-        if segments and segments[-1]["end"] is not None:
-            duration = segments[-1]["end"]
+        parsed = processor.decode(generated[0][prompt_len:], return_format="parsed")
+        full_text = (parsed.get("transcription") or "").strip()
+        detected_language = parsed.get("language") or lang_code or None
+
+        segments = [{"id": 0, "start": None, "end": None, "text": full_text}] if full_text else []
 
         return JSONResponse({
             "backend": "qwen3",
             "model": model,
-            "language": lang_code,
-            "duration": duration,
+            "language": detected_language,
+            "duration": None,
             "text": full_text,
             "segments": segments,
             "diarized": False,
@@ -173,6 +202,23 @@ async def transcribe(
         raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}")
     finally:
         tmp_path.unlink(missing_ok=True)
+        if wav_path is not None:
+            wav_path.unlink(missing_ok=True)
+        # Long files blow up KV-cache/activation memory; without this, that
+        # memory stays reserved by this process and starves the other backends
+        # sharing the same GPU until the container restarts. (del locals()[...]
+        # does NOT work in CPython, hence the explicit names.)
+        try:
+            del inputs
+        except NameError:
+            pass
+        try:
+            del generated
+        except NameError:
+            pass
+        _free_gpu()
+        _active_requests -= 1
+        _last_used = time.monotonic()
 
 
 if __name__ == "__main__":

app/asr/workers/vibevoice_worker.py

@@ -0,0 +1,235 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import gc
+import tempfile
+import time
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import uvicorn
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.responses import JSONResponse
+
+import sys
+sys.path.insert(0, "/app")
+from asr.config import DEVICE, MODEL_CACHE, ensure_runtime_dirs
+
+import os
+# 컨테이너는 983 유저로 실행되는데 HOME(/app)이 root 소유라 numba/matplotlib/torch가
+# 각자 ~/.cache, ~/.config 아래에 쓰려다 실패한다. HOME을 쓰기 가능한 곳으로 돌린다.
+os.environ["HOME"] = "/tmp"  # 컨테이너 기본 HOME=/app은 983 유저가 쓰기 불가 (setdefault로는 덮어쓰기 안 됨)
+os.environ.setdefault("NUMBA_CACHE_DIR", "/tmp/numba_cache")
+
+app = FastAPI(title="ASR VibeVoice Worker")
+
+_MODEL_CACHE: Dict[str, Any] = {}
+
+# Idle-unload / model-switch eviction: only one model stays resident at a
+# time, and the whole cache is dropped after IDLE_UNLOAD_SECONDS of no
+# requests, so this backend doesn't permanently hog GPU memory shared with
+# the other ASR workers.
+IDLE_UNLOAD_SECONDS = 120
+_last_used: float = 0.0
+_active_requests: int = 0
+
+
+def _log(msg: str) -> None:
+    print(f"[vibevoice] {msg}", flush=True)
+
+
+def _free_gpu() -> None:
+    gc.collect()
+    try:
+        import torch
+        if DEVICE == "cuda" and torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+
+
+def _unload_models() -> None:
+    if not _MODEL_CACHE:
+        return
+    _log(f"unloading {list(_MODEL_CACHE.keys())}")
+    _MODEL_CACHE.clear()
+    _free_gpu()
+
+
+async def _idle_unload_loop() -> None:
+    while True:
+        await asyncio.sleep(30)
+        if _active_requests == 0 and _last_used and (time.monotonic() - _last_used) >= IDLE_UNLOAD_SECONDS:
+            _unload_models()
+
+
+def _load_model(model_id: str) -> Any:
+    if model_id in _MODEL_CACHE:
+        return _MODEL_CACHE[model_id]
+
+    if _MODEL_CACHE:
+        # Only one model resident at a time — switching models frees the old one.
+        _unload_models()
+
+    import torch
+    from vibevoice.modular.modeling_vibevoice_asr import VibeVoiceASRForConditionalGeneration
+    from vibevoice.processor.vibevoice_asr_processor import VibeVoiceASRProcessor
+
+    _log(f"loading model {model_id}")
+    try:
+        processor = VibeVoiceASRProcessor.from_pretrained(
+            model_id,
+            language_model_pretrained_name="Qwen/Qwen2.5-1.5B",
+            cache_dir=str(MODEL_CACHE),
+        )
+        model = VibeVoiceASRForConditionalGeneration.from_pretrained(
+            model_id,
+            dtype=torch.bfloat16 if DEVICE == "cuda" else torch.float32,
+            attn_implementation="sdpa",
+            trust_remote_code=True,
+            cache_dir=str(MODEL_CACHE),
+        )
+        model.to(DEVICE)
+        model.eval()
+    except Exception as e:
+        # A load interrupted partway (e.g. OOM) can leave CUDA memory
+        # fragmented/leaked in ways gc.collect()+empty_cache() don't reliably
+        # reclaim, and since _MODEL_CACHE never got populated the idle-unload
+        # loop has nothing to clean up either. Restarting the whole process
+        # is the only guaranteed way to get that memory back — supervisord's
+        # autorestart=true respawns it immediately.
+        _log(f"model load failed, restarting process to reclaim GPU memory: {type(e).__name__}: {e}")
+        os._exit(1)
+
+    _MODEL_CACHE[model_id] = (model, processor)
+    _log(f"model {model_id} loaded")
+    return _MODEL_CACHE[model_id]
+
+
+@app.on_event("startup")
+async def startup() -> None:
+    ensure_runtime_dirs()
+    asyncio.create_task(_idle_unload_loop())
+
+
+@app.get("/health")
+def health() -> Dict[str, Any]:
+    return {"status": "ok", "device": DEVICE, "loaded_models": list(_MODEL_CACHE.keys())}
+
+
+@app.post("/transcribe")
+async def transcribe(
+    file: UploadFile = File(...),
+    model: str = Form("microsoft/VibeVoice-ASR"),
+    max_new_tokens: int = Form(4096),
+) -> JSONResponse:
+    ensure_runtime_dirs()
+    suffix = Path(file.filename or "audio.bin").suffix or ".wav"
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+        tmp.write(await file.read())
+        tmp_path = Path(tmp.name)
+
+    global _active_requests, _last_used
+    _active_requests += 1
+    try:
+        import torch
+
+        vv_model, processor = _load_model(model)
+
+        inputs = processor(
+            audio=[str(tmp_path)],
+            sampling_rate=None,
+            return_tensors="pt",
+            padding=True,
+            add_generation_prompt=True,
+        )
+        inputs = {k: v.to(DEVICE) if isinstance(v, torch.Tensor) else v for k, v in inputs.items()}
+
+        gen_config = {
+            "max_new_tokens": max_new_tokens,
+            "pad_token_id": processor.pad_id,
+            "eos_token_id": processor.tokenizer.eos_token_id,
+            "do_sample": False,
+        }
+
+        _log(f"transcribing model={model}")
+        with torch.no_grad():
+            output_ids = vv_model.generate(**inputs, **gen_config)
+
+        input_length = inputs["input_ids"].shape[1]
+        generated_ids = output_ids[0, input_length:]
+        raw_text = processor.decode(generated_ids, skip_special_tokens=True)
+
+        try:
+            raw_segments = processor.post_process_transcription(raw_text)
+        except Exception as e:
+            _log(f"post_process_transcription failed: {e}")
+            raw_segments = []
+
+        segments: List[Dict[str, Any]] = []
+        full_text_parts: List[str] = []
+        for i, seg in enumerate(raw_segments):
+            text = (seg.get("text") or "").strip()
+            speaker_id = seg.get("speaker_id")
+            start = seg.get("start_time")
+            end = seg.get("end_time")
+            segments.append({
+                "id": i,
+                "start": round(float(start), 3) if start is not None else None,
+                "end": round(float(end), 3) if end is not None else None,
+                "text": text,
+                "speaker": f"SPEAKER_{speaker_id:02d}" if speaker_id is not None else "UNKNOWN",
+            })
+            if text:
+                full_text_parts.append(text)
+
+        duration = segments[-1]["end"] if segments and segments[-1]["end"] is not None else None
+        full_text = " ".join(full_text_parts)
+
+        return JSONResponse({
+            "backend": "vibevoice",
+            "model": model,
+            "language": None,
+            "duration": duration,
+            "text": full_text,
+            "segments": segments,
+            "diarized": True,
+        })
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        _log(f"error: {type(e).__name__}: {e}")
+        raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}")
+    finally:
+        tmp_path.unlink(missing_ok=True)
+        # Long files blow up KV-cache/activation memory; without this, that
+        # memory stays reserved by this process and starves the other backends
+        # sharing the same GPU until the container restarts. (del locals()[...]
+        # does NOT work in CPython, hence the explicit names.)
+        try:
+            del inputs
+        except NameError:
+            pass
+        try:
+            del output_ids
+        except NameError:
+            pass
+        try:
+            del generated_ids
+        except NameError:
+            pass
+        _free_gpu()
+        _active_requests -= 1
+        _last_used = time.monotonic()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="0.0.0.0")
+    parser.add_argument("--port", type=int, default=8006)
+    args = parser.parse_args()
+    _log(f"starting host={args.host} port={args.port} device={DEVICE}")
+    uvicorn.run(app, host=args.host, port=args.port)

app/supervisord.conf

@@ -6,6 +6,7 @@ pidfile=/tmp/supervisord.pid
 [program:faster_whisper]
 command=/opt/venvs/faster_whisper/bin/python /app/asr/workers/faster_whisper_worker.py --host 0.0.0.0 --port 8001
 directory=/app
+environment=LD_LIBRARY_PATH="/opt/venvs/faster_whisper/lib/python3.11/site-packages/nvidia/cudnn/lib:%(ENV_LD_LIBRARY_PATH)s"
 autostart=true
 autorestart=true
 stdout_logfile=/dev/fd/1
@@ -25,6 +26,17 @@ stderr_logfile=/dev/fd/2
 stderr_logfile_maxbytes=0
 priority=20
 
+[program:vibevoice]
+command=/opt/venvs/vibevoice/bin/python /app/asr/workers/vibevoice_worker.py --host 0.0.0.0 --port 8006
+directory=/app
+autostart=true
+autorestart=true
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/fd/2
+stderr_logfile_maxbytes=0
+priority=25
+
 [program:xtts]
 command=/opt/venvs/xtts/bin/python /app/tts/workers/xtts_worker.py --host 0.0.0.0 --port 8005
 directory=/app

app/tts/workers/xtts_worker.py

@@ -1,8 +1,11 @@
 from __future__ import annotations
 
 import argparse
+import asyncio
+import gc
 import os
 import tempfile
+import time
 from pathlib import Path
 from typing import Any, Dict
 
@@ -28,6 +31,13 @@ app = FastAPI(title="TTS XTTS Worker")
 MODEL_NAME = "tts_models/multilingual/multi-dataset/xtts_v2"
 _MODEL_CACHE: Dict[str, Any] = {}
 
+# Idle-unload: the model is dropped after IDLE_UNLOAD_SECONDS of no synthesis
+# requests, so this backend doesn't permanently hog GPU memory shared with
+# the ASR workers.
+IDLE_UNLOAD_SECONDS = 120
+_last_used: float = 0.0
+_active_requests: int = 0
+
 
 def _log(msg: str) -> None:
     print(f"[xtts] {msg}", flush=True)
@@ -41,6 +51,31 @@ def _device() -> str:
         return "cpu"
 
 
+def _free_gpu() -> None:
+    gc.collect()
+    try:
+        import torch
+        if torch.cuda.is_available():
+            torch.cuda.empty_cache()
+    except Exception:
+        pass
+
+
+def _unload_model() -> None:
+    if not _MODEL_CACHE:
+        return
+    _log("unloading model (idle)")
+    _MODEL_CACHE.clear()
+    _free_gpu()
+
+
+async def _idle_unload_loop() -> None:
+    while True:
+        await asyncio.sleep(30)
+        if _active_requests == 0 and _last_used and (time.monotonic() - _last_used) >= IDLE_UNLOAD_SECONDS:
+            _unload_model()
+
+
 def _load_model() -> Any:
     if MODEL_NAME in _MODEL_CACHE:
         return _MODEL_CACHE[MODEL_NAME]
@@ -48,15 +83,26 @@ def _load_model() -> Any:
     from TTS.api import TTS
 
     _log(f"loading model {MODEL_NAME}")
+    try:
         tts = TTS(MODEL_NAME).to(_device())
+    except Exception as e:
+        # A load interrupted partway (e.g. OOM) can leave CUDA memory
+        # fragmented/leaked in ways gc.collect()+empty_cache() don't reliably
+        # reclaim, and since _MODEL_CACHE never got populated the idle-unload
+        # loop has nothing to clean up either. Restarting the whole process
+        # is the only guaranteed way to get that memory back — supervisord's
+        # autorestart=true respawns it immediately.
+        _log(f"model load failed, restarting process to reclaim GPU memory: {type(e).__name__}: {e}")
+        os._exit(1)
     _MODEL_CACHE[MODEL_NAME] = tts
     _log("model loaded")
     return tts
 
 
 @app.on_event("startup")
-def startup() -> None:
+async def startup() -> None:
     ensure_runtime_dirs()
+    asyncio.create_task(_idle_unload_loop())
 
 
 @app.get("/health")
@@ -74,10 +120,14 @@ def synthesize(
     if not speaker_path.exists():
         raise HTTPException(status_code=400, detail=f"speaker_wav not found: {speaker_wav}")
 
+    global _active_requests, _last_used
+    _active_requests += 1
     try:
         tts = _load_model()
     except Exception as e:
         _log(f"model load error: {type(e).__name__}: {e}")
+        _active_requests -= 1
+        _last_used = time.monotonic()
         raise HTTPException(status_code=500, detail=f"모델 로드 실패: {type(e).__name__}: {e}")
 
     with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
@@ -105,6 +155,9 @@ def synthesize(
         raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}")
     finally:
         out_path.unlink(missing_ok=True)
+        _free_gpu()
+        _active_requests -= 1
+        _last_used = time.monotonic()
 
 
 if __name__ == "__main__":

app/ui/assets/js/asr.js

@@ -7,15 +7,13 @@ async function loadConfig() {
     const r = await fetch('/asr/config');
     SERVER_CONFIG = await r.json();
   } catch (e) {
-    SERVER_CONFIG = {
-      default_backend: 'faster-whisper',
-      default_model: 'large-v3',
-      default_language: 'ko',
-      backends: {
-        'faster-whisper': { models: ['tiny','base','small','medium','large-v3','large-v2','turbo'] },
-        'qwen3': { models: ['Qwen/Qwen3-ASR-2B','Qwen/Qwen3-ASR-8B'] },
-      },
-    };
+    // No hardcoded fallback list here on purpose: the server's /asr/config is
+    // the single source of truth for backend/model names. A duplicated guess
+    // here would silently drift out of sync with it (this has already bitten
+    // us once — see qwen3 model IDs). Surface the failure instead.
+    SERVER_CONFIG = null;
+    const el = document.getElementById('file-status');
+    if (el) el.textContent = '설정을 불러오지 못했습니다. 페이지를 새로고침해주세요.';
   }
   initAsrUI();
 }
@@ -52,8 +50,8 @@ function populateModels(backend) {
 }
 
 function onBackendChange() {
-  const isQwen = backendSel.value === 'qwen3';
-  fwOptions.style.display = isQwen ? 'none' : '';
+  const isFasterWhisper = backendSel.value === 'faster-whisper';
+  fwOptions.style.display = isFasterWhisper ? '' : 'none';
   populateModels(backendSel.value);
 }
 

app/ui/index.html

@@ -47,6 +47,7 @@
                     <select id="backend" name="backend">
                       <option value="faster-whisper" selected>faster-whisper</option>
                       <option value="qwen3">Qwen3-ASR</option>
+                      <option value="vibevoice">VibeVoice-ASR</option>
                     </select>
                   </label>
                   <label>모델

envs/faster_whisper.txt

@@ -2,5 +2,5 @@ fastapi==0.115.0
 uvicorn[standard]==0.30.6
 python-multipart==0.0.9
 faster-whisper==1.1.1
-pyannote.audio>=3.1
+pyannote.audio==4.0.7
 numpy

envs/qwen3.txt

@@ -2,7 +2,7 @@
 fastapi==0.115.0
 uvicorn[standard]==0.30.6
 python-multipart==0.0.9
-transformers>=4.45.0
+transformers==5.14.1
 accelerate>=0.30.0
 librosa>=0.10.0
 soundfile>=0.12.0

scripts/smoke_test.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Minimal post-deploy smoke test. Hits every ASR/TTS worker's own endpoint
+# directly (bypassing the gateway's OIDC auth) with a synthetic test clip, to
+# catch startup/dependency/API-compat regressions before a user has to find
+# them (qwen3's wrong model IDs, pyannote's use_auth_token/token rename, and
+# the cuDNN LD_LIBRARY_PATH conflict all would have shown up here as an
+# immediate FAIL instead of a silent break discovered later).
+#
+# Run this after every `bash build.sh` + redeploy.
+set -uo pipefail
+
+CONTAINER="${SPEECH_CONTAINER:-speech}"
+TMP_WAV="/tmp/smoke_test_tone.wav"
+FAIL=0
+
+pass() { printf '  OK  %s\n' "$1"; }
+fail() { printf 'FAIL  %s: %s\n' "$1" "$2"; FAIL=1; }
+
+echo "=== generating synthetic test clip ==="
+podman exec "$CONTAINER" ffmpeg -y -f lavfi -i "sine=frequency=440:duration=3" -ar 16000 -ac 1 "$TMP_WAV" >/dev/null 2>&1
+
+echo "=== faster-whisper (8001) ==="
+resp=$(podman exec "$CONTAINER" curl -s -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:8001/transcribe \
+  -F "file=@${TMP_WAV};type=audio/wav" -F "model=tiny" -F "language=ko")
+[ "$resp" = "200" ] && pass "faster-whisper" || fail "faster-whisper" "HTTP $resp"
+
+echo "=== qwen3 (8004) ==="
+resp=$(podman exec "$CONTAINER" curl -s -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:8004/transcribe \
+  -F "file=@${TMP_WAV};type=audio/wav" -F "model=Qwen/Qwen3-ASR-0.6B-hf" -F "language=ko")
+[ "$resp" = "200" ] && pass "qwen3" || fail "qwen3" "HTTP $resp"
+
+echo "=== vibevoice (8006) ==="
+resp=$(podman exec "$CONTAINER" curl -s -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:8006/transcribe \
+  -F "file=@${TMP_WAV};type=audio/wav" -F "model=microsoft/VibeVoice-ASR")
+[ "$resp" = "200" ] && pass "vibevoice" || fail "vibevoice" "HTTP $resp"
+
+echo "=== xtts (8005) ==="
+resp=$(podman exec "$CONTAINER" curl -s -o /dev/null -w "%{http_code}" -X POST http://127.0.0.1:8005/synthesize \
+  -F "text=스모크 테스트입니다." -F "language=ko" -F "speaker_wav=${TMP_WAV}")
+[ "$resp" = "200" ] && pass "xtts" || fail "xtts" "HTTP $resp"
+
+podman exec "$CONTAINER" rm -f "$TMP_WAV"
+
+echo
+if [ "$FAIL" = "0" ]; then
+  echo "all backends OK"
+else
+  echo "one or more backends FAILED — check: podman logs $CONTAINER"
+fi
+exit "$FAIL"