Fix broken file transcription and speaker diarization

qwen3 backend: model list pointed at nonexistent HF repos (Qwen3-ASR-2B/8B
don't exist), and inference went through the generic HF ASR pipeline with
Whisper-only options (return_timestamps, task/language generate_kwargs) that
Qwen3-ASR's chat-style architecture doesn't support. Switched to the real
0.6B/1.7B-hf checkpoints and drive them via
processor.apply_transcription_request() + model.generate(). Also added an
ffmpeg pre-conversion step since the model's feature extractor can't decode
m4a via librosa.

faster-whisper backend: speaker diarization was broken by two pyannote.audio
API changes it hadn't caught up with (use_auth_token= renamed to token=, and
pipeline() now returns a DiarizeOutput wrapper instead of an Annotation
directly). Also pinned LD_LIBRARY_PATH for that worker so it picks up its own
venv's cuDNN instead of the older one shadowing it via the container's global
LD_LIBRARY_PATH, which crashed pyannote's GPU init.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

app/asr/router.py

@@ -48,7 +48,7 @@ def config() -> Dict[str, Any]:
                 "models": ["tiny", "base", "small", "medium", "large-v3", "large-v2", "turbo"],
             },
             "qwen3": {
-                "models": ["Qwen/Qwen3-ASR-2B", "Qwen/Qwen3-ASR-8B"],
+                "models": ["Qwen/Qwen3-ASR-0.6B-hf", "Qwen/Qwen3-ASR-1.7B-hf"],
             },
         },
     }

app/asr/workers/faster_whisper_worker.py

@@ -209,7 +209,7 @@ def _get_diarization_pipeline():
             raise RuntimeError("화자 분리를 사용하려면 PYANNOTE_HF_TOKEN 환경변수를 설정하세요.")
         _DIARIZATION_PIPELINE = Pipeline.from_pretrained(
             "pyannote/speaker-diarization-3.1",
-            use_auth_token=PYANNOTE_HF_TOKEN,
+            token=PYANNOTE_HF_TOKEN,
         )
         if DEVICE == "cuda":
             import torch
@@ -234,7 +234,9 @@ def _apply_diarization(
         if max_speakers is not None:
             kwargs["max_speakers"] = max_speakers
 
-    diarization = pipeline(str(audio_path), **kwargs)
+    # pyannote.audio 4.x: pipeline() returns a DiarizeOutput dataclass instead of
+    # an Annotation directly; the itertracks-capable Annotation is .speaker_diarization.
+    diarization = pipeline(str(audio_path), **kwargs).speaker_diarization
     segments = result.get("segments", [])
     for seg in segments:
         seg_start = float(seg.get("start", 0))

app/asr/workers/qwen3_worker.py

@@ -2,9 +2,10 @@ from __future__ import annotations
 
 import argparse
 import gc
+import subprocess
 import tempfile
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional, Tuple
 
 import uvicorn
 from fastapi import FastAPI, File, Form, HTTPException, UploadFile
@@ -22,7 +23,11 @@ os.environ.setdefault("NUMBA_CACHE_DIR", "/tmp/numba_cache")
 
 app = FastAPI(title="ASR Qwen3 Worker")
 
-_PIPE_CACHE: Dict[str, Any] = {}
+# model_id -> (model, processor). Qwen3-ASR is a chat-style audio LM, not a
+# Whisper/CTC model, so it cannot go through the generic HF ASR `pipeline()`
+# (no return_timestamps, no task/language generate_kwargs). Load it directly
+# and drive it through Qwen3ASRProcessor.apply_transcription_request().
+_MODEL_CACHE: Dict[str, Tuple[Any, Any]] = {}
 
 LANG_MAP = {
     "ko": "korean", "en": "english", "ja": "japanese", "zh": "chinese",
@@ -51,47 +56,43 @@ def _free_gpu(*objs: Any) -> None:
         pass
 
 
-def _load_pipe(model_id: str) -> Any:
+def _to_wav(src_path: Path) -> Path:
-    if model_id in _PIPE_CACHE:
+    """librosa/soundfile (used by Qwen3ASRFeatureExtractor) can't decode m4a/mp3/etc,
-        return _PIPE_CACHE[model_id]
+    so normalize everything to 16kHz mono WAV via ffmpeg before handing it to the
+    processor, the same way faster-whisper does internally."""
+    wav_path = src_path.with_suffix(".conv.wav")
+    result = subprocess.run(
+        ["ffmpeg", "-y", "-i", str(src_path), "-ar", "16000", "-ac", "1", "-f", "wav", str(wav_path)],
+        capture_output=True,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"ffmpeg conversion failed: {result.stderr.decode(errors='replace')[-500:]}")
+    return wav_path
+
+
+def _load_model(model_id: str) -> Tuple[Any, Any]:
+    if model_id in _MODEL_CACHE:
+        return _MODEL_CACHE[model_id]
 
     import torch
-    from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
+    from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
 
     _log(f"loading model {model_id}")
     dtype = torch.float16 if DEVICE == "cuda" else torch.float32
 
-    try:
+    processor = AutoProcessor.from_pretrained(model_id, cache_dir=str(MODEL_CACHE))
-        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+    model = AutoModelForSpeechSeq2Seq.from_pretrained(
-            model_id,
+        model_id,
-            torch_dtype=dtype,
+        dtype=dtype,
-            low_cpu_mem_usage=True,
+        low_cpu_mem_usage=True,
-            cache_dir=str(MODEL_CACHE),
+        cache_dir=str(MODEL_CACHE),
-        )
+    )
-        model.to(DEVICE)
+    model.to(DEVICE)
-        processor = AutoProcessor.from_pretrained(model_id, cache_dir=str(MODEL_CACHE))
+    model.eval()
-        pipe = pipeline(
-            "automatic-speech-recognition",
-            model=model,
-            tokenizer=processor.tokenizer,
-            feature_extractor=processor.feature_extractor,
-            torch_dtype=dtype,
-            device=DEVICE,
-        )
-    except Exception:
-        # Fallback: let pipeline handle loading (uses device_map)
-        _log("direct load failed, trying pipeline auto-load")
-        pipe = pipeline(
-            "automatic-speech-recognition",
-            model=model_id,
-            torch_dtype=torch.float16 if DEVICE == "cuda" else torch.float32,
-            device_map="auto" if DEVICE == "cuda" else None,
-            model_kwargs={"cache_dir": str(MODEL_CACHE)},
-        )
 
-    _PIPE_CACHE[model_id] = pipe
+    _MODEL_CACHE[model_id] = (model, processor)
     _log(f"model {model_id} loaded")
-    return pipe
+    return _MODEL_CACHE[model_id]
 
 
 @app.on_event("startup")
@@ -101,15 +102,14 @@ def startup() -> None:
 
 @app.get("/health")
 def health() -> Dict[str, Any]:
-    return {"status": "ok", "device": DEVICE, "loaded_models": list(_PIPE_CACHE.keys())}
+    return {"status": "ok", "device": DEVICE, "loaded_models": list(_MODEL_CACHE.keys())}
 
 
 @app.post("/transcribe")
 async def transcribe(
     file: UploadFile = File(...),
-    model: str = Form("Qwen/Qwen3-ASR-2B"),
+    model: str = Form("Qwen/Qwen3-ASR-1.7B-hf"),
     language: Optional[str] = Form("ko"),
-    task: str = Form("transcribe"),
 ) -> JSONResponse:
     ensure_runtime_dirs()
     suffix = Path(file.filename or "audio.bin").suffix or ".wav"
@@ -118,49 +118,36 @@ async def transcribe(
         tmp.write(await file.read())
         tmp_path = Path(tmp.name)
 
+    wav_path: Optional[Path] = None
     try:
-        pipe = _load_pipe(model)
+        import torch
 
-        lang_code = (language or "ko").strip().lower()
+        model_obj, processor = _load_model(model)
-        lang_name = LANG_MAP.get(lang_code, lang_code)
 
-        generate_kwargs: Dict[str, Any] = {"task": task}
+        lang_code = (language or "").strip().lower()
-        if lang_code:
+        lang_name = LANG_MAP.get(lang_code, lang_code) if lang_code else None
-            generate_kwargs["language"] = lang_name
 
-        _log(f"transcribing language={lang_code} model={model}")
+        _log(f"transcribing language={lang_code or 'auto'} model={model}")
-        raw = pipe(
-            str(tmp_path),
-            return_timestamps=True,
-            generate_kwargs=generate_kwargs,
-        )
 
-        # raw may be {"text": "...", "chunks": [...]} or {"text": "..."}
+        wav_path = _to_wav(tmp_path)
-        full_text: str = raw.get("text", "").strip()
+        inputs = processor.apply_transcription_request(audio=str(wav_path), language=lang_name)
-        chunks: List[Dict[str, Any]] = raw.get("chunks", [])
+        inputs = inputs.to(model_obj.device, dtype=model_obj.dtype)
+        prompt_len = inputs["input_ids"].shape[1]
 
-        segments = []
+        with torch.no_grad():
-        for i, chunk in enumerate(chunks):
+            generated = model_obj.generate(**inputs)
-            ts = chunk.get("timestamp") or (None, None)
-            start = float(ts[0]) if ts[0] is not None else None
-            end = float(ts[1]) if ts[1] is not None else None
-            segments.append({
-                "id": i,
-                "start": round(start, 3) if start is not None else None,
-                "end": round(end, 3) if end is not None else None,
-                "text": chunk.get("text", "").strip(),
-            })
 
-        # Estimate duration from last segment end
+        parsed = processor.decode(generated[0][prompt_len:], return_format="parsed")
-        duration = None
+        full_text = (parsed.get("transcription") or "").strip()
-        if segments and segments[-1]["end"] is not None:
+        detected_language = parsed.get("language") or lang_code or None
-            duration = segments[-1]["end"]
+
+        segments = [{"id": 0, "start": None, "end": None, "text": full_text}] if full_text else []
 
         return JSONResponse({
             "backend": "qwen3",
             "model": model,
-            "language": lang_code,
+            "language": detected_language,
-            "duration": duration,
+            "duration": None,
             "text": full_text,
             "segments": segments,
             "diarized": False,
@@ -173,6 +160,8 @@ async def transcribe(
         raise HTTPException(status_code=500, detail=f"{type(e).__name__}: {e}")
     finally:
         tmp_path.unlink(missing_ok=True)
+        if wav_path is not None:
+            wav_path.unlink(missing_ok=True)
 
 
 if __name__ == "__main__":

app/supervisord.conf

@@ -6,6 +6,7 @@ pidfile=/tmp/supervisord.pid
 [program:faster_whisper]
 command=/opt/venvs/faster_whisper/bin/python /app/asr/workers/faster_whisper_worker.py --host 0.0.0.0 --port 8001
 directory=/app
+environment=LD_LIBRARY_PATH="/opt/venvs/faster_whisper/lib/python3.11/site-packages/nvidia/cudnn/lib:%(ENV_LD_LIBRARY_PATH)s"
 autostart=true
 autorestart=true
 stdout_logfile=/dev/fd/1

app/ui/assets/js/asr.js

@@ -13,7 +13,7 @@ async function loadConfig() {
       default_language: 'ko',
       backends: {
         'faster-whisper': { models: ['tiny','base','small','medium','large-v3','large-v2','turbo'] },
-        'qwen3': { models: ['Qwen/Qwen3-ASR-2B','Qwen/Qwen3-ASR-8B'] },
+        'qwen3': { models: ['Qwen/Qwen3-ASR-0.6B-hf','Qwen/Qwen3-ASR-1.7B-hf'] },
       },
     };
   }