Files
kunthawat 261b0f3e91 feat: wire Opus decode + VAD turn detection to firmware-compatible reply
Device streams continuous 20ms Opus frames with no turn markers. The loop
now decodes Opus, feeds the VAD, fires one turn on end-of-speech, and
replies with the firmware JSON contract (tts.state / stt.text) + Opus audio.

- app/audio/opus.py: RealOpus/Opus16kEncoder/PassthroughOpus; fix add_dll_directory
  to use absolute paths (WinError 87)
- app/xiaozhi/websocket.py: DeviceLoop.run() = decode -> VAD -> run_turn -> reply
- app/config.py: AudioConfig (opus kind + VAD thresholds)
- config.yaml: audio.opus=real, VAD 5/8/3000 frames
- tests/test_app.py: utterance = loud burst + silence; asserts stt + audio + tts stop
- requirements.txt: opuslib>=3

21/21 tests; live handshake verified via zhi.moreminimore.com on venv python.
2026-10-03 20:35:38 +07:00

86 lines
2.4 KiB
Python

"""Event/message frames the bridge sends to the device (text, not audio).
These drive the device UI: "listening" / "thinking" / "speaking" states and
transcript display. FIRMWARE: exact field names may vary by build.
"""
from __future__ import annotations
from typing import Optional
class MsgType:
LISTENING = "listening"
THINKING = "thinking"
SPEAKING = "speaking"
TEXT = "text" # assistant transcript to display
STT = "stt" # user transcript to display
TTS = "tts" # assistant text + optional audio flag
STATE = "state" # listening / speaking state transitions
def state(msg: str) -> dict:
return {"type": MsgType.STATE, "state": msg}
def text(msg: str, role: str = "assistant") -> dict:
return {"type": MsgType.TEXT, "role": role, "text": msg}
def stt(text: str) -> dict:
return {"type": MsgType.STT, "text": text}
def tts(text: str, audio: bool = True) -> dict:
return {"type": MsgType.TTS, "text": text, "audio": audio}
def listen_start() -> dict:
return state("listening")
def speak_start() -> dict:
return state("speaking")
def thinking() -> dict:
return state("thinking")
# ---------------------------------------------------------------------------
# Firmware reply shapes (xiaozhi-esp32 v2.5.0, verified in application.cc):
# the device understands type "stt" (text), "tts" (state start /
# sentence_start + text / stop) and "llm" (emotion). "state"/"text" above
# are kept for synthetic clients that display transcripts.
# ---------------------------------------------------------------------------
def tts_start() -> dict:
return {"type": MsgType.TTS, "state": "start"}
def tts_sentence_start(text: str) -> dict:
return {"type": MsgType.TTS, "state": "sentence_start", "text": text}
def tts_stop() -> dict:
return {"type": MsgType.TTS, "state": "stop"}
def llm(emotion: str) -> dict:
return {"type": "llm", "emotion": emotion}
def build_message(kind: str, text: str = "", **extra) -> dict:
"""Small dispatcher used by the session to emit device-facing messages."""
if kind == "text":
return text(text)
if kind == "stt":
return stt(text)
if kind == "tts":
return tts(text)
if kind == "listening":
return listen_start()
if kind == "speaking":
return speak_start()
if kind == "thinking":
return thinking()
raise ValueError(f"unknown message kind: {kind}")