Device streams continuous 20ms Opus frames with no turn markers. The loop now decodes Opus, feeds the VAD, fires one turn on end-of-speech, and replies with the firmware JSON contract (tts.state / stt.text) + Opus audio. - app/audio/opus.py: RealOpus/Opus16kEncoder/PassthroughOpus; fix add_dll_directory to use absolute paths (WinError 87) - app/xiaozhi/websocket.py: DeviceLoop.run() = decode -> VAD -> run_turn -> reply - app/config.py: AudioConfig (opus kind + VAD thresholds) - config.yaml: audio.opus=real, VAD 5/8/3000 frames - tests/test_app.py: utterance = loud burst + silence; asserts stt + audio + tts stop - requirements.txt: opuslib>=3 21/21 tests; live handshake verified via zhi.moreminimore.com on venv python.
86 lines
2.4 KiB
Python
86 lines
2.4 KiB
Python
"""Event/message frames the bridge sends to the device (text, not audio).
|
|
|
|
These drive the device UI: "listening" / "thinking" / "speaking" states and
|
|
transcript display. FIRMWARE: exact field names may vary by build.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Optional
|
|
|
|
|
|
class MsgType:
|
|
LISTENING = "listening"
|
|
THINKING = "thinking"
|
|
SPEAKING = "speaking"
|
|
TEXT = "text" # assistant transcript to display
|
|
STT = "stt" # user transcript to display
|
|
TTS = "tts" # assistant text + optional audio flag
|
|
STATE = "state" # listening / speaking state transitions
|
|
|
|
|
|
def state(msg: str) -> dict:
|
|
return {"type": MsgType.STATE, "state": msg}
|
|
|
|
|
|
def text(msg: str, role: str = "assistant") -> dict:
|
|
return {"type": MsgType.TEXT, "role": role, "text": msg}
|
|
|
|
|
|
def stt(text: str) -> dict:
|
|
return {"type": MsgType.STT, "text": text}
|
|
|
|
|
|
def tts(text: str, audio: bool = True) -> dict:
|
|
return {"type": MsgType.TTS, "text": text, "audio": audio}
|
|
|
|
|
|
def listen_start() -> dict:
|
|
return state("listening")
|
|
|
|
|
|
def speak_start() -> dict:
|
|
return state("speaking")
|
|
|
|
|
|
def thinking() -> dict:
|
|
return state("thinking")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Firmware reply shapes (xiaozhi-esp32 v2.5.0, verified in application.cc):
|
|
# the device understands type "stt" (text), "tts" (state start /
|
|
# sentence_start + text / stop) and "llm" (emotion). "state"/"text" above
|
|
# are kept for synthetic clients that display transcripts.
|
|
# ---------------------------------------------------------------------------
|
|
def tts_start() -> dict:
|
|
return {"type": MsgType.TTS, "state": "start"}
|
|
|
|
|
|
def tts_sentence_start(text: str) -> dict:
|
|
return {"type": MsgType.TTS, "state": "sentence_start", "text": text}
|
|
|
|
|
|
def tts_stop() -> dict:
|
|
return {"type": MsgType.TTS, "state": "stop"}
|
|
|
|
|
|
def llm(emotion: str) -> dict:
|
|
return {"type": "llm", "emotion": emotion}
|
|
|
|
|
|
def build_message(kind: str, text: str = "", **extra) -> dict:
|
|
"""Small dispatcher used by the session to emit device-facing messages."""
|
|
if kind == "text":
|
|
return text(text)
|
|
if kind == "stt":
|
|
return stt(text)
|
|
if kind == "tts":
|
|
return tts(text)
|
|
if kind == "listening":
|
|
return listen_start()
|
|
if kind == "speaking":
|
|
return speak_start()
|
|
if kind == "thinking":
|
|
return thinking()
|
|
raise ValueError(f"unknown message kind: {kind}")
|