feat: wire Opus decode + VAD turn detection to firmware-compatible reply

Device streams continuous 20ms Opus frames with no turn markers. The loop
now decodes Opus, feeds the VAD, fires one turn on end-of-speech, and
replies with the firmware JSON contract (tts.state / stt.text) + Opus audio.

- app/audio/opus.py: RealOpus/Opus16kEncoder/PassthroughOpus; fix add_dll_directory
  to use absolute paths (WinError 87)
- app/xiaozhi/websocket.py: DeviceLoop.run() = decode -> VAD -> run_turn -> reply
- app/config.py: AudioConfig (opus kind + VAD thresholds)
- config.yaml: audio.opus=real, VAD 5/8/3000 frames
- tests/test_app.py: utterance = loud burst + silence; asserts stt + audio + tts stop
- requirements.txt: opuslib>=3

21/21 tests; live handshake verified via zhi.moreminimore.com on venv python.
This commit is contained in:
kunthawat committed 2026-10-03 20:35:38 +07:00
1 parent 3bfeff29bd
commit 261b0f3e91
7 files changed
+293 -40

No files matched your search

+22 -10
View File
@@ -78,6 +78,8 @@ def test_ws_header_auth_rejects_bad_token():
def test_ws_handshake_and_turn():
"""One spoken utterance (burst of speech frames + trailing silence)
-> firmware-compatible reply: stt text, tts start/sentence/stop, audio."""
client = TestClient(make_app(_cfg()))
with client.websocket_connect("/ws/xiaozhi") as ws:
# hello frame
@@ -85,16 +87,26 @@ def test_ws_handshake_and_turn():
reply = json.loads(ws.receive_text())
assert reply["type"] == "hello"
assert reply["session_id"]
# one utterance -> expect stt + text + audio back
ws.send_bytes(b"\x00\x00" * 64)
got_text = got_audio = False
# one utterance: 6 loud frames (16-bit, RMS >= 300), then silence
# frames that push the VAD past min_silence_frames.
loud = b"\x10\x0f" * 320 # 20 ms @ 16 kHz mono, amplitude 3856
silent = b"\x00\x00" * 320
for _ in range(6):
ws.send_bytes(loud)
for _ in range(12):
try:
kind = ws.receive()
except Exception:
break
if kind.get("text") is not None:
got_text = True
ws.send_bytes(silent)
got_stt = got_audio = False
tts_stop = False
deadline = 0
while not tts_stop and deadline < 256:
frame = ws.receive()
deadline += 1
if frame.get("text") is not None:
payload = json.loads(frame["text"])
if payload.get("type") == "stt":
got_stt = True
if payload.get("type") == "tts" and payload.get("state") == "stop":
tts_stop = True
else:
got_audio = True
assert got_text and got_audio
assert got_stt and got_audio and tts_stop