feat: wire Opus decode + VAD turn detection to firmware-compatible reply
Device streams continuous 20ms Opus frames with no turn markers. The loop now decodes Opus, feeds the VAD, fires one turn on end-of-speech, and replies with the firmware JSON contract (tts.state / stt.text) + Opus audio. - app/audio/opus.py: RealOpus/Opus16kEncoder/PassthroughOpus; fix add_dll_directory to use absolute paths (WinError 87) - app/xiaozhi/websocket.py: DeviceLoop.run() = decode -> VAD -> run_turn -> reply - app/config.py: AudioConfig (opus kind + VAD thresholds) - config.yaml: audio.opus=real, VAD 5/8/3000 frames - tests/test_app.py: utterance = loud burst + silence; asserts stt + audio + tts stop - requirements.txt: opuslib>=3 21/21 tests; live handshake verified via zhi.moreminimore.com on venv python.
This commit is contained in:
1 parent
3bfeff29bd
commit
261b0f3e91
7 files changed
+293
-40
No files matched your search
+22
-10
@@ -78,6 +78,8 @@ def test_ws_header_auth_rejects_bad_token():
|
||||
|
||||
|
||||
def test_ws_handshake_and_turn():
|
||||
"""One spoken utterance (burst of speech frames + trailing silence)
|
||||
-> firmware-compatible reply: stt text, tts start/sentence/stop, audio."""
|
||||
client = TestClient(make_app(_cfg()))
|
||||
with client.websocket_connect("/ws/xiaozhi") as ws:
|
||||
# hello frame
|
||||
@@ -85,16 +87,26 @@ def test_ws_handshake_and_turn():
|
||||
reply = json.loads(ws.receive_text())
|
||||
assert reply["type"] == "hello"
|
||||
assert reply["session_id"]
|
||||
# one utterance -> expect stt + text + audio back
|
||||
ws.send_bytes(b"\x00\x00" * 64)
|
||||
got_text = got_audio = False
|
||||
# one utterance: 6 loud frames (16-bit, RMS >= 300), then silence
|
||||
# frames that push the VAD past min_silence_frames.
|
||||
loud = b"\x10\x0f" * 320 # 20 ms @ 16 kHz mono, amplitude 3856
|
||||
silent = b"\x00\x00" * 320
|
||||
for _ in range(6):
|
||||
ws.send_bytes(loud)
|
||||
for _ in range(12):
|
||||
try:
|
||||
kind = ws.receive()
|
||||
except Exception:
|
||||
break
|
||||
if kind.get("text") is not None:
|
||||
got_text = True
|
||||
ws.send_bytes(silent)
|
||||
got_stt = got_audio = False
|
||||
tts_stop = False
|
||||
deadline = 0
|
||||
while not tts_stop and deadline < 256:
|
||||
frame = ws.receive()
|
||||
deadline += 1
|
||||
if frame.get("text") is not None:
|
||||
payload = json.loads(frame["text"])
|
||||
if payload.get("type") == "stt":
|
||||
got_stt = True
|
||||
if payload.get("type") == "tts" and payload.get("state") == "stop":
|
||||
tts_stop = True
|
||||
else:
|
||||
got_audio = True
|
||||
assert got_text and got_audio
|
||||
assert got_stt and got_audio and tts_stop
|
||||
Reference in new issue
Block a user