ESP32-S3 -> Wi-Fi -> TLS/WebSocket -> Realtime 模型接口

板载 I2S 数字麦克风（16kHz）
        ↓
32-bit I2S sample 转 mono PCM16
        ↓
ESP-SR AFE AEC（麦克风 M + 播放参考 R）
        ↓
上行音频队列
        ↓
Base64 + JSON + WebSocket
        ↓
Qwen Omni Realtime / 豆包 Realtime
        ↓
模型返回 24kHz mono PCM16
        ↓
PSRAM 播放抖动缓冲
        ↓
音量缩放 + stereo I2S
        ↓
MAX98357 + 喇叭

上行：麦克风 -> AEC -> WebSocket -> 模型
下行：模型 -> WebSocket -> 抖动缓冲 -> MAX98357

16000 * 1 * 2 = 32000 bytes/s

24000 * 1 * 2 = 48000 bytes/s

录音 -> 上传 -> commit -> response -> 播放

每次读取 512 个 16kHz sample
512 / 16000 = 32ms

qwen3.5-omni-plus-realtime

wss://dashscope.aliyuncs.com/api-ws/v1/realtime?model=qwen3.5-omni-plus-realtime

Authorization: Bearer <DashScope API Key>

{
  "type": "session.update",
  "session": {
    "modalities": ["text", "audio"],
    "input_audio_format": "pcm",
    "output_audio_format": "pcm",
    "voice": "Tina",
    "turn_detection": {
      "type": "semantic_vad",
      "threshold": 0.1,
      "prefix_padding_ms": 500,
      "silence_duration_ms": 900
    }
  }
}

{
  "type": "input_audio_buffer.append",
  "audio": "<Base64 PCM>"
}

response.audio.delta
response.audio.done
response.done

wss://openspeech.bytedance.com/api/v3/duplex/realtime/dialogue

X-Api-Key: <API Key>
X-Api-Connect-Id: <每次连接唯一的随机 ID>

1.2.6.0

{
  "type": "session.create",
  "session": {
    "model": "1.2.6.0",
    "instructions": "你是一位耐心、简洁的中文老师。",
    "audio": {
      "input": {
        "format": { "type": "pcm", "sample_rate": 16000 }
      },
      "output": {
        "format": { "type": "pcm_s16le", "sample_rate": 24000 },
        "voice": "zh_female_vv_jupiter_bigtts"
      }
    }
  }
}

response.output_audio.started
response.output_audio.delta
response.output_audio.done
response.done

conversation.item.input_audio_transcription.started
conversation.item.input_audio_transcription.delta
conversation.item.input_audio_transcription.completed

I2S -> PCM -> AEC -> Queue -> WebSocket
WebSocket -> Base64 -> PCM -> Ring Buffer -> I2S

websocket text event too large: payload_len=20680
audio delta base64 decode failed: -42

data_len       当前分片长度
payload_len    完整 WebSocket payload 长度
payload_offset 当前分片在完整 payload 中的偏移

memcpy(s_ws_rx_text + s_ws_rx_text_len,
       data->data_ptr,
       data->data_len);
s_ws_rx_text_len += data->data_len;

if (data->payload_offset + data->data_len >= data->payload_len) {
    s_ws_rx_text[s_ws_rx_text_len] = '\0';
    enqueue_ws_rx_message((uint8_t *)s_ws_rx_text,
                          s_ws_rx_text_len,
                          false);
    s_ws_rx_text_len = 0;
}

读取 I2S -> AEC -> Base64 -> WebSocket send

收到一包 -> 播放完 -> 等网络 -> 收到下一包 -> 再播放

WebSocket RX -> playback ring -> I2S playback task

audio delta gap: 380ms
playback underflow

s_afe_aec = afe_aec_create(
    "MR",
    AEC_FILTER_LENGTH,
    AFE_TYPE_FD,
    AFE_MODE_LOW_COST
);

M：麦克风采到的“用户声音 + 喇叭回声”
R：设备实际送给喇叭的播放参考

M0, R0, M1, R1, M2, R2 ...

afe_aec_process(s_afe_aec, s_aec_input_frame, s_aec_out_frame);

模型 PCM
  -> 播放音量缩放
  -> 一份写入 MAX98357
  -> 同一份转成 16kHz，写入 AEC 参考环

#define AEC_REF_DELAY_MS 80

#define MIC_SAMPLE_RATE_HZ 16000
#define MIC_READ_FRAMES 512

#define ENABLE_AEC 1
#define AEC_FILTER_LENGTH 4
#define AEC_REF_RING_SAMPLES (MIC_SAMPLE_RATE_HZ * 2)
#define AEC_REF_DELAY_MS 80

// 远端播放参考足够大，而 AEC 输出很小时，把它视为残余回声。
#define AEC_FAR_END_GUARD 1
#define AEC_REF_ACTIVE_ABS_LEVEL 180
#define AEC_NEAR_END_KEEP_ABS_LEVEL 1800

static esp_err_t init_aec(void)
{
#if ENABLE_AEC
    s_aec_ref_ring_mutex = xSemaphoreCreateMutex();
    ESP_RETURN_ON_FALSE(s_aec_ref_ring_mutex != NULL,
                        ESP_ERR_NO_MEM, TAG,
                        "create AEC reference mutex failed");

    s_aec_ref_ring = heap_caps_malloc(
        AEC_REF_RING_SAMPLES * sizeof(s_aec_ref_ring[0]),
        MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT
    );
    ESP_RETURN_ON_FALSE(s_aec_ref_ring != NULL,
                        ESP_ERR_NO_MEM, TAG,
                        "alloc AEC reference ring failed");

    // MR：麦克风 M + 播放参考 R；FD：全双工模式。
    s_afe_aec = afe_aec_create(
        "MR",
        AEC_FILTER_LENGTH,
        AFE_TYPE_FD,
        AFE_MODE_LOW_COST
    );
    ESP_RETURN_ON_FALSE(s_afe_aec != NULL,
                        ESP_ERR_NO_MEM, TAG,
                        "create ESP-SR AFE AEC failed");

    s_aec_frame_samples = afe_aec_get_chunksize(s_afe_aec);
    ESP_RETURN_ON_FALSE(s_aec_frame_samples == MIC_READ_FRAMES,
                        ESP_ERR_INVALID_SIZE, TAG,
                        "AEC frame size mismatch: aec=%d mic=%d",
                        s_aec_frame_samples,
                        MIC_READ_FRAMES);

    s_aec_ref_frame = heap_caps_aligned_calloc(
        16, s_aec_frame_samples, sizeof(int16_t),
        MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT
    );
    s_aec_input_frame = heap_caps_aligned_calloc(
        16, s_aec_frame_samples * 2, sizeof(int16_t),
        MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT
    );
    s_aec_out_frame = heap_caps_aligned_calloc(
        16, s_aec_frame_samples, sizeof(int16_t),
        MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT
    );

    ESP_RETURN_ON_FALSE(
        s_aec_ref_frame != NULL &&
        s_aec_input_frame != NULL &&
        s_aec_out_frame != NULL,
        ESP_ERR_NO_MEM, TAG,
        "alloc aligned AEC frame buffers failed"
    );

    aec_ref_ring_clear();
#endif
    return ESP_OK;
}

for (size_t i = 0; i < frames_now; i++) {
    int32_t scaled = (int32_t)mono_pcm[offset + i]
                   * PLAYBACK_VOLUME_LEVEL
                   / PLAYBACK_VOLUME_DENOMINATOR;
    int16_t sample = clamp_i16(scaled);

    // AEC 参考使用音量缩放后的真实播放 sample。
    s_amp_mono_reference_24k[i] = sample;

    // MAX98357 使用 stereo I2S，左右 slot 写同一份 mono。
    s_amp_stereo_samples[i * 2] = sample;
    s_amp_stereo_samples[i * 2 + 1] = sample;
}

if (s_afe_aec != NULL) {
    aec_ref_write_24k_mono(s_amp_mono_reference_24k,
                           frames_now);
}

i2s_channel_write(s_amp_tx_chan,
                  s_amp_stereo_samples,
                  frames_now * 2 * sizeof(int16_t),
                  &bytes_written,
                  pdMS_TO_TICKS(2000));

static void aec_ref_write_24k_mono(const int16_t *samples,
                                   size_t sample_count)
{
    size_t out_count = 0;

    for (size_t i = 0; i < sample_count; i++) {
        s_aec_resample_group[s_aec_resample_group_count++] = samples[i];

        if (s_aec_resample_group_count == 3) {
            s_aec_ref_resampled[out_count++] =
                s_aec_resample_group[0];
            s_aec_ref_resampled[out_count++] =
                (int16_t)(((int32_t)s_aec_resample_group[1]
                         + s_aec_resample_group[2]) / 2);
            s_aec_resample_group_count = 0;
        }
    }

    aec_ref_ring_write(s_aec_ref_resampled, out_count);
}

static void aec_ref_ring_read_delay_or_zero(int16_t *out,
                                            size_t sample_count)
{
    if (s_aec_ref_ring == NULL || sample_count == 0) {
        if (sample_count > 0) {
            memset(out, 0, sample_count * sizeof(out[0]));
        }
        return;
    }

    xSemaphoreTake(s_aec_ref_ring_mutex, portMAX_DELAY);

    size_t delay_samples =
        MIC_SAMPLE_RATE_HZ * AEC_REF_DELAY_MS / 1000;
    size_t readable = s_aec_ref_ring_filled > delay_samples
                    ? s_aec_ref_ring_filled - delay_samples
                    : 0;
    size_t read_count = sample_count < readable
                      ? sample_count
                      : readable;

    size_t read_pos =
        (s_aec_ref_ring_tail + delay_samples)
        % AEC_REF_RING_SAMPLES;
    size_t first = AEC_REF_RING_SAMPLES - read_pos;
    if (first > read_count) {
        first = read_count;
    }

    memcpy(out,
           s_aec_ref_ring + read_pos,
           first * sizeof(out[0]));

    size_t second = read_count - first;
    if (second > 0) {
        memcpy(out + first,
               s_aec_ref_ring,
               second * sizeof(out[0]));
    }

    if (read_count > 0) {
        s_aec_ref_ring_tail =
            (s_aec_ref_ring_tail + read_count)
            % AEC_REF_RING_SAMPLES;
        s_aec_ref_ring_filled -= read_count;
    }

    xSemaphoreGive(s_aec_ref_ring_mutex);

    if (read_count < sample_count) {
        memset(out + read_count,
               0,
               (sample_count - read_count) * sizeof(out[0]));
    }
}

const int16_t *pcm_to_send = s_mic_pcm16;
uint32_t mic_abs = mean_abs_i16(s_mic_pcm16, frames_read);
uint32_t ref_abs = 0;
uint32_t out_abs = mic_abs;
bool playback_active =
    s_response_playing || playback_ring_level() > 0;

if (s_afe_aec != NULL &&
    frames_read == (size_t)s_aec_frame_samples) {
    aec_ref_ring_read_delay_or_zero(s_aec_ref_frame,
                                    frames_read);

    for (size_t i = 0; i < frames_read; i++) {
        s_aec_input_frame[i * 2] = s_mic_pcm16[i];
        s_aec_input_frame[i * 2 + 1] = s_aec_ref_frame[i];
    }

    size_t out_bytes = afe_aec_process(
        s_afe_aec,
        s_aec_input_frame,
        s_aec_out_frame
    );
    if (out_bytes != frames_read * sizeof(int16_t)) {
        ESP_LOGW(TAG, "AFE AEC output size mismatch");
        continue;
    }

    mic_abs = mean_abs_i16(s_mic_pcm16, frames_read);
    ref_abs = mean_abs_i16(s_aec_ref_frame, frames_read);
    out_abs = mean_abs_i16(s_aec_out_frame, frames_read);

    // 播放参考明显存在、AEC 输出又很小时，
    // 剩下的更可能是远端回声，而不是真人插话。
    if (playback_active &&
        ref_abs >= AEC_REF_ACTIVE_ABS_LEVEL &&
        out_abs < AEC_NEAR_END_KEEP_ABS_LEVEL) {
        memset(s_aec_out_frame,
               0,
               frames_read * sizeof(int16_t));
        out_abs = 0;
    }

    pcm_to_send = s_aec_out_frame;
}

// 无论当前有没有人说话，每个 32ms 帧仍然连续入队上传。
enqueue_uplink_audio(pcm_to_send, frames_read);

mic：原始麦克风能量
ref：播放参考能量
out：AEC 输出能量

AudioServerNoAudioInputTooLongError

有人声：连续上传 AEC 输出
无人声：仍按实时节奏连续上传音频/静音
轮次判断：交给豆包 Realtime 接口

conversation.item.input_audio_transcription.started

if (playback_active && out_abs >= BARGE_IN_ABS_LEVEL) {
    s_barge_in_voice_frames++;
    if (s_barge_in_voice_frames == 2) {
        mark_user_turn_and_interrupt("local AEC near-end");
    }
} else {
    s_barge_in_voice_frames = 0;
}

停止接受当前回复音频
清空本地播放环
清空旧的 AEC 参考
发送 response.cancel
继续上传麦克风 PCM
丢弃取消生效前仍在路上的旧 audio delta

#define BARGE_IN_ABS_LEVEL AEC_NEAR_END_KEEP_ABS_LEVEL
#define BARGE_IN_CONFIRM_FRAMES 2

static uint32_t s_barge_in_voice_frames;

static volatile bool s_user_turn_pending;
static volatile bool s_response_playing;
static volatile bool s_accepting_response_audio;
static volatile bool s_cancel_response_requested;
static volatile bool s_drop_response_audio_until_done;

if (playback_active && out_abs >= BARGE_IN_ABS_LEVEL) {
    s_barge_in_voice_frames++;

    if (s_barge_in_voice_frames == BARGE_IN_CONFIRM_FRAMES) {
        mark_user_turn_and_interrupt("local AEC near-end");
    }
} else {
    s_barge_in_voice_frames = 0;
}

static void mark_user_turn_and_interrupt(const char *source)
{
    // 标记已经有一轮真实用户输入，后续模型回复可以被接收。
    s_user_turn_pending = true;

    if (!s_response_playing && playback_ring_level() == 0) {
        return;
    }

    // 1. 立即停止本地播放状态。
    s_response_playing = false;
    s_accepting_response_audio = false;

    // 2. response.cancel 到达云端需要时间，期间可能还有旧 delta 在路上。
    //    在收到当前 response.done/canceled 之前全部丢弃。
    s_drop_response_audio_until_done = true;

    // 3. 清除已经缓存、但还没有从喇叭播放的旧回答。
    playback_ring_clear();

    // 4. 老师声音已被打断，旧参考也不能带入下一轮 AEC。
    aec_ref_ring_clear();

    // 5. 让独立上行任务发送 response.cancel，
    //    不在音频任务里直接等待 WebSocket/TLS。
    s_cancel_response_requested = true;

    if (s_playback_task_handle != NULL) {
        xTaskNotifyGive(s_playback_task_handle);
    }

    ESP_LOGI(TAG,
             "user barge-in, interrupt playback: source=%s",
             source);
}

static bool mark_response_audio_started(void)
{
    // 当前回复已经被用户取消，丢弃仍在路上的旧音频。
    if (s_drop_response_audio_until_done) {
        return false;
    }

    if (!s_response_playing) {
        // 没有用户输入却出现新回复，视为回声自触发。
        if (!s_user_turn_pending) {
            s_accepting_response_audio = false;
            s_cancel_response_requested = true;
            ESP_LOGW(TAG, "drop self-triggered response audio");
            return false;
        }

        s_user_turn_pending = false;
        s_accepting_response_audio = true;
        s_last_audio_delta_ms = 0;
        aec_ref_start_response();
    }

    s_response_playing = true;
    return s_accepting_response_audio;
}

static void handle_audio_delta(cJSON *root)
{
    cJSON *delta = cJSON_GetObjectItem(root, "delta");
    if (!cJSON_IsString(delta) || delta->valuestring == NULL) {
        return;
    }

    size_t out_len = 0;
    int ret = mbedtls_base64_decode(
        s_ws_audio_binary,
        WS_AUDIO_BINARY_BUFFER_SIZE,
        &out_len,
        (const unsigned char *)delta->valuestring,
        strlen(delta->valuestring)
    );
    if (ret != 0 || !mark_response_audio_started()) {
        return;
    }

    playback_ring_write(s_ws_audio_binary, out_len);
}

if (strcmp(type->valuestring, "response.output_audio.done") == 0 ||
    strcmp(type->valuestring, "response.done") == 0) {
    s_response_playing = false;
    s_accepting_response_audio = false;
    s_drop_response_audio_until_done = false;
}

if (strcmp(type->valuestring, "response.canceled") == 0) {
    s_drop_response_audio_until_done = false;
}

if (strcmp(type->valuestring,
           "conversation.item.input_audio_transcription.started") == 0) {
    s_input_transcript_len = 0;
    s_input_transcript[0] = '\0';
}

if (strcmp(type->valuestring,
           "conversation.item.input_audio_transcription.delta") == 0) {
    cJSON *delta = cJSON_GetObjectItem(root, "text");
    if (!cJSON_IsString(delta)) {
        delta = cJSON_GetObjectItem(root, "delta");
    }

    if (cJSON_IsString(delta) && delta->valuestring != NULL) {
        size_t delta_len = strlen(delta->valuestring);

        if (delta_len > 0 && !s_user_turn_pending) {
            mark_user_turn_and_interrupt("realtime transcript");
        }

        // 后面继续把 delta 追加到转写字符串，代码略。
    }
}

本地 AEC 能量检测：尽快让喇叭停下来
response.cancel：通知 Realtime 接口停止当前回复
Realtime 接口：继续识别用户内容和语音轮次
非空 transcription.delta：本地检测漏判时兜底

空闲
等待首次预缓冲
播放欠载后的重新缓冲
用户打断后

static void write_silence(size_t frame_count)
{
    size_t frames_left = frame_count;

    while (frames_left > 0) {
        size_t frames_now = frames_left > AMP_WRITE_FRAMES
                          ? AMP_WRITE_FRAMES
                          : frames_left;

        memset(s_amp_stereo_samples,
               0,
               frames_now * 2 * sizeof(int16_t));

        size_t bytes_written = 0;
        i2s_channel_write(
            s_amp_tx_chan,
            s_amp_stereo_samples,
            frames_now * 2 * sizeof(int16_t),
            &bytes_written,
            pdMS_TO_TICKS(1000)
        );

        frames_left -= frames_now;
    }
}

if (!playing) {
    size_t threshold = recovering_from_underflow
                     ? PLAYBACK_REBUFFER_BYTES
                     : PLAYBACK_PREBUFFER_BYTES;

    if (level >= threshold ||
        (!s_response_playing && level > 0)) {
        playing = true;
        recovering_from_underflow = false;
    } else {
        // 没有语音数据时也保持 BCLK、LRCLK 和零 sample 连续输出。
        write_silence(AMP_WRITE_FRAMES);
        continue;
    }
}

size_t read_bytes = playback_ring_read(
    (uint8_t *)s_playback_mono_samples,
    AMP_WRITE_FRAMES * sizeof(int16_t)
);

if (read_bytes == 0) {
    if (s_response_playing) {
        recovering_from_underflow = true;
    }
    playing = false;
    write_silence(AMP_WRITE_FRAMES);
    continue;
}

idf.py menuconfig

Qwen Omni Realtime Demo

Doubao Realtime Demo

cd projects/03_qwen_omni_realtime_ptt
idf.py set-target esp32s3
idf.py build
idf.py -p /dev/cu.usbserial-10 flash monitor

cd projects/05_doubao_realtime
idf.py set-target esp32s3
idf.py build
idf.py -p /dev/cu.usbserial-10 flash monitor

Wi-Fi connected
websocket connected
session.created / session.updated
user transcript: ...
response.output_audio.started
playback start: buffered=...
response.output_audio.done
response.done

websocket text event too large
base64 decode failed
uplink queue full
audio delta gap
playback underflow
AudioServerNoAudioInputTooLongError

连接 WebSocket
上传 PCM
播放返回 PCM