esp_uart_dma.cpp

#include "esp_uart.hpp"
 
#if SOC_GDMA_SUPPORTED && SOC_UHCI_SUPPORTED
 
#include <algorithm>
#include <array>
 
#include "esp_attr.h"
#include "esp_err.h"
#include "esp_heap_caps.h"
#include "esp_memory_utils.h"
#include "esp_private/periph_ctrl.h"
#include "hal/uhci_ll.h"
#include "soc/ext_mem_defs.h"
 
namespace
{
// RX uses a circular DMA descriptor ring, similar to STM/CH circular RX DMA
// behavior (continuous receive + software consumer index).
// RX 使用循环 DMA 描述符环，语义接近 STM/CH 的循环 RX DMA：持续接收，
// 软件侧维护消费索引。
constexpr uint32_t DMA_RX_NODE_COUNT = 8;
 
// Current ESP GDMA link items cannot describe more than 4095 bytes in one node.
// 当前 ESP GDMA link item 单节点最多只能描述 4095 字节。
constexpr size_t DMA_MAX_BUFFER_SIZE_PER_LINK_ITEM = 4095U;
 
// Minimal local view of the GDMA link descriptor layout used for in-place patching.
// 为就地修改描述符长度而保留的 GDMA link descriptor 最小本地视图。
struct GdmaLinkItem
{
  struct
  {
    uint32_t size : 12;
    uint32_t length : 12;
    uint32_t reserved24 : 4;
    uint32_t err_eof : 1;
    uint32_t reserved29 : 1;
    uint32_t suc_eof : 1;
    uint32_t owner : 1;
  } dw0;
  void* buffer;
  GdmaLinkItem* next;
};
 
constexpr uint32_t GDMA_OWNER_CPU = 0U;
constexpr uint32_t GDMA_OWNER_DMA = 1U;
 
// Helper used for DMA storage and node-size alignment calculations.
// 用于 DMA storage 和 node 大小对齐计算的辅助函数。
size_t AlignUp(size_t value, size_t align)
{
  if (align <= 1)
  {
    return value;
  }
  return ((value + align - 1) / align) * align;
}
 
// Convert the cached address returned by ESP-IDF into the non-cache alias used
// by the GDMA descriptors when the target SoC exposes one.
// 当目标 SoC 提供 non-cache alias 时，把 ESP-IDF 返回的 cache 地址转换成
// GDMA 描述符使用的 non-cache 地址。
uintptr_t CacheAddrToNonCache(uintptr_t addr)
{
#if SOC_NON_CACHEABLE_OFFSET
  return addr + SOC_NON_CACHEABLE_OFFSET;
#else
  return addr;
#endif
}
 
// Recover the first link item from a GDMA list head address.
// 从 GDMA list head 地址恢复首个 link item。
GdmaLinkItem* LinkItemFromHeadAddr(uintptr_t head_addr)
{
  return reinterpret_cast<GdmaLinkItem*>(CacheAddrToNonCache(head_addr));
}
 
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE || SOC_PSRAM_DMA_CAPABLE
extern "C" esp_err_t esp_cache_msync(void* addr, size_t size, int flags);
 
constexpr int CACHE_SYNC_FLAG_UNALIGNED = (1 << 1);
constexpr int CACHE_SYNC_FLAG_DIR_C2M = (1 << 2);
constexpr int CACHE_SYNC_FLAG_DIR_M2C = (1 << 3);
 
// Synchronize one DMA window when the active memory region is cacheable.
// 当当前内存区域可缓存时，同步一个 DMA 窗口。
bool CacheSyncDmaBuffer(const void* addr, size_t size, bool cache_to_mem)
{
  if ((addr == nullptr) || (size == 0U))
  {
    return true;
  }
 
#if SOC_PSRAM_DMA_CAPABLE && !SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE
  if (!esp_ptr_external_ram(addr))
  {
    return true;
  }
#endif
 
  int flags = cache_to_mem ? CACHE_SYNC_FLAG_DIR_C2M : CACHE_SYNC_FLAG_DIR_M2C;
  flags |= CACHE_SYNC_FLAG_UNALIGNED;
 
  const esp_err_t ret = esp_cache_msync(const_cast<void*>(addr), size, flags);
  // Non-cacheable regions can return ESP_ERR_INVALID_ARG; treat as no-op success.
  return (ret == ESP_OK) || (ret == ESP_ERR_INVALID_ARG);
}
#endif
}  // namespace
 
namespace LibXR
{
 
// TX EOF means the current staged active payload has fully left the DMA engine.
// TX EOF 表示当前暂存的 active payload 已经完整离开 DMA 引擎。
bool IRAM_ATTR ESP32UART::DmaTxEofCallback(gdma_channel_handle_t, gdma_event_data_t*,
                                           void* user_data)
{
  auto* uart = static_cast<ESP32UART*>(user_data);
  if (uart != nullptr)
  {
    uart->tx_dma_model_.OnTransferDone(true);
  }
  return false;
}
 
// TX descriptor error is surfaced as a backend TX failure.
// TX 描述符错误会被上报为后端 TX 失败。
bool IRAM_ATTR ESP32UART::DmaTxDescrErrCallback(gdma_channel_handle_t, gdma_event_data_t*,
                                                void* user_data)
{
  auto* uart = static_cast<ESP32UART*>(user_data);
  if (uart != nullptr)
  {
    uart->HandleDmaTxError();
  }
  return false;
}
 
// RX done callback only forwards the event into the UART object state machine.
// RX 完成回调只负责把事件转发到 UART 对象状态机。
bool IRAM_ATTR ESP32UART::DmaRxDoneCallback(gdma_channel_handle_t,
                                            gdma_event_data_t* event_data,
                                            void* user_data)
{
  auto* uart = static_cast<ESP32UART*>(user_data);
  if ((uart != nullptr) && uart->rx_config_gate_.TryEnterRx())
  {
    uart->HandleDmaRxDone(event_data);
    if (uart->rx_config_gate_.LeaveRx())
    {
      uart->tx_dma_model_.RequestConfig(true);
    }
  }
  return false;
}
 
// RX descriptor error requests a full RX ring recovery.
// RX 描述符错误要求完整恢复 RX 环。
bool IRAM_ATTR ESP32UART::DmaRxDescrErrCallback(gdma_channel_handle_t, gdma_event_data_t*,
                                                void* user_data)
{
  auto* uart = static_cast<ESP32UART*>(user_data);
  if ((uart != nullptr) && uart->rx_config_gate_.TryEnterRx())
  {
    uart->HandleDmaRxError();
    if (uart->rx_config_gate_.LeaveRx())
    {
      uart->tx_dma_model_.RequestConfig(true);
    }
  }
  return false;
}
 
// DMA backend bring-up does three things:
// 1. Bind UHCI to the selected UART.
// 2. Prepare two TX descriptor lists, one per double-buffer half.
// 3. Prepare one circular RX descriptor ring.
// DMA 后端初始化做三件事：
// 1. 把 UHCI 绑定到选定 UART。
// 2. 为双缓冲两半各准备一条 TX 描述符链。
// 3. 准备一条循环 RX 描述符环。
ErrorCode ESP32UART::InitDmaBackend()
{
  if (dma_backend_enabled_)
  {
    return ErrorCode::OK;
  }
 
  periph_module_enable(PERIPH_UHCI0_MODULE);
  periph_module_reset(PERIPH_UHCI0_MODULE);
 
  uhci_hal_init(&uhci_hal_, 0);
  uhci_ll_attach_uart_port(uhci_hal_.dev, uart_num_);
 
  uhci_seper_chr_t sep_chr = {};
  sep_chr.sub_chr_en = 0;
  uhci_ll_set_seper_chr(uhci_hal_.dev, &sep_chr);
  uhci_ll_rx_set_eof_mode(uhci_hal_.dev, UHCI_RX_IDLE_EOF);
 
  gdma_channel_alloc_config_t tx_cfg = {
      .sibling_chan = nullptr,
      .direction = GDMA_CHANNEL_DIRECTION_TX,
      .flags = {},
  };
  if (gdma_new_ahb_channel(&tx_cfg, &tx_dma_channel_) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  if (gdma_connect(tx_dma_channel_, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_UHCI, 0)) !=
      ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  gdma_transfer_config_t transfer_cfg = {
      .max_data_burst_size = 0,
      .access_ext_mem = true,
  };
  if (gdma_config_transfer(tx_dma_channel_, &transfer_cfg) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  size_t tx_int_alignment = 1;
  size_t tx_ext_alignment = 1;
  if (gdma_get_alignment_constraints(tx_dma_channel_, &tx_int_alignment,
                                     &tx_ext_alignment) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
  const size_t tx_dma_alignment =
      std::max<size_t>(1, std::max(tx_int_alignment, tx_ext_alignment));
 
  gdma_strategy_config_t tx_strategy = {
      .owner_check = true,
      .auto_update_desc = true,
      .eof_till_data_popped = true,
  };
  if (gdma_apply_strategy(tx_dma_channel_, &tx_strategy) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  gdma_link_list_config_t tx_link_cfg = {
      .num_items = 1,
      .item_alignment = 4,
      .flags = {},
  };
  gdma_link_list_handle_t tx_dma_links[2] = {nullptr, nullptr};
 
  for (int i = 0; i < 2; ++i)
  {
    if (gdma_new_link_list(&tx_link_cfg, &tx_dma_links[i]) != ESP_OK)
    {
      return ErrorCode::INIT_ERR;
    }
 
    gdma_buffer_mount_config_t tx_mount = {
        .buffer = tx_dma_model_.Buffer(i),
        .buffer_alignment = tx_dma_alignment,
        .length = 1,
        .flags =
            {
                .mark_eof = 1,
                .mark_final = 1,
                .bypass_buffer_align_check = 0,
            },
    };
 
    if (gdma_link_mount_buffers(tx_dma_links[i], 0, &tx_mount, 1, nullptr) != ESP_OK)
    {
      return ErrorCode::INIT_ERR;
    }
 
    tx_dma_head_addr_[i] = gdma_link_get_head_addr(tx_dma_links[i]);
    if (tx_dma_head_addr_[i] == 0U)
    {
      return ErrorCode::INIT_ERR;
    }
  }
 
  gdma_tx_event_callbacks_t tx_callbacks = {
      .on_trans_eof = DmaTxEofCallback,
      .on_descr_err = DmaTxDescrErrCallback,
  };
  if (gdma_register_tx_event_callbacks(tx_dma_channel_, &tx_callbacks, this) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  gdma_channel_alloc_config_t rx_cfg = {
      .sibling_chan = nullptr,
      .direction = GDMA_CHANNEL_DIRECTION_RX,
      .flags = {},
  };
  if (gdma_new_ahb_channel(&rx_cfg, &rx_dma_channel_) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  if (gdma_connect(rx_dma_channel_, GDMA_MAKE_TRIGGER(GDMA_TRIG_PERIPH_UHCI, 0)) !=
      ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  if (gdma_config_transfer(rx_dma_channel_, &transfer_cfg) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  size_t rx_int_alignment = 1;
  size_t rx_ext_alignment = 1;
  if (gdma_get_alignment_constraints(rx_dma_channel_, &rx_int_alignment,
                                     &rx_ext_alignment) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
  const size_t rx_dma_alignment =
      std::max<size_t>(1, std::max(rx_int_alignment, rx_ext_alignment));
 
  gdma_link_list_config_t rx_link_cfg = {
      .num_items = DMA_RX_NODE_COUNT,
      .item_alignment = 4,
      .flags = {},
  };
  if (gdma_new_link_list(&rx_link_cfg, &rx_dma_link_) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  // Keep one ring window reasonably large to lower ISR pressure at high baud.
  // 保持单个环窗口适度偏大，以降低高波特率下的 ISR 压力。
  const size_t rx_chunk_target = std::min<size_t>(
      std::max<size_t>(32, rx_isr_buffer_size_ / DMA_RX_NODE_COUNT), 512);
  rx_dma_chunk_size_ = std::max<size_t>(AlignUp(rx_chunk_target, 4), 32);
  const size_t rx_storage_alignment = std::max<size_t>(4, rx_dma_alignment);
  const size_t rx_storage_bytes =
      AlignUp(rx_dma_chunk_size_ * DMA_RX_NODE_COUNT, rx_storage_alignment);
 
  rx_dma_storage_ = static_cast<uint8_t*>(
      heap_caps_aligned_alloc(rx_storage_alignment, rx_storage_bytes,
                              MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA | MALLOC_CAP_8BIT));
  if (rx_dma_storage_ == nullptr)
  {
    return ErrorCode::NO_MEM;
  }
 
  std::array<gdma_buffer_mount_config_t, DMA_RX_NODE_COUNT> rx_mount = {};
  for (uint32_t i = 0; i < DMA_RX_NODE_COUNT; ++i)
  {
    rx_mount[i] = gdma_buffer_mount_config_t{
        .buffer = rx_dma_storage_ + (static_cast<size_t>(i) * rx_dma_chunk_size_),
        .buffer_alignment = rx_dma_alignment,
        .length = rx_dma_chunk_size_,
        .flags =
            {
                .mark_eof = 0,
                .mark_final = 0,
                .bypass_buffer_align_check = 0,
            },
    };
  }
 
  if (gdma_link_mount_buffers(rx_dma_link_, 0, rx_mount.data(), DMA_RX_NODE_COUNT,
                              nullptr) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  gdma_rx_event_callbacks_t rx_callbacks = {
      .on_recv_eof = nullptr,
      .on_descr_err = DmaRxDescrErrCallback,
      .on_recv_done = DmaRxDoneCallback,
  };
  if (gdma_register_rx_event_callbacks(rx_dma_channel_, &rx_callbacks, this) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  if (gdma_reset(rx_dma_channel_) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  if (gdma_start(rx_dma_channel_, gdma_link_get_head_addr(rx_dma_link_)) != ESP_OK)
  {
    return ErrorCode::INIT_ERR;
  }
 
  rx_dma_node_index_ = 0;
  dma_backend_enabled_ = true;
  return ErrorCode::OK;
}
 
// TX DMA start only patches the dynamic fields of the pre-mounted descriptor
// list, so the hot path avoids rebuilding descriptors for every request.
// TX DMA 启动时只修改预挂载描述符链的动态字段，避免每次请求都重建描述符。
bool IRAM_ATTR ESP32UART::StartDmaTx(uint8_t* data, size_t size, int block)
{
  if ((tx_dma_channel_ == nullptr) || (data == nullptr) || (size == 0U) ||
      (size > DMA_MAX_BUFFER_SIZE_PER_LINK_ITEM) || ((block != 0) && (block != 1)) ||
      (tx_dma_head_addr_[block] == 0U))
  {
    return false;
  }
 
  auto* desc = LinkItemFromHeadAddr(tx_dma_head_addr_[block]);
  if (desc == nullptr)
  {
    return false;
  }
 
  desc->buffer = data;
  desc->dw0.size = static_cast<uint32_t>(size);
  desc->dw0.length = static_cast<uint32_t>(size);
  desc->dw0.err_eof = 0U;
  desc->dw0.suc_eof = 1U;
  desc->dw0.owner = GDMA_OWNER_DMA;
  desc->next = nullptr;
  std::atomic_thread_fence(std::memory_order_release);
 
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE || SOC_PSRAM_DMA_CAPABLE
  if (!CacheSyncDmaBuffer(data, size, true))
  {
    return false;
  }
#endif
 
  return gdma_start(tx_dma_channel_, tx_dma_head_addr_[block]) == ESP_OK;
}
 
// RX DMA completion can span multiple ring nodes, so consume at most one full
// ring window per callback and advance the software node cursor in lockstep.
// 一次 RX DMA 完成可能跨越多个环节点，因此每次回调最多消费一个完整环窗口，
// 并同步推进软件节点游标。
void IRAM_ATTR ESP32UART::PushDmaRxData(size_t recv_size, bool in_isr)
{
  if ((rx_dma_storage_ == nullptr) || (rx_dma_chunk_size_ == 0))
  {
    return;
  }
 
  const size_t max_window = rx_dma_chunk_size_ * DMA_RX_NODE_COUNT;
  size_t remaining = std::min(recv_size, max_window);
 
  while (remaining > 0)
  {
    const size_t offset = static_cast<size_t>(rx_dma_node_index_) * rx_dma_chunk_size_;
    const size_t chunk = std::min(remaining, rx_dma_chunk_size_);
    auto* chunk_ptr = rx_dma_storage_ + offset;
 
#if SOC_CACHE_INTERNAL_MEM_VIA_L1CACHE || SOC_PSRAM_DMA_CAPABLE
    if (!CacheSyncDmaBuffer(chunk_ptr, chunk, false))
    {
      HandleDmaRxError();
      return;
    }
#endif
    PushRxBytes(chunk_ptr, chunk, in_isr);
    remaining -= chunk;
    rx_dma_node_index_ = (rx_dma_node_index_ + 1U) % DMA_RX_NODE_COUNT;
  }
}
 
// RX DMA completion either reports a full node or the final EOF-sized tail.
// RX DMA 完成要么报告整节点，要么报告带 EOF 的尾段长度。
void IRAM_ATTR ESP32UART::HandleDmaRxDone(gdma_event_data_t* event_data)
{
  if ((rx_dma_storage_ == nullptr) || (rx_dma_chunk_size_ == 0))
  {
    return;
  }
 
  if ((event_data != nullptr) && event_data->flags.abnormal_eof)
  {
    HandleDmaRxError();
    return;
  }
 
  size_t recv_size = rx_dma_chunk_size_;
  if ((event_data != nullptr) && event_data->flags.normal_eof)
  {
    const size_t eof_size = gdma_link_count_buffer_size_till_eof(
        rx_dma_link_, static_cast<int>(rx_dma_node_index_));
    if (eof_size > 0)
    {
      recv_size = eof_size;
    }
  }
 
  PushDmaRxData(recv_size, true);
}
 
// RX DMA recovery restarts the circular ring from node zero.
// RX DMA 恢复会从节点零重新启动整个环。
void IRAM_ATTR ESP32UART::HandleDmaRxError()
{
  if ((rx_dma_channel_ == nullptr) || (rx_dma_link_ == nullptr))
  {
    return;
  }
 
  gdma_stop(rx_dma_channel_);
  gdma_reset(rx_dma_channel_);
  rx_dma_node_index_ = 0;
  (void)gdma_start(rx_dma_channel_, gdma_link_get_head_addr(rx_dma_link_));
}
 
// TX DMA recovery aborts the current hardware transfer before the common terminal
// path advances a ready pending request.
// TX DMA 恢复先中止当前硬件传输，再由公共终止路径推进 ready pending 请求。
void IRAM_ATTR ESP32UART::HandleDmaTxError()
{
  if (tx_dma_channel_ != nullptr)
  {
    gdma_stop(tx_dma_channel_);
    gdma_reset(tx_dma_channel_);
  }
  tx_dma_model_.OnTransferError(true);
}
 
}  // namespace LibXR
 
#endif