[XDP] AIE debug and status improvements in support of VE2 (#8757)

src/runtime_src/xdp/profile/plugin/CMakeLists.txt

@@ -2,7 +2,8 @@
 # Copyright (C) 2022-2025 Advanced Micro Devices, Inc.  All rights reserved.
 #
 
-if(XDP_VE2_BUILD_CMAKE STREQUAL "yes")
+if (XDP_VE2_BUILD_CMAKE STREQUAL "yes")
+  add_subdirectory(aie_status)
   add_subdirectory(aie_profile)  
   add_subdirectory(aie_trace)
   add_subdirectory(aie_debug)

src/runtime_src/xdp/profile/plugin/aie_debug/CMakeLists.txt

@@ -37,6 +37,7 @@ if (XDP_VE2_BUILD_CMAKE STREQUAL "yes")
   target_link_libraries(xdp_aie_debug_plugin PRIVATE xdp_core xrt_coreutil xaiengine)
   target_compile_definitions(xdp_aie_debug_plugin PRIVATE FAL_LINUX="on" XDP_VE2_BUILD=1)
   target_include_directories(xdp_aie_debug_plugin PRIVATE ${CMAKE_SOURCE_DIR}/src)
+
   install (TARGETS xdp_aie_debug_plugin
     LIBRARY DESTINATION ${XDP_PLUGIN_INSTALL_DIR}
   )

src/runtime_src/xdp/profile/plugin/aie_debug/aie_debug_metadata.h

@@ -228,6 +228,7 @@ class AIE1UsedRegisters : public UsedRegisters {
     core_addresses.emplace(aie1::cm_performance_counter1_event_value);
     core_addresses.emplace(aie1::cm_performance_counter2_event_value);
     core_addresses.emplace(aie1::cm_performance_counter3_event_value);
+    core_addresses.emplace(aie1::cm_ecc_scrubbing_event);
 
     // Memory modules
     memory_addresses.emplace(aie1::mm_performance_control0);
@@ -1589,6 +1590,7 @@ class AIE2UsedRegisters : public UsedRegisters {
     core_addresses.emplace(aie2::cm_performance_counter1_event_value);
     core_addresses.emplace(aie2::cm_performance_counter2_event_value);
     core_addresses.emplace(aie2::cm_performance_counter3_event_value);
+    core_addresses.emplace(aie2::cm_ecc_scrubbing_event);
 
     // Memory modules
     memory_addresses.emplace(aie2::mm_performance_control0);
@@ -4548,6 +4550,7 @@ class AIE2psUsedRegisters : public UsedRegisters {
     core_addresses.emplace(aie2ps::cm_performance_counter1_event_value);
     core_addresses.emplace(aie2ps::cm_performance_counter2_event_value);
     core_addresses.emplace(aie2ps::cm_performance_counter3_event_value);
+    core_addresses.emplace(aie2ps::cm_ecc_scrubbing_event);
 
     // Memory modules
     memory_addresses.emplace(aie2ps::mm_performance_control0);
@@ -4576,6 +4579,7 @@ class AIE2psUsedRegisters : public UsedRegisters {
     interface_addresses.emplace(aie2ps::shim_performance_counter5);
     interface_addresses.emplace(aie2ps::shim_performance_counter0_event_value);
     interface_addresses.emplace(aie2ps::shim_performance_counter1_event_value);
+    interface_addresses.emplace(aie2ps::uc_memory_privileged);
 
     // Memory tiles
     memory_tile_addresses.emplace(aie2ps::mem_performance_control0);

src/runtime_src/xdp/profile/plugin/aie_debug/aie_debug_plugin.cpp

@@ -205,11 +205,16 @@ namespace xdp {
    ***************************************************************************/
   void AieDebugPlugin::endAIEDebugRead(void* handle)
   {
-    xrt_core::message::send(severity_level::info, "XRT", "AIE Debug endAIEDebugRead");
+    if (!mPollRegisters)
+      return;
+
     auto deviceID = getDeviceIDFromHandle(handle);
     xrt_core::message::send(severity_level::debug, "XRT",
-      "AieDebugPlugin::endAIEDebugRead deviceID is " + std::to_string(deviceID));
+      "AieDebugPlugin::endAIEDebugRead - polling registers for device " + std::to_string(deviceID));
+
+    // Poll all requested AIE registers
     handleToAIEData[handle].implementation->poll(deviceID, handle);
+    mPollRegisters = false;
   }
 
   /****************************************************************************
@@ -225,7 +230,15 @@ namespace xdp {
     if (!AIEData.valid)
       return;
 
-    //AIEData.implementation->poll(0, handle);
+    // Poll all requested AIE registers (if not done already)
+    if (mPollRegisters) {
+      auto deviceID = getDeviceIDFromHandle(handle);
+      xrt_core::message::send(severity_level::debug, "XRT",
+        "AieDebugPlugin::endPollforDevice - polling registers for device " + std::to_string(deviceID));
+
+      AIEData.implementation->poll(deviceID, handle);
+      mPollRegisters = false;
+    }
 
     handleToAIEData.erase(handle);
   }

src/runtime_src/xdp/profile/plugin/aie_debug/aie_debug_plugin.h

@@ -42,6 +42,7 @@ namespace xdp {
 
   private:
     static bool live;
+    bool mPollRegisters = true;
     uint32_t mIndex = 0;
 
     // This struct and handleToAIEData map is created to provision multiple AIEs

src/runtime_src/xdp/profile/plugin/aie_debug/edge/aie_debug.cpp

@@ -97,7 +97,7 @@ namespace xdp {
    ***************************************************************************/
   void AieDebug_EdgeImpl::poll(const uint64_t deviceID, void* handle)
   {
-    xrt_core::message::send(severity_level::debug, "XRT", "Calling AIE Poll.");
+    xrt_core::message::send(severity_level::debug, "XRT", "Calling AIE Debug Poll.");
 
     if (!(db->getStaticInfo().isDeviceReady(deviceID))) {
       xrt_core::message::send(severity_level::debug, "XRT", 
@@ -113,6 +113,9 @@ namespace xdp {
       return;
     }
 
+    xrt_core::message::send(severity_level::debug, "XRT", 
+      "Debugging registers for " + std::to_string(debugTileMap.size()) + " AIE tiles.");
+
     for (auto& tileAddr : debugTileMap) {
       tileAddr.second->readValues(aieDevInst);
       tileAddr.second->printValues(deviceID, db);

src/runtime_src/xdp/profile/plugin/aie_debug/edge/aie_debug.h

@@ -26,6 +26,7 @@
 #include "xdp/profile/plugin/vp_base/vp_base_plugin.h"
 #include "xdp/profile/plugin/aie_debug/aie_debug_plugin.h"
 
+#include "core/common/message.h"
 #include "core/edge/common/aie_parser.h"
 #include "xaiefal/xaiefal.hpp"
 
@@ -62,6 +63,11 @@ namespace xdp {
     }
 
     void readValues(XAie_DevInst* aieDevInst) {
+      std::stringstream msg;
+      msg << "Debugging " << relativeOffsets.size() << " registers for tile " 
+          << +col << "," << +row;
+      xrt_core::message::send(xrt_core::message::severity_level::debug, "XRT", msg.str());
+
       for (auto& offset : relativeOffsets) {
         uint32_t val = 0;
         XAie_Read32(aieDevInst, offset + tileOffset, &val);

src/runtime_src/xdp/profile/plugin/aie_debug/ve2/aie_debug.cpp

@@ -165,8 +165,7 @@ namespace xdp {
 
       // Traverse all active and/or requested tiles
       for (auto& tileMetric : configMetrics) {
-        //auto& metricSet  = tileMetric.second;
-        auto tile        = tileMetric.first;
+        auto tile       = tileMetric.first;
         auto tileOffset = XAie_GetTileAddr(aieDevInst, tile.row, tile.col);
 
         // Traverse all registers within tile

src/runtime_src/xdp/profile/plugin/aie_debug/ve2/aie_debug.h

@@ -26,6 +26,7 @@
 #include "xdp/profile/plugin/vp_base/vp_base_plugin.h"
 #include "xdp/profile/plugin/aie_debug/aie_debug_plugin.h"
 
+#include "core/common/message.h"
 #include "core/edge/common/aie_parser.h"
 #include "xaiefal/xaiefal.hpp"
 
@@ -62,6 +63,11 @@ namespace xdp {
     }
 
     void readValues(XAie_DevInst* aieDevInst) {
+      std::stringstream msg;
+      msg << "Debugging " << relativeOffsets.size() << " registers for tile " 
+          << +col << "," << +row;
+      xrt_core::message::send(xrt_core::message::severity_level::debug, "XRT", msg.str());
+
       for (auto& offset : relativeOffsets) {
         uint32_t val = 0;
         XAie_Read32(aieDevInst, offset + tileOffset, &val);

src/runtime_src/xdp/profile/plugin/aie_profile/aie_profile_defs.h

@@ -50,6 +50,7 @@ constexpr uint32_t UC_MDM_PCCMDR                    = 0x000b5480;
 constexpr uint32_t UC_MDM_PCSR                      = 0x000b54c0;
 constexpr uint32_t UC_MDM_PCDRR                     = 0x000b5580;
 constexpr uint32_t UC_MDM_PCWR                      = 0x000b55c0;
+constexpr uint32_t UC_MEMORY_PRIVILEGED             = 0x000c0034;
 constexpr uint32_t UC_NUM_EVENT_COUNTERS            = 5;
 constexpr uint32_t UC_NUM_LATENCY_COUNTERS          = 1;
 constexpr uint32_t UC_MDM_PCCMDR_CLEAR_BIT          = 4;

src/runtime_src/xdp/profile/plugin/aie_profile/util/aie_profile_util.cpp

@@ -719,12 +719,30 @@ namespace xdp::aie::profile {
    * Configure counters in Microblaze Debug Module (MDM)
    * TODO: convert to using XAie_Events once support is available from driver
    ***************************************************************************/
-  void configMDMCounters(XAie_DevInst* aieDevInst, uint8_t col, uint8_t row, 
+  void configMDMCounters(XAie_DevInst* aieDevInst, int hwGen, uint8_t col, uint8_t row, 
                          const std::vector<uint32_t> events)
   {
+    // Ensure supported generation and not privileged
+    if (hwGen < 5) 
+      return;
+
+    uint32_t val;
+    static bool showWarning = true;
     auto tileOffset = XAie_GetTileAddr(aieDevInst, row, col);
+    XAie_Read32(aieDevInst, tileOffset + UC_MEMORY_PRIVILEGED, &val);
+
+    if (val) {
+      if (showWarning) {
+        xrt_core::message::send(severity_level::warning, "XRT", 
+          "Unable to configure MDM counters since debug port is privileged.");
+        showWarning = false;
+      }
+      return;
+    }
 
+    //
     // Use MDM protocol to configure counters
+    //
     // 1. Reset to first counter
     XAie_Write32(aieDevInst, tileOffset + UC_MDM_PCCMDR, 1 << UC_MDM_PCCMDR_RESET_BIT);
 
@@ -740,12 +758,30 @@ namespace xdp::aie::profile {
   /****************************************************************************
    * Read counters in Microblaze Debug Module (MDM)
    ***************************************************************************/
-  void readMDMCounters(XAie_DevInst* aieDevInst, uint8_t col, uint8_t row, 
+  void readMDMCounters(XAie_DevInst* aieDevInst, int hwGen, uint8_t col, uint8_t row, 
                        std::vector<uint64_t>& values)
   {
+    // Ensure supported generation and not privileged
+    if (hwGen < 5) 
+      return;
+
+    uint32_t val;
+    static bool showWarning = true;
     auto tileOffset = XAie_GetTileAddr(aieDevInst, row, col);
+    XAie_Read32(aieDevInst, tileOffset + UC_MEMORY_PRIVILEGED, &val);
 
+    if (val) {
+      if (showWarning) {
+        xrt_core::message::send(severity_level::warning, "XRT", 
+          "Unable to read MDM counters since debug port is privileged.");
+        showWarning = false;
+      }
+      return;
+    }
+
+    //
     // Use MDM protocol to read counters
+    //
     // 1. Sample and stop counters
     XAie_Write32(aieDevInst, tileOffset + UC_MDM_PCCMDR, 1 << UC_MDM_PCCMDR_SAMPLE_BIT);
     // NOTE: Do counters need to be stopped before reading? If so, uncomment line below.
@@ -769,15 +805,18 @@ namespace xdp::aie::profile {
       }
     }
 
-    // 4. Read values of event counters
+    // 4. Reset to first counter
+    XAie_Write32(aieDevInst, tileOffset + UC_MDM_PCCMDR, 1 << UC_MDM_PCCMDR_RESET_BIT);
+
+    // 5. Read values of event counters
     for (uint32_t c=0; c < UC_NUM_EVENT_COUNTERS; ++c) {
       uint32_t val;
       XAie_Read32(aieDevInst, tileOffset + UC_MDM_PCDRR, &val);
       uint64_t val2 = (overflows.at(c)) ? (val + OVERFLOW_32BIT) : val;
       values.push_back(val2);
     }
 
-    // 5. Read four values from latency counter
+    // 6. Read four values from latency counter
     //    Read 1 - The number of times the event occurred
     //    Read 2 - The sum of each event latency
     //    Read 3 - The sum of each event latency squared
@@ -791,7 +830,7 @@ namespace xdp::aie::profile {
       latencyValues.push_back(val2);
     }
 
-    // 6. Calculate average latency
+    // 7. Calculate average latency
     // NOTE: for now, only report average (we also have min and max; see above)
     auto numValues = latencyValues.at(0);
     auto totalLatency = latencyValues.at(1);

src/runtime_src/xdp/profile/plugin/aie_profile/util/aie_profile_util.h

@@ -233,7 +233,7 @@ namespace xdp::aie::profile {
    * @param row tile row
    * @param events vector of events to use in counters
    */
-  void configMDMCounters(XAie_DevInst* aieDevInst, uint8_t col, uint8_t row, 
+  void configMDMCounters(XAie_DevInst* aieDevInst, int hwGen, uint8_t col, uint8_t row, 
                          const std::vector<uint32_t> events);
 
   /**
@@ -243,7 +243,7 @@ namespace xdp::aie::profile {
    * @param row tile row
    * @param values vector of values from counters
    */
-  void readMDMCounters(XAie_DevInst* aieDevInst, uint8_t col, uint8_t row, 
+  void readMDMCounters(XAie_DevInst* aieDevInst, int hwGen, uint8_t col, uint8_t row, 
                        std::vector<uint64_t>& values);
 
 }  // namespace xdp::aie::profile

src/runtime_src/xdp/profile/plugin/aie_profile/ve2/aie_profile.cpp

@@ -280,6 +280,7 @@ namespace xdp {
     bool runtimeCounters = false;
 
     auto stats = aieDevice->getRscStat(XAIEDEV_DEFAULT_GROUP_AVAIL);
+    auto hwGen = metadata->getHardwareGen();
     auto configChannel0 = metadata->getConfigChannel0();
     auto configChannel1 = metadata->getConfigChannel1();
     uint8_t startColShift = metadata->getPartitionOverlayStartCols().front();
@@ -308,7 +309,7 @@ namespace xdp {
         if (module == static_cast<int>(module_type::uc)) {
           // Configure
           auto events = microcontrollerEvents[metricSet];
-          aie::profile::configMDMCounters(aieDevInst, col, row, events);
+          aie::profile::configMDMCounters(aieDevInst, hwGen, col, row, events);
           // Record
           tile_type recordTile;
           recordTile.col = col;
@@ -517,6 +518,7 @@ namespace xdp {
     uint32_t prevColumn = 0;
     uint32_t prevRow = 0;
     uint64_t timerValue = 0;
+    auto hwGen = metadata->getHardwareGen();
 
     // Iterate over all AIE Counters & Timers
     auto numCounters = db->getStaticInfo().getNumAIECounter(index);
@@ -608,7 +610,7 @@ namespace xdp {
       auto events = ucTile.second;
 
       std::vector<uint64_t> counterValues;
-      aie::profile::readMDMCounters(aieDevInst, tile.col, tile.row, counterValues);
+      aie::profile::readMDMCounters(aieDevInst, hwGen, tile.col, tile.row, counterValues);
 
       double timestamp = xrt_core::time_ns() / 1.0e6;
 

src/runtime_src/xdp/profile/plugin/aie_status/CMakeLists.txt

@@ -1,30 +1,41 @@
 # SPDX-License-Identifier: Apache-2.0
-# Copyright (C) 2022 Advanced Micro Devices, Inc.  All rights reserved.
+# Copyright (C) 2022-2025 Advanced Micro Devices, Inc.  All rights reserved.
 #
 
 # ====================================================================
-# This builds the AIE Status plugin.  It is currently only built
-# on Edge platforms that support AIE.
+# This builds the AIE Status plugin.  It is currently built on
+# Edge and VE2 platforms that support AIE.
 # ====================================================================
 
-if (DEFINED XRT_AIE_BUILD AND ${XRT_NATIVE_BUILD} STREQUAL "no")
-
 file(GLOB AIE_STATUS_PLUGIN_FILES
   "${PROFILE_DIR}/plugin/aie_status/*.h"
   "${PROFILE_DIR}/plugin/aie_status/*.cpp"
   "${PROFILE_DIR}/writer/aie_status/*.h"
   "${PROFILE_DIR}/writer/aie_status/*.cpp"
 )
 
-add_library(xdp_aie_status_plugin MODULE ${AIE_STATUS_PLUGIN_FILES})
-add_dependencies(xdp_aie_status_plugin xdp_core)
-target_link_libraries(xdp_aie_status_plugin PRIVATE xdp_core xaiengine)
-target_compile_definitions(xdp_aie_status_plugin PRIVATE FAL_LINUX="on")
+if (XDP_VE2_BUILD_CMAKE STREQUAL "yes")
+  add_library(xdp_aie_status_plugin MODULE ${AIE_STATUS_PLUGIN_FILES})
+  add_dependencies(xdp_aie_status_plugin xdp_core)
+  target_link_libraries(xdp_aie_status_plugin PRIVATE xdp_core xaiengine)
+  target_compile_definitions(xdp_aie_status_plugin PRIVATE XDP_VE2_BUILD=1 FAL_LINUX="on")
+  target_include_directories(xdp_aie_status_plugin PRIVATE ${CMAKE_SOURCE_DIR}/src)
+  set_target_properties(xdp_aie_status_plugin PROPERTIES VERSION ${XRT_VERSION_STRING} SOVERSION ${XRT_SOVERSION})
 
-set_target_properties(xdp_aie_status_plugin PROPERTIES VERSION ${XRT_VERSION_STRING} SOVERSION ${XRT_SOVERSION})
+  install (TARGETS xdp_aie_status_plugin
+    LIBRARY DESTINATION ${XDP_PLUGIN_INSTALL_DIR} COMPONENT ${XRT_COMPONENT}
+  )
 
-install (TARGETS xdp_aie_status_plugin
-  LIBRARY DESTINATION ${XDP_PLUGIN_INSTALL_DIR} COMPONENT ${XRT_COMPONENT}
-)
+elseif (DEFINED XRT_AIE_BUILD AND ${XRT_NATIVE_BUILD} STREQUAL "no")
+  add_library(xdp_aie_status_plugin MODULE ${AIE_STATUS_PLUGIN_FILES})
+  add_dependencies(xdp_aie_status_plugin xdp_core)
+  target_link_libraries(xdp_aie_status_plugin PRIVATE xdp_core xaiengine)
+  target_compile_definitions(xdp_aie_status_plugin PRIVATE FAL_LINUX="on")
+
+  set_target_properties(xdp_aie_status_plugin PROPERTIES VERSION ${XRT_VERSION_STRING} SOVERSION ${XRT_SOVERSION})
+
+  install (TARGETS xdp_aie_status_plugin
+    LIBRARY DESTINATION ${XDP_PLUGIN_INSTALL_DIR} COMPONENT ${XRT_COMPONENT}
+  )
 
 endif()