瀏覽代碼

enhance nvlink detection

AlpinDale 9 月之前
父節點
當前提交
096d9eb6c5
共有 1 個文件被更改,包括 4 次插入2 次删除
  1. 4 2
      aphrodite/distributed/device_communicators/custom_all_reduce.py

+ 4 - 2
aphrodite/distributed/device_communicators/custom_all_reduce.py

@@ -141,8 +141,10 @@ def _is_full_nvlink(rank, world_size):
     for i in range(world_size):
         if i != rank:
             try:
-                link_state = pynvml.nvmlDeviceGetNvLinkState(handle, i)
-                if not link_state:
+                peer_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
+                p2p_status = pynvml.nvmlDeviceGetP2PStatus(
+                    handle, peer_handle, pynvml.NVML_P2P_CAPS_INDEX_NVLINK)
+                if p2p_status != pynvml.NVML_P2P_STATUS_OK:
                     return False
             except pynvml.NVMLError as error:
                 logger.info(