[RFC] Support dynamically loading LTTng tracing

debian/control

@@ -18,6 +18,7 @@ Build-Depends: cmake (>= 2.8.11),
                pkg-config,
                python3-dev,
                python3-docutils,
+               liblttng-ust-dev (>= 2.13.0),
                valgrind [amd64 arm64 armhf i386 mips mips64el mipsel powerpc ppc64 ppc64el s390x]
 Rules-Requires-Root: no
 Standards-Version: 4.7.0
@@ -116,6 +117,25 @@ Description: Examples for the libibverbs library
  This package contains useful libibverbs1 example programs such as
  ibv_devinfo, which displays information about InfiniBand devices.
 
+Package: ibverbs-trace
+Section: libs
+Architecture: linux-any
+Multi-Arch: same
+Depends: liblttng-ust1 (>= 2.13.0)
+         ${misc:Depends},
+         ${shlibs:Depends}
+Description: Trace for the libibverbs library
+ libibverbs is a library that allows userspace processes to use RDMA
+ "verbs" as described in the InfiniBand Architecture Specification and
+ the RDMA Protocol Verbs Specification.  iWARP ethernet NICs support
+ RDMA over hardware-offloaded TCP/IP, while InfiniBand is a
+ high-throughput, low-latency networking technology.  InfiniBand host
+ channel adapters (HCAs) and iWARP NICs commonly support direct
+ hardware access from userspace (kernel bypass), and libibverbs
+ supports this when available.
+ .
+ This package contains trace libraries for libibverbs1.
+
 Package: libibverbs-dev
 Section: libdevel
 Architecture: linux-any

debian/ibverbs-providers.install

@@ -1,7 +1,23 @@
 etc/libibverbs.d/
 usr/lib/*/libefa.so.*
 usr/lib/*/libhns.so.*
-usr/lib/*/libibverbs/lib*-rdmav*.so
 usr/lib/*/libmana.so.*
 usr/lib/*/libmlx4.so.*
 usr/lib/*/libmlx5.so.*
+usr/lib/*/libibverbs/libbnxt_re-rdmav*.so
+usr/lib/*/libibverbs/libcxgb4-rdmav*.so
+usr/lib/*/libibverbs/libefa-rdmav*.so
+usr/lib/*/libibverbs/liberdma-rdmav*.so
+usr/lib/*/libibverbs/libhfi1verbs-rdmav*.so
+usr/lib/*/libibverbs/libhns-rdmav*.so
+usr/lib/*/libibverbs/libipathverbs-rdmav*.so
+usr/lib/*/libibverbs/libirdma-rdmav*.so
+usr/lib/*/libibverbs/libmana-rdmav*.so
+usr/lib/*/libibverbs/libmlx4-rdmav*.so
+usr/lib/*/libibverbs/libmlx5-rdmav*.so
+usr/lib/*/libibverbs/libmthca-rdmav*.so
+usr/lib/*/libibverbs/libocrdma-rdmav*.so
+usr/lib/*/libibverbs/libqedr-rdmav*.so
+usr/lib/*/libibverbs/librxe-rdmav*.so
+usr/lib/*/libibverbs/libsiw-rdmav*.so
+usr/lib/*/libibverbs/libvmw_pvrdma-rdmav*.so

debian/ibverbs-trace.install

@@ -0,0 +1 @@
+usr/lib/*/libibverbs/lib*trace-rdmav*.so

debian/rules

@@ -37,6 +37,7 @@ DH_AUTO_CONFIGURE := "--" \
 		     "-DCMAKE_INSTALL_UDEV_RULESDIR:PATH=/lib/udev/rules.d" \
 		     "-DCMAKE_INSTALL_PERLDIR:PATH=/usr/share/perl5" \
 		     "-DENABLE_STATIC=1" \
+		     "-DENABLE_LTTNG=1" \
 		     $(EXTRA_CMAKE_FLAGS)
 
 override_dh_auto_configure:
@@ -49,7 +50,6 @@ override_dh_auto_configure:
 			        -DNO_PYVERBS=1; \
 	fi
 
-
 override_dh_auto_build:
 	ninja -C build-deb -v
 

libibverbs/CMakeLists.txt

@@ -14,6 +14,7 @@ publish_internal_headers(infiniband
   driver.h
   kern-abi.h
   marshall.h
+  trace.h
   )
 
 configure_file("libibverbs.map.in"

libibverbs/trace.h

@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+/*
+ * Copyright (c) 2025 Hisilicon Limited.
+ */
+
+#ifndef _TRACE_H
+#define _TRACE_H
+
+#if defined(LTTNG_ENABLED)
+
+#include <lttng/tracepoint.h>
+
+#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
+
+#else
+
+#define rdma_tracepoint(arg...)
+
+#endif /* defined(LTTNG_ENABLED) */
+
+#endif /* _TRACE_H */

libibverbs/verbs.h

@@ -2284,6 +2284,10 @@ extern const struct verbs_device_ops verbs_provider_qedr;
 extern const struct verbs_device_ops verbs_provider_rxe;
 extern const struct verbs_device_ops verbs_provider_siw;
 extern const struct verbs_device_ops verbs_provider_vmw_pvrdma;
+extern const struct verbs_device_ops verbs_provider_efa_trace;
+extern const struct verbs_device_ops verbs_provider_hns_trace;
+extern const struct verbs_device_ops verbs_provider_mlx5_trace;
+extern const struct verbs_device_ops verbs_provider_rxe_trace;
 extern const struct verbs_device_ops verbs_provider_all;
 extern const struct verbs_device_ops verbs_provider_none;
 void ibv_static_providers(void *unused, ...);

providers/efa/CMakeLists.txt

@@ -1,10 +1,11 @@
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-  set(TRACE_FILE efa_trace.c)
+  rdma_provider(efa_trace
+    efa_trace.c
+  )
 endif()
 
 rdma_shared_provider(efa libefa.map
 	1 1.3.${PACKAGE_VERSION}
-	${TRACE_FILE}
 	efa.c
 	verbs.c
 )
@@ -16,6 +17,12 @@ publish_headers(infiniband
 rdma_pkg_config("efa" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
 
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-	target_include_directories(efa PUBLIC ".")
 	target_link_libraries(efa LINK_PRIVATE LTTng::UST)
+	target_include_directories(efa_trace-rdmav${IBVERBS_PABI_VERSION} PUBLIC ".")
+	target_link_libraries(efa_trace-rdmav${IBVERBS_PABI_VERSION} LINK_PRIVATE LTTng::UST)
+	if (ENABLE_STATIC)
+		target_link_libraries(efa-static LINK_PRIVATE LTTng::UST)
+		target_include_directories(efa_trace PUBLIC ".")
+		target_link_libraries(efa_trace LINK_PRIVATE LTTng::UST)
+	endif()
 endif()

providers/efa/efa_trace.c

@@ -4,6 +4,5 @@
  */
 
 #define LTTNG_UST_TRACEPOINT_CREATE_PROBES
-#define LTTNG_UST_TRACEPOINT_DEFINE
 
 #include "efa_trace.h"

providers/efa/efa_trace.h

@@ -103,8 +103,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 	)
 )
 
-#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
-
 #endif /* __EFA_TRACE_H__*/
 
 #include <lttng/tracepoint-event.h>
@@ -114,8 +112,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 #ifndef __EFA_TRACE_H__
 #define __EFA_TRACE_H__
 
-#define rdma_tracepoint(arg...)
-
 #endif /* __EFA_TRACE_H__*/
 
 #endif /* defined(LTTNG_ENABLED) */

providers/efa/verbs.c

@@ -22,7 +22,11 @@
 #include "efa_io_regs_defs.h"
 #include "efadv.h"
 #include "verbs.h"
+
+#define LTTNG_UST_TRACEPOINT_DEFINE
+#define LTTNG_UST_TRACEPOINT_PROBE_DYNAMIC_LINKAGE
 #include "efa_trace.h"
+#include <infiniband/trace.h>
 
 #define EFA_DEV_CAP(ctx, cap) \
 	((ctx)->device_caps & EFA_QUERY_DEVICE_CAPS_##cap)

providers/hns/CMakeLists.txt

@@ -1,10 +1,11 @@
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-  set(TRACE_FILE hns_roce_u_trace.c)
+  rdma_provider(hns_trace
+    hns_roce_u_trace.c
+    )
 endif()
 
 rdma_shared_provider(hns libhns.map
   1 1.0.${PACKAGE_VERSION}
-  ${TRACE_FILE}
   hns_roce_u.c
   hns_roce_u_buf.c
   hns_roce_u_db.c
@@ -19,6 +20,12 @@ publish_headers(infiniband
 rdma_pkg_config("hns" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
 
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-  target_include_directories(hns PUBLIC ".")
   target_link_libraries(hns LINK_PRIVATE LTTng::UST)
+  target_include_directories(hns_trace-rdmav${IBVERBS_PABI_VERSION} PUBLIC ".")
+  target_link_libraries(hns_trace-rdmav${IBVERBS_PABI_VERSION} LINK_PRIVATE LTTng::UST)
+  if (ENABLE_STATIC)
+    target_link_libraries(hns-static LINK_PRIVATE LTTng::UST)
+    target_include_directories(hns_trace PUBLIC ".")
+    target_link_libraries(hns_trace LINK_PRIVATE LTTng::UST)
+  endif()
 endif()

providers/hns/hns_roce_u_hw_v2.c

@@ -38,7 +38,11 @@
 #include "hns_roce_u.h"
 #include "hns_roce_u_db.h"
 #include "hns_roce_u_hw_v2.h"
+
+#define LTTNG_UST_TRACEPOINT_DEFINE
+#define LTTNG_UST_TRACEPOINT_PROBE_DYNAMIC_LINKAGE
 #include "hns_roce_u_trace.h"
+#include <infiniband/trace.h>
 
 #define HR_IBV_OPC_MAP(ib_key, hr_key) \
 		[IBV_WR_ ## ib_key] = HNS_ROCE_WQE_OP_ ## hr_key

providers/hns/hns_roce_u_trace.c

@@ -4,6 +4,5 @@
  */
 
 #define LTTNG_UST_TRACEPOINT_CREATE_PROBES
-#define LTTNG_UST_TRACEPOINT_DEFINE
 
 #include "hns_roce_u_trace.h"

providers/hns/hns_roce_u_trace.h

@@ -115,8 +115,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 	)
 )
 
-#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
-
 #endif /* __HNS_TRACE_H__*/
 
 #include <lttng/tracepoint-event.h>
@@ -126,8 +124,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 #ifndef __HNS_TRACE_H__
 #define __HNS_TRACE_H__
 
-#define rdma_tracepoint(arg...)
-
 #endif /* __HNS_TRACE_H__*/
 
 #endif /* defined(LTTNG_ENABLED) */

providers/mlx5/CMakeLists.txt

@@ -11,12 +11,13 @@ if (MLX5_MW_DEBUG)
 endif()
 
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-  set(TRACE_FILE mlx5_trace.c)
+  rdma_provider(mlx5_trace
+    mlx5_trace.c
+)
 endif()
 
 rdma_shared_provider(mlx5 libmlx5.map
   1 1.25.${PACKAGE_VERSION}
-  ${TRACE_FILE}
   buf.c
   cq.c
   dbrec.c
@@ -55,6 +56,12 @@ publish_headers(infiniband
 rdma_pkg_config("mlx5" "libibverbs" "${CMAKE_THREAD_LIBS_INIT}")
 
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-	target_include_directories(mlx5 PUBLIC ".")
 	target_link_libraries(mlx5 LINK_PRIVATE LTTng::UST)
+	target_include_directories(mlx5_trace-rdmav${IBVERBS_PABI_VERSION} PUBLIC ".")
+	target_link_libraries(mlx5_trace-rdmav${IBVERBS_PABI_VERSION} LINK_PRIVATE LTTng::UST)
+	if (ENABLE_STATIC)
+		target_link_libraries(mlx5-static LINK_PRIVATE LTTng::UST)
+		target_include_directories(mlx5_trace PUBLIC ".")
+		target_link_libraries(mlx5_trace LINK_PRIVATE LTTng::UST)
+	endif()
 endif()

providers/mlx5/mlx5_trace.c

@@ -4,6 +4,5 @@
  */
 
 #define LTTNG_UST_TRACEPOINT_CREATE_PROBES
-#define LTTNG_UST_TRACEPOINT_DEFINE
 
 #include "mlx5_trace.h"

providers/mlx5/mlx5_trace.h

@@ -41,8 +41,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 	)
 )
 
-#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
-
 #endif /* __MLX5_TRACE_H__*/
 
 #include <lttng/tracepoint-event.h>
@@ -52,8 +50,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 #ifndef __MLX5_TRACE_H__
 #define __MLX5_TRACE_H__
 
-#define rdma_tracepoint(arg...)
-
 #endif /* __MLX5_TRACE_H__*/
 
 #endif /* defined(LTTNG_ENABLED) */

providers/mlx5/qp.c

@@ -42,7 +42,11 @@
 
 #include "mlx5.h"
 #include "mlx5_ifc.h"
+
+#define LTTNG_UST_TRACEPOINT_DEFINE
+#define LTTNG_UST_TRACEPOINT_PROBE_DYNAMIC_LINKAGE
 #include "mlx5_trace.h"
+#include <infiniband/trace.h>
 #include "wqe.h"
 
 #define MLX5_ATOMIC_SIZE 8

providers/rxe/CMakeLists.txt

@@ -1,13 +1,20 @@
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-  set(TRACE_FILE rxe_trace.c)
+  rdma_provider(rxe_trace
+    rxe_trace.c
+    )
 endif()
 
 rdma_provider(rxe
-  ${TRACE_FILE}
   rxe.c
   )
 
 if (ENABLE_LTTNG AND LTTNGUST_FOUND)
-	target_include_directories("rxe-rdmav${IBVERBS_PABI_VERSION}" PUBLIC ".")
 	target_link_libraries("rxe-rdmav${IBVERBS_PABI_VERSION}" LINK_PRIVATE LTTng::UST)
+	target_include_directories("rxe_trace-rdmav${IBVERBS_PABI_VERSION}" PUBLIC ".")
+	target_link_libraries("rxe_trace-rdmav${IBVERBS_PABI_VERSION}" LINK_PRIVATE LTTng::UST)
+	if (ENABLE_STATIC)
+		target_link_libraries(rxe LINK_PRIVATE LTTng::UST)
+		target_include_directories("rxe_trace" PUBLIC ".")
+		target_link_libraries("rxe_trace" LINK_PRIVATE LTTng::UST)
+	endif()
 endif()

providers/rxe/rxe.c

@@ -55,7 +55,11 @@
 #include "rxe_queue.h"
 #include "rxe-abi.h"
 #include "rxe.h"
+
+#define LTTNG_UST_TRACEPOINT_DEFINE
+#define LTTNG_UST_TRACEPOINT_PROBE_DYNAMIC_LINKAGE
 #include "rxe_trace.h"
+#include <infiniband/trace.h>
 
 static void rxe_free_context(struct ibv_context *ibctx);
 

providers/rxe/rxe_trace.c

@@ -4,6 +4,5 @@
  */
 
 #define LTTNG_UST_TRACEPOINT_CREATE_PROBES
-#define LTTNG_UST_TRACEPOINT_DEFINE
 
 #include "rxe_trace.h"

providers/rxe/rxe_trace.h

@@ -41,8 +41,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 	)
 )
 
-#define rdma_tracepoint(arg...) lttng_ust_tracepoint(arg)
-
 #endif /* __RXE_TRACE_H__*/
 
 #include <lttng/tracepoint-event.h>
@@ -52,8 +50,6 @@ LTTNG_UST_TRACEPOINT_EVENT(
 #ifndef __RXE_TRACE_H__
 #define __RXE_TRACE_H__
 
-#define rdma_tracepoint(arg...)
-
 #endif /* __RXE_TRACE_H__*/
 
 #endif /* defined(LTTNG_ENABLED) */