Commit 5a93929d authored by Santosh Pradhan's avatar Santosh Pradhan Committed by Leon Romanovsky
Browse files

RDMA/rtrs-clt: Add event tracing support 

Add event tracing mechanism for following routines:
- rtrs_clt_reconnect_work()
- rtrs_clt_close_conns()
- rtrs_rdma_error_recovery()

How to use:
1. Load the rtrs_client module
2. cd /sys/kernel/debug/tracing
3. If all the events need to be enabled:
        echo 1 > events/rtrs_clt/enable
4. OR only speific routine/event needs to be enabled e.g.
        echo 1 > events/rtrs_clt/rtrs_clt_close_conns/enable
5. cat trace
6. Run some workload which can trigger rtrs_clt_close_conns()

Link: https://lore.kernel.org/r/20220818105240.110234-2-haris.iqbal@ionos.com


Signed-off-by: default avatarSantosh Pradhan <santosh.pradhan@ionos.com>
Signed-off-by: default avatarJack Wang <jinpu.wang@ionos.com>
Signed-off-by: default avatarMd Haris Iqbal <haris.iqbal@ionos.com>
Signed-off-by: default avatarLeon Romanovsky <leon@kernel.org>
parent 13ad1125

drivers/infiniband/ulp/rtrs/Makefile

+4 −1
Original line number Diff line number Diff line 
# SPDX-License-Identifier: GPL-2.0-or-later



CFLAGS_rtrs-clt-trace.o = -I$(src)



rtrs-client-y := rtrs-clt.o \

		  rtrs-clt-stats.o \

		  rtrs-clt-sysfs.o

		  rtrs-clt-sysfs.o \

		  rtrs-clt-trace.o



rtrs-server-y := rtrs-srv.o \

		  rtrs-srv-stats.o \

drivers/infiniband/ulp/rtrs/rtrs-clt-trace.c

0 → 100644
+15 −0
Original line number Diff line number Diff line 
// SPDX-License-Identifier: GPL-2.0+

/*

 * RDMA Network Block Driver

 *

 * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.

 */

#include "rtrs.h"

#include "rtrs-clt.h"



/*

 * We include this last to have the helpers above available for the trace

 * event implementations.

 */

#define CREATE_TRACE_POINTS

#include "rtrs-clt-trace.h"

drivers/infiniband/ulp/rtrs/rtrs-clt-trace.h

0 → 100644
+86 −0
Original line number Diff line number Diff line 
/* SPDX-License-Identifier: GPL-2.0+ */

/*

 * RDMA Network Block Driver

 *

 * Copyright (c) 2022 1&1 IONOS SE. All rights reserved.

 */

#undef TRACE_SYSTEM

#define TRACE_SYSTEM rtrs_clt



#if !defined(_TRACE_RTRS_CLT_H) || defined(TRACE_HEADER_MULTI_READ)

#define _TRACE_RTRS_CLT_H



#include <linux/tracepoint.h>



struct rtrs_clt_path;

struct rtrs_clt_sess;



TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING);

TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTING_ERR);

TRACE_DEFINE_ENUM(RTRS_CLT_RECONNECTING);

TRACE_DEFINE_ENUM(RTRS_CLT_CONNECTED);

TRACE_DEFINE_ENUM(RTRS_CLT_CLOSING);

TRACE_DEFINE_ENUM(RTRS_CLT_CLOSED);

TRACE_DEFINE_ENUM(RTRS_CLT_DEAD);



#define show_rtrs_clt_state(x) \

	__print_symbolic(x, \

		{ RTRS_CLT_CONNECTING,		"CONNECTING" }, \

		{ RTRS_CLT_CONNECTING_ERR,	"CONNECTING_ERR" }, \

		{ RTRS_CLT_RECONNECTING,	"RECONNECTING" }, \

		{ RTRS_CLT_CONNECTED,		"CONNECTED" }, \

		{ RTRS_CLT_CLOSING,		"CLOSING" }, \

		{ RTRS_CLT_CLOSED,		"CLOSED" }, \

		{ RTRS_CLT_DEAD,		"DEAD" })



DECLARE_EVENT_CLASS(rtrs_clt_conn_class,

	TP_PROTO(struct rtrs_clt_path *clt_path),



	TP_ARGS(clt_path),



	TP_STRUCT__entry(

		__field(int, state)

		__field(int, reconnect_attempts)

		__field(int, max_reconnect_attempts)

		__field(int, fail_cnt)

		__field(int, success_cnt)

		__array(char, sessname, NAME_MAX)

	),



	TP_fast_assign(

		struct rtrs_clt_sess *clt = clt_path->clt;



		__entry->state = clt_path->state;

		__entry->reconnect_attempts = clt_path->reconnect_attempts;

		__entry->max_reconnect_attempts = clt->max_reconnect_attempts;

		__entry->fail_cnt = clt_path->stats->reconnects.fail_cnt;

		__entry->success_cnt = clt_path->stats->reconnects.successful_cnt;

		memcpy(__entry->sessname, kobject_name(&clt_path->kobj), NAME_MAX);

	),



	TP_printk("RTRS-CLT: sess='%s' state=%s attempts='%d' max-attempts='%d' fail='%d' success='%d'",

		   __entry->sessname,

		   show_rtrs_clt_state(__entry->state),

		   __entry->reconnect_attempts,

		   __entry->max_reconnect_attempts,

		   __entry->fail_cnt,

		   __entry->success_cnt

	)

);



#define DEFINE_CLT_CONN_EVENT(name) \

DEFINE_EVENT(rtrs_clt_conn_class, rtrs_##name, \

	TP_PROTO(struct rtrs_clt_path *clt_path), \

	TP_ARGS(clt_path))



DEFINE_CLT_CONN_EVENT(clt_reconnect_work);

DEFINE_CLT_CONN_EVENT(clt_close_conns);

DEFINE_CLT_CONN_EVENT(rdma_error_recovery);



#endif /* _TRACE_RTRS_CLT_H */



#undef TRACE_INCLUDE_PATH

#define TRACE_INCLUDE_PATH .

#define TRACE_INCLUDE_FILE rtrs-clt-trace

#include <trace/define_trace.h>

drivers/infiniband/ulp/rtrs/rtrs-clt.c

+7 −0
Original line number Diff line number Diff line
@@ -16,6 +16,7 @@ 


#include "rtrs-clt.h"

#include "rtrs-log.h"

#include "rtrs-clt-trace.h"



#define RTRS_CONNECT_TIMEOUT_MS 30000

/*
@@ -302,6 +303,8 @@ static void rtrs_rdma_error_recovery(struct rtrs_clt_con *con) 
{

	struct rtrs_clt_path *clt_path = to_clt_path(con->c.path);



	trace_rtrs_rdma_error_recovery(clt_path);



	if (rtrs_clt_change_state_from_to(clt_path,

					   RTRS_CLT_CONNECTED,

					   RTRS_CLT_RECONNECTING)) {
@@ -1942,6 +1945,8 @@ static int rtrs_rdma_conn_rejected(struct rtrs_clt_con *con, 


void rtrs_clt_close_conns(struct rtrs_clt_path *clt_path, bool wait)

{

	trace_rtrs_clt_close_conns(clt_path);



	if (rtrs_clt_change_state_get_old(clt_path, RTRS_CLT_CLOSING, NULL))

		queue_work(rtrs_wq, &clt_path->close_work);

	if (wait)
@@ -2648,6 +2653,8 @@ static void rtrs_clt_reconnect_work(struct work_struct *work) 
				reconnect_dwork);

	clt = clt_path->clt;



	trace_rtrs_clt_reconnect_work(clt_path);



	if (READ_ONCE(clt_path->state) != RTRS_CLT_RECONNECTING)

		return;