rpma: adopt usage_to_access to support iWARP specificity

src/mr.c

@@ -58,46 +58,6 @@ struct rpma_mr_remote {
 	int usage; /* usage of the memory region */
 };
 
-/* helper functions */
-
-/*
- * usage_to_access -- convert usage to access
- *
- * Note: APM type of flush requires the same access as RPMA_MR_USAGE_READ_SRC
- */
-static int
-usage_to_access(int usage)
-{
-	int access = 0;
-
-	if (usage & (RPMA_MR_USAGE_READ_SRC |\
-			RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY |\
-			RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT))
-		access |= IBV_ACCESS_REMOTE_READ;
-
-	if (usage & RPMA_MR_USAGE_READ_DST)
-		access |= IBV_ACCESS_LOCAL_WRITE;
-
-	if (usage & RPMA_MR_USAGE_WRITE_SRC)
-		access |= IBV_ACCESS_LOCAL_WRITE;
-
-	if (usage & RPMA_MR_USAGE_WRITE_DST)
-		/*
-		 * If IBV_ACCESS_REMOTE_WRITE is set, then
-		 * IBV_ACCESS_LOCAL_WRITE must be set too.
-		 */
-		access |= IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
-
-	if (usage & RPMA_MR_USAGE_RECV)
-		access |= IBV_ACCESS_LOCAL_WRITE;
-
-	/*
-	 * There is no IBV_ACCESS_* value to be set for RPMA_MR_USAGE_SEND.
-	 */
-
-	return access;
-}
-
 /* internal librpma API */
 
 /*
@@ -324,6 +284,8 @@ int
 rpma_mr_reg(struct rpma_peer *peer, void *ptr, size_t size, int usage,
 		struct rpma_mr_local **mr_ptr)
 {
+	int ret;
+
 	if (peer == NULL || ptr == NULL || size == 0 || mr_ptr == NULL)
 		return RPMA_E_INVAL;
 
@@ -336,9 +298,7 @@ rpma_mr_reg(struct rpma_peer *peer, void *ptr, size_t size, int usage,
 		return RPMA_E_NOMEM;
 
 	struct ibv_mr *ibv_mr;
-	int ret = rpma_peer_mr_reg(peer, &ibv_mr, ptr, size,
-			usage_to_access(usage));
-	if (ret) {
+	if ((ret = rpma_peer_mr_reg(peer, &ibv_mr, ptr, size, usage))) {
 		free(mr);
 		return ret;
 	}

src/peer.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * peer.c -- librpma peer-related implementations
@@ -31,6 +31,54 @@ struct rpma_peer {
 
 /* internal librpma API */
 
+/*
+ * rpma_peer_usage_to_access -- convert usage to access
+ *
+ * Note: APM type of flush requires the same access as RPMA_MR_USAGE_READ_SRC
+ */
+static int
+rpma_peer_usage_to_access(struct rpma_peer *peer, int usage)
+{
+	enum ibv_transport_type type =
+			peer->pd->context->device->transport_type;
+	int access = 0;
+
+	if (usage & (RPMA_MR_USAGE_READ_SRC |\
+			RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY |\
+			RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT))
+		access |= IBV_ACCESS_REMOTE_READ;
+
+	if (usage & RPMA_MR_USAGE_READ_DST) {
+		access |= IBV_ACCESS_LOCAL_WRITE;
+
+		/*
+		 * iWARP implements the READ operation as the WRITE operation
+		 * in the opposite direction.
+		 */
+		if (type == IBV_TRANSPORT_IWARP)
+			access |= IBV_ACCESS_REMOTE_WRITE;
+	}
+
+	if (usage & RPMA_MR_USAGE_WRITE_SRC)
+		access |= IBV_ACCESS_LOCAL_WRITE;
+
+	if (usage & RPMA_MR_USAGE_WRITE_DST)
+		/*
+		 * If IBV_ACCESS_REMOTE_WRITE is set, then
+		 * IBV_ACCESS_LOCAL_WRITE must be set too.
+		 */
+		access |= IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE;
+
+	if (usage & RPMA_MR_USAGE_RECV)
+		access |= IBV_ACCESS_LOCAL_WRITE;
+
+	/*
+	 * There is no IBV_ACCESS_* value to be set for RPMA_MR_USAGE_SEND.
+	 */
+
+	return access;
+}
+
 /*
  * rpma_peer_create_qp -- allocate a QP associated with the CM ID
  *
@@ -108,8 +156,10 @@ rpma_peer_create_qp(struct rpma_peer *peer, struct rdma_cm_id *id,
  */
 int
 rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr,
-		void *addr, size_t length, int access)
+		void *addr, size_t length, int usage)
 {
+	int access = rpma_peer_usage_to_access(peer, usage);
+
 	*ibv_mr_ptr = ibv_reg_mr(peer->pd, addr, length,
 					RPMA_IBV_ACCESS(access));
 	if (*ibv_mr_ptr != NULL)

src/peer.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause */
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * peer.h -- librpma peer-related internal definitions
@@ -33,6 +33,6 @@ int rpma_peer_create_qp(struct rpma_peer *peer, struct rdma_cm_id *id,
  * - RPMA_E_PROVIDER - registering the memory region failed
  */
 int rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr,
-		void *addr, size_t length, int access);
+		void *addr, size_t length, int usage);
 
 #endif /* LIBRPMA_PEER_H */

tests/integration/common/mocks.c

@@ -23,6 +23,10 @@ struct ibv_mr Ibv_mr_flush;	/* mock IBV MR FLUSH */
 struct rdma_cm_id Cm_id;	/* mock CM ID */
 struct ibv_qp Ibv_qp;		/* mock IBV QP */
 
+struct ibv_device Ibv_device = {{0}, 0, IBV_TRANSPORT_IB};
+struct ibv_context Ibv_context = {&Ibv_device};
+struct ibv_pd Ibv_pd = {&Ibv_context, 0};
+
 #ifdef ON_DEMAND_PAGING_SUPPORTED
 /* predefined IBV On-demand Paging caps */
 struct ibv_odp_caps Ibv_odp_capable_caps = {

tests/integration/common/mocks.h

@@ -12,15 +12,15 @@
 #define MOCK_PORT		"1234"
 #define MOCK_MR			(&Ibv_mr)
 #define MOCK_MR_RAW		(&Ibv_mr_raw)
-#define MOCK_MR_FLUSH	(&Ibv_mr_flush)
+#define MOCK_MR_FLUSH		(&Ibv_mr_flush)
 #define MOCK_RAW_SIZE		8 /* memory region size */
 #define MOCK_COMP_CHANNEL	(&Ibv_comp_channel)
+#define MOCK_IBV_PD		(&Ibv_pd)
 #define MOCK_CQ			(&Ibv_cq)
 #define MOCK_VERBS		(&Verbs_context.context)
 #define MOCK_EVCH		((struct rdma_event_channel *)0xE4C4)
 #define MOCK_SRC_ADDR		((struct sockaddr *)0x0ADD)
 #define MOCK_DST_ADDR		((struct sockaddr *)0x0ADE)
-#define MOCK_IBV_PD		((struct ibv_pd *)0x00D0)
 #define MOCK_QP			((struct ibv_qp *)0xD56A)
 #define MOCK_OP_CONTEXT		((void *)0xC417)
 #define MOCK_RKEY		((uint32_t)0x10111213)
@@ -48,12 +48,15 @@ struct common_data {
 extern struct verbs_context Verbs_context;
 /* mock IBV completion channel */
 extern struct ibv_comp_channel Ibv_comp_channel;
+extern struct ibv_context Ibv_context;	/* mock IBV context */
+extern struct ibv_device Ibv_device;	/* mock IBV device */
 extern struct ibv_cq Ibv_cq;		/* mock IBV CQ */
 extern struct ibv_mr Ibv_mr;		/* mock IBV MR */
 extern struct ibv_mr Ibv_mr_raw;	/* mock IBV MR RAW */
 extern struct ibv_mr Ibv_mr_flush;	/* mock IBV MR FLUSH */
-extern struct rdma_cm_id Cm_id;	/* mock CM ID */
+extern struct rdma_cm_id Cm_id;		/* mock CM ID */
 extern struct ibv_qp Ibv_qp;		/* mock IBV QP */
+extern struct ibv_pd Ibv_pd;		/* mock IBV PD */
 
 /* predefined IBV On-demand Paging caps */
 extern struct ibv_odp_caps Ibv_odp_capable_caps;

tests/unit/common/mocks-ibverbs.c

@@ -15,6 +15,9 @@
 /* mocked IBV entities */
 struct verbs_context Verbs_context;
 struct ibv_comp_channel Ibv_comp_channel;
+struct ibv_device Ibv_device;
+struct ibv_context Ibv_context = {&Ibv_device};
+struct ibv_pd Ibv_pd = {&Ibv_context, 0};
 struct ibv_cq Ibv_cq;
 struct ibv_qp Ibv_qp;
 struct ibv_mr Ibv_mr;

tests/unit/common/mocks-ibverbs.h

@@ -13,6 +13,9 @@
 /* mocked IBV entities */
 extern struct verbs_context Verbs_context;
 extern struct ibv_comp_channel Ibv_comp_channel;
+extern struct ibv_context Ibv_context;
+extern struct ibv_device Ibv_device;
+extern struct ibv_pd Ibv_pd;
 extern struct ibv_cq Ibv_cq;
 extern struct ibv_qp Ibv_qp;
 extern struct ibv_mr Ibv_mr;
@@ -21,7 +24,7 @@ extern struct ibv_mr Ibv_mr;
 #define MOCK_VERBS		(&Verbs_context.context)
 #define MOCK_COMP_CHANNEL	(struct ibv_comp_channel *)&Ibv_comp_channel
 #define MOCK_IBV_CQ		(struct ibv_cq *)&Ibv_cq
-#define MOCK_IBV_PD		(struct ibv_pd *)0x00D0
+#define MOCK_IBV_PD		(struct ibv_pd *)&Ibv_pd
 #define MOCK_QP			(struct ibv_qp *)&Ibv_qp
 #define MOCK_MR			(struct ibv_mr *)&Ibv_mr
 

tests/unit/common/mocks-rpma-peer.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * mocks-rpma-peer.c -- librpma peer.c module mocks
@@ -38,7 +38,7 @@ rpma_peer_create_qp(struct rpma_peer *peer, struct rdma_cm_id *id,
  */
 int
 rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr,
-		void *addr, size_t length, int access)
+		void *addr, size_t length, int usage)
 {
 	/*
 	 * rpma_peer_mr_reg() and malloc() may be called in any order.
@@ -51,7 +51,7 @@ rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr,
 	assert_ptr_equal(peer, MOCK_PEER);
 	assert_ptr_equal(addr, MOCK_PTR);
 	assert_int_equal(length, MOCK_SIZE);
-	assert_int_equal(access, args->access);
+	assert_int_equal(usage, args->usage);
 
 	*ibv_mr_ptr = args->mr;
 	if (*ibv_mr_ptr == NULL) {

tests/unit/mr/mr-reg.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: BSD-3-Clause
-/* Copyright 2020, Intel Corporation */
+/* Copyright 2020-2021, Intel Corporation */
 
 /*
  * mr-reg.c -- the memory region registration/deregistration unit tests
@@ -24,33 +24,8 @@ static struct prestate prestates[] = {
 	/* values used in reg_dereg__success called with (prestates + 0) */
 	{RPMA_MR_USAGE_READ_SRC, IBV_ACCESS_REMOTE_READ, NULL},
 	/* values used in reg_dereg__success called with (prestates + 1) */
-	{RPMA_MR_USAGE_READ_DST, IBV_ACCESS_LOCAL_WRITE, NULL},
-	/* values used in reg_dereg__success called with (prestates + 2) */
 	{(RPMA_MR_USAGE_READ_SRC | RPMA_MR_USAGE_READ_DST),
 		(IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE), NULL},
-	/* values used in reg_dereg__success called with (prestates + 3) */
-	{RPMA_MR_USAGE_WRITE_SRC, IBV_ACCESS_LOCAL_WRITE, NULL},
-	/* values used in reg_dereg__success called with (prestates + 4) */
-	{RPMA_MR_USAGE_WRITE_DST,
-		(IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE), NULL},
-	/* values used in reg_dereg__success called with (prestates + 5) */
-	{RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT, IBV_ACCESS_REMOTE_READ, NULL},
-	/* values used in reg_dereg__success called with (prestates + 6) */
-	{(RPMA_MR_USAGE_WRITE_SRC | RPMA_MR_USAGE_WRITE_DST |
-	RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT),
-		(IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE |
-		IBV_ACCESS_REMOTE_READ), NULL},
-	/* values used in reg_dereg__success called with (prestates + 7) */
-	{RPMA_MR_USAGE_RECV, IBV_ACCESS_LOCAL_WRITE, NULL},
-	/* values used in reg_dereg__success called with (prestates + 8) */
-	{RPMA_MR_USAGE_SEND, 0, NULL},
-	/* values used in reg_dereg__success called with (prestates + 9) */
-	{(RPMA_MR_USAGE_READ_SRC | RPMA_MR_USAGE_READ_DST |
-	RPMA_MR_USAGE_WRITE_SRC | RPMA_MR_USAGE_WRITE_DST |
-	RPMA_MR_USAGE_RECV | RPMA_MR_USAGE_SEND |
-	RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT),
-		(IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE |
-		IBV_ACCESS_LOCAL_WRITE), NULL},
 };
 
 /*
@@ -285,32 +260,14 @@ static const struct CMUnitTest tests_reg[] = {
 	cmocka_unit_test(reg__wrong_usage),
 	cmocka_unit_test(reg__failed_E_NOMEM),
 	cmocka_unit_test(reg__peer_mr_reg_failed_E_PROVIDER),
-	{ "reg_dereg__USAGE_READ_SRC", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 0},
-	{ "reg_dereg__USAGE_READ_DST", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 1},
-	{ "reg_dereg__USAGE_READ_SRC_DST", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 2},
-	{ "reg_dereg__USAGE_WRITE_SRC", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 3},
-	{ "reg_dereg__USAGE_WRITE_DST", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 4},
-	{ "reg_dereg__USAGE_FLUSHABLE", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 5},
-	{ "reg_dereg__USAGE_WRITE_SRC_DST_FLUSHABLE", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 6},
-	{ "reg_dereg__USAGE_RECV", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 7},
-	{ "reg_dereg__USAGE_SEND", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 8},
-	{ "reg_dereg__USAGE_ALL", reg_dereg__success,
-		setup__reg_success, teardown__dereg_success, prestates + 9},
+	cmocka_unit_test_prestate_setup_teardown(reg_dereg__success,
+		setup__reg_success, teardown__dereg_success, prestates),
 
 	/* rpma_mr_dereg() unit tests */
 	cmocka_unit_test(dereg__NULL_mr_ptr),
 	cmocka_unit_test(dereg__NULL_mr),
 	cmocka_unit_test_prestate(dereg__failed_E_PROVIDER,
-		prestates + 2),
+		prestates + 1),
 	cmocka_unit_test(NULL)
 };