From e442b98cf980ec9a37ba5d476698e76a0ac59d7e Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 6 May 2021 13:51:12 +0200 Subject: [PATCH] rpma: adopt usage_to_access to support iWARP specificity --- src/mr.c | 46 +----------- src/peer.c | 54 +++++++++++++- src/peer.h | 4 +- tests/integration/common/mocks.c | 4 + tests/integration/common/mocks.h | 9 ++- tests/unit/common/mocks-ibverbs.c | 3 + tests/unit/common/mocks-ibverbs.h | 5 +- tests/unit/common/mocks-rpma-peer.c | 6 +- tests/unit/mr/mr-reg.c | 51 +------------ tests/unit/peer/peer-common.c | 59 ++++++++++++++- tests/unit/peer/peer-common.h | 32 +++++++- tests/unit/peer/peer-mr_reg.c | 109 ++++++++++++++++++++++++---- 12 files changed, 264 insertions(+), 118 deletions(-) diff --git a/src/mr.c b/src/mr.c index 5345aa1e57..1da178af60 100644 --- a/src/mr.c +++ b/src/mr.c @@ -58,46 +58,6 @@ struct rpma_mr_remote { int usage; /* usage of the memory region */ }; -/* helper functions */ - -/* - * usage_to_access -- convert usage to access - * - * Note: APM type of flush requires the same access as RPMA_MR_USAGE_READ_SRC - */ -static int -usage_to_access(int usage) -{ - int access = 0; - - if (usage & (RPMA_MR_USAGE_READ_SRC |\ - RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY |\ - RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT)) - access |= IBV_ACCESS_REMOTE_READ; - - if (usage & RPMA_MR_USAGE_READ_DST) - access |= IBV_ACCESS_LOCAL_WRITE; - - if (usage & RPMA_MR_USAGE_WRITE_SRC) - access |= IBV_ACCESS_LOCAL_WRITE; - - if (usage & RPMA_MR_USAGE_WRITE_DST) - /* - * If IBV_ACCESS_REMOTE_WRITE is set, then - * IBV_ACCESS_LOCAL_WRITE must be set too. - */ - access |= IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE; - - if (usage & RPMA_MR_USAGE_RECV) - access |= IBV_ACCESS_LOCAL_WRITE; - - /* - * There is no IBV_ACCESS_* value to be set for RPMA_MR_USAGE_SEND. - */ - - return access; -} - /* internal librpma API */ /* @@ -324,6 +284,8 @@ int rpma_mr_reg(struct rpma_peer *peer, void *ptr, size_t size, int usage, struct rpma_mr_local **mr_ptr) { + int ret; + if (peer == NULL || ptr == NULL || size == 0 || mr_ptr == NULL) return RPMA_E_INVAL; @@ -336,9 +298,7 @@ rpma_mr_reg(struct rpma_peer *peer, void *ptr, size_t size, int usage, return RPMA_E_NOMEM; struct ibv_mr *ibv_mr; - int ret = rpma_peer_mr_reg(peer, &ibv_mr, ptr, size, - usage_to_access(usage)); - if (ret) { + if ((ret = rpma_peer_mr_reg(peer, &ibv_mr, ptr, size, usage))) { free(mr); return ret; } diff --git a/src/peer.c b/src/peer.c index 7ca4b4aafd..4fa2770106 100644 --- a/src/peer.c +++ b/src/peer.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * peer.c -- librpma peer-related implementations @@ -31,6 +31,54 @@ struct rpma_peer { /* internal librpma API */ +/* + * rpma_peer_usage_to_access -- convert usage to access + * + * Note: APM type of flush requires the same access as RPMA_MR_USAGE_READ_SRC + */ +static int +rpma_peer_usage_to_access(struct rpma_peer *peer, int usage) +{ + enum ibv_transport_type type = + peer->pd->context->device->transport_type; + int access = 0; + + if (usage & (RPMA_MR_USAGE_READ_SRC |\ + RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY |\ + RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT)) + access |= IBV_ACCESS_REMOTE_READ; + + if (usage & RPMA_MR_USAGE_READ_DST) { + access |= IBV_ACCESS_LOCAL_WRITE; + + /* + * iWARP implements the READ operation as the WRITE operation + * in the opposite direction. + */ + if (type == IBV_TRANSPORT_IWARP) + access |= IBV_ACCESS_REMOTE_WRITE; + } + + if (usage & RPMA_MR_USAGE_WRITE_SRC) + access |= IBV_ACCESS_LOCAL_WRITE; + + if (usage & RPMA_MR_USAGE_WRITE_DST) + /* + * If IBV_ACCESS_REMOTE_WRITE is set, then + * IBV_ACCESS_LOCAL_WRITE must be set too. + */ + access |= IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE; + + if (usage & RPMA_MR_USAGE_RECV) + access |= IBV_ACCESS_LOCAL_WRITE; + + /* + * There is no IBV_ACCESS_* value to be set for RPMA_MR_USAGE_SEND. + */ + + return access; +} + /* * rpma_peer_create_qp -- allocate a QP associated with the CM ID * @@ -108,8 +156,10 @@ rpma_peer_create_qp(struct rpma_peer *peer, struct rdma_cm_id *id, */ int rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr, - void *addr, size_t length, int access) + void *addr, size_t length, int usage) { + int access = rpma_peer_usage_to_access(peer, usage); + *ibv_mr_ptr = ibv_reg_mr(peer->pd, addr, length, RPMA_IBV_ACCESS(access)); if (*ibv_mr_ptr != NULL) diff --git a/src/peer.h b/src/peer.h index 50fb7d6c07..9273546a6a 100644 --- a/src/peer.h +++ b/src/peer.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * peer.h -- librpma peer-related internal definitions @@ -33,6 +33,6 @@ int rpma_peer_create_qp(struct rpma_peer *peer, struct rdma_cm_id *id, * - RPMA_E_PROVIDER - registering the memory region failed */ int rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr, - void *addr, size_t length, int access); + void *addr, size_t length, int usage); #endif /* LIBRPMA_PEER_H */ diff --git a/tests/integration/common/mocks.c b/tests/integration/common/mocks.c index 89557ae8fd..41e7fa97d5 100644 --- a/tests/integration/common/mocks.c +++ b/tests/integration/common/mocks.c @@ -23,6 +23,10 @@ struct ibv_mr Ibv_mr_flush; /* mock IBV MR FLUSH */ struct rdma_cm_id Cm_id; /* mock CM ID */ struct ibv_qp Ibv_qp; /* mock IBV QP */ +struct ibv_device Ibv_device = {{0}, 0, IBV_TRANSPORT_IB}; +struct ibv_context Ibv_context = {&Ibv_device}; +struct ibv_pd Ibv_pd = {&Ibv_context, 0}; + #ifdef ON_DEMAND_PAGING_SUPPORTED /* predefined IBV On-demand Paging caps */ struct ibv_odp_caps Ibv_odp_capable_caps = { diff --git a/tests/integration/common/mocks.h b/tests/integration/common/mocks.h index 7fac29ad85..ba4f8ed44c 100644 --- a/tests/integration/common/mocks.h +++ b/tests/integration/common/mocks.h @@ -12,15 +12,15 @@ #define MOCK_PORT "1234" #define MOCK_MR (&Ibv_mr) #define MOCK_MR_RAW (&Ibv_mr_raw) -#define MOCK_MR_FLUSH (&Ibv_mr_flush) +#define MOCK_MR_FLUSH (&Ibv_mr_flush) #define MOCK_RAW_SIZE 8 /* memory region size */ #define MOCK_COMP_CHANNEL (&Ibv_comp_channel) +#define MOCK_IBV_PD (&Ibv_pd) #define MOCK_CQ (&Ibv_cq) #define MOCK_VERBS (&Verbs_context.context) #define MOCK_EVCH ((struct rdma_event_channel *)0xE4C4) #define MOCK_SRC_ADDR ((struct sockaddr *)0x0ADD) #define MOCK_DST_ADDR ((struct sockaddr *)0x0ADE) -#define MOCK_IBV_PD ((struct ibv_pd *)0x00D0) #define MOCK_QP ((struct ibv_qp *)0xD56A) #define MOCK_OP_CONTEXT ((void *)0xC417) #define MOCK_RKEY ((uint32_t)0x10111213) @@ -48,12 +48,15 @@ struct common_data { extern struct verbs_context Verbs_context; /* mock IBV completion channel */ extern struct ibv_comp_channel Ibv_comp_channel; +extern struct ibv_context Ibv_context; /* mock IBV context */ +extern struct ibv_device Ibv_device; /* mock IBV device */ extern struct ibv_cq Ibv_cq; /* mock IBV CQ */ extern struct ibv_mr Ibv_mr; /* mock IBV MR */ extern struct ibv_mr Ibv_mr_raw; /* mock IBV MR RAW */ extern struct ibv_mr Ibv_mr_flush; /* mock IBV MR FLUSH */ -extern struct rdma_cm_id Cm_id; /* mock CM ID */ +extern struct rdma_cm_id Cm_id; /* mock CM ID */ extern struct ibv_qp Ibv_qp; /* mock IBV QP */ +extern struct ibv_pd Ibv_pd; /* mock IBV PD */ /* predefined IBV On-demand Paging caps */ extern struct ibv_odp_caps Ibv_odp_capable_caps; diff --git a/tests/unit/common/mocks-ibverbs.c b/tests/unit/common/mocks-ibverbs.c index 31dc07f596..5a2a997576 100644 --- a/tests/unit/common/mocks-ibverbs.c +++ b/tests/unit/common/mocks-ibverbs.c @@ -15,6 +15,9 @@ /* mocked IBV entities */ struct verbs_context Verbs_context; struct ibv_comp_channel Ibv_comp_channel; +struct ibv_device Ibv_device; +struct ibv_context Ibv_context = {&Ibv_device}; +struct ibv_pd Ibv_pd = {&Ibv_context, 0}; struct ibv_cq Ibv_cq; struct ibv_qp Ibv_qp; struct ibv_mr Ibv_mr; diff --git a/tests/unit/common/mocks-ibverbs.h b/tests/unit/common/mocks-ibverbs.h index 11aeb994a5..c8706b0df9 100644 --- a/tests/unit/common/mocks-ibverbs.h +++ b/tests/unit/common/mocks-ibverbs.h @@ -13,6 +13,9 @@ /* mocked IBV entities */ extern struct verbs_context Verbs_context; extern struct ibv_comp_channel Ibv_comp_channel; +extern struct ibv_context Ibv_context; +extern struct ibv_device Ibv_device; +extern struct ibv_pd Ibv_pd; extern struct ibv_cq Ibv_cq; extern struct ibv_qp Ibv_qp; extern struct ibv_mr Ibv_mr; @@ -21,7 +24,7 @@ extern struct ibv_mr Ibv_mr; #define MOCK_VERBS (&Verbs_context.context) #define MOCK_COMP_CHANNEL (struct ibv_comp_channel *)&Ibv_comp_channel #define MOCK_IBV_CQ (struct ibv_cq *)&Ibv_cq -#define MOCK_IBV_PD (struct ibv_pd *)0x00D0 +#define MOCK_IBV_PD (struct ibv_pd *)&Ibv_pd #define MOCK_QP (struct ibv_qp *)&Ibv_qp #define MOCK_MR (struct ibv_mr *)&Ibv_mr diff --git a/tests/unit/common/mocks-rpma-peer.c b/tests/unit/common/mocks-rpma-peer.c index f1e8332f46..3076cdc4b7 100644 --- a/tests/unit/common/mocks-rpma-peer.c +++ b/tests/unit/common/mocks-rpma-peer.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * mocks-rpma-peer.c -- librpma peer.c module mocks @@ -38,7 +38,7 @@ rpma_peer_create_qp(struct rpma_peer *peer, struct rdma_cm_id *id, */ int rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr, - void *addr, size_t length, int access) + void *addr, size_t length, int usage) { /* * rpma_peer_mr_reg() and malloc() may be called in any order. @@ -51,7 +51,7 @@ rpma_peer_mr_reg(struct rpma_peer *peer, struct ibv_mr **ibv_mr_ptr, assert_ptr_equal(peer, MOCK_PEER); assert_ptr_equal(addr, MOCK_PTR); assert_int_equal(length, MOCK_SIZE); - assert_int_equal(access, args->access); + assert_int_equal(usage, args->usage); *ibv_mr_ptr = args->mr; if (*ibv_mr_ptr == NULL) { diff --git a/tests/unit/mr/mr-reg.c b/tests/unit/mr/mr-reg.c index dead9cbeca..0861a8a3e9 100644 --- a/tests/unit/mr/mr-reg.c +++ b/tests/unit/mr/mr-reg.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * mr-reg.c -- the memory region registration/deregistration unit tests @@ -24,33 +24,8 @@ static struct prestate prestates[] = { /* values used in reg_dereg__success called with (prestates + 0) */ {RPMA_MR_USAGE_READ_SRC, IBV_ACCESS_REMOTE_READ, NULL}, /* values used in reg_dereg__success called with (prestates + 1) */ - {RPMA_MR_USAGE_READ_DST, IBV_ACCESS_LOCAL_WRITE, NULL}, - /* values used in reg_dereg__success called with (prestates + 2) */ {(RPMA_MR_USAGE_READ_SRC | RPMA_MR_USAGE_READ_DST), (IBV_ACCESS_REMOTE_READ | IBV_ACCESS_LOCAL_WRITE), NULL}, - /* values used in reg_dereg__success called with (prestates + 3) */ - {RPMA_MR_USAGE_WRITE_SRC, IBV_ACCESS_LOCAL_WRITE, NULL}, - /* values used in reg_dereg__success called with (prestates + 4) */ - {RPMA_MR_USAGE_WRITE_DST, - (IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE), NULL}, - /* values used in reg_dereg__success called with (prestates + 5) */ - {RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT, IBV_ACCESS_REMOTE_READ, NULL}, - /* values used in reg_dereg__success called with (prestates + 6) */ - {(RPMA_MR_USAGE_WRITE_SRC | RPMA_MR_USAGE_WRITE_DST | - RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT), - (IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE | - IBV_ACCESS_REMOTE_READ), NULL}, - /* values used in reg_dereg__success called with (prestates + 7) */ - {RPMA_MR_USAGE_RECV, IBV_ACCESS_LOCAL_WRITE, NULL}, - /* values used in reg_dereg__success called with (prestates + 8) */ - {RPMA_MR_USAGE_SEND, 0, NULL}, - /* values used in reg_dereg__success called with (prestates + 9) */ - {(RPMA_MR_USAGE_READ_SRC | RPMA_MR_USAGE_READ_DST | - RPMA_MR_USAGE_WRITE_SRC | RPMA_MR_USAGE_WRITE_DST | - RPMA_MR_USAGE_RECV | RPMA_MR_USAGE_SEND | - RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT), - (IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE | - IBV_ACCESS_LOCAL_WRITE), NULL}, }; /* @@ -285,32 +260,14 @@ static const struct CMUnitTest tests_reg[] = { cmocka_unit_test(reg__wrong_usage), cmocka_unit_test(reg__failed_E_NOMEM), cmocka_unit_test(reg__peer_mr_reg_failed_E_PROVIDER), - { "reg_dereg__USAGE_READ_SRC", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 0}, - { "reg_dereg__USAGE_READ_DST", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 1}, - { "reg_dereg__USAGE_READ_SRC_DST", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 2}, - { "reg_dereg__USAGE_WRITE_SRC", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 3}, - { "reg_dereg__USAGE_WRITE_DST", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 4}, - { "reg_dereg__USAGE_FLUSHABLE", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 5}, - { "reg_dereg__USAGE_WRITE_SRC_DST_FLUSHABLE", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 6}, - { "reg_dereg__USAGE_RECV", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 7}, - { "reg_dereg__USAGE_SEND", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 8}, - { "reg_dereg__USAGE_ALL", reg_dereg__success, - setup__reg_success, teardown__dereg_success, prestates + 9}, + cmocka_unit_test_prestate_setup_teardown(reg_dereg__success, + setup__reg_success, teardown__dereg_success, prestates), /* rpma_mr_dereg() unit tests */ cmocka_unit_test(dereg__NULL_mr_ptr), cmocka_unit_test(dereg__NULL_mr), cmocka_unit_test_prestate(dereg__failed_E_PROVIDER, - prestates + 2), + prestates + 1), cmocka_unit_test(NULL) }; diff --git a/tests/unit/peer/peer-common.c b/tests/unit/peer/peer-common.c index e75aa07b82..8dc15769be 100644 --- a/tests/unit/peer/peer-common.c +++ b/tests/unit/peer/peer-common.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * peer-common.c -- the common part of the peer unit test @@ -49,7 +49,7 @@ setup__peer(void **in_out) } /* - * delete__teardown -- delete the rpma_peer object + * teardown__peer -- delete the rpma_peer object */ int teardown__peer(void **peer_ptr) @@ -71,3 +71,58 @@ teardown__peer(void **peer_ptr) return 0; } + +/* + * setup__peer_prestates -- prepare a valid rpma_peer object + * (encapsulating the MOCK_IBV_PD) + */ +int +setup__peer_prestates(void **pprestate) +{ + struct prestate *prestate = *pprestate; + + /* + * configure mocks for rpma_peer_new(): + * NOTE: it is not allowed to call ibv_dealloc_pd() if ibv_alloc_pd() + * succeeded. + */ + will_return(rpma_utils_ibv_context_is_odp_capable, + prestate->is_odp_capable); + struct ibv_alloc_pd_mock_args alloc_args = {MOCK_VALIDATE, MOCK_IBV_PD}; + will_return(ibv_alloc_pd, &alloc_args); + expect_value(ibv_alloc_pd, ibv_ctx, MOCK_VERBS); + will_return(__wrap__test_malloc, MOCK_OK); + + /* setup */ + int ret = rpma_peer_new(MOCK_VERBS, &prestate->peer); + assert_int_equal(ret, 0); + assert_non_null(prestate->peer); + + return 0; +} + +/* + * teardown__peer_prestates -- delete the rpma_peer object + */ +int +teardown__peer_prestates(void **pprestate) +{ + struct prestate *prestate = *pprestate; + + /* + * configure mocks for rpma_peer_delete(): + * NOTE: it is not allowed to call ibv_alloc_pd() nor malloc() in + * rpma_peer_delete(). + */ + struct ibv_dealloc_pd_mock_args dealloc_args = + {MOCK_VALIDATE, MOCK_OK}; + will_return(ibv_dealloc_pd, &dealloc_args); + expect_value(ibv_dealloc_pd, pd, MOCK_IBV_PD); + + /* teardown */ + int ret = rpma_peer_delete(&prestate->peer); + assert_int_equal(ret, MOCK_OK); + assert_null(prestate->peer); + + return 0; +} diff --git a/tests/unit/peer/peer-common.h b/tests/unit/peer/peer-common.h index 65d80fef1b..fd1b4d9cf0 100644 --- a/tests/unit/peer/peer-common.h +++ b/tests/unit/peer/peer-common.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * peer-common.h -- the header of the common part of the peer unit test @@ -19,7 +19,24 @@ * is added dynamically during the fall-back to using On-Demand Paging * registration type. */ -#define MOCK_ACCESS (unsigned)7 +#define MOCK_ACCESS (unsigned)(\ + IBV_ACCESS_LOCAL_WRITE |\ + IBV_ACCESS_REMOTE_WRITE |\ + IBV_ACCESS_REMOTE_READ) + +/* + * The test usage value is a combination of all possible + * RPMA_MR_USAGE_* values. + */ +#define MOCK_USAGE (unsigned)(\ + RPMA_MR_USAGE_READ_SRC |\ + RPMA_MR_USAGE_READ_DST |\ + RPMA_MR_USAGE_WRITE_SRC |\ + RPMA_MR_USAGE_WRITE_DST |\ + RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY |\ + RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT |\ + RPMA_MR_USAGE_SEND |\ + RPMA_MR_USAGE_RECV) extern int OdpCapable; extern int OdpIncapable; @@ -27,4 +44,15 @@ extern int OdpIncapable; int setup__peer(void **in_out); int teardown__peer(void **peer_ptr); +struct prestate { + enum ibv_transport_type transport_type; + int usage; + unsigned access; + int is_odp_capable; + struct rpma_peer *peer; +}; + +int setup__peer_prestates(void **pprestate); +int teardown__peer_prestates(void **pprestate); + #endif /* PEER_COMMON_H */ diff --git a/tests/unit/peer/peer-mr_reg.c b/tests/unit/peer/peer-mr_reg.c index 91105bf4c9..163cec53dd 100644 --- a/tests/unit/peer/peer-mr_reg.c +++ b/tests/unit/peer/peer-mr_reg.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2020, Intel Corporation */ +/* Copyright 2020-2021, Intel Corporation */ /* * peer-mr_reg.c -- a peer unit test @@ -18,6 +18,58 @@ #include "peer-common.h" #include "test-common.h" +static struct prestate prestates[] = { + /* 0-1) non-iWARP and iWARP are the same */ + {IBV_TRANSPORT_IB, + (RPMA_MR_USAGE_READ_SRC | + RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY | + RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT), + IBV_ACCESS_REMOTE_READ, + MOCK_ODP_CAPABLE}, + {IBV_TRANSPORT_IWARP, + (RPMA_MR_USAGE_READ_SRC | + RPMA_MR_USAGE_FLUSH_TYPE_VISIBILITY | + RPMA_MR_USAGE_FLUSH_TYPE_PERSISTENT), + IBV_ACCESS_REMOTE_READ, + MOCK_ODP_CAPABLE}, + /* 2-3) non-iWARP and iWARP differs */ + {IBV_TRANSPORT_IB, + RPMA_MR_USAGE_READ_DST, + IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, + {IBV_TRANSPORT_IWARP, + RPMA_MR_USAGE_READ_DST, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE, + MOCK_ODP_CAPABLE}, + /* 4-5) non-iWARP and iWARP are the same */ + {IBV_TRANSPORT_IB, + RPMA_MR_USAGE_WRITE_SRC, + IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, + {IBV_TRANSPORT_IWARP, + RPMA_MR_USAGE_WRITE_SRC, + IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, + /* 6-7) non-iWARP and iWARP are the same */ + {IBV_TRANSPORT_IB, + RPMA_MR_USAGE_WRITE_DST, + IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, + {IBV_TRANSPORT_IWARP, + RPMA_MR_USAGE_WRITE_DST, + IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, + /* 8-9) non-iWARP and iWARP are the same */ + {IBV_TRANSPORT_IB, + RPMA_MR_USAGE_RECV, + IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, + {IBV_TRANSPORT_IWARP, + RPMA_MR_USAGE_RECV, + IBV_ACCESS_LOCAL_WRITE, + MOCK_ODP_CAPABLE}, +}; + /* * mr_reg__fail_ENOMEM -- ibv_reg_mr() failed with ENOMEM */ @@ -37,7 +89,7 @@ mr_reg__fail_ENOMEM(void **peer_ptr) /* run test */ struct ibv_mr *mr = NULL; int ret = rpma_peer_mr_reg(peer, &mr, MOCK_ADDR, - MOCK_LEN, MOCK_ACCESS); + MOCK_LEN, MOCK_USAGE); /* verify the results */ assert_int_equal(ret, RPMA_E_PROVIDER); @@ -63,7 +115,7 @@ mr_reg__fail_EOPNOTSUPP_no_odp(void **peer_ptr) /* run test */ struct ibv_mr *mr = NULL; int ret = rpma_peer_mr_reg(peer, &mr, MOCK_ADDR, - MOCK_LEN, MOCK_ACCESS); + MOCK_LEN, MOCK_USAGE); /* verify the results */ assert_int_equal(ret, RPMA_E_PROVIDER); @@ -99,7 +151,7 @@ mr_reg__fail_EOPNOTSUPP_EAGAIN(void **peer_ptr) /* run test */ struct ibv_mr *mr = NULL; int ret = rpma_peer_mr_reg(peer, &mr, MOCK_ADDR, - MOCK_LEN, MOCK_ACCESS); + MOCK_LEN, MOCK_USAGE); /* verify the results */ assert_int_equal(ret, RPMA_E_PROVIDER); @@ -110,21 +162,25 @@ mr_reg__fail_EOPNOTSUPP_EAGAIN(void **peer_ptr) * mr_reg__success -- happy day scenario */ static void -mr_reg__success(void **peer_ptr) +mr_reg__success(void **pprestate) { - struct rpma_peer *peer = *peer_ptr; + struct prestate *prestate = *pprestate; + + struct rpma_peer *peer = prestate->peer; + struct ibv_pd *mock_ibv_pd = MOCK_IBV_PD; /* configure mocks */ - expect_value(ibv_reg_mr, pd, MOCK_IBV_PD); + mock_ibv_pd->context->device->transport_type = prestate->transport_type; + expect_value(ibv_reg_mr, pd, mock_ibv_pd); expect_value(ibv_reg_mr, addr, MOCK_ADDR); expect_value(ibv_reg_mr, length, MOCK_LEN); - expect_value(ibv_reg_mr, access, MOCK_ACCESS); + expect_value(ibv_reg_mr, access, prestate->access); will_return(ibv_reg_mr, MOCK_MR); /* run test */ struct ibv_mr *mr; int ret = rpma_peer_mr_reg(peer, &mr, MOCK_ADDR, - MOCK_LEN, MOCK_ACCESS); + MOCK_LEN, prestate->usage); /* verify the results */ assert_int_equal(ret, MOCK_OK); @@ -158,7 +214,7 @@ mr_reg__success_odp(void **peer_ptr) /* run test */ struct ibv_mr *mr; int ret = rpma_peer_mr_reg(peer, &mr, MOCK_ADDR, - MOCK_LEN, MOCK_ACCESS); + MOCK_LEN, MOCK_USAGE); /* verify the results */ #ifdef ON_DEMAND_PAGING_SUPPORTED @@ -185,9 +241,36 @@ main(int argc, char *argv[]) cmocka_unit_test_prestate_setup_teardown( mr_reg__fail_EOPNOTSUPP_EAGAIN, setup__peer, teardown__peer, &OdpCapable), - cmocka_unit_test_prestate_setup_teardown( - mr_reg__success, - setup__peer, teardown__peer, &OdpCapable), + { "mr_reg__USAGE_READ_SRC_IB", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 0}, + { "mr_reg__USAGE_READ_SRC_iWARP", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 1}, + { "mr_reg__USAGE_READ_DST_IB", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 2}, + { "mr_reg__USAGE_READ_DST_iWARP", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 3}, + { "mr_reg__USAGE_WRITE_SRC_IB", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 4}, + { "mr_reg__USAGE_WRITE_SRC_iWARP", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 5}, + { "mr_reg__USAGE_WRITE_DST_IB", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 6}, + { "mr_reg__USAGE_WRITE_DST_iWARP", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 7}, + { "mr_reg__USAGE_RECV_IB", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 8}, + { "mr_reg__USAGE_RECV_iWARP", mr_reg__success, + setup__peer_prestates, + teardown__peer_prestates, prestates + 9}, cmocka_unit_test_prestate_setup_teardown( mr_reg__success_odp, setup__peer, teardown__peer, &OdpCapable),