diff --git a/examples/11-write-with-imm/CMakeLists.txt b/examples/11-write-with-imm/CMakeLists.txt new file mode 100644 index 0000000000..652a65e7b0 --- /dev/null +++ b/examples/11-write-with-imm/CMakeLists.txt @@ -0,0 +1,44 @@ +# +# SPDX-License-Identifier: BSD-3-Clause +# Copyright (c) 2020-2021 Fujitsu +# + +cmake_minimum_required(VERSION 3.3) +project(write-with-imm C) + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} + ${CMAKE_SOURCE_DIR}/../cmake + ${CMAKE_SOURCE_DIR}/../../cmake) + +include(${CMAKE_SOURCE_DIR}/../../cmake/functions.cmake) +# set LIBRT_LIBRARIES if linking with librt is required +check_if_librt_is_required() + +find_package(PkgConfig QUIET) + +if(PKG_CONFIG_FOUND) + pkg_check_modules(LIBRPMA librpma) + pkg_check_modules(LIBIBVERBS libibverbs) +endif() +if(NOT LIBRPMA_FOUND) + find_package(LIBRPMA REQUIRED librpma) +endif() +if(NOT LIBIBVERBS_FOUND) + find_package(LIBIBVERBS REQUIRED libibverbs) +endif() + +link_directories(${LIBRPMA_LIBRARY_DIRS} ${LIBIBVERBS_LIBRARY_DIRS}) + +function(add_example name) + set(srcs ${ARGN}) + add_executable(${name} ${srcs}) + target_include_directories(${name} + PRIVATE + ${LIBRPMA_INCLUDE_DIRS} + ${LIBIBVERBS_INCLUDE_DIRS} + ../common) + target_link_libraries(${name} rpma ${LIBIBVERBS_LIBRARIES} ${LIBRT_LIBRARIES}) +endfunction() + +add_example(server server.c ../common/common-conn.c) +add_example(client client.c ../common/common-conn.c) diff --git a/examples/11-write-with-imm/README.md b/examples/11-write-with-imm/README.md new file mode 100644 index 0000000000..f817861c6f --- /dev/null +++ b/examples/11-write-with-imm/README.md @@ -0,0 +1,20 @@ +Example of rpma write with immediate data +=== + +The rpma write with immediate data implements two parts of the process: +- The client connects to the server and writes a message with immediate +data to the server. +- The server receives a message with immediate data from client. +The immediate data is compared with the expected immediate data sent +by the client as the private data during establishing the connection. +The argument 'operation' only supports write or write_atomic. + +## Usage + +```bash +[user@server]$ ./server $server_address $port +``` + +```bash +[user@client]$ ./client $server_address $port $operation $word $imm +``` diff --git a/examples/11-write-with-imm/client.c b/examples/11-write-with-imm/client.c new file mode 100644 index 0000000000..cbda7657b1 --- /dev/null +++ b/examples/11-write-with-imm/client.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright (c) 2020-2021 Fujitsu */ + +/* + * client.c -- a client of the write-with-imm example + * + * Please see README.md for a detailed description of this example. + */ + +#include +#include +#include +#include +#include + +#include "common-conn.h" + +#define USAGE_STR "usage: %s \n" + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 6) { + fprintf(stderr, USAGE_STR, argv[0]); + return -1; + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + char *operation = argv[3]; + char *word = argv[4]; + + if (strcmp(operation, "write") && strcmp(operation, "write_atomic")) { + fprintf(stderr, + "'operation' argument only supports write or write_atomic\n"); + return -1; + } + + uint64_t imm = strtoul(argv[5], NULL, 10); + if (imm == ULONG_MAX && errno == ERANGE) { + fprintf(stderr, "strtoul() overflowed\n"); + return -1; + } + + if (imm > UINT32_MAX) { + fprintf(stderr, + "the provided immediate data is too big(%lu > %u)\n", + imm, UINT32_MAX); + return -1; + } + + /* RPMA resources - general */ + struct rpma_peer *peer = NULL; + struct rpma_mr_local *src_mr = NULL; + struct rpma_mr_remote *dst_mr = NULL; + struct rpma_conn *conn = NULL; + struct rpma_completion cmpl; + int ret; + + /* prepare memory */ + char *src = malloc_aligned(KILOBYTE); + if (!src) + return -1; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = client_peer_via_address(addr, &peer); + if (ret) + goto err_mr_free; + + /* register the memory */ + ret = rpma_mr_reg(peer, src, KILOBYTE, RPMA_MR_USAGE_SEND, &src_mr); + if (ret) + goto err_peer_delete; + + /* + * establish a new connection to a server and send an immediate + * data for validation on the sever side + */ + struct rpma_conn_private_data src_pdata; + src_pdata.ptr = (uint32_t *)&imm; + src_pdata.len = sizeof(uint32_t); + ret = client_connect(peer, addr, port, &src_pdata, &conn); + if (ret) + goto err_mr_dereg; + + /* obtain the remote memory description */ + struct rpma_conn_private_data dst_pdata; + ret = rpma_conn_get_private_data(conn, &dst_pdata); + if (ret) + goto err_conn_disconnect; + if (dst_pdata.len < sizeof(struct common_data)) { + fprintf(stderr, + "received connection's private data is too small (%u < %zu)\n", + dst_pdata.len, sizeof(struct common_data)); + ret = -1; + goto err_conn_disconnect; + } + + struct common_data *dst_data = dst_pdata.ptr; + ret = rpma_mr_remote_from_descriptor(&dst_data->descriptors[0], + dst_data->mr_desc_size, &dst_mr); + if (ret) + goto err_conn_disconnect; + + /* send a message with immediate data to the server */ + strcpy(src, word); + if (strcmp(operation, "write") == 0) { + fprintf(stdout, "write a value %s with immediate data %u\n", + src, (uint32_t)imm); + ret = rpma_write_with_imm(conn, dst_mr, dst_data->data_offset, + src_mr, 0, KILOBYTE, RPMA_F_COMPLETION_ALWAYS, + (uint32_t)imm, NULL); + } else { + fprintf(stdout, + "atomic write a value %s with immediate data %u\n", + src, (uint32_t)imm); + if (strlen(src) > RPMA_ATOMIC_WRITE_ALIGNMENT) + fprintf(stdout, + "atomic write %s will be truncated to %d bytes\n", + src, RPMA_ATOMIC_WRITE_ALIGNMENT); + ret = rpma_write_atomic_with_imm(conn, dst_mr, + dst_data->data_offset, src_mr, 0, + RPMA_F_COMPLETION_ALWAYS, (uint32_t)imm, NULL); + } + if (ret) + goto err_conn_disconnect; + + /* prepare completions, get one and validate it */ + ret = rpma_conn_completion_wait(conn); + if (ret) + goto err_conn_disconnect; + + ret = rpma_conn_completion_get(conn, &cmpl); + if (ret) + goto err_conn_disconnect; + + if (cmpl.op_status != IBV_WC_SUCCESS) { + fprintf(stderr, + "an unexpected completion: %s\n", + ibv_wc_status_str(cmpl.op_status)); + ret = -1; + } + + if (cmpl.op != RPMA_OP_WRITE) { + fprintf(stderr, + "an unexpected type of operation (%d != %d)\n", + cmpl.op, RPMA_OP_SEND); + ret = -1; + } + +err_conn_disconnect: + common_disconnect_and_wait_for_conn_close(&conn); + +err_mr_dereg: + /* deregister the memory regions */ + rpma_mr_dereg(&src_mr); + +err_peer_delete: + /* delete the peer object */ + rpma_peer_delete(&peer); + +err_mr_free: + /* free the memory */ + free(src); + + return ret; +} diff --git a/examples/11-write-with-imm/server.c b/examples/11-write-with-imm/server.c new file mode 100644 index 0000000000..b8ff11e4bf --- /dev/null +++ b/examples/11-write-with-imm/server.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* Copyright (c) 2020-2021 Fujitsu */ + +/* + * server.c -- a server of the write-with-imm example + * + * Please see README.md for a detailed description of this example. + */ + +#include +#include +#include +#include + +#include "common-conn.h" + +#define USAGE_STR "usage: %s \n" + +int +main(int argc, char *argv[]) +{ + /* validate parameters */ + if (argc < 3) { + fprintf(stderr, USAGE_STR, argv[0]); + return -1; + } + + /* configure logging thresholds to see more details */ + rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO); + rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO); + + /* read common parameters */ + char *addr = argv[1]; + char *port = argv[2]; + + /* prepare memory */ + char *dst = malloc_aligned(KILOBYTE); + if (!dst) + return -1; + + /* RPMA resources */ + struct rpma_peer *peer = NULL; + struct rpma_mr_local *dst_mr = NULL; + struct rpma_ep *ep = NULL; + struct rpma_conn_req *req = NULL; + struct rpma_conn *conn = NULL; + enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED; + struct rpma_completion cmpl; + int ret; + + /* + * lookup an ibv_context via the address and create a new peer using it + */ + ret = server_peer_via_address(addr, &peer); + if (ret) + goto err_mr_free; + + /* register the memory */ + ret = rpma_mr_reg(peer, dst, KILOBYTE, RPMA_MR_USAGE_WRITE_DST, + &dst_mr); + if (ret) + goto err_peer_delete; + + /* get size of the memory region's descriptor */ + size_t mr_desc_size; + ret = rpma_mr_get_descriptor_size(dst_mr, &mr_desc_size); + if (ret) + goto err_mr_dereg; + + /* calculate data for the client write */ + struct common_data dst_data; + dst_data.data_offset = 0; + dst_data.mr_desc_size = mr_desc_size; + + /* get the memory region's descriptor */ + ret = rpma_mr_get_descriptor(dst_mr, &dst_data.descriptors[0]); + if (ret) + goto err_mr_dereg; + + /* start a listening endpoint at addr:port */ + ret = rpma_ep_listen(peer, addr, port, &ep); + if (ret) + goto err_mr_dereg; + + /* receive an incoming connection request */ + ret = rpma_ep_next_conn_req(ep, NULL, &req); + if (ret) + goto err_ep_shutdown; + + /* prepare to receive a message with immediate data from the client */ + ret = rpma_conn_req_recv(req, dst_mr, 0, KILOBYTE, NULL); + if (ret) { + rpma_conn_req_delete(&req); + goto err_ep_shutdown; + } + + /* accept the connection request and obtain the connection object */ + struct rpma_conn_private_data dst_pdata; + dst_pdata.ptr = &dst_data; + dst_pdata.len = sizeof(struct common_data); + ret = rpma_conn_req_connect(&req, &dst_pdata, &conn); + if (ret) { + rpma_conn_req_delete(&req); + goto err_ep_shutdown; + } + + /* wait for the connection to be established */ + ret = rpma_conn_next_event(conn, &conn_event); + if (ret) + goto err_conn_disconnect; + if (conn_event != RPMA_CONN_ESTABLISHED) { + fprintf(stderr, + "rpma_conn_next_event returned an unexptected event\n"); + ret = -1; + goto err_conn_disconnect; + } + + /* get the expected immediate data from the connection's private data */ + struct rpma_conn_private_data src_pdata; + ret = rpma_conn_get_private_data(conn, &src_pdata); + if (ret) + goto err_conn_disconnect; + if (src_pdata.len < sizeof(uint32_t)) { + fprintf(stderr, + "received connection's private data is too small (%u < %zu)\n", + src_pdata.len, sizeof(uint32_t)); + ret = -1; + goto err_conn_disconnect; + } + uint32_t *exp_imm = src_pdata.ptr; + + /* prepare completions, get one and validate it */ + ret = rpma_conn_completion_wait(conn); + if (ret) + goto err_conn_disconnect; + + ret = rpma_conn_completion_get(conn, &cmpl); + if (ret) + goto err_conn_disconnect; + + if (cmpl.op_status != IBV_WC_SUCCESS) { + fprintf(stderr, + "an unexpected completion %s\n", + ibv_wc_status_str(cmpl.op_status)); + ret = -1; + goto err_conn_disconnect; + } + + if (cmpl.op != RPMA_OP_RECV_RDMA_WITH_IMM) { + fprintf(stderr, + "an unexpected type of operation (%d != %d)\n", + cmpl.op, RPMA_OP_RECV_RDMA_WITH_IMM); + ret = -1; + goto err_conn_disconnect; + } + + if (!(cmpl.flags & IBV_WC_WITH_IMM)) { + fprintf(stderr, + "an unexpected completion flag (no IBV_WC_WITH_IMM)\n"); + ret = -1; + goto err_conn_disconnect; + } + + if (cmpl.imm != *exp_imm) { + fprintf(stderr, + "an unexpected immediate data (%u != %u)\n", + cmpl.imm, *exp_imm); + ret = -1; + } else { + printf("receive a value %s with immediate data %u\n", dst, + cmpl.imm); + } + +err_conn_disconnect: + common_wait_for_conn_close_and_disconnect(&conn); + +err_ep_shutdown: + rpma_ep_shutdown(&ep); + +err_mr_dereg: + /* deregister the memory regions */ + rpma_mr_dereg(&dst_mr); + +err_peer_delete: + /* delete the peer object */ + rpma_peer_delete(&peer); + +err_mr_free: + /* free the memory */ + free(dst); + + return ret; +} diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 49fa40b712..98d7ce670a 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -133,6 +133,10 @@ add_example(NAME 10-send-with-imm BIN server USE_LIBIBVERBS SRCS 10-send-with-imm/server.c common/common-conn.c) add_example(NAME 10-send-with-imm BIN client USE_LIBIBVERBS SRCS 10-send-with-imm/client.c common/common-conn.c) +add_example(NAME 11-write-with-imm BIN server USE_LIBIBVERBS + SRCS 11-write-with-imm/server.c common/common-conn.c) +add_example(NAME 11-write-with-imm BIN client USE_LIBIBVERBS + SRCS 11-write-with-imm/client.c common/common-conn.c) add_example(NAME log BIN log SRCS log/log-example.c diff --git a/examples/run-all-on-SoftRoCE.sh b/examples/run-all-on-SoftRoCE.sh index 8d25a81cc2..2576f3eaf4 100755 --- a/examples/run-all-on-SoftRoCE.sh +++ b/examples/run-all-on-SoftRoCE.sh @@ -102,6 +102,11 @@ function run_example() { $VLD_CCMD $DIR/client $IP_ADDRESS $PORT "1st_word" "1234" RV=$? ;; + 11-write-with-imm) + echo "Starting the client ..." + $VLD_CCMD $DIR/client $IP_ADDRESS $PORT "write" "1st_word" "1234" + RV=$? + ;; *) echo "Starting the client ..." $VLD_CCMD $DIR/client $IP_ADDRESS $PORT