Try to implement rpma send with immediate data

doc/manuals_3.txt

@@ -50,6 +50,7 @@ rpma_peer_new.3
 rpma_read.3
 rpma_recv.3
 rpma_send.3
+rpma_send_with_imm.3
 rpma_utils_conn_event_2str.3
 rpma_utils_get_ibv_context.3
 rpma_utils_ibv_context_is_odp_capable.3

examples/10-send-with-imm/CMakeLists.txt

@@ -0,0 +1,44 @@
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2020 Fujitsu
+#
+
+cmake_minimum_required(VERSION 3.3)
+project(send-with-imm C)
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
+	${CMAKE_SOURCE_DIR}/../cmake
+	${CMAKE_SOURCE_DIR}/../../cmake)
+
+include(${CMAKE_SOURCE_DIR}/../../cmake/functions.cmake)
+# set LIBRT_LIBRARIES if linking with librt is required
+check_if_librt_is_required()
+
+find_package(PkgConfig QUIET)
+
+if(PKG_CONFIG_FOUND)
+	pkg_check_modules(LIBRPMA librpma)
+	pkg_check_modules(LIBIBVERBS libibverbs)
+endif()
+if(NOT LIBRPMA_FOUND)
+	find_package(LIBRPMA REQUIRED librpma)
+endif()
+if(NOT LIBIBVERBS_FOUND)
+	find_package(LIBIBVERBS REQUIRED libibverbs)
+endif()
+
+link_directories(${LIBRPMA_LIBRARY_DIRS} ${LIBIBVERBS_LIBRARY_DIRS})
+
+function(add_example name)
+	set(srcs ${ARGN})
+	add_executable(${name} ${srcs})
+	target_include_directories(${name}
+		PRIVATE
+			${LIBRPMA_INCLUDE_DIRS}
+			${LIBIBVERBS_INCLUDE_DIRS}
+			../common)
+	target_link_libraries(${name} rpma ${LIBIBVERBS_LIBRARIES})
+endfunction()
+
+add_example(server server.c ../common/common-conn.c)
+add_example(client client.c ../common/common-conn.c)

examples/10-send-with-imm/README.md

@@ -0,0 +1,19 @@
+Example of rpma send with immediate data
+===
+
+The rpma send with immediate data implements two parts of the process:
+- The client connects to the server and sends a message with immediate
+data to the server.
+- The server receives a message with immediate data from client.
+The immediate data is compared with the expected immediate data sent
+by the client as the private data during establishing the connection.
+
+## Usage
+
+```bash
+[user@server]$ ./server $server_address $port
+```
+
+```bash
+[user@client]$ ./client $server_address $port $word $imm
+```

examples/10-send-with-imm/client.c

@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/* Copyright (c) 2020 Fujitsu */
+
+/*
+ * client.c -- a client of the send-with-imm example
+ *
+ * Please see README.md for a detailed description of this example.
+ */
+
+#include <librpma.h>
+#include <limits.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "common-conn.h"
+
+#define USAGE_STR "usage: %s <server_address> <port> <word> <imm>\n"
+
+int
+main(int argc, char *argv[])
+{
+	/* validate parameters */
+	if (argc < 5) {
+		fprintf(stderr, USAGE_STR, argv[0]);
+		return -1;
+	}
+
+	/* configure logging thresholds to see more details */
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO);
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO);
+
+	/* read common parameters */
+	char *addr = argv[1];
+	char *port = argv[2];
+	char *word = argv[3];
+
+	uint64_t imm = strtoul(argv[4], NULL, 10);
+	if (imm == ULONG_MAX && errno == ERANGE) {
+		fprintf(stderr, "strtoul() overflowed\n");
+		return -1;
+	}
+
+	if (imm > UINT32_MAX) {
+		fprintf(stderr,
+			"the provided immediate data is too big(%lu > %u)\n",
+			imm, UINT32_MAX);
+		return -1;
+	}
+
+	/* RPMA resources - general */
+	struct rpma_peer *peer = NULL;
+	struct rpma_mr_local *send_mr = NULL;
+	struct rpma_conn *conn = NULL;
+	struct rpma_completion cmpl;
+	int ret;
+
+	/* prepare memory */
+	char *send = malloc_aligned(KILOBYTE);
+	if (!send)
+		return -1;
+
+	/*
+	 * lookup an ibv_context via the address and create a new peer using it
+	 */
+	ret = client_peer_via_address(addr, &peer);
+	if (ret)
+		goto err_mr_free;
+
+	/* register the memory */
+	ret = rpma_mr_reg(peer, send, KILOBYTE, RPMA_MR_USAGE_SEND, &send_mr);
+	if (ret)
+		goto err_peer_delete;
+
+	/*
+	 * establish a new connection to a server and send an immediate
+	 * data for validation on the sever side
+	 */
+	struct rpma_conn_private_data pdata;
+	pdata.ptr = (uint32_t *)&imm;
+	pdata.len = sizeof(uint32_t);
+	ret = client_connect(peer, addr, port, &pdata, &conn);
+	if (ret)
+		goto err_mr_dereg;
+
+	/* send a message with immediate data to the server */
+	fprintf(stdout, "send value %s with immediate data %u\n",
+		word, (uint32_t)imm);
+	strcpy(send, word);
+	ret = rpma_send_with_imm(conn, send_mr, 0, KILOBYTE,
+		RPMA_F_COMPLETION_ALWAYS, (uint32_t)imm, NULL);
+	if (ret)
+		goto err_conn_disconnect;
+
+	/* prepare completions, get one and validate it */
+	ret = rpma_conn_completion_wait(conn);
+	if (ret)
+		goto err_conn_disconnect;
+
+	ret = rpma_conn_completion_get(conn, &cmpl);
+	if (ret)
+		goto err_conn_disconnect;
+
+	if (cmpl.op_status != IBV_WC_SUCCESS) {
+		fprintf(stderr,
+			"an unexpected completion: %s\n",
+			ibv_wc_status_str(cmpl.op_status));
+		ret = -1;
+	}
+
+	if (cmpl.op != RPMA_OP_SEND) {
+		fprintf(stderr,
+			"an unexpected type of operation (%d != %d)\n",
+			cmpl.op, RPMA_OP_SEND);
+		ret = -1;
+	}
+
+err_conn_disconnect:
+	common_disconnect_and_wait_for_conn_close(&conn);
+
+err_mr_dereg:
+	/* deregister the memory regions */
+	rpma_mr_dereg(&send_mr);
+
+err_peer_delete:
+	/* delete the peer object */
+	rpma_peer_delete(&peer);
+
+err_mr_free:
+	/* free the memory */
+	free(send);
+
+	return ret;
+}

examples/10-send-with-imm/server.c

@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/* Copyright (c) 2020 Fujitsu */
+
+/*
+ * server.c -- a server of the send-with-imm example
+ *
+ * Please see README.md for a detailed description of this example.
+ */
+
+#include <inttypes.h>
+#include <librpma.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "common-conn.h"
+
+#define USAGE_STR "usage: %s <server_address> <port>\n"
+
+int
+main(int argc, char *argv[])
+{
+	/* validate parameters */
+	if (argc < 3) {
+		fprintf(stderr, USAGE_STR, argv[0]);
+		return -1;
+	}
+
+	/* configure logging thresholds to see more details */
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO);
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO);
+
+	/* read common parameters */
+	char *addr = argv[1];
+	char *port = argv[2];
+
+	/* prepare memory */
+	char *recv = malloc_aligned(KILOBYTE);
+	if (!recv)
+		return -1;
+
+	/* RPMA resources */
+	struct rpma_peer *peer = NULL;
+	struct rpma_mr_local *recv_mr = NULL;
+	struct rpma_ep *ep = NULL;
+	struct rpma_conn_req *req = NULL;
+	struct rpma_conn *conn = NULL;
+	enum rpma_conn_event conn_event = RPMA_CONN_UNDEFINED;
+	struct rpma_completion cmpl;
+	int ret;
+
+	/*
+	 * lookup an ibv_context via the address and create a new peer using it
+	 */
+	ret = server_peer_via_address(addr, &peer);
+	if (ret)
+		goto err_mr_free;
+
+	/* register the memory */
+	ret = rpma_mr_reg(peer, recv, KILOBYTE, RPMA_MR_USAGE_RECV, &recv_mr);
+	if (ret)
+		goto err_peer_delete;
+
+	/* start a listening endpoint at addr:port */
+	ret = rpma_ep_listen(peer, addr, port, &ep);
+	if (ret)
+		goto err_mr_dereg;
+
+	/* receive an incoming connection request */
+	ret = rpma_ep_next_conn_req(ep, NULL, &req);
+	if (ret)
+		goto err_ep_shutdown;
+
+	/* prepare to receive a message with immediate data from the client */
+	ret = rpma_conn_req_recv(req, recv_mr, 0, KILOBYTE, NULL);
+	if (ret) {
+		rpma_conn_req_delete(&req);
+		goto err_ep_shutdown;
+	}
+
+	/* accept the connection request and obtain the connection object */
+	ret = rpma_conn_req_connect(&req, NULL, &conn);
+	if (ret) {
+		rpma_conn_req_delete(&req);
+		goto err_ep_shutdown;
+	}
+
+	/* wait for the connection to be established */
+	ret = rpma_conn_next_event(conn, &conn_event);
+	if (ret)
+		goto err_conn_disconnect;
+	if (conn_event != RPMA_CONN_ESTABLISHED) {
+		fprintf(stderr,
+			"rpma_conn_next_event returned an unexptected event\n");
+		ret = -1;
+		goto err_conn_disconnect;
+	}
+
+	/* get the expected immediate data from the connection's private data */
+	struct rpma_conn_private_data pdata;
+	ret = rpma_conn_get_private_data(conn, &pdata);
+	if (ret)
+		goto err_conn_disconnect;
+	if (pdata.len < sizeof(uint32_t)) {
+		fprintf(stderr,
+			"received connection's private data is too small (%u < %zu)\n",
+			pdata.len, sizeof(uint32_t));
+		ret = -1;
+		goto err_conn_disconnect;
+	}
+	uint32_t *exp_imm = pdata.ptr;
+
+	/* prepare completions, get one and validate it */
+	ret = rpma_conn_completion_wait(conn);
+	if (ret)
+		goto err_conn_disconnect;
+
+	ret = rpma_conn_completion_get(conn, &cmpl);
+	if (ret)
+		goto err_conn_disconnect;
+
+	if (cmpl.op_status != IBV_WC_SUCCESS) {
+		fprintf(stderr,
+			"an unexpected completion %s\n",
+			ibv_wc_status_str(cmpl.op_status));
+		ret = -1;
+		goto err_conn_disconnect;
+	}
+
+	if (cmpl.op != RPMA_OP_RECV) {
+		fprintf(stderr,
+			"an unexpected type of operation (%d != %d)\n",
+			cmpl.op, RPMA_OP_RECV);
+		ret = -1;
+		goto err_conn_disconnect;
+	}
+
+	if (!(cmpl.flags & IBV_WC_WITH_IMM)) {
+		fprintf(stderr,
+			"an unexpected completion flag (no IBV_WC_WITH_IMM)\n");
+		ret = -1;
+		goto err_conn_disconnect;
+	}
+
+	if (cmpl.imm != *exp_imm) {
+		fprintf(stderr,
+			"an unexpected immediate data (%u != %u)\n",
+			cmpl.imm, *exp_imm);
+		ret = -1;
+	} else {
+		printf("receive a value %s with immediate data %u\n", recv,
+			cmpl.imm);
+	}
+
+err_conn_disconnect:
+	common_wait_for_conn_close_and_disconnect(&conn);
+
+err_ep_shutdown:
+	rpma_ep_shutdown(&ep);
+
+err_mr_dereg:
+	/* deregister the memory regions */
+	rpma_mr_dereg(&recv_mr);
+
+err_peer_delete:
+	/* delete the peer object */
+	rpma_peer_delete(&peer);
+
+err_mr_free:
+	/* free the memory */
+	free(recv);
+
+	return ret;
+}

examples/CMakeLists.txt

@@ -129,6 +129,10 @@ add_example(NAME 09-flush-to-persistent-GPSPM BIN server USE_LIBPMEM_IF_FOUND US
 add_example(NAME 09-flush-to-persistent-GPSPM BIN client USE_LIBPMEM_IF_FOUND USE_LIBPROTOBUFC
 	SRCS 09-flush-to-persistent-GPSPM/client.c common/common-conn.c
 	09-flush-to-persistent-GPSPM/GPSPM_flush.pb-c.c)
+add_example(NAME 10-send-with-imm BIN server USE_LIBIBVERBS
+	SRCS 10-send-with-imm/server.c common/common-conn.c)
+add_example(NAME 10-send-with-imm BIN client USE_LIBIBVERBS
+	SRCS 10-send-with-imm/client.c common/common-conn.c)
 
 add_example(NAME log BIN log SRCS
 	log/log-example.c