Server select timeout to prevent server to get stuck because of clien…

…t or network errors (esnet#1101)

src/iperf.h

@@ -160,6 +160,7 @@ struct iperf_settings
     EVP_PKEY  *client_rsa_pubkey;
 #endif // HAVE_SSL
     int	      connect_timeout;	    /* socket connection timeout, in ms */
+    int       idle_timeout;         /* server idle time timeout */
 };
 
 struct iperf_test;
@@ -344,6 +345,11 @@ struct iperf_test
     iperf_size_t bitrate_limit_last_interval_index;       /* Index of the last interval traffic insrted into the cyclic array */
     int          bitrate_limit_exceeded;                  /* Set by callback routine when average data rate exceeded the server's bitrate limit */
 
+    int server_last_run_rc;                      /* Save last server run rc for next test */
+    uint server_forced_idle_restarts_count;      /* count number of forced server restarts to make sure it is not stack */
+    uint server_forced_no_msg_restarts_count;    /* count number of forced server restarts to make sure it is not stack */
+    uint server_test_number;                     /* count number of tests performed by a server */
+
     char      cookie[COOKIE_SIZE];
 //    struct iperf_stream *streams;               /* pointer to list of struct stream */
     SLIST_HEAD(slisthead, iperf_stream) streams;
@@ -400,6 +406,7 @@ struct iperf_test
 #define MAX_BURST 1000
 #define MAX_MSS (9 * 1024)
 #define MAX_STREAMS 128
+#define NO_MSG_RCVD_TIMEOUT 120
 
 #define TIMESTAMP_FORMAT "%c "
 

src/iperf_api.c

@@ -388,6 +388,12 @@ iperf_get_test_connect_timeout(struct iperf_test *ipt)
     return ipt->settings->connect_timeout;
 }
 
+int
+iperf_get_test_idle_timeout(struct iperf_test *ipt)
+{
+    return ipt->settings->idle_timeout;
+}
+
 /************** Setter routines for some fields inside iperf_test *************/
 
 void
@@ -719,6 +725,12 @@ iperf_set_test_connect_timeout(struct iperf_test* ipt, int ct)
     ipt->settings->connect_timeout = ct;
 }
 
+void
+iperf_set_test_idle_timeout(struct iperf_test* ipt, int to)
+{
+    ipt->settings->idle_timeout = to;
+}
+
 
 /********************** Get/set test protocol structure ***********************/
 
@@ -954,6 +966,7 @@ iperf_parse_arguments(struct iperf_test *test, int argc, char **argv)
 	{"fq-rate", required_argument, NULL, OPT_FQ_RATE},
 	{"pacing-timer", required_argument, NULL, OPT_PACING_TIMER},
 	{"connect-timeout", required_argument, NULL, OPT_CONNECT_TIMEOUT},
+        {"idle-timeout", required_argument, NULL, OPT_IDLE_TIMEOUT},
         {"debug", no_argument, NULL, 'd'},
         {"help", no_argument, NULL, 'h'},
         {NULL, 0, NULL, 0}
@@ -1275,6 +1288,14 @@ iperf_parse_arguments(struct iperf_test *test, int argc, char **argv)
             case 'F':
                 test->diskfile_name = optarg;
                 break;
+            case OPT_IDLE_TIMEOUT:
+                test->settings->idle_timeout = atoi(optarg);
+                if (test->settings->idle_timeout < 1 || test->settings->idle_timeout > MAX_TIME) {
+                    i_errno = IEIDLETIMEOUT;
+                    return -1;
+                }
+		server_flag = 1;
+	        break;
             case 'A':
 #if defined(HAVE_CPU_AFFINITY)
                 test->affinity = strtol(optarg, &endptr, 0);

src/iperf_api.h

@@ -81,6 +81,7 @@ typedef uint64_t iperf_size_t;
 #define OPT_TIMESTAMPS 22
 #define OPT_SERVER_SKEW_THRESHOLD 23
 #define OPT_BIND_DEV 24
+#define OPT_IDLE_TIMEOUT 25
 
 /* states */
 #define TEST_START 1
@@ -365,6 +366,7 @@ enum {
     IETOTALRATE = 27,       // Total required bandwidth is larger than server's limit
     IETOTALINTERVAL = 28,   // Invalid time interval for calculating average data rate
     IESKEWTHRESHOLD = 29,   // Invalid value specified as skew threshold
+    IEIDLETIMEOUT = 30,     // Invalid value specified as idle state timeout
     /* Test errors */
     IENEWTEST = 100,        // Unable to create a new test (check perror)
     IEINITTEST = 101,       // Test initialization failed (check perror)
@@ -410,6 +412,7 @@ enum {
     IESETBUF2= 141,	    // Socket buffer size incorrect (written value != read value)
     IEAUTHTEST = 142,       // Test authorization failed
     IEBINDDEV = 143,        // Unable to bind-to-device (check perror, maybe permissions?)
+    IENOMSG = 144,          // No message was received for NO_MSG_RCVD_TIMEOUT time period
     /* Stream errors */
     IECREATESTREAM = 200,   // Unable to create a new stream (check herror/perror)
     IEINITSTREAM = 201,     // Unable to initialize stream (check herror/perror)

src/iperf_error.c

@@ -427,6 +427,12 @@ iperf_strerror(int int_errno)
     case IESKEWTHRESHOLD:
 	    snprintf(errstr, len, "skew threshold must be a positive number");
             break;
+	case IEIDLETIMEOUT:
+	    snprintf(errstr, len, "idle timeout parameter is not positive or larget then allowed limit");
+            break;
+	case IENOMSG:
+	    snprintf(errstr, len, "no message was received for %d seconds", NO_MSG_RCVD_TIMEOUT);
+            break;
 	default:
 	    snprintf(errstr, len, "int_errno=%d", int_errno);
 	    perr = 1;

src/iperf_locale.c

@@ -126,6 +126,8 @@ const char usage_longstr[] = "Usage: iperf3 [-s|-c host] [options]\n"
 			   "  --server-bitrate-limit #[KMG][/#]   server's total bit rate limit (default 0 = no limit)\n"
 			   "                            (optional slash and number of secs interval for averaging\n"
 			   "                            total data rate.  Default is 5 seconds)\n"
+                           "  --idle-timeout #         restart server if is idle for #[sec] to overcome\n"
+                           "                           stacked server for different reasone\n"
 #if defined(HAVE_SSL)
                            "  --rsa-private-key-path    path to the RSA private key used to decrypt\n"
 			   "                            authentication credentials\n"

src/iperf_server_api.c

@@ -87,11 +87,15 @@ iperf_server_listen(struct iperf_test *test)
     }
 
     if (!test->json_output) {
-	iperf_printf(test, "-----------------------------------------------------------\n");
-	iperf_printf(test, "Server listening on %d\n", test->server_port);
-	iperf_printf(test, "-----------------------------------------------------------\n");
-	if (test->forceflush)
-	    iflush(test);
+        if (test->server_last_run_rc != 2)
+            test->server_test_number +=1;
+        if (test->debug || test->server_last_run_rc != 2) {
+	    iperf_printf(test, "-----------------------------------------------------------\n");
+	    iperf_printf(test, "Server listening on %d (test #%d)\n", test->server_port, test->server_test_number);
+	    iperf_printf(test, "-----------------------------------------------------------\n");
+	    if (test->forceflush)
+	        iflush(test);
+        }
     }
 
     FD_ZERO(&test->read_set);
@@ -417,8 +421,12 @@ iperf_run_server(struct iperf_test *test)
     fd_set read_set, write_set;
     struct iperf_stream *sp;
     struct iperf_time now;
+    struct iperf_time last_receive_time;
+    struct iperf_time diff_time;
     struct timeval* timeout;
+    struct timeval default_timeout;
     int flag;
+    int64_t t_usecs;
 
     if (test->logfile)
         if (iperf_open_logfile(test) < 0)
@@ -447,6 +455,8 @@ iperf_run_server(struct iperf_test *test)
         return -2;
     }
 
+    iperf_time_now(&last_receive_time); // Initialize last time something was received
+
     test->state = IPERF_START;
     send_streams_accepted = 0;
     rec_streams_accepted = 0;
@@ -465,14 +475,59 @@ iperf_run_server(struct iperf_test *test)
 
 	iperf_time_now(&now);
 	timeout = tmr_timeout(&now);
-        result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout);
 
+        // Ensure select() will timeout to allow handling error cases that require server restart
+        if (timeout == NULL) {
+            default_timeout.tv_sec = 0;
+            if (test->state == IPERF_START) {
+                if (test-> settings->idle_timeout > 0)
+                    default_timeout.tv_sec = test-> settings->idle_timeout;
+            } else {
+                default_timeout.tv_sec = NO_MSG_RCVD_TIMEOUT;
+            }
+            if (default_timeout.tv_sec > 0) {
+                default_timeout.tv_usec = 0;
+                timeout = &default_timeout;
+            }
+        }
+
+        result = select(test->max_fd + 1, &read_set, &write_set, NULL, timeout);
         if (result < 0 && errno != EINTR) {
-	    cleanup_server(test);
+            cleanup_server(test);
             i_errno = IESELECT;
             return -1;
+        } else if (result == 0) {
+            // If nothing was received during the last ... time
+            // then probably something got stack either at the client, server or network,
+            // and Test should forced to end.
+            iperf_time_now(&now);
+            t_usecs = 0;
+            if (iperf_time_diff(&now, &last_receive_time, &diff_time) == 0) {
+                t_usecs = iperf_time_in_usecs(&diff_time);
+                if (test->state == IPERF_START) {
+                    if (test->settings->idle_timeout > 0 && t_usecs >= test->settings->idle_timeout * SEC_TO_US) {
+                        test->server_forced_idle_restarts_count += 1;
+                        if (test->debug)
+                            printf("Server restart (#%d) in idle state as no connection request was received for %d sec\n",
+                                test->server_forced_idle_restarts_count, test-> settings->idle_timeout);
+                        cleanup_server(test);
+                        return 2;
+                    }
+                }
+                else if (t_usecs > NO_MSG_RCVD_TIMEOUT * SEC_TO_US) {
+                    test->server_forced_no_msg_restarts_count += 1;
+                    i_errno = IENOMSG;
+                    iperf_err(test, "Server restart (#%d) in active test as no message was received for %d sec",
+                        test->server_forced_no_msg_restarts_count, NO_MSG_RCVD_TIMEOUT);
+                    cleanup_server(test);
+                    return -1;
+                }
+
+            }
         }
+
 	if (result > 0) {
+            iperf_time_now(&last_receive_time);
             if (FD_ISSET(test->listener, &read_set)) {
                 if (test->state != CREATE_STREAMS) {
                     if (iperf_accept(test) < 0) {

src/main.c

@@ -151,14 +151,15 @@ run(struct iperf_test *test)
             for (;;) {
 		int rc;
 		rc = iperf_run_server(test);
+                test->server_last_run_rc =rc;
 		if (rc < 0) {
 		    iperf_err(test, "error - %s", iperf_strerror(i_errno));
 		    if (rc < -1) {
 		        iperf_errexit(test, "exiting");
 		    }
                 }
                 iperf_reset_test(test);
-                if (iperf_get_test_one_off(test)) {
+                if (iperf_get_test_one_off(test) && rc != 2) {
 		    /* Authentication failure doesn't count for 1-off test */
 		    if (rc < 0 && i_errno == IEAUTHTEST) {
 			continue;