From b091942bcc0dd413e892474a3fe1804a443a4543 Mon Sep 17 00:00:00 2001 From: "Edward A. Lee" Date: Sun, 21 May 2023 12:30:05 -0700 Subject: [PATCH 01/25] Very first pass on refactoring the RTI for enclaves --- core/federated/RTI/CMakeLists.txt | 1 + core/federated/RTI/enclave.c | 43 ++++ core/federated/RTI/enclave.h | 51 ++++ core/federated/RTI/rti_lib.c | 375 ++++++++++++++---------------- core/federated/RTI/rti_lib.h | 41 +--- 5 files changed, 275 insertions(+), 236 deletions(-) create mode 100644 core/federated/RTI/enclave.c create mode 100644 core/federated/RTI/enclave.h diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt index 800eb4ccb..37ae14ee2 100644 --- a/core/federated/RTI/CMakeLists.txt +++ b/core/federated/RTI/CMakeLists.txt @@ -60,6 +60,7 @@ include_directories(${IncludeDir}/utils) # Declare a new executable target and list all its sources add_executable( RTI + enclave.c rti.c rti_lib.c ${CoreLib}/trace.c diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c new file mode 100644 index 000000000..13d740aff --- /dev/null +++ b/core/federated/RTI/enclave.c @@ -0,0 +1,43 @@ +#include "enclave.h" + +// FIXME: This should not be here. +#include "rti_lib.h" + +extern RTI_instance_t _RTI; + +// Global variables defined in tag.c: +extern instant_t start_time; + +// FIXME: rename "federate" everywhere in this file. + +void logical_tag_complete(enclave_t* enclave, tag_t completed) { + // FIXME: Consolidate this message with NET to get NMR (Next Message Request). + // Careful with handling startup and shutdown. + lf_mutex_lock(&rti_mutex); + + enclave->completed = completed; + if (_RTI.tracing_enabled) { + tracepoint_RTI_from_federate(receive_LTC, enclave->id, &(enclave->completed)); + } + + LF_PRINT_LOG("RTI received from federate %d the Logical Tag Complete (LTC) (%lld, %u).", + enclave->id, enclave->completed.time - start_time, enclave->completed.microstep); + + // See if we can remove any of the recorded in-transit messages for this. + // FIXME: Should this be here? + // clean_in_transit_message_record_up_to_tag(enclave->in_transit_message_tags, enclave->completed); + + // Check downstream federates to see whether they should now be granted a TAG. + for (int i = 0; i < enclave->num_downstream; i++) { + // FIXME: Shouldn't use federate_t here. + federate_t* downstream = &_RTI.federates[enclave->downstream[i]]; + // FIXME: Does this need to have two implementations?: + send_advance_grant_if_safe(downstream); + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + // FIXME: Does this need to have two implementations?: + send_downstream_advance_grants_if_safe(downstream, visited); + free(visited); + } + + lf_mutex_unlock(&rti_mutex); +} diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h new file mode 100644 index 000000000..6bfefb898 --- /dev/null +++ b/core/federated/RTI/enclave.h @@ -0,0 +1,51 @@ +#ifndef ENCLAVE_H +#define ENCLAVE_H + +#include +#include +#include // Defines perror(), errno +#include +#include "platform.h" // Platform-specific types and functions +#include "util.h" // Defines print functions (e.g., lf_print). +#include "net_util.h" // Defines network functions. +#include "net_common.h" // Defines message types, etc. Includes and "reactor.h". +#include "tag.h" // Time-related types and functions. +#include "trace.h" // Tracing related functions + + +/** Mode of execution of a federate. */ +typedef enum execution_mode_t { + FAST, + REALTIME +} execution_mode_t; + +/** State of a federate during execution. */ +typedef enum fed_state_t { + NOT_CONNECTED, // The federate has not connected. + GRANTED, // Most recent MSG_TYPE_NEXT_EVENT_TAG has been granted. + PENDING // Waiting for upstream federates. +} fed_state_t; + +typedef struct enclave_t { + uint16_t id; // ID of this enclave. + tag_t completed; // The largest logical tag completed by the federate (or NEVER if no LTC has been received). + tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) + tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) + tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). + fed_state_t state; // State of the federate. + int* upstream; // Array of upstream federate ids. + interval_t* upstream_delay; // Minimum delay on connections from upstream federates. + // Here, NEVER encodes no delay. 0LL is a microstep delay. + int num_upstream; // Size of the array of upstream federates and delays. + int* downstream; // Array of downstream federate ids. + int num_downstream; // Size of the array of downstream federates. + execution_mode_t mode; // FAST or REALTIME. + bool requested_stop; // Indicates that the federate has requested stop or has replied + // to a request for stop from the RTI. Used to prevent double-counting + // a federate when handling lf_request_stop(). +} enclave_t; + +// FIXME: Docs +void logical_tag_complete(enclave_t* enclave, tag_t completed); + +#endif // ENCLAVE_H \ No newline at end of file diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index a5bfc9909..bbd3d663e 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -192,15 +192,15 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty } void send_tag_advance_grant(federate_t* fed, tag_t tag) { - if (fed->state == NOT_CONNECTED - || lf_tag_compare(tag, fed->last_granted) <= 0 - || lf_tag_compare(tag, fed->last_provisionally_granted) < 0 + if (fed->enclave.state == NOT_CONNECTED + || lf_tag_compare(tag, fed->enclave.last_granted) <= 0 + || lf_tag_compare(tag, fed->enclave.last_provisionally_granted) < 0 ) { return; } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[fed->id].state == PENDING) { + while (_RTI.federates[fed->enclave.id].enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -211,35 +211,35 @@ void send_tag_advance_grant(federate_t* fed, tag_t tag) { encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_TAG, fed->id, &tag); + tracepoint_RTI_to_federate(send_TAG, fed->enclave.id, &tag); } // This function is called in send_advance_grant_if_safe(), which is a long // function. During this call, the socket might close, causing the following write_to_socket // to fail. Consider a failure here a soft failure and update the federate's status. ssize_t bytes_written = write_to_socket(fed->socket, message_length, buffer); if (bytes_written < (ssize_t)message_length) { - lf_print_error("RTI failed to send tag advance grant to federate %d.", fed->id); + lf_print_error("RTI failed to send tag advance grant to federate %d.", fed->enclave.id); if (bytes_written < 0) { - fed->state = NOT_CONNECTED; + fed->enclave.state = NOT_CONNECTED; // FIXME: We need better error handling, but don't stop other execution here. // mark_federate_requesting_stop(fed); } } else { - fed->last_granted = tag; + fed->enclave.last_granted = tag; LF_PRINT_LOG("RTI sent to federate %d the tag advance grant (TAG) (%lld, %u).", - fed->id, tag.time - start_time, tag.microstep); + fed->enclave.id, tag.time - start_time, tag.microstep); } } tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]) { - if (visited[fed->id] || fed->state == NOT_CONNECTED) { + if (visited[fed->enclave.id] || fed->enclave.state == NOT_CONNECTED) { // Federate has stopped executing or we have visited it before. // No point in checking upstream federates. return candidate; } - visited[fed->id] = true; - tag_t result = fed->next_event; + visited[fed->enclave.id] = true; + tag_t result = fed->enclave.next_event; // If the candidate is less than this federate's next_event, use the candidate. if (lf_tag_compare(candidate, result) < 0) { @@ -254,34 +254,34 @@ tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]) { // Check upstream federates to see whether any of them might send // an event that would result in an earlier next event. - for (int i = 0; i < fed->num_upstream; i++) { + for (int i = 0; i < fed->enclave.num_upstream; i++) { tag_t upstream_result = transitive_next_event( - &_RTI.federates[fed->upstream[i]], result, visited); + &_RTI.federates[fed->enclave.upstream[i]], result, visited); // Add the "after" delay of the connection to the result. - upstream_result = lf_delay_tag(upstream_result, fed->upstream_delay[i]); + upstream_result = lf_delay_tag(upstream_result, fed->enclave.upstream_delay[i]); // If the adjusted event time is less than the result so far, update the result. if (lf_tag_compare(upstream_result, result) < 0) { result = upstream_result; } } - if (lf_tag_compare(result, fed->completed) < 0) { - result = fed->completed; + if (lf_tag_compare(result, fed->enclave.completed) < 0) { + result = fed->enclave.completed; } return result; } void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { - if (fed->state == NOT_CONNECTED - || lf_tag_compare(tag, fed->last_granted) <= 0 - || lf_tag_compare(tag, fed->last_provisionally_granted) <= 0 + if (fed->enclave.state == NOT_CONNECTED + || lf_tag_compare(tag, fed->enclave.last_granted) <= 0 + || lf_tag_compare(tag, fed->enclave.last_provisionally_granted) <= 0 ) { return; } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[fed->id].state == PENDING) { + while (_RTI.federates[fed->enclave.id].enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -292,7 +292,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); if (_RTI.tracing_enabled){ - tracepoint_RTI_to_federate(send_PTAG, fed->id, &tag); + tracepoint_RTI_to_federate(send_PTAG, fed->enclave.id, &tag); } // This function is called in send_advance_grant_if_safe(), which is a long // function. During this call, the socket might close, causing the following write_to_socket @@ -300,16 +300,16 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { ssize_t bytes_written = write_to_socket(fed->socket, message_length, buffer); if (bytes_written < (ssize_t)message_length) { - lf_print_error("RTI failed to send tag advance grant to federate %d.", fed->id); + lf_print_error("RTI failed to send tag advance grant to federate %d.", fed->enclave.id); if (bytes_written < 0) { - fed->state = NOT_CONNECTED; + fed->enclave.state = NOT_CONNECTED; // FIXME: We need better error handling, but don't stop other execution here. // mark_federate_requesting_stop(fed); } } else { - fed->last_provisionally_granted = tag; + fed->enclave.last_provisionally_granted = tag; LF_PRINT_LOG("RTI sent to federate %d the Provisional Tag Advance Grant (PTAG) (%lld, %u).", - fed->id, tag.time - start_time, tag.microstep); + fed->enclave.id, tag.time - start_time, tag.microstep); // Send PTAG to all upstream federates, if they have not had // a later or equal PTAG or TAG sent previously and if their transitive @@ -317,11 +317,11 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { // NOTE: This could later be replaced with a TNET mechanism once // we have an available encoding of causality interfaces. // That might be more efficient. - for (int j = 0; j < fed->num_upstream; j++) { - federate_t* upstream = &_RTI.federates[fed->upstream[j]]; + for (int j = 0; j < fed->enclave.num_upstream; j++) { + federate_t* upstream = &_RTI.federates[fed->enclave.upstream[j]]; // Ignore this federate if it has resigned. - if (upstream->state == NOT_CONNECTED) continue; + if (upstream->enclave.state == NOT_CONNECTED) continue; // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. @@ -329,7 +329,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( - upstream, upstream->next_event, visited); + upstream, upstream->enclave.next_event, visited); free(visited); // If these tags are equal, then @@ -348,13 +348,13 @@ bool send_advance_grant_if_safe(federate_t* fed) { // Find the earliest LTC of upstream federates. tag_t min_upstream_completed = FOREVER_TAG; - for (int j = 0; j < fed->num_upstream; j++) { - federate_t* upstream = &_RTI.federates[fed->upstream[j]]; + for (int j = 0; j < fed->enclave.num_upstream; j++) { + federate_t* upstream = &_RTI.federates[fed->enclave.upstream[j]]; // Ignore this federate if it has resigned. - if (upstream->state == NOT_CONNECTED) continue; + if (upstream->enclave.state == NOT_CONNECTED) continue; - tag_t candidate = lf_delay_tag(upstream->completed, fed->upstream_delay[j]); + tag_t candidate = lf_delay_tag(upstream->enclave.completed, fed->enclave.upstream_delay[j]); if (lf_tag_compare(candidate, min_upstream_completed) < 0) { min_upstream_completed = candidate; @@ -362,10 +362,10 @@ bool send_advance_grant_if_safe(federate_t* fed) { } LF_PRINT_LOG("Minimum upstream LTC for fed %d is (%lld, %u) " "(adjusted by after delay).", - fed->id, + fed->enclave.id, min_upstream_completed.time - start_time, min_upstream_completed.microstep); - if (lf_tag_compare(min_upstream_completed, fed->last_granted) > 0 - && lf_tag_compare(min_upstream_completed, fed->next_event) >= 0 // The federate has to advance its tag + if (lf_tag_compare(min_upstream_completed, fed->enclave.last_granted) > 0 + && lf_tag_compare(min_upstream_completed, fed->enclave.next_event) >= 0 // The federate has to advance its tag ) { send_tag_advance_grant(fed, min_upstream_completed); return true; @@ -389,25 +389,25 @@ bool send_advance_grant_if_safe(federate_t* fed) { FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, NEVER - start_time, 0u); - for (int j = 0; j < fed->num_upstream; j++) { - federate_t* upstream = &_RTI.federates[fed->upstream[j]]; + for (int j = 0; j < fed->enclave.num_upstream; j++) { + federate_t* upstream = &_RTI.federates[fed->enclave.upstream[j]]; // Ignore this federate if it has resigned. - if (upstream->state == NOT_CONNECTED) continue; + if (upstream->enclave.state == NOT_CONNECTED) continue; // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( - upstream, upstream->next_event, visited); + upstream, upstream->enclave.next_event, visited); LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag (%lld, %u).", - fed->id, - upstream->id, + fed->enclave.id, + upstream->enclave.id, upstream_next_event.time - start_time, upstream_next_event.microstep); // Adjust by the "after" delay. // Note that "no delay" is encoded as NEVER, // whereas one microstep delay is encoded as 0LL. - tag_t candidate = lf_delay_tag(upstream_next_event, fed->upstream_delay[j]); + tag_t candidate = lf_delay_tag(upstream_next_event, fed->enclave.upstream_delay[j]); if (lf_tag_compare(candidate, t_d) < 0) { t_d = candidate; @@ -419,30 +419,30 @@ bool send_advance_grant_if_safe(federate_t* fed) { t_d.time - start_time, t_d.microstep); if ( - lf_tag_compare(t_d, fed->next_event) > 0 // The federate has something to do. - && lf_tag_compare(t_d, fed->last_provisionally_granted) >= 0 // The grant is not redundant + lf_tag_compare(t_d, fed->enclave.next_event) > 0 // The federate has something to do. + && lf_tag_compare(t_d, fed->enclave.last_provisionally_granted) >= 0 // The grant is not redundant // (equal is important to override any previous // PTAGs). - && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. + && lf_tag_compare(t_d, fed->enclave.last_granted) > 0 // The grant is not redundant. ) { // All upstream federates have events with a larger tag than fed, so it is safe to send a TAG. LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG "(adjusted by after delay). Granting tag advance for " PRINTF_TAG, - fed->id, + fed->enclave.id, t_d.time - lf_time_start(), t_d.microstep, - fed->next_event.time - lf_time_start(), - fed->next_event.microstep); - send_tag_advance_grant(fed, fed->next_event); + fed->enclave.next_event.time - lf_time_start(), + fed->enclave.next_event.microstep); + send_tag_advance_grant(fed, fed->enclave.next_event); } else if ( - lf_tag_compare(t_d, fed->next_event) == 0 // The federate has something to do. - && lf_tag_compare(t_d, fed->last_provisionally_granted) > 0 // The grant is not redundant. - && lf_tag_compare(t_d, fed->last_granted) > 0 // The grant is not redundant. + lf_tag_compare(t_d, fed->enclave.next_event) == 0 // The federate has something to do. + && lf_tag_compare(t_d, fed->enclave.last_provisionally_granted) > 0 // The grant is not redundant. + && lf_tag_compare(t_d, fed->enclave.last_granted) > 0 // The grant is not redundant. ) { // Some upstream federate has an event that has the same tag as fed's next event, so we can only provisionally // grant a TAG (via a PTAG). LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG " (adjusted by after delay). Granting provisional tag advance.", - fed->id, + fed->enclave.id, t_d.time - start_time, t_d.microstep); send_provisional_tag_advance_grant(fed, t_d); @@ -451,10 +451,10 @@ bool send_advance_grant_if_safe(federate_t* fed) { } void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { - visited[fed->id] = true; - for (int i = 0; i < fed->num_downstream; i++) { - federate_t* downstream = &_RTI.federates[fed->downstream[i]]; - if (visited[downstream->id]) continue; + visited[fed->enclave.id] = true; + for (int i = 0; i < fed->enclave.num_downstream; i++) { + federate_t* downstream = &_RTI.federates[fed->enclave.downstream[i]]; + if (visited[downstream->enclave.id]) continue; send_advance_grant_if_safe(downstream); send_downstream_advance_grants_if_safe(downstream, visited); } @@ -470,7 +470,7 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even next_event_tag = min_in_transit_tag; } - _RTI.federates[federate_id].next_event = next_event_tag; + _RTI.federates[federate_id].enclave.next_event = next_event_tag; LF_PRINT_DEBUG( "RTI: Updated the recorded next event tag for federate %d to " PRINTF_TAG, @@ -482,7 +482,7 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even // Check to see whether we can reply now with a tag advance grant. // If the federate has no upstream federates, then it does not wait for // nor expect a reply. It just proceeds to advance time. - if (_RTI.federates[federate_id].num_upstream > 0) { + if (_RTI.federates[federate_id].enclave.num_upstream > 0) { send_advance_grant_if_safe(&_RTI.federates[federate_id]); } // Check downstream federates to see whether they should now be granted a TAG. @@ -498,7 +498,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf read_from_socket_errexit(sending_federate->socket, message_size, &(buffer[1]), " RTI failed to read port absent message from federate %u.", - sending_federate->id); + sending_federate->enclave.id); uint16_t reactor_port_id = extract_uint16(&(buffer[1])); uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); @@ -515,7 +515,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // If the destination federate is no longer connected, issue a warning // and return. - if (_RTI.federates[federate_id].state == NOT_CONNECTED) { + if (_RTI.federates[federate_id].enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); @@ -523,14 +523,14 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf "completed (%lld, %d), " "last_granted (%lld, %d), " "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].next_event.time - start_time, - _RTI.federates[federate_id].next_event.microstep, - _RTI.federates[federate_id].completed.time - start_time, - _RTI.federates[federate_id].completed.microstep, - _RTI.federates[federate_id].last_granted.time - start_time, - _RTI.federates[federate_id].last_granted.microstep, - _RTI.federates[federate_id].last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].last_provisionally_granted.microstep + _RTI.federates[federate_id].enclave.next_event.time - start_time, + _RTI.federates[federate_id].enclave.next_event.microstep, + _RTI.federates[federate_id].enclave.completed.time - start_time, + _RTI.federates[federate_id].enclave.completed.microstep, + _RTI.federates[federate_id].enclave.last_granted.time - start_time, + _RTI.federates[federate_id].enclave.last_granted.microstep, + _RTI.federates[federate_id].enclave.last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].enclave.last_provisionally_granted.microstep ); return; } @@ -541,7 +541,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[federate_id].state == PENDING) { + while (_RTI.federates[federate_id].enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -585,7 +585,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { LF_PRINT_LOG("RTI received message from federate %d for federate %u port %u with intended tag " PRINTF_TAG ". Forwarding.", - sending_federate->id, federate_id, reactor_port_id, + sending_federate->enclave.id, federate_id, reactor_port_id, intended_tag.time - lf_time_start(), intended_tag.microstep); read_from_socket_errexit(sending_federate->socket, bytes_to_read, &(buffer[header_size]), @@ -595,7 +595,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); if (_RTI.tracing_enabled) { - tracepoint_RTI_from_federate(receive_TAGGED_MSG, sending_federate->id, &intended_tag); + tracepoint_RTI_from_federate(receive_TAGGED_MSG, sending_federate->enclave.id, &intended_tag); } // Need to acquire the mutex lock to ensure that the thread handling @@ -605,7 +605,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // If the destination federate is no longer connected, issue a warning // and return. - if (_RTI.federates[federate_id].state == NOT_CONNECTED) { + if (_RTI.federates[federate_id].enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); @@ -613,14 +613,14 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { "completed (%lld, %d), " "last_granted (%lld, %d), " "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].next_event.time - start_time, - _RTI.federates[federate_id].next_event.microstep, - _RTI.federates[federate_id].completed.time - start_time, - _RTI.federates[federate_id].completed.microstep, - _RTI.federates[federate_id].last_granted.time - start_time, - _RTI.federates[federate_id].last_granted.microstep, - _RTI.federates[federate_id].last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].last_provisionally_granted.microstep + _RTI.federates[federate_id].enclave.next_event.time - start_time, + _RTI.federates[federate_id].enclave.next_event.microstep, + _RTI.federates[federate_id].enclave.completed.time - start_time, + _RTI.federates[federate_id].enclave.completed.microstep, + _RTI.federates[federate_id].enclave.last_granted.time - start_time, + _RTI.federates[federate_id].enclave.last_granted.microstep, + _RTI.federates[federate_id].enclave.last_provisionally_granted.time - start_time, + _RTI.federates[federate_id].enclave.last_provisionally_granted.microstep ); return; } @@ -636,7 +636,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { ); // Record this in-transit message in federate's in-transit message queue. - if (lf_tag_compare(_RTI.federates[federate_id].completed, intended_tag) < 0) { + if (lf_tag_compare(_RTI.federates[federate_id].enclave.completed, intended_tag) < 0) { // Add a record of this message to the list of in-transit messages to this federate. add_in_transit_message_record( _RTI.federates[federate_id].in_transit_message_tags, @@ -654,18 +654,18 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { ", but there is an in-transit message with tag " PRINTF_TAG " from federate %hu. " "This is going to cause an STP violation under centralized coordination.", federate_id, - _RTI.federates[federate_id].completed.time - lf_time_start(), - _RTI.federates[federate_id].completed.microstep, + _RTI.federates[federate_id].enclave.completed.time - lf_time_start(), + _RTI.federates[federate_id].enclave.completed.microstep, intended_tag.time - lf_time_start(), intended_tag.microstep, - sending_federate->id + sending_federate->enclave.id ); // FIXME: Drop the federate? } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[federate_id].state == PENDING) { + while (_RTI.federates[federate_id].enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -706,40 +706,21 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { void handle_logical_tag_complete(federate_t* fed) { unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)]; read_from_socket_errexit(fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, - "RTI failed to read the content of the logical tag complete from federate %d.", fed->id); + "RTI failed to read the content of the logical tag complete from federate %d.", fed->enclave.id); + tag_t completed = extract_tag(buffer); + logical_tag_complete(&fed->enclave, completed); - // FIXME: Consolidate this message with NET to get NMR (Next Message Request). - // Careful with handling startup and shutdown. + // FIXME: Should this function be in the enclave version? lf_mutex_lock(&rti_mutex); - - fed->completed = extract_tag(buffer); - if (_RTI.tracing_enabled) { - tracepoint_RTI_from_federate(receive_LTC, fed->id, &(fed->completed)); - } - - LF_PRINT_LOG("RTI received from federate %d the Logical Tag Complete (LTC) (%lld, %u).", - fed->id, fed->completed.time - start_time, fed->completed.microstep); - - // See if we can remove any of the recorded in-transit messages for this. - clean_in_transit_message_record_up_to_tag(fed->in_transit_message_tags, fed->completed); - - // Check downstream federates to see whether they should now be granted a TAG. - for (int i = 0; i < fed->num_downstream; i++) { - federate_t* downstream = &_RTI.federates[fed->downstream[i]]; - send_advance_grant_if_safe(downstream); - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(downstream, visited); - free(visited); - } - + clean_in_transit_message_record_up_to_tag(fed->in_transit_message_tags, fed->enclave.completed); lf_mutex_unlock(&rti_mutex); } void handle_next_event_tag(federate_t* fed) { unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)]; read_from_socket_errexit(fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, - "RTI failed to read the content of the next event tag from federate %d.", fed->id); + "RTI failed to read the content of the next event tag from federate %d.", fed->enclave.id); // Acquire a mutex lock to ensure that this state does not change while a // message is in transport or being used to determine a TAG. @@ -751,13 +732,13 @@ void handle_next_event_tag(federate_t* fed) { tag_t intended_tag = extract_tag(buffer); if (_RTI.tracing_enabled) { - tracepoint_RTI_from_federate(receive_NET, fed->id, &intended_tag); + tracepoint_RTI_from_federate(receive_NET, fed->enclave.id, &intended_tag); } LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) " PRINTF_TAG, - fed->id, intended_tag.time - start_time, + fed->enclave.id, intended_tag.time - start_time, intended_tag.microstep); update_federate_next_event_tag_locked( - fed->id, + fed->enclave.id, intended_tag ); lf_mutex_unlock(&rti_mutex); @@ -781,18 +762,18 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { // Iterate over federates and send each the message. for (int i = 0; i < _RTI.number_of_federates; i++) { - if (_RTI.federates[i].state == NOT_CONNECTED) { + if (_RTI.federates[i].enclave.state == NOT_CONNECTED) { continue; } - if (lf_tag_compare(_RTI.federates[i].next_event, _RTI.max_stop_tag) >= 0) { + if (lf_tag_compare(_RTI.federates[i].enclave.next_event, _RTI.max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. - _RTI.federates[i].next_event = _RTI.max_stop_tag; + _RTI.federates[i].enclave.next_event = _RTI.max_stop_tag; } if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_GRN, _RTI.federates[i].id, &_RTI.max_stop_tag); + tracepoint_RTI_to_federate(send_STOP_GRN, _RTI.federates[i].enclave.id, &_RTI.max_stop_tag); } write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, - "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", _RTI.federates[i].id); + "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", _RTI.federates[i].enclave.id); } LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag (%lld, %u).", @@ -802,11 +783,11 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { } void mark_federate_requesting_stop(federate_t* fed) { - if (!fed->requested_stop) { + if (!fed->enclave.requested_stop) { // Assume that the federate // has requested stop _RTI.num_feds_handling_stop++; - fed->requested_stop = true; + fed->enclave.requested_stop = true; } if (_RTI.num_feds_handling_stop == _RTI.number_of_federates) { // We now have information about the stop time of all @@ -816,12 +797,12 @@ void mark_federate_requesting_stop(federate_t* fed) { } void handle_stop_request_message(federate_t* fed) { - LF_PRINT_DEBUG("RTI handling stop_request from federate %d.", fed->id); + LF_PRINT_DEBUG("RTI handling stop_request from federate %d.", fed->enclave.id); size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; unsigned char buffer[bytes_to_read]; read_from_socket_errexit(fed->socket, bytes_to_read, buffer, - "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", fed->id); + "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", fed->enclave.id); // Acquire a mutex lock to ensure that this state does change while a // message is in transport or being used to determine a TAG. @@ -829,7 +810,7 @@ void handle_stop_request_message(federate_t* fed) { // Check whether we have already received a stop_tag // from this federate - if (_RTI.federates[fed->id].requested_stop) { + if (_RTI.federates[fed->enclave.id].enclave.requested_stop) { // Ignore this request lf_mutex_unlock(&rti_mutex); return; @@ -839,7 +820,7 @@ void handle_stop_request_message(federate_t* fed) { tag_t proposed_stop_tag = extract_tag(buffer); if (_RTI.tracing_enabled) { - tracepoint_RTI_from_federate(receive_STOP_REQ, fed->id, &proposed_stop_tag); + tracepoint_RTI_from_federate(receive_STOP_REQ, fed->enclave.id, &proposed_stop_tag); } // Update the maximum stop tag received from federates @@ -848,7 +829,7 @@ void handle_stop_request_message(federate_t* fed) { } LF_PRINT_LOG("RTI received from federate %d a MSG_TYPE_STOP_REQUEST message with tag (%lld, %u).", - fed->id, proposed_stop_tag.time - start_time, proposed_stop_tag.microstep); + fed->enclave.id, proposed_stop_tag.time - start_time, proposed_stop_tag.microstep); // If this federate has not already asked // for a stop, add it to the tally. @@ -869,18 +850,18 @@ void handle_stop_request_message(federate_t* fed) { // Iterate over federates and send each the MSG_TYPE_STOP_REQUEST message // if we do not have a stop_time already for them. for (int i = 0; i < _RTI.number_of_federates; i++) { - if (_RTI.federates[i].id != fed->id && _RTI.federates[i].requested_stop == false) { - if (_RTI.federates[i].state == NOT_CONNECTED) { + if (_RTI.federates[i].enclave.id != fed->enclave.id && _RTI.federates[i].enclave.requested_stop == false) { + if (_RTI.federates[i].enclave.state == NOT_CONNECTED) { mark_federate_requesting_stop(&_RTI.federates[i]); continue; } if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, _RTI.federates[i].id, &_RTI.max_stop_tag); + tracepoint_RTI_to_federate(send_STOP_REQ, _RTI.federates[i].enclave.id, &_RTI.max_stop_tag); } write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, - "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", _RTI.federates[i].id); + "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", _RTI.federates[i].enclave.id); if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, _RTI.federates[i].id, &_RTI.max_stop_tag); + tracepoint_RTI_to_federate(send_STOP_REQ, _RTI.federates[i].enclave.id, &_RTI.max_stop_tag); } } } @@ -894,15 +875,15 @@ void handle_stop_request_reply(federate_t* fed) { size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_REPLY_LENGTH - 1; unsigned char buffer_stop_time[bytes_to_read]; read_from_socket_errexit(fed->socket, bytes_to_read, buffer_stop_time, - "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", fed->id); + "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", fed->enclave.id); tag_t federate_stop_tag = extract_tag(buffer_stop_time); if (_RTI.tracing_enabled) { - tracepoint_RTI_from_federate(receive_STOP_REQ_REP, fed->id, &federate_stop_tag); + tracepoint_RTI_from_federate(receive_STOP_REQ_REP, fed->enclave.id, &federate_stop_tag); } - LF_PRINT_LOG("RTI received from federate %d STOP reply tag (%lld, %u).", fed->id, + LF_PRINT_LOG("RTI received from federate %d STOP reply tag (%lld, %u).", fed->enclave.id, federate_stop_tag.time - start_time, federate_stop_tag.microstep); @@ -963,7 +944,7 @@ void handle_address_ad(uint16_t federate_id) { ssize_t bytes_read = read_from_socket(_RTI.federates[federate_id].socket, sizeof(int32_t), (unsigned char *)buffer); if (bytes_read < (ssize_t)sizeof(int32_t)) { - LF_PRINT_DEBUG("Error reading port data from federate %d.", _RTI.federates[federate_id].id); + LF_PRINT_DEBUG("Error reading port data from federate %d.", _RTI.federates[federate_id].enclave.id); // Leave the server port at -1, which means "I don't know". return; } @@ -986,13 +967,13 @@ void handle_timestamp(federate_t *my_fed) { // Read bytes from the socket. We need 8 bytes. ssize_t bytes_read = read_from_socket(my_fed->socket, sizeof(int64_t), (unsigned char*)&buffer); if (bytes_read < (ssize_t)sizeof(int64_t)) { - lf_print_error("ERROR reading timestamp from federate %d.\n", my_fed->id); + lf_print_error("ERROR reading timestamp from federate %d.\n", my_fed->enclave.id); } int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t *)(&buffer))); if (_RTI.tracing_enabled) { tag_t tag = {.time = timestamp, .microstep = 0}; - tracepoint_RTI_from_federate(receive_TIMESTAMP, my_fed->id, &tag); + tracepoint_RTI_from_federate(receive_TIMESTAMP, my_fed->enclave.id, &tag); } LF_PRINT_LOG("RTI received timestamp message: %lld.", timestamp); @@ -1025,30 +1006,30 @@ void handle_timestamp(federate_t *my_fed) { if (_RTI.tracing_enabled) { tag_t tag = {.time = start_time, .microstep = 0}; - tracepoint_RTI_to_federate(send_TIMESTAMP, my_fed->id, &tag); + tracepoint_RTI_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } ssize_t bytes_written = write_to_socket( my_fed->socket, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer ); if (bytes_written < MSG_TYPE_TIMESTAMP_LENGTH) { - lf_print_error("Failed to send the starting time to federate %d.", my_fed->id); + lf_print_error("Failed to send the starting time to federate %d.", my_fed->enclave.id); } lf_mutex_lock(&rti_mutex); // Update state for the federate to indicate that the MSG_TYPE_TIMESTAMP // message has been sent. That MSG_TYPE_TIMESTAMP message grants time advance to // the federate to the start time. - my_fed->state = GRANTED; + my_fed->enclave.state = GRANTED; lf_cond_broadcast(&sent_start_time); - LF_PRINT_LOG("RTI sent start time %lld to federate %d.", start_time, my_fed->id); + LF_PRINT_LOG("RTI sent start time %lld to federate %d.", start_time, my_fed->enclave.id); lf_mutex_unlock(&rti_mutex); } void send_physical_clock(unsigned char message_type, federate_t* fed, socket_type_t socket_type) { - if (fed->state == NOT_CONNECTED) { + if (fed->enclave.state == NOT_CONNECTED) { lf_print_warning("Clock sync: RTI failed to send physical time to federate %d. Socket not connected.\n", - fed->id); + fed->enclave.id); return; } unsigned char buffer[sizeof(int64_t) + 1]; @@ -1064,7 +1045,7 @@ void send_physical_clock(unsigned char message_type, federate_t* fed, socket_typ (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr)); if (bytes_written < (ssize_t)sizeof(int64_t) + 1) { lf_print_warning("Clock sync: RTI failed to send physical time to federate %d: %s\n", - fed->id, + fed->enclave.id, strerror(errno)); return; } @@ -1072,12 +1053,12 @@ void send_physical_clock(unsigned char message_type, federate_t* fed, socket_typ LF_PRINT_DEBUG("Clock sync: RTI sending TCP message type %u.", buffer[0]); write_to_socket_errexit(fed->socket, 1 + sizeof(int64_t), buffer, "Clock sync: RTI failed to send physical time to federate %d: %s.", - fed->id, + fed->enclave.id, strerror(errno)); } LF_PRINT_DEBUG("Clock sync: RTI sent PHYSICAL_TIME_SYNC_MESSAGE with timestamp %lld to federate %d.", current_physical_time, - fed->id); + fed->enclave.id); } void handle_physical_clock_sync_message(federate_t* my_fed, socket_type_t socket_type) { @@ -1125,10 +1106,10 @@ void* clock_synchronization_thread(void* noargs) { nanosleep(&sleep_time, &remaining_time); // Can be interrupted any_federates_connected = false; for (int fed = 0; fed < _RTI.number_of_federates; fed++) { - if (_RTI.federates[fed].state == NOT_CONNECTED) { + if (_RTI.federates[fed].enclave.state == NOT_CONNECTED) { // FIXME: We need better error handling here, but clock sync failure // should not stop execution. - lf_print_error("Clock sync failed with federate %d. Not connected.", _RTI.federates[fed].id); + lf_print_error("Clock sync failed with federate %d. Not connected.", _RTI.federates[fed].enclave.id); // mark_federate_requesting_stop(&_RTI.federates[fed]); continue; } else if (!_RTI.federates[fed].clock_synchronization_enabled) { @@ -1154,11 +1135,11 @@ void* clock_synchronization_thread(void* noargs) { if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { int32_t fed_id = extract_int32(&(buffer[1])); // Check that this message came from the correct federate. - if (fed_id != _RTI.federates[fed].id) { + if (fed_id != _RTI.federates[fed].enclave.id) { // Message is from the wrong federate. Discard the message. lf_print_warning("Clock sync: Received T3 message from federate %d, " "but expected one from %d. Discarding message.", - fed_id, _RTI.federates[fed].id); + fed_id, _RTI.federates[fed].enclave.id); continue; } LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id); @@ -1172,14 +1153,14 @@ void* clock_synchronization_thread(void* noargs) { "Discarding message.", buffer[0], MSG_TYPE_CLOCK_SYNC_T3, - _RTI.federates[fed].id); + _RTI.federates[fed].enclave.id); continue; } } else { lf_print_warning("Clock sync: Read from UDP socket failed: %s. " "Skipping clock sync round for federate %d.", strerror(errno), - _RTI.federates[fed].id); + _RTI.federates[fed].enclave.id); remaining_attempts = -1; } } @@ -1203,15 +1184,15 @@ void handle_federate_resign(federate_t *my_fed) { "RTI failed to read the timed message header from remote federate."); // Extract the tag sent by the resigning federate tag_t tag = extract_tag(&(buffer[1])); - tracepoint_RTI_from_federate(receive_RESIGN, my_fed->id, &tag); + tracepoint_RTI_from_federate(receive_RESIGN, my_fed->enclave.id, &tag); } - my_fed->state = NOT_CONNECTED; + my_fed->enclave.state = NOT_CONNECTED; // FIXME: The following results in spurious error messages. // mark_federate_requesting_stop(my_fed); // Indicate that there will no further events from this federate. - my_fed->next_event = FOREVER_TAG; + my_fed->enclave.next_event = FOREVER_TAG; // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket, // the close should happen when receiving a 0 length message from the other end. @@ -1222,7 +1203,7 @@ void handle_federate_resign(federate_t *my_fed) { // an orderly shutdown. // close(my_fed->socket); // from unistd.h - lf_print("Federate %d has resigned.", my_fed->id); + lf_print("Federate %d has resigned.", my_fed->enclave.id); // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep @@ -1243,28 +1224,28 @@ void* federate_thread_TCP(void* fed) { unsigned char buffer[FED_COM_BUFFER_SIZE]; // Listen for messages from the federate. - while (my_fed->state != NOT_CONNECTED) { + while (my_fed->enclave.state != NOT_CONNECTED) { // Read no more than one byte to get the message type. ssize_t bytes_read = read_from_socket(my_fed->socket, 1, buffer); if (bytes_read < 1) { // Socket is closed - lf_print_warning("RTI: Socket to federate %d is closed. Exiting the thread.", my_fed->id); - my_fed->state = NOT_CONNECTED; + lf_print_warning("RTI: Socket to federate %d is closed. Exiting the thread.", my_fed->enclave.id); + my_fed->enclave.state = NOT_CONNECTED; my_fed->socket = -1; // FIXME: We need better error handling here, but this is probably not the right thing to do. // mark_federate_requesting_stop(my_fed); break; } - LF_PRINT_DEBUG("RTI: Received message type %u from federate %d.", buffer[0], my_fed->id); + LF_PRINT_DEBUG("RTI: Received message type %u from federate %d.", buffer[0], my_fed->enclave.id); switch(buffer[0]) { case MSG_TYPE_TIMESTAMP: handle_timestamp(my_fed); break; case MSG_TYPE_ADDRESS_QUERY: - handle_address_query(my_fed->id); + handle_address_query(my_fed->enclave.id); break; case MSG_TYPE_ADDRESS_ADVERTISEMENT: - handle_address_ad(my_fed->id); + handle_address_ad(my_fed->enclave.id); break; case MSG_TYPE_TAGGED_MESSAGE: handle_timed_message(my_fed, buffer); @@ -1292,9 +1273,9 @@ void* federate_thread_TCP(void* fed) { handle_port_absent_message(my_fed, buffer); break; default: - lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->id, buffer[0]); + lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->enclave.id, buffer[0]); if (_RTI.tracing_enabled) { - tracepoint_RTI_from_federate(receive_UNIDENTIFIED, my_fed->id, NULL); + tracepoint_RTI_from_federate(receive_UNIDENTIFIED, my_fed->enclave.id, NULL); } } } @@ -1389,7 +1370,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { - if (_RTI.federates[fed_id].state != NOT_CONNECTED) { + if (_RTI.federates[fed_id].enclave.state != NOT_CONNECTED) { lf_print_error("RTI received duplicate federate ID: %d.", fed_id); if (_RTI.tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); @@ -1424,7 +1405,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // Set the federate's state as pending // because it is waiting for the start time to be // sent by the RTI before beginning its execution. - _RTI.federates[fed_id].state = PENDING; + _RTI.federates[fed_id].enclave.state = PENDING; LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); // Send an MSG_TYPE_ACK message. @@ -1456,26 +1437,26 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { return 0; } else { // Read the number of upstream and downstream connections - _RTI.federates[fed_id].num_upstream = extract_int32(&(connection_info_header[1])); - _RTI.federates[fed_id].num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); + _RTI.federates[fed_id].enclave.num_upstream = extract_int32(&(connection_info_header[1])); + _RTI.federates[fed_id].enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); LF_PRINT_DEBUG( "RTI got %d upstreams and %d downstreams from federate %d.", - _RTI.federates[fed_id].num_upstream, - _RTI.federates[fed_id].num_downstream, + _RTI.federates[fed_id].enclave.num_upstream, + _RTI.federates[fed_id].enclave.num_downstream, fed_id); // Allocate memory for the upstream and downstream pointers - _RTI.federates[fed_id].upstream = (int*)malloc(sizeof(federate_t*) * _RTI.federates[fed_id].num_upstream); - _RTI.federates[fed_id].downstream = (int*)malloc(sizeof(federate_t*) * _RTI.federates[fed_id].num_downstream); + _RTI.federates[fed_id].enclave.upstream = (int*)malloc(sizeof(federate_t*) * _RTI.federates[fed_id].enclave.num_upstream); + _RTI.federates[fed_id].enclave.downstream = (int*)malloc(sizeof(federate_t*) * _RTI.federates[fed_id].enclave.num_downstream); // Allocate memory for the upstream delay pointers - _RTI.federates[fed_id].upstream_delay = + _RTI.federates[fed_id].enclave.upstream_delay = (interval_t*)malloc( - sizeof(interval_t) * _RTI.federates[fed_id].num_upstream + sizeof(interval_t) * _RTI.federates[fed_id].enclave.num_upstream ); size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) * - _RTI.federates[fed_id].num_upstream) + (sizeof(uint16_t) * _RTI.federates[fed_id].num_downstream); + _RTI.federates[fed_id].enclave.num_upstream) + (sizeof(uint16_t) * _RTI.federates[fed_id].enclave.num_downstream); unsigned char* connections_info_body = (unsigned char*)malloc(connections_info_body_size); read_from_socket_errexit( socket_id, @@ -1488,16 +1469,16 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { // Keep track of where we are in the buffer size_t message_head = 0; // First, read the info about upstream federates - for (int i=0; i<_RTI.federates[fed_id].num_upstream; i++) { - _RTI.federates[fed_id].upstream[i] = extract_uint16(&(connections_info_body[message_head])); + for (int i=0; i<_RTI.federates[fed_id].enclave.num_upstream; i++) { + _RTI.federates[fed_id].enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head])); message_head += sizeof(uint16_t); - _RTI.federates[fed_id].upstream_delay[i] = extract_int64(&(connections_info_body[message_head])); + _RTI.federates[fed_id].enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head])); message_head += sizeof(int64_t); } // Next, read the info about downstream federates - for (int i=0; i<_RTI.federates[fed_id].num_downstream; i++) { - _RTI.federates[fed_id].downstream[i] = extract_uint16(&(connections_info_body[message_head])); + for (int i=0; i<_RTI.federates[fed_id].enclave.num_downstream; i++) { + _RTI.federates[fed_id].enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head])); message_head += sizeof(uint16_t); } @@ -1729,25 +1710,25 @@ void* respond_to_erroneous_connections(void* nothing) { } void initialize_federate(uint16_t id) { - _RTI.federates[id].id = id; + _RTI.federates[id].enclave.id = id; _RTI.federates[id].socket = -1; // No socket. _RTI.federates[id].clock_synchronization_enabled = true; - _RTI.federates[id].completed = NEVER_TAG; - _RTI.federates[id].last_granted = NEVER_TAG; - _RTI.federates[id].last_provisionally_granted = NEVER_TAG; - _RTI.federates[id].next_event = NEVER_TAG; + _RTI.federates[id].enclave.completed = NEVER_TAG; + _RTI.federates[id].enclave.last_granted = NEVER_TAG; + _RTI.federates[id].enclave.last_provisionally_granted = NEVER_TAG; + _RTI.federates[id].enclave.next_event = NEVER_TAG; _RTI.federates[id].in_transit_message_tags = initialize_in_transit_message_q(); - _RTI.federates[id].state = NOT_CONNECTED; - _RTI.federates[id].upstream = NULL; - _RTI.federates[id].upstream_delay = NULL; - _RTI.federates[id].num_upstream = 0; - _RTI.federates[id].downstream = NULL; - _RTI.federates[id].num_downstream = 0; - _RTI.federates[id].mode = REALTIME; + _RTI.federates[id].enclave.state = NOT_CONNECTED; + _RTI.federates[id].enclave.upstream = NULL; + _RTI.federates[id].enclave.upstream_delay = NULL; + _RTI.federates[id].enclave.num_upstream = 0; + _RTI.federates[id].enclave.downstream = NULL; + _RTI.federates[id].enclave.num_downstream = 0; + _RTI.federates[id].enclave.mode = REALTIME; strncpy(_RTI.federates[id].server_hostname ,"localhost", INET_ADDRSTRLEN); _RTI.federates[id].server_ip_addr.s_addr = 0; _RTI.federates[id].server_port = -1; - _RTI.federates[id].requested_stop = false; + _RTI.federates[id].enclave.requested_stop = false; } int32_t start_rti_server(uint16_t port) { @@ -1785,10 +1766,10 @@ void wait_for_federates(int socket_descriptor) { // Wait for federate threads to exit. void* thread_exit_status; for (int i = 0; i < _RTI.number_of_federates; i++) { - lf_print("RTI: Waiting for thread handling federate %d.", _RTI.federates[i].id); + lf_print("RTI: Waiting for thread handling federate %d.", _RTI.federates[i].enclave.id); lf_thread_join(_RTI.federates[i].thread_id, &thread_exit_status); free_in_transit_message_q(_RTI.federates[i].in_transit_message_tags); - lf_print("RTI: Federate %d thread exited.", _RTI.federates[i].id); + lf_print("RTI: Federate %d thread exited.", _RTI.federates[i].enclave.id); } _RTI.all_federates_exited = true; diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index 411232850..2da11779d 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -11,9 +11,6 @@ #ifndef RTI_LIB_H #define RTI_LIB_H -#include -#include -#include // Defines perror(), errno #include #include // Provides select() function to read from multiple sockets. #include // Defines struct sockaddr_in @@ -21,15 +18,9 @@ #include // Defines read(), write(), and close() #include // Defines gethostbyname(). #include // Defines bzero(). -#include #include // Defines wait() for process to change state. -#include "platform.h" // Platform-specific types and functions -#include "util.h" // Defines print functions (e.g., lf_print). -#include "net_util.h" // Defines network functions. -#include "net_common.h" // Defines message types, etc. Includes and "reactor.h". -#include "tag.h" // Time-related types and functions. -#include "trace.h" // Tracing related functions +#include "enclave.h" #ifdef __RTI_AUTH__ #include // For secure random number generation. @@ -47,19 +38,6 @@ typedef enum socket_type_t { UDP } socket_type_t; -/** Mode of execution of a federate. */ -typedef enum execution_mode_t { - FAST, - REALTIME -} execution_mode_t; - -/** State of a federate during execution. */ -typedef enum fed_state_t { - NOT_CONNECTED, // The federate has not connected. - GRANTED, // Most recent MSG_TYPE_NEXT_EVENT_TAG has been granted. - PENDING // Waiting for upstream federates. -} fed_state_t; - /** * Information about a federate known to the RTI, including its runtime state, * mode of execution, and connectivity with other federates. @@ -69,27 +47,15 @@ typedef enum fed_state_t { * any scheduling constraints. */ typedef struct federate_t { - uint16_t id; // ID of this federate. + enclave_t enclave; lf_thread_t thread_id; // The ID of the thread handling communication with this federate. int socket; // The TCP socket descriptor for communicating with this federate. struct sockaddr_in UDP_addr; // The UDP address for the federate. bool clock_synchronization_enabled; // Indicates the status of clock synchronization // for this federate. Enabled by default. - tag_t completed; // The largest logical tag completed by the federate (or NEVER if no LTC has been received). - tag_t last_granted; // The maximum TAG that has been granted so far (or NEVER if none granted) - tag_t last_provisionally_granted; // The maximum PTAG that has been provisionally granted (or NEVER if none granted) - tag_t next_event; // Most recent NET received from the federate (or NEVER if none received). in_transit_message_record_q_t* in_transit_message_tags; // Record of in-transit messages to this federate that are not // yet processed. This record is ordered based on the time // value of each message for a more efficient access. - fed_state_t state; // State of the federate. - int* upstream; // Array of upstream federate ids. - interval_t* upstream_delay; // Minimum delay on connections from upstream federates. - // Here, NEVER encodes no delay. 0LL is a microstep delay. - int num_upstream; // Size of the array of upstream federates and delays. - int* downstream; // Array of downstream federate ids. - int num_downstream; // Size of the array of downstream federates. - execution_mode_t mode; // FAST or REALTIME. char server_hostname[INET_ADDRSTRLEN]; // Human-readable IP address and int32_t server_port; // port number of the socket server of the federate // if it has any incoming direct connections from other federates. @@ -97,9 +63,6 @@ typedef struct federate_t { // RTI has not been informed of the port number. struct in_addr server_ip_addr; // Information about the IP address of the socket // server of the federate. - bool requested_stop; // Indicates that the federate has requested stop or has replied - // to a request for stop from the RTI. Used to prevent double-counting - // a federate when handling lf_request_stop(). } federate_t; /** From 0a1327a516c2549cb5007a80aeebc5fc84095b42 Mon Sep 17 00:00:00 2001 From: "Edward A. Lee" Date: Sun, 21 May 2023 15:02:40 -0700 Subject: [PATCH 02/25] Refactored more functions for enclave use. --- core/federated/RTI/enclave.c | 126 ++++++++++++++++++++++++++++++-- core/federated/RTI/enclave.h | 56 +++++++++++++++ core/federated/RTI/rti_lib.c | 134 ++++------------------------------- core/federated/RTI/rti_lib.h | 43 ----------- 4 files changed, 192 insertions(+), 167 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 13d740aff..bed8625a3 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -31,13 +31,131 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { for (int i = 0; i < enclave->num_downstream; i++) { // FIXME: Shouldn't use federate_t here. federate_t* downstream = &_RTI.federates[enclave->downstream[i]]; - // FIXME: Does this need to have two implementations?: - send_advance_grant_if_safe(downstream); + // Notify downstream federate if appropriate. + notify_advance_grant_if_safe((enclave_t*)downstream); bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - // FIXME: Does this need to have two implementations?: - send_downstream_advance_grants_if_safe(downstream, visited); + // Notify enclaves downstream of downstream if appropriate. + notify_downstream_advance_grant_if_safe((enclave_t*)downstream, visited); free(visited); } lf_mutex_unlock(&rti_mutex); } + +tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { + tag_advance_grant_t result = {.tag = NEVER_TAG, .is_provisional = false}; + + // Find the earliest LTC of upstream federates (M). + tag_t min_upstream_completed = FOREVER_TAG; + + for (int j = 0; j < e->num_upstream; j++) { + federate_t* upstream = &_RTI.federates[e->upstream[j]]; + + // Ignore this federate if it has resigned. + if (upstream->enclave.state == NOT_CONNECTED) continue; + + tag_t candidate = lf_delay_tag(upstream->enclave.completed, e->upstream_delay[j]); + + if (lf_tag_compare(candidate, min_upstream_completed) < 0) { + min_upstream_completed = candidate; + } + } + LF_PRINT_LOG("Minimum upstream LTC for fed %d is (%lld, %u) " + "(adjusted by after delay).", + e->id, + min_upstream_completed.time - start_time, min_upstream_completed.microstep); + if (lf_tag_compare(min_upstream_completed, e->last_granted) > 0 + && lf_tag_compare(min_upstream_completed, e->next_event) >= 0 // The federate has to advance its tag + ) { + result.tag = min_upstream_completed; + return result; + } + + // Can't make progress based only on upstream LTCs. + // If all (transitive) upstream federates of the federate + // have earliest event tags such that the + // federate can now advance its tag, then send it a TAG message. + // Find the earliest event time of each such upstream federate, + // adjusted by delays on the connections. + + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + + // Find the tag of the earliest possible incoming message from + // upstream federates. + tag_t t_d = FOREVER_TAG; + LF_PRINT_DEBUG("NOTE: FOREVER is displayed as (%lld, %u) and NEVER as (%lld, %u)", + FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, + NEVER - start_time, 0u); + + for (int j = 0; j < e->num_upstream; j++) { + federate_t* upstream = &_RTI.federates[e->upstream[j]]; + + // Ignore this federate if it has resigned. + if (upstream->enclave.state == NOT_CONNECTED) continue; + + // Find the (transitive) next event tag upstream. + tag_t upstream_next_event = transitive_next_event( + upstream, upstream->enclave.next_event, visited); + + LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag (%lld, %u).", + e->id, + upstream->enclave.id, + upstream_next_event.time - start_time, upstream_next_event.microstep); + + // Adjust by the "after" delay. + // Note that "no delay" is encoded as NEVER, + // whereas one microstep delay is encoded as 0LL. + tag_t candidate = lf_delay_tag(upstream_next_event, e->upstream_delay[j]); + + if (lf_tag_compare(candidate, t_d) < 0) { + t_d = candidate; + } + } + free(visited); + + LF_PRINT_LOG("Earliest next event upstream has tag (%lld, %u).", + t_d.time - start_time, t_d.microstep); + + if ( + lf_tag_compare(t_d, e->next_event) > 0 // The federate has something to do. + && lf_tag_compare(t_d, e->last_provisionally_granted) >= 0 // The grant is not redundant + // (equal is important to override any previous + // PTAGs). + && lf_tag_compare(t_d, e->last_granted) > 0 // The grant is not redundant. + ) { + // All upstream federates have events with a larger tag than fed, so it is safe to send a TAG. + LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG + "(adjusted by after delay). Granting tag advance for " PRINTF_TAG, + e->id, + t_d.time - lf_time_start(), t_d.microstep, + e->next_event.time - lf_time_start(), + e->next_event.microstep); + result.tag = e->next_event; + } else if ( + lf_tag_compare(t_d, e->next_event) == 0 // The federate has something to do. + && lf_tag_compare(t_d, e->last_provisionally_granted) > 0 // The grant is not redundant. + && lf_tag_compare(t_d, e->last_granted) > 0 // The grant is not redundant. + ) { + // Some upstream federate has an event that has the same tag as fed's next event, so we can only provisionally + // grant a TAG (via a PTAG). + LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG + " (adjusted by after delay). Granting provisional tag advance.", + e->id, + t_d.time - start_time, t_d.microstep); + result.tag = t_d; + result.is_provisional = true; + } + return result; +} + +void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { + visited[e->id] = true; + for (int i = 0; i < e->num_downstream; i++) { + enclave_t* downstream = (enclave_t*)&_RTI.federates[e->downstream[i]]; + if (visited[downstream->id]) continue; + notify_advance_grant_if_safe(downstream); + notify_downstream_advance_grant_if_safe(downstream, visited); + } +} diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index 6bfefb898..dbd83a3ae 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -48,4 +48,60 @@ typedef struct enclave_t { // FIXME: Docs void logical_tag_complete(enclave_t* enclave, tag_t completed); +typedef struct { + tag_t tag; // NEVER if there is no tag advance grant. + bool is_provisional; // True for PTAG, false for TAG. +} tag_advance_grant_t; + +/** + * For all enclaves downstream of the specified enclave, determine + * whether they should be notified of a TAG or PTAG and notify them if so. + * + * This assumes the caller holds the mutex. + * + * @param e The upstream enclave. + * @param visited An array of booleans used to determine whether an enclave has + * been visited (initially all false). + */ +void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]); + +/** + * @brief Either send to a federate or unblock an enclave to give it a tag. + * This function requires two different implementations, one for enclaves + * and one for federates. + * FIXME: This is only implemented for federates right now. + * @param e The enclave. + */ +void notify_advance_grant_if_safe(enclave_t* e); + +/** + * Determine whether the specified enclave is eligible for a tag advance grant, + * (TAG) and, if so, return the details. This is called upon receiving a LTC, NET + * or resign from an upstream enclave. + * + * This function calculates the minimum M over + * all upstream enclaves of the "after" delay plus the most recently + * received LTC from that enclave. If M is greater than the + * most recent TAG to e or greater than or equal to the most + * recent PTAG, then return TAG(M). + * + * If the above conditions do not result in returning a TAG, then find the + * minimum M of the earliest possible future message from upstream federates. + * This is calculated by transitively looking at the most recently received + * NET calls from upstream enclaves. + * If M is greater than the NET of e or the most recent PTAG to e, then + * return a TAG with tag equal to the NET of e or the PTAG. + * If M is equal to the NET of the federate, then return PTAG(M). + * + * This should be called whenever an immediately upstream federate sends to + * the RTI an LTC (Logical Tag Complete), or when a transitive upstream + * federate sends a NET (Next Event Tag) message. + * It is also called when an upstream federate resigns from the federation. + * + * This function assumes that the caller holds the mutex lock. + * + * @return True if the TAG message is sent and false otherwise. + */ +tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e); + #endif // ENCLAVE_H \ No newline at end of file diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index bbd3d663e..114d06e5f 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -213,7 +213,7 @@ void send_tag_advance_grant(federate_t* fed, tag_t tag) { if (_RTI.tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, fed->enclave.id, &tag); } - // This function is called in send_advance_grant_if_safe(), which is a long + // This function is called in notify_advance_grant_if_safe(), which is a long // function. During this call, the socket might close, causing the following write_to_socket // to fail. Consider a failure here a soft failure and update the federate's status. ssize_t bytes_written = write_to_socket(fed->socket, message_length, buffer); @@ -294,7 +294,7 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { if (_RTI.tracing_enabled){ tracepoint_RTI_to_federate(send_PTAG, fed->enclave.id, &tag); } - // This function is called in send_advance_grant_if_safe(), which is a long + // This function is called in notify_advance_grant_if_safe(), which is a long // function. During this call, the socket might close, causing the following write_to_socket // to fail. Consider a failure here a soft failure and update the federate's status. ssize_t bytes_written = write_to_socket(fed->socket, message_length, buffer); @@ -343,121 +343,15 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { } } -bool send_advance_grant_if_safe(federate_t* fed) { - - // Find the earliest LTC of upstream federates. - tag_t min_upstream_completed = FOREVER_TAG; - - for (int j = 0; j < fed->enclave.num_upstream; j++) { - federate_t* upstream = &_RTI.federates[fed->enclave.upstream[j]]; - - // Ignore this federate if it has resigned. - if (upstream->enclave.state == NOT_CONNECTED) continue; - - tag_t candidate = lf_delay_tag(upstream->enclave.completed, fed->enclave.upstream_delay[j]); - - if (lf_tag_compare(candidate, min_upstream_completed) < 0) { - min_upstream_completed = candidate; - } - } - LF_PRINT_LOG("Minimum upstream LTC for fed %d is (%lld, %u) " - "(adjusted by after delay).", - fed->enclave.id, - min_upstream_completed.time - start_time, min_upstream_completed.microstep); - if (lf_tag_compare(min_upstream_completed, fed->enclave.last_granted) > 0 - && lf_tag_compare(min_upstream_completed, fed->enclave.next_event) >= 0 // The federate has to advance its tag - ) { - send_tag_advance_grant(fed, min_upstream_completed); - return true; - } - - // Can't make progress based only on upstream LTCs. - // If all (transitive) upstream federates of the federate - // have earliest event tags such that the - // federate can now advance its tag, then send it a TAG message. - // Find the earliest event time of each such upstream federate, - // adjusted by delays on the connections. - - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - - // Find the tag of the earliest possible incoming message from - // upstream federates. - tag_t t_d = FOREVER_TAG; - LF_PRINT_DEBUG("NOTE: FOREVER is displayed as (%lld, %u) and NEVER as (%lld, %u)", - FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, - NEVER - start_time, 0u); - - for (int j = 0; j < fed->enclave.num_upstream; j++) { - federate_t* upstream = &_RTI.federates[fed->enclave.upstream[j]]; - - // Ignore this federate if it has resigned. - if (upstream->enclave.state == NOT_CONNECTED) continue; - - // Find the (transitive) next event tag upstream. - tag_t upstream_next_event = transitive_next_event( - upstream, upstream->enclave.next_event, visited); - - LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag (%lld, %u).", - fed->enclave.id, - upstream->enclave.id, - upstream_next_event.time - start_time, upstream_next_event.microstep); - - // Adjust by the "after" delay. - // Note that "no delay" is encoded as NEVER, - // whereas one microstep delay is encoded as 0LL. - tag_t candidate = lf_delay_tag(upstream_next_event, fed->enclave.upstream_delay[j]); - - if (lf_tag_compare(candidate, t_d) < 0) { - t_d = candidate; +void notify_advance_grant_if_safe(enclave_t* e) { + tag_advance_grant_t grant = tag_advance_grant_if_safe(e); + if (lf_tag_compare(grant.tag, NEVER_TAG) != 0) { + if (grant.is_provisional) { + send_provisional_tag_advance_grant((federate_t*)e, grant.tag); + } else { + send_tag_advance_grant((federate_t*)e, grant.tag); } } - free(visited); - - LF_PRINT_LOG("Earliest next event upstream has tag (%lld, %u).", - t_d.time - start_time, t_d.microstep); - - if ( - lf_tag_compare(t_d, fed->enclave.next_event) > 0 // The federate has something to do. - && lf_tag_compare(t_d, fed->enclave.last_provisionally_granted) >= 0 // The grant is not redundant - // (equal is important to override any previous - // PTAGs). - && lf_tag_compare(t_d, fed->enclave.last_granted) > 0 // The grant is not redundant. - ) { - // All upstream federates have events with a larger tag than fed, so it is safe to send a TAG. - LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG - "(adjusted by after delay). Granting tag advance for " PRINTF_TAG, - fed->enclave.id, - t_d.time - lf_time_start(), t_d.microstep, - fed->enclave.next_event.time - lf_time_start(), - fed->enclave.next_event.microstep); - send_tag_advance_grant(fed, fed->enclave.next_event); - } else if ( - lf_tag_compare(t_d, fed->enclave.next_event) == 0 // The federate has something to do. - && lf_tag_compare(t_d, fed->enclave.last_provisionally_granted) > 0 // The grant is not redundant. - && lf_tag_compare(t_d, fed->enclave.last_granted) > 0 // The grant is not redundant. - ) { - // Some upstream federate has an event that has the same tag as fed's next event, so we can only provisionally - // grant a TAG (via a PTAG). - LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG - " (adjusted by after delay). Granting provisional tag advance.", - fed->enclave.id, - t_d.time - start_time, t_d.microstep); - - send_provisional_tag_advance_grant(fed, t_d); - } - return false; -} - -void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]) { - visited[fed->enclave.id] = true; - for (int i = 0; i < fed->enclave.num_downstream; i++) { - federate_t* downstream = &_RTI.federates[fed->enclave.downstream[i]]; - if (visited[downstream->enclave.id]) continue; - send_advance_grant_if_safe(downstream); - send_downstream_advance_grants_if_safe(downstream, visited); - } } void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { @@ -483,13 +377,13 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even // If the federate has no upstream federates, then it does not wait for // nor expect a reply. It just proceeds to advance time. if (_RTI.federates[federate_id].enclave.num_upstream > 0) { - send_advance_grant_if_safe(&_RTI.federates[federate_id]); + notify_advance_grant_if_safe(&_RTI.federates[federate_id]); } // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(&_RTI.federates[federate_id], visited); + notify_downstream_advance_grant_if_safe(&_RTI.federates[federate_id], visited); free(visited); } @@ -1209,7 +1103,7 @@ void handle_federate_resign(federate_t *my_fed) { // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - send_downstream_advance_grants_if_safe(my_fed, visited); + notify_downstream_advance_grant_if_safe(my_fed, visited); free(visited); lf_mutex_unlock(&rti_mutex); @@ -1263,8 +1157,8 @@ void* federate_thread_TCP(void* fed) { case MSG_TYPE_STOP_REQUEST: handle_stop_request_message(my_fed); // FIXME: Reviewed until here. // Need to also look at - // send_advance_grant_if_safe() - // and send_downstream_advance_grants_if_safe() + // notify_advance_grant_if_safe() + // and notify_downstream_advance_grant_if_safe() break; case MSG_TYPE_STOP_REQUEST_REPLY: handle_stop_request_reply(my_fed); diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index 2da11779d..ce918c57c 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -259,49 +259,6 @@ tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]); */ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag); -/** - * Determine whether the specified federate fed is eligible for a tag advance grant, - * (TAG) and, if so, send it one. This is called upon receiving a LTC, NET - * or resign from an upstream federate. - * - * This function calculates the minimum M over - * all upstream federates of the "after" delay plus the most recently - * received LTC from that federate. If M is greater than the - * most recently sent TAG to fed or greater than or equal to the most - * recently sent PTAG, then send a TAG(M) to fed and return. - * - * If the above conditions do not result in sending a TAG, then find the - * minimum M of the earliest possible future message from upstream federates. - * This is calculated by transitively looking at the most recently received - * NET message from upstream federates. - * If M is greater than the NET of the federate fed or the most recently - * sent PTAG to that federate, then - * send TAG to the federate with tag equal to the NET of fed or the PTAG. - * If M is equal to the NET of the federate, then send PTAG(M). - * - * This should be called whenever an immediately upstream federate sends to - * the RTI an LTC (Logical Tag Complete), or when a transitive upstream - * federate sends a NET (Next Event Tag) message. - * It is also called when an upstream federate resigns from the federation. - * - * This function assumes that the caller holds the mutex lock. - * - * @return True if the TAG message is sent and false otherwise. - */ -bool send_advance_grant_if_safe(federate_t* fed); - -/** - * For all federates downstream of the specified federate, determine - * whether they should be sent a TAG or PTAG and send it if so. - * - * This assumes the caller holds the mutex. - * - * @param fed The upstream federate. - * @param visited An array of booleans used to determine whether a federate has - * been visited (initially all false). - */ -void send_downstream_advance_grants_if_safe(federate_t* fed, bool visited[]); - /** * @brief Update the next event tag of federate `federate_id`. * From 82cccd9bd6722a36c98fc8b6ff1c6159b4688bc3 Mon Sep 17 00:00:00 2001 From: "Edward A. Lee" Date: Mon, 22 May 2023 05:59:35 -0700 Subject: [PATCH 03/25] Added candidate next_event_tag function --- core/federated/RTI/enclave.c | 57 ++++++++++++++++++++++++++++++++++++ core/federated/RTI/enclave.h | 32 ++++++++++++++++++++ core/federated/RTI/rti_lib.c | 25 ++-------------- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index bed8625a3..82adfbc75 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -159,3 +159,60 @@ void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { notify_downstream_advance_grant_if_safe(downstream, visited); } } + +void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag) { + e->next_event = next_event_tag; + + LF_PRINT_DEBUG( + "RTI: Updated the recorded next event tag for enclave %d to " PRINTF_TAG, + e->id, + next_event_tag.time - lf_time_start(), + next_event_tag.microstep + ); + + // Check to see whether we can reply now with a tag advance grant. + // If the federate has no upstream federates, then it does not wait for + // nor expect a reply. It just proceeds to advance time. + if (e->num_upstream > 0) { + notify_advance_grant_if_safe(e); + } + // Check downstream federates to see whether they should now be granted a TAG. + // To handle cycles, need to create a boolean array to keep + // track of which upstream federates have been visited. + bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + notify_downstream_advance_grant_if_safe(e, visited); + free(visited); +} + +tag_advance_grant_t next_event_tag(enclave_t* e, tag_t next_event_tag) { + tag_advance_grant_t result; + + // First, update the enclave data structure to record this next_event_tag, + // and notify any downstream enclaves, and unblock them if appropriate. + lf_mutex_lock(&rti_mutex); + + tag_t previous_tag = e->last_granted; + tag_t previous_ptag = e->last_provisionally_granted; + + update_enclave_next_event_tag_locked(e, next_event_tag); + + while(true) { + // Determine whether the above call notified e of a TAG or PTAG. + // If so, return that value. + if (lf_tag_compare(previous_tag, e->last_granted) < 0) { + result.tag = e->last_granted; + result.is_provisional = false; + lf_mutex_unlock(&rti_mutex); + return result; + } + if (lf_tag_compare(previous_ptag, e->last_provisionally_granted) < 0) { + result.tag = e->last_provisionally_granted; + result.is_provisional = true; + lf_mutex_unlock(&rti_mutex); + return result; + } + + // If not, block. + lf_cond_wait(&e->next_event_condition); + } +} diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index dbd83a3ae..72b4dc163 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -43,6 +43,10 @@ typedef struct enclave_t { bool requested_stop; // Indicates that the federate has requested stop or has replied // to a request for stop from the RTI. Used to prevent double-counting // a federate when handling lf_request_stop(). + lf_cond_t next_event_condition; // Condition variable used by enclaves to notify an enclave + // that it's call to next_event_tag() should unblock. +// FIXME: This has to be initialized! +// lf_cond_init(&next_event_condition, &rti_mutex); } enclave_t; // FIXME: Docs @@ -104,4 +108,32 @@ void notify_advance_grant_if_safe(enclave_t* e); */ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e); +/** + * @brief Get the tag to advance to. + * + * An enclave should call this function when it is ready to advance its tag, + * passing as the second argument the tag of the earliest event on its event queue. + * The returned tag may be less than or equal to the argument tag and is interpreted + * by the enclave as the tag to which it can advance. + * + * This will also notify downstream enclaves with a TAG or PTAG if appropriate, + * possibly unblocking their own calls to this same function. + * + * @param e The enclave. + * @param next_event_tag The next event tag for e. + */ +tag_advance_grant_t next_event_tag(enclave_t* e, tag_t next_event_tag); + +/** + * @brief Update the next event tag of an enclave. + * + * This will notify downstream enclaves with a TAG or PTAG if appropriate. + * + * This function assumes that the caller is holding the rti_mutex. + * + * @param e The enclave. + * @param next_event_tag The next event tag for e. + */ +void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag); + #endif // ENCLAVE_H \ No newline at end of file diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 114d06e5f..1920cf034 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -363,28 +363,7 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even ) { next_event_tag = min_in_transit_tag; } - - _RTI.federates[federate_id].enclave.next_event = next_event_tag; - - LF_PRINT_DEBUG( - "RTI: Updated the recorded next event tag for federate %d to " PRINTF_TAG, - federate_id, - next_event_tag.time - lf_time_start(), - next_event_tag.microstep - ); - - // Check to see whether we can reply now with a tag advance grant. - // If the federate has no upstream federates, then it does not wait for - // nor expect a reply. It just proceeds to advance time. - if (_RTI.federates[federate_id].enclave.num_upstream > 0) { - notify_advance_grant_if_safe(&_RTI.federates[federate_id]); - } - // Check downstream federates to see whether they should now be granted a TAG. - // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - notify_downstream_advance_grant_if_safe(&_RTI.federates[federate_id], visited); - free(visited); + update_enclave_next_event_tag_locked(&_RTI.federates[federate_id].enclave, next_event_tag); } void handle_port_absent_message(federate_t* sending_federate, unsigned char* buffer) { @@ -1103,7 +1082,7 @@ void handle_federate_resign(federate_t *my_fed) { // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. - notify_downstream_advance_grant_if_safe(my_fed, visited); + notify_downstream_advance_grant_if_safe(&my_fed->enclave, visited); free(visited); lf_mutex_unlock(&rti_mutex); From 533002c66ef4fa77cb3cde8151091b85106b07d5 Mon Sep 17 00:00:00 2001 From: "Edward A. Lee" Date: Mon, 22 May 2023 06:15:17 -0700 Subject: [PATCH 04/25] Added .vscode to .gitignore --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8fc845c40..df202f15d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ /.idea/ /docs/_build /docs/api -/.vscode/ +**/.vscode +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json /build/ **/.DS_Store /core/federated/RTI/build/ From af9c247f169517e5f9b82c6a9c49fa91d8b812a6 Mon Sep 17 00:00:00 2001 From: "Edward A. Lee" Date: Mon, 22 May 2023 10:32:30 -0700 Subject: [PATCH 05/25] More refactoring --- core/federated/RTI/enclave.c | 32 +++++++++++ core/federated/RTI/enclave.h | 45 ++++++++++++++-- core/federated/RTI/enclave_impl.c | 19 +++++++ core/federated/RTI/rti.c | 2 +- core/federated/RTI/rti_lib.c | 89 ++++++++++++------------------- core/federated/RTI/rti_lib.h | 31 +---------- 6 files changed, 128 insertions(+), 90 deletions(-) create mode 100644 core/federated/RTI/enclave_impl.c diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 82adfbc75..1de23a7cf 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -10,6 +10,25 @@ extern instant_t start_time; // FIXME: rename "federate" everywhere in this file. +void initialize_enclave(enclave_t* e, uint16_t id) { + e->id = id; + e->completed = NEVER_TAG; + e->last_granted = NEVER_TAG; + e->last_provisionally_granted = NEVER_TAG; + e->next_event = NEVER_TAG; + e->state = NOT_CONNECTED; + e->upstream = NULL; + e->upstream_delay = NULL; + e->num_upstream = 0; + e->downstream = NULL; + e->num_downstream = 0; + e->mode = REALTIME; + e->requested_stop = false; + + // Initialize the next event condition variable. + lf_cond_init(&e->next_event_condition, &rti_mutex); +} + void logical_tag_complete(enclave_t* enclave, tag_t completed) { // FIXME: Consolidate this message with NET to get NMR (Next Message Request). // Careful with handling startup and shutdown. @@ -191,6 +210,8 @@ tag_advance_grant_t next_event_tag(enclave_t* e, tag_t next_event_tag) { // and notify any downstream enclaves, and unblock them if appropriate. lf_mutex_lock(&rti_mutex); + // FIXME: If last_granted is already greater than next_event_tag, return next_event_tag. + tag_t previous_tag = e->last_granted; tag_t previous_ptag = e->last_provisionally_granted; @@ -216,3 +237,14 @@ tag_advance_grant_t next_event_tag(enclave_t* e, tag_t next_event_tag) { lf_cond_wait(&e->next_event_condition); } } + +void notify_advance_grant_if_safe(enclave_t* e) { + tag_advance_grant_t grant = tag_advance_grant_if_safe(e); + if (lf_tag_compare(grant.tag, NEVER_TAG) != 0) { + if (grant.is_provisional) { + notify_provisional_tag_advance_grant(e, grant.tag); + } else { + notify_tag_advance_grant(e, grant.tag); + } + } +} diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index 72b4dc163..aa4c8d23f 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -45,8 +45,6 @@ typedef struct enclave_t { // a federate when handling lf_request_stop(). lf_cond_t next_event_condition; // Condition variable used by enclaves to notify an enclave // that it's call to next_event_tag() should unblock. -// FIXME: This has to be initialized! -// lf_cond_init(&next_event_condition, &rti_mutex); } enclave_t; // FIXME: Docs @@ -57,6 +55,12 @@ typedef struct { bool is_provisional; // True for PTAG, false for TAG. } tag_advance_grant_t; +/** + * Initialize the enclave with the specified ID. + * @param id The enclave ID. + */ +void initialize_enclave(enclave_t* e, uint16_t id); + /** * For all enclaves downstream of the specified enclave, determine * whether they should be notified of a TAG or PTAG and notify them if so. @@ -69,15 +73,50 @@ typedef struct { */ void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]); +/** + * Notify a tag advance grant (TAG) message to the specified federate. + * Do not notify it if a previously sent PTAG was greater or if a + * previously sent TAG was greater or equal. + * + * This function will keep a record of this TAG in the federate's last_granted + * field. + * + * This function assumes that the caller holds the mutex lock. + * + * FIXME: This needs two implementations, one for enclaves and one for federates. + * + * @param e The enclave. + * @param tag The tag to grant. + */ +void notify_tag_advance_grant(enclave_t* e, tag_t tag); + /** * @brief Either send to a federate or unblock an enclave to give it a tag. * This function requires two different implementations, one for enclaves * and one for federates. - * FIXME: This is only implemented for federates right now. + * + * This assumes the caller holds the mutex. + * * @param e The enclave. */ void notify_advance_grant_if_safe(enclave_t* e); +/** + * Nontify a provisional tag advance grant (PTAG) message to the specified enclave. + * Do not notify it if a previously sent PTAG or TAG was greater or equal. + * + * This function will keep a record of this PTAG in the federate's last_provisionally_granted + * field. + * + * This function assumes that the caller holds the mutex lock. + * + * FIXME: This needs two implementations, one for enclaves and one for federates. + * + * @param e The enclave. + * @param tag The tag to grant. + */ +void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag); + /** * Determine whether the specified enclave is eligible for a tag advance grant, * (TAG) and, if so, return the details. This is called upon receiving a LTC, NET diff --git a/core/federated/RTI/enclave_impl.c b/core/federated/RTI/enclave_impl.c new file mode 100644 index 000000000..427ec8e85 --- /dev/null +++ b/core/federated/RTI/enclave_impl.c @@ -0,0 +1,19 @@ +#include "enclave.h" + +// FIXME: This should not be here. +#include "rti_lib.h" +#include "platform.h" + +void notify_tag_advance_grant(enclave_t* e, tag_t tag) { + if (e->state == NOT_CONNECTED + || lf_tag_compare(tag, e->last_granted) <= 0 + || lf_tag_compare(tag, e->last_provisionally_granted) < 0 + ) { + return; + } + if (_RTI.tracing_enabled) { + tracepoint_RTI_to_federate(send_TAG, e->id, &tag); + } + e->last_granted = tag; + lf_cond_signal(&e->next_event_condition); +} diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 0997f9f52..60ba8f8df 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -75,7 +75,7 @@ int main(int argc, const char* argv[]) { assert(_RTI.number_of_federates < UINT16_MAX); _RTI.federates = (federate_t*)calloc(_RTI.number_of_federates, sizeof(federate_t)); for (uint16_t i = 0; i < _RTI.number_of_federates; i++) { - initialize_federate(i); + initialize_federate(&_RTI.federates[i], i); } int socket_descriptor = start_rti_server(_RTI.user_specified_port); wait_for_federates(socket_descriptor); diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 1920cf034..45a0c87e8 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -191,16 +191,16 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty return socket_descriptor; } -void send_tag_advance_grant(federate_t* fed, tag_t tag) { - if (fed->enclave.state == NOT_CONNECTED - || lf_tag_compare(tag, fed->enclave.last_granted) <= 0 - || lf_tag_compare(tag, fed->enclave.last_provisionally_granted) < 0 +void notify_tag_advance_grant(enclave_t* e, tag_t tag) { + if (e->state == NOT_CONNECTED + || lf_tag_compare(tag, e->last_granted) <= 0 + || lf_tag_compare(tag, e->last_provisionally_granted) < 0 ) { return; } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[fed->enclave.id].enclave.state == PENDING) { + while (_RTI.federates[e->id].enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -211,23 +211,23 @@ void send_tag_advance_grant(federate_t* fed, tag_t tag) { encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_TAG, fed->enclave.id, &tag); + tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long // function. During this call, the socket might close, causing the following write_to_socket // to fail. Consider a failure here a soft failure and update the federate's status. - ssize_t bytes_written = write_to_socket(fed->socket, message_length, buffer); + ssize_t bytes_written = write_to_socket(((federate_t*)e)->socket, message_length, buffer); if (bytes_written < (ssize_t)message_length) { - lf_print_error("RTI failed to send tag advance grant to federate %d.", fed->enclave.id); + lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id); if (bytes_written < 0) { - fed->enclave.state = NOT_CONNECTED; + e->state = NOT_CONNECTED; // FIXME: We need better error handling, but don't stop other execution here. // mark_federate_requesting_stop(fed); } } else { - fed->enclave.last_granted = tag; + e->last_granted = tag; LF_PRINT_LOG("RTI sent to federate %d the tag advance grant (TAG) (%lld, %u).", - fed->enclave.id, tag.time - start_time, tag.microstep); + e->id, tag.time - start_time, tag.microstep); } } @@ -272,16 +272,16 @@ tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]) { return result; } -void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { - if (fed->enclave.state == NOT_CONNECTED - || lf_tag_compare(tag, fed->enclave.last_granted) <= 0 - || lf_tag_compare(tag, fed->enclave.last_provisionally_granted) <= 0 +void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { + if (e->state == NOT_CONNECTED + || lf_tag_compare(tag, e->last_granted) <= 0 + || lf_tag_compare(tag, e->last_provisionally_granted) <= 0 ) { return; } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[fed->enclave.id].enclave.state == PENDING) { + while (_RTI.federates[e->id].enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -292,24 +292,24 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); if (_RTI.tracing_enabled){ - tracepoint_RTI_to_federate(send_PTAG, fed->enclave.id, &tag); + tracepoint_RTI_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long // function. During this call, the socket might close, causing the following write_to_socket // to fail. Consider a failure here a soft failure and update the federate's status. - ssize_t bytes_written = write_to_socket(fed->socket, message_length, buffer); + ssize_t bytes_written = write_to_socket(((federate_t*)e)->socket, message_length, buffer); if (bytes_written < (ssize_t)message_length) { - lf_print_error("RTI failed to send tag advance grant to federate %d.", fed->enclave.id); + lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id); if (bytes_written < 0) { - fed->enclave.state = NOT_CONNECTED; + e->state = NOT_CONNECTED; // FIXME: We need better error handling, but don't stop other execution here. // mark_federate_requesting_stop(fed); } } else { - fed->enclave.last_provisionally_granted = tag; + e->last_provisionally_granted = tag; LF_PRINT_LOG("RTI sent to federate %d the Provisional Tag Advance Grant (PTAG) (%lld, %u).", - fed->enclave.id, tag.time - start_time, tag.microstep); + e->id, tag.time - start_time, tag.microstep); // Send PTAG to all upstream federates, if they have not had // a later or equal PTAG or TAG sent previously and if their transitive @@ -317,8 +317,8 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { // NOTE: This could later be replaced with a TNET mechanism once // we have an available encoding of causality interfaces. // That might be more efficient. - for (int j = 0; j < fed->enclave.num_upstream; j++) { - federate_t* upstream = &_RTI.federates[fed->enclave.upstream[j]]; + for (int j = 0; j < e->num_upstream; j++) { + federate_t* upstream = &_RTI.federates[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->enclave.state == NOT_CONNECTED) continue; @@ -337,23 +337,12 @@ void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag) { // in which case, another will not be sent. But it // may not have been already granted. if (lf_tag_compare(upstream_next_event, tag) >= 0) { - send_provisional_tag_advance_grant(upstream, tag); + notify_provisional_tag_advance_grant(&upstream->enclave, tag); } } } } -void notify_advance_grant_if_safe(enclave_t* e) { - tag_advance_grant_t grant = tag_advance_grant_if_safe(e); - if (lf_tag_compare(grant.tag, NEVER_TAG) != 0) { - if (grant.is_provisional) { - send_provisional_tag_advance_grant((federate_t*)e, grant.tag); - } else { - send_tag_advance_grant((federate_t*)e, grant.tag); - } - } -} - void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(_RTI.federates[federate_id].in_transit_message_tags); if (lf_tag_compare( @@ -1582,26 +1571,14 @@ void* respond_to_erroneous_connections(void* nothing) { return NULL; } -void initialize_federate(uint16_t id) { - _RTI.federates[id].enclave.id = id; - _RTI.federates[id].socket = -1; // No socket. - _RTI.federates[id].clock_synchronization_enabled = true; - _RTI.federates[id].enclave.completed = NEVER_TAG; - _RTI.federates[id].enclave.last_granted = NEVER_TAG; - _RTI.federates[id].enclave.last_provisionally_granted = NEVER_TAG; - _RTI.federates[id].enclave.next_event = NEVER_TAG; - _RTI.federates[id].in_transit_message_tags = initialize_in_transit_message_q(); - _RTI.federates[id].enclave.state = NOT_CONNECTED; - _RTI.federates[id].enclave.upstream = NULL; - _RTI.federates[id].enclave.upstream_delay = NULL; - _RTI.federates[id].enclave.num_upstream = 0; - _RTI.federates[id].enclave.downstream = NULL; - _RTI.federates[id].enclave.num_downstream = 0; - _RTI.federates[id].enclave.mode = REALTIME; - strncpy(_RTI.federates[id].server_hostname ,"localhost", INET_ADDRSTRLEN); - _RTI.federates[id].server_ip_addr.s_addr = 0; - _RTI.federates[id].server_port = -1; - _RTI.federates[id].enclave.requested_stop = false; +void initialize_federate(federate_t* fed, uint16_t id) { + initialize_enclave(&fed->enclave, id); + fed->socket = -1; // No socket. + fed->clock_synchronization_enabled = true; + fed->in_transit_message_tags = initialize_in_transit_message_q(); + strncpy(fed->server_hostname ,"localhost", INET_ADDRSTRLEN); + fed->server_ip_addr.s_addr = 0; + fed->server_port = -1; } int32_t start_rti_server(uint16_t port) { diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index ce918c57c..0f8939a27 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -206,21 +206,6 @@ extern int lf_critical_section_exit(); */ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_type); -/** - * Send a tag advance grant (TAG) message to the specified federate. - * Do not send it if a previously sent PTAG was greater or if a - * previously sent TAG was greater or equal. - * - * This function will keep a record of this TAG in the federate's last_granted - * field. - * - * This function assumes that the caller holds the mutex lock. - * - * @param fed The federate. - * @param tag The tag to grant. - */ -void send_tag_advance_grant(federate_t* fed, tag_t tag); - /** * Find the earliest tag at which the specified federate may * experience its next event. This is the least next event tag (NET) @@ -245,20 +230,6 @@ void send_tag_advance_grant(federate_t* fed, tag_t tag); */ tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]); -/** - * Send a provisional tag advance grant (PTAG) message to the specified federate. - * Do not send it if a previously sent PTAG or TAG was greater or equal. - * - * This function will keep a record of this PTAG in the federate's last_provisionally_granted - * field. - * - * This function assumes that the caller holds the mutex lock. - * - * @param fed The federate. - * @param tag The tag to grant. - */ -void send_provisional_tag_advance_grant(federate_t* fed, tag_t tag); - /** * @brief Update the next event tag of federate `federate_id`. * @@ -534,7 +505,7 @@ void* respond_to_erroneous_connections(void* nothing); * Initialize the federate with the specified ID. * @param id The federate ID. */ -void initialize_federate(uint16_t id); +void initialize_federate(federate_t* fed, uint16_t id); /** * Start the socket server for the runtime infrastructure (RTI) and From d14bc6f2472d7f7d01d211910e5794f05a2c0c74 Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Mon, 22 May 2023 18:58:56 -0700 Subject: [PATCH 06/25] Bringing commit 'Add enclaves RTI' from rti-enclaves to rti-refactoring --- core/federated/RTI/enclave.c | 75 ++++-- core/federated/RTI/enclave.h | 48 ++++ core/federated/RTI/enclave_impl.c | 2 +- core/federated/RTI/rti.c | 20 +- core/federated/RTI/rti_lib.c | 394 ++++++++++++++---------------- core/federated/RTI/rti_lib.h | 49 +--- 6 files changed, 309 insertions(+), 279 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 1de23a7cf..5283df46d 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -1,13 +1,13 @@ #include "enclave.h" -// FIXME: This should not be here. -#include "rti_lib.h" - -extern RTI_instance_t _RTI; +extern enclave_RTI_t _RTI; // Global variables defined in tag.c: extern instant_t start_time; +// RTI mutex, which is the main lock +extern lf_mutex_t rti_mutex; + // FIXME: rename "federate" everywhere in this file. void initialize_enclave(enclave_t* e, uint16_t id) { @@ -48,11 +48,11 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { // Check downstream federates to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { - // FIXME: Shouldn't use federate_t here. - federate_t* downstream = &_RTI.federates[enclave->downstream[i]]; + // FIXME: Shouldn't use enclave_t here. + enclave_t* downstream = &_RTI.enclaves[enclave->downstream[i]]; // Notify downstream federate if appropriate. notify_advance_grant_if_safe((enclave_t*)downstream); - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. // Notify enclaves downstream of downstream if appropriate. notify_downstream_advance_grant_if_safe((enclave_t*)downstream, visited); free(visited); @@ -68,12 +68,12 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < e->num_upstream; j++) { - federate_t* upstream = &_RTI.federates[e->upstream[j]]; + enclave_t* upstream = &_RTI.enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. - if (upstream->enclave.state == NOT_CONNECTED) continue; + if (upstream->state == NOT_CONNECTED) continue; - tag_t candidate = lf_delay_tag(upstream->enclave.completed, e->upstream_delay[j]); + tag_t candidate = lf_delay_tag(upstream->completed, e->upstream_delay[j]); if (lf_tag_compare(candidate, min_upstream_completed) < 0) { min_upstream_completed = candidate; @@ -99,7 +99,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the tag of the earliest possible incoming message from // upstream federates. @@ -109,18 +109,18 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { NEVER - start_time, 0u); for (int j = 0; j < e->num_upstream; j++) { - federate_t* upstream = &_RTI.federates[e->upstream[j]]; + enclave_t* upstream = &_RTI.enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. - if (upstream->enclave.state == NOT_CONNECTED) continue; + if (upstream->state == NOT_CONNECTED) continue; // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( - upstream, upstream->enclave.next_event, visited); + upstream, upstream->next_event, visited); LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag (%lld, %u).", e->id, - upstream->enclave.id, + upstream->id, upstream_next_event.time - start_time, upstream_next_event.microstep); // Adjust by the "after" delay. @@ -172,7 +172,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { visited[e->id] = true; for (int i = 0; i < e->num_downstream; i++) { - enclave_t* downstream = (enclave_t*)&_RTI.federates[e->downstream[i]]; + enclave_t* downstream = (enclave_t*)&_RTI.enclaves[e->downstream[i]]; if (visited[downstream->id]) continue; notify_advance_grant_if_safe(downstream); notify_downstream_advance_grant_if_safe(downstream, visited); @@ -198,7 +198,7 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(e, visited); free(visited); } @@ -248,3 +248,44 @@ void notify_advance_grant_if_safe(enclave_t* e) { } } } + +tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { + if (visited[e->id] || e->state == NOT_CONNECTED) { + // Federate has stopped executing or we have visited it before. + // No point in checking upstream federates. + return candidate; + } + + visited[e->id] = true; + tag_t result = e->next_event; + + // If the candidate is less than this federate's next_event, use the candidate. + if (lf_tag_compare(candidate, result) < 0) { + result = candidate; + } + + // The result cannot be earlier than the start time. + if (result.time < start_time) { + // Earliest next event cannot be before the start time. + result = (tag_t){.time = start_time, .microstep = 0u}; + } + + // Check upstream federates to see whether any of them might send + // an event that would result in an earlier next event. + for (int i = 0; i < e->num_upstream; i++) { + tag_t upstream_result = transitive_next_event( + &_RTI.enclaves[e->upstream[i]], result, visited); + + // Add the "after" delay of the connection to the result. + upstream_result = lf_delay_tag(upstream_result, e->upstream_delay[i]); + + // If the adjusted event time is less than the result so far, update the result. + if (lf_tag_compare(upstream_result, result) < 0) { + result = upstream_result; + } + } + if (lf_tag_compare(result, e->completed) < 0) { + result = e->completed; + } + return result; +} diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index aa4c8d23f..c51f8a25c 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -47,6 +47,30 @@ typedef struct enclave_t { // that it's call to next_event_tag() should unblock. } enclave_t; +/** + * Structure that an enclave RTI instance uses to keep track of its own and its + * corresponding enclaves'state. + */ +typedef struct enclave_RTI_t { + // The enclaves. + enclave_t **enclaves; + + // Number of enclaves + int32_t number_of_enclaves; + + // RTI's decided stop tag for federates + tag_t max_stop_tag; + + // Number of federates handling stop + int num_feds_handling_stop; + + /** + * Boolean indicating that tracing is enabled. + */ + bool tracing_enabled; +} enclave_RTI_t; + + // FIXME: Docs void logical_tag_complete(enclave_t* enclave, tag_t completed); @@ -175,4 +199,28 @@ tag_advance_grant_t next_event_tag(enclave_t* e, tag_t next_event_tag); */ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag); +/** + * Find the earliest tag at which the specified federate may + * experience its next event. This is the least next event tag (NET) + * of the specified federate and (transitively) upstream federates + * (with delays of the connections added). For upstream federates, + * we assume (conservatively) that federate upstream of those + * may also send an event. The result will never be less than + * the completion time of the federate (which may be NEVER, + * if the federate has not yet completed a logical time). + * + * FIXME: This could be made less conservative by building + * at code generation time a causality interface table indicating + * which outputs can be triggered by which inputs. For now, we + * assume any output can be triggered by any input. + * + * @param fed The federate. + * @param candidate A candidate tag (for the first invocation, + * this should be fed->next_event). + * @param visited An array of booleans indicating which federates + * have been visited (for the first invocation, this should be + * an array of falses of size _RTI.number_of_federates). + */ +tag_t transitive_next_event(enclave_t *e, tag_t candidate, bool visited[]); + #endif // ENCLAVE_H \ No newline at end of file diff --git a/core/federated/RTI/enclave_impl.c b/core/federated/RTI/enclave_impl.c index 427ec8e85..12d4b0cd7 100644 --- a/core/federated/RTI/enclave_impl.c +++ b/core/federated/RTI/enclave_impl.c @@ -1,7 +1,7 @@ #include "enclave.h" // FIXME: This should not be here. -#include "rti_lib.h" +#include "enclave_impl.h" #include "platform.h" void notify_tag_advance_grant(enclave_t* e, tag_t tag) { diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 60ba8f8df..b4b68af0f 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -49,7 +49,9 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "rti_lib.h" unsigned int _lf_number_of_workers = 0u; -extern RTI_instance_t _RTI; +extern federation_RTI_t _RTI; + +extern lf_mutex_t rti_mutex; /** * RTI trace file name @@ -66,20 +68,20 @@ int main(int argc, const char* argv[]) { // Processing command-line arguments failed. return -1; } - if (_RTI.tracing_enabled) { - _lf_number_of_workers = _RTI.number_of_federates; + if (_RTI.enclave_rti.tracing_enabled) { + _lf_number_of_workers = _RTI.enclave_rti.number_of_enclaves; start_trace(rti_trace_file_name); printf("Tracing the RTI execution in %s file.\n", rti_trace_file_name); } - printf("Starting RTI for %d federates in federation ID %s\n", _RTI.number_of_federates, _RTI.federation_id); - assert(_RTI.number_of_federates < UINT16_MAX); - _RTI.federates = (federate_t*)calloc(_RTI.number_of_federates, sizeof(federate_t)); - for (uint16_t i = 0; i < _RTI.number_of_federates; i++) { - initialize_federate(&_RTI.federates[i], i); + printf("Starting RTI for %d federates in federation ID %s\n", _RTI.enclave_rti.number_of_enclaves, _RTI.federation_id); + assert(_RTI.enclave_rti.number_of_enclaves < UINT16_MAX); + _RTI.enclave_rti.enclaves = (federate_t*)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(federate_t)); + for (uint16_t i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { + initialize_federate((federate_t*)&_RTI.enclave_rti.enclaves[i], i); } int socket_descriptor = start_rti_server(_RTI.user_specified_port); wait_for_federates(socket_descriptor); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { stop_trace(); } printf("RTI is exiting.\n"); diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 45a0c87e8..548d5ead5 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -32,12 +32,12 @@ extern instant_t start_time; /** * The state of this RTI instance. */ -RTI_instance_t _RTI = { - .max_stop_tag = NEVER_TAG, +federation_RTI_t _RTI = { + .enclave_rti.max_stop_tag = NEVER_TAG, .max_start_time = 0LL, - .number_of_federates = 0, + .enclave_rti.number_of_enclaves = 0, .num_feds_proposed_start = 0, - .num_feds_handling_stop = 0, + .enclave_rti.num_feds_handling_stop = 0, .all_federates_exited = false, .federation_id = "Unidentified Federation", .user_specified_port = 0, @@ -200,7 +200,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[e->id].enclave.state == PENDING) { + while (e->state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -210,7 +210,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -231,47 +231,6 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { } } -tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]) { - if (visited[fed->enclave.id] || fed->enclave.state == NOT_CONNECTED) { - // Federate has stopped executing or we have visited it before. - // No point in checking upstream federates. - return candidate; - } - - visited[fed->enclave.id] = true; - tag_t result = fed->enclave.next_event; - - // If the candidate is less than this federate's next_event, use the candidate. - if (lf_tag_compare(candidate, result) < 0) { - result = candidate; - } - - // The result cannot be earlier than the start time. - if (result.time < start_time) { - // Earliest next event cannot be before the start time. - result = (tag_t){.time = start_time, .microstep = 0u}; - } - - // Check upstream federates to see whether any of them might send - // an event that would result in an earlier next event. - for (int i = 0; i < fed->enclave.num_upstream; i++) { - tag_t upstream_result = transitive_next_event( - &_RTI.federates[fed->enclave.upstream[i]], result, visited); - - // Add the "after" delay of the connection to the result. - upstream_result = lf_delay_tag(upstream_result, fed->enclave.upstream_delay[i]); - - // If the adjusted event time is less than the result so far, update the result. - if (lf_tag_compare(upstream_result, result) < 0) { - result = upstream_result; - } - } - if (lf_tag_compare(result, fed->enclave.completed) < 0) { - result = fed->enclave.completed; - } - return result; -} - void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { if (e->state == NOT_CONNECTED || lf_tag_compare(tag, e->last_granted) <= 0 @@ -281,7 +240,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { } // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[e->id].enclave.state == PENDING) { + while (e->state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } @@ -291,7 +250,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_RTI.tracing_enabled){ + if (_RTI.enclave_rti.tracing_enabled){ tracepoint_RTI_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -318,14 +277,14 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { // we have an available encoding of causality interfaces. // That might be more efficient. for (int j = 0; j < e->num_upstream; j++) { - federate_t* upstream = &_RTI.federates[e->upstream[j]]; + federate_t* upstream = (federate_t*)&_RTI.enclave_rti.enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->enclave.state == NOT_CONNECTED) continue; // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( @@ -344,7 +303,8 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { } void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { - tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(_RTI.federates[federate_id].in_transit_message_tags); + federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[federate_id]; + tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(fed->in_transit_message_tags); if (lf_tag_compare( min_in_transit_tag, next_event_tag @@ -352,7 +312,7 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even ) { next_event_tag = min_in_transit_tag; } - update_enclave_next_event_tag_locked(&_RTI.federates[federate_id].enclave, next_event_tag); + update_enclave_next_event_tag_locked(&(fed->enclave), next_event_tag); } void handle_port_absent_message(federate_t* sending_federate, unsigned char* buffer) { @@ -366,7 +326,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); tag_t tag = extract_tag(&(buffer[1 + 2 * sizeof(uint16_t)])); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_PORT_ABS, federate_id, &tag); } @@ -377,22 +337,23 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // If the destination federate is no longer connected, issue a warning // and return. - if (_RTI.federates[federate_id].enclave.state == NOT_CONNECTED) { + federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[federate_id]; + if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); - LF_PRINT_LOG("Fed status: next_event (%lld, %d), " - "completed (%lld, %d), " - "last_granted (%lld, %d), " - "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].enclave.next_event.time - start_time, - _RTI.federates[federate_id].enclave.next_event.microstep, - _RTI.federates[federate_id].enclave.completed.time - start_time, - _RTI.federates[federate_id].enclave.completed.microstep, - _RTI.federates[federate_id].enclave.last_granted.time - start_time, - _RTI.federates[federate_id].enclave.last_granted.microstep, - _RTI.federates[federate_id].enclave.last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].enclave.last_provisionally_granted.microstep + LF_PRINT_LOG("Fed status: next_event (" PRINTF_TIME ", %d), " + "completed (" PRINTF_TIME ", %d), " + "last_granted (" PRINTF_TIME ", %d), " + "last_provisionally_granted (" PRINTF_TIME ", %d).", + fed->enclave.next_event.time - start_time, + fed->enclave.next_event.microstep, + fed->enclave.completed.time - start_time, + fed->enclave.completed.microstep, + fed->enclave.last_granted.time - start_time, + fed->enclave.last_granted.microstep, + fed->enclave.last_provisionally_granted.time - start_time, + fed->enclave.last_provisionally_granted.microstep ); return; } @@ -403,14 +364,14 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[federate_id].enclave.state == PENDING) { + while (fed->enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } // Forward the message. - int destination_socket = _RTI.federates[federate_id].socket; - if (_RTI.tracing_enabled) { + int destination_socket = fed->socket; + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_PORT_ABS, federate_id, &tag); } write_to_socket_errexit(destination_socket, message_size + 1, buffer, @@ -456,7 +417,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // Following only works for string messages. // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_TAGGED_MSG, sending_federate->enclave.id, &intended_tag); } @@ -467,28 +428,29 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // If the destination federate is no longer connected, issue a warning // and return. - if (_RTI.federates[federate_id].enclave.state == NOT_CONNECTED) { + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[federate_id]; + if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", federate_id); - LF_PRINT_LOG("Fed status: next_event (%lld, %d), " - "completed (%lld, %d), " - "last_granted (%lld, %d), " - "last_provisionally_granted (%lld, %d).", - _RTI.federates[federate_id].enclave.next_event.time - start_time, - _RTI.federates[federate_id].enclave.next_event.microstep, - _RTI.federates[federate_id].enclave.completed.time - start_time, - _RTI.federates[federate_id].enclave.completed.microstep, - _RTI.federates[federate_id].enclave.last_granted.time - start_time, - _RTI.federates[federate_id].enclave.last_granted.microstep, - _RTI.federates[federate_id].enclave.last_provisionally_granted.time - start_time, - _RTI.federates[federate_id].enclave.last_provisionally_granted.microstep + LF_PRINT_LOG("Fed status: next_event (" PRINTF_TIME ", %d), " + "completed (" PRINTF_TIME ", %d), " + "last_granted (" PRINTF_TIME ", %d), " + "last_provisionally_granted (" PRINTF_TIME ", %d).", + fed->enclave.next_event.time - start_time, + fed->enclave.next_event.microstep, + fed->enclave.completed.time - start_time, + fed->enclave.completed.microstep, + fed->enclave.last_granted.time - start_time, + fed->enclave.last_granted.microstep, + fed->enclave.last_provisionally_granted.time - start_time, + fed->enclave.last_provisionally_granted.microstep ); return; } // Forward the message or message chunk. - int destination_socket = _RTI.federates[federate_id].socket; + int destination_socket = fed->socket; LF_PRINT_DEBUG( "RTI forwarding message to port %d of federate %hu of length %zu.", @@ -498,10 +460,10 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { ); // Record this in-transit message in federate's in-transit message queue. - if (lf_tag_compare(_RTI.federates[federate_id].enclave.completed, intended_tag) < 0) { + if (lf_tag_compare(fed->enclave.completed, intended_tag) < 0) { // Add a record of this message to the list of in-transit messages to this federate. add_in_transit_message_record( - _RTI.federates[federate_id].in_transit_message_tags, + fed->in_transit_message_tags, intended_tag ); LF_PRINT_DEBUG( @@ -516,8 +478,8 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { ", but there is an in-transit message with tag " PRINTF_TAG " from federate %hu. " "This is going to cause an STP violation under centralized coordination.", federate_id, - _RTI.federates[federate_id].enclave.completed.time - lf_time_start(), - _RTI.federates[federate_id].enclave.completed.microstep, + fed->enclave.completed.time - lf_time_start(), + fed->enclave.completed.microstep, intended_tag.time - lf_time_start(), intended_tag.microstep, sending_federate->enclave.id @@ -527,12 +489,12 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // Need to make sure that the destination federate's thread has already // sent the starting MSG_TYPE_TIMESTAMP message. - while (_RTI.federates[federate_id].enclave.state == PENDING) { + while (fed->enclave.state == PENDING) { // Need to wait here. lf_cond_wait(&sent_start_time); } - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_TAGGED_MSG, federate_id, &intended_tag); } @@ -593,7 +555,7 @@ void handle_next_event_tag(federate_t* fed) { tag_t intended_tag = extract_tag(buffer); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_NET, fed->enclave.id, &intended_tag); } LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) " PRINTF_TAG, @@ -620,27 +582,28 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { } // Reply with a stop granted to all federates unsigned char outgoing_buffer[MSG_TYPE_STOP_GRANTED_LENGTH]; - ENCODE_STOP_GRANTED(outgoing_buffer, _RTI.max_stop_tag.time, _RTI.max_stop_tag.microstep); + ENCODE_STOP_GRANTED(outgoing_buffer, _RTI.enclave_rti.max_stop_tag.time, _RTI.enclave_rti.max_stop_tag.microstep); // Iterate over federates and send each the message. - for (int i = 0; i < _RTI.number_of_federates; i++) { - if (_RTI.federates[i].enclave.state == NOT_CONNECTED) { + for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[i]; + if (fed->enclave.state == NOT_CONNECTED) { continue; } - if (lf_tag_compare(_RTI.federates[i].enclave.next_event, _RTI.max_stop_tag) >= 0) { + if (lf_tag_compare(fed->enclave.next_event, _RTI.enclave_rti.max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. - _RTI.federates[i].enclave.next_event = _RTI.max_stop_tag; + fed->enclave.next_event = _RTI.enclave_rti.max_stop_tag; } - if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_GRN, _RTI.federates[i].enclave.id, &_RTI.max_stop_tag); + if (_RTI.enclave_rti.tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_RTI.enclave_rti.max_stop_tag); } - write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, - "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", _RTI.federates[i].enclave.id); + write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, + "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); } - LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag (%lld, %u).", - _RTI.max_stop_tag.time - start_time, - _RTI.max_stop_tag.microstep); + LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag (" PRINTF_TIME ", %u).", + _RTI.enclave_rti.max_stop_tag.time - start_time, + _RTI.enclave_rti.max_stop_tag.microstep); _lf_rti_stop_granted_already_sent_to_federates = true; } @@ -648,10 +611,10 @@ void mark_federate_requesting_stop(federate_t* fed) { if (!fed->enclave.requested_stop) { // Assume that the federate // has requested stop - _RTI.num_feds_handling_stop++; + _RTI.enclave_rti.num_feds_handling_stop++; fed->enclave.requested_stop = true; } - if (_RTI.num_feds_handling_stop == _RTI.number_of_federates) { + if (_RTI.enclave_rti.num_feds_handling_stop == _RTI.enclave_rti.number_of_enclaves) { // We now have information about the stop time of all // federates. _lf_rti_broadcast_stop_time_to_federates_already_locked(); @@ -672,7 +635,7 @@ void handle_stop_request_message(federate_t* fed) { // Check whether we have already received a stop_tag // from this federate - if (_RTI.federates[fed->enclave.id].enclave.requested_stop) { + if (fed->enclave.requested_stop) { // Ignore this request lf_mutex_unlock(&rti_mutex); return; @@ -681,13 +644,13 @@ void handle_stop_request_message(federate_t* fed) { // Extract the proposed stop tag for the federate tag_t proposed_stop_tag = extract_tag(buffer); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ, fed->enclave.id, &proposed_stop_tag); } // Update the maximum stop tag received from federates - if (lf_tag_compare(proposed_stop_tag, _RTI.max_stop_tag) > 0) { - _RTI.max_stop_tag = proposed_stop_tag; + if (lf_tag_compare(proposed_stop_tag, _RTI.enclave_rti.max_stop_tag) > 0) { + _RTI.enclave_rti.max_stop_tag = proposed_stop_tag; } LF_PRINT_LOG("RTI received from federate %d a MSG_TYPE_STOP_REQUEST message with tag (%lld, %u).", @@ -697,7 +660,7 @@ void handle_stop_request_message(federate_t* fed) { // for a stop, add it to the tally. mark_federate_requesting_stop(fed); - if (_RTI.num_feds_handling_stop == _RTI.number_of_federates) { + if (_RTI.enclave_rti.num_feds_handling_stop == _RTI.enclave_rti.number_of_enclaves) { // We now have information about the stop time of all // federates. This is extremely unlikely, but it can occur // all federates call lf_request_stop() at the same tag. @@ -707,29 +670,30 @@ void handle_stop_request_message(federate_t* fed) { // Forward the stop request to all other federates that have not // also issued a stop request. unsigned char stop_request_buffer[MSG_TYPE_STOP_REQUEST_LENGTH]; - ENCODE_STOP_REQUEST(stop_request_buffer, _RTI.max_stop_tag.time, _RTI.max_stop_tag.microstep); + ENCODE_STOP_REQUEST(stop_request_buffer, _RTI.enclave_rti.max_stop_tag.time, _RTI.enclave_rti.max_stop_tag.microstep); // Iterate over federates and send each the MSG_TYPE_STOP_REQUEST message // if we do not have a stop_time already for them. - for (int i = 0; i < _RTI.number_of_federates; i++) { - if (_RTI.federates[i].enclave.id != fed->enclave.id && _RTI.federates[i].enclave.requested_stop == false) { - if (_RTI.federates[i].enclave.state == NOT_CONNECTED) { - mark_federate_requesting_stop(&_RTI.federates[i]); + for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[i]; + if (fed->enclave.id != fed->enclave.id && fed->enclave.requested_stop == false) { + if (fed->enclave.state == NOT_CONNECTED) { + mark_federate_requesting_stop(fed); continue; } - if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, _RTI.federates[i].enclave.id, &_RTI.max_stop_tag); + if (_RTI.enclave_rti.tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI.enclave_rti.max_stop_tag); } - write_to_socket_errexit(_RTI.federates[i].socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, - "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", _RTI.federates[i].enclave.id); - if (_RTI.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, _RTI.federates[i].enclave.id, &_RTI.max_stop_tag); + write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, + "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", fed->enclave.id); + if (_RTI.enclave_rti.tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI.enclave_rti.max_stop_tag); } } } - LF_PRINT_LOG("RTI forwarded to federates MSG_TYPE_STOP_REQUEST with tag (%lld, %u).", - _RTI.max_stop_tag.time - start_time, - _RTI.max_stop_tag.microstep); + LF_PRINT_LOG("RTI forwarded to federates MSG_TYPE_STOP_REQUEST with tag (" PRINTF_TIME ", %u).", + _RTI.enclave_rti.max_stop_tag.time - start_time, + _RTI.enclave_rti.max_stop_tag.microstep); lf_mutex_unlock(&rti_mutex); } @@ -741,7 +705,7 @@ void handle_stop_request_reply(federate_t* fed) { tag_t federate_stop_tag = extract_tag(buffer_stop_time); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ_REP, fed->enclave.id, &federate_stop_tag); } @@ -752,8 +716,8 @@ void handle_stop_request_reply(federate_t* fed) { // Acquire the mutex lock so that we can change the state of the RTI lf_mutex_lock(&rti_mutex); // If the federate has not requested stop before, count the reply - if (lf_tag_compare(federate_stop_tag, _RTI.max_stop_tag) > 0) { - _RTI.max_stop_tag = federate_stop_tag; + if (lf_tag_compare(federate_stop_tag, _RTI.enclave_rti.max_stop_tag) > 0) { + _RTI.enclave_rti.max_stop_tag = federate_stop_tag; } mark_federate_requesting_stop(fed); lf_mutex_unlock(&rti_mutex); @@ -762,16 +726,17 @@ void handle_stop_request_reply(federate_t* fed) { ////////////////////////////////////////////////// void handle_address_query(uint16_t fed_id) { + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[fed_id]; // Use buffer both for reading and constructing the reply. // The length is what is needed for the reply. unsigned char buffer[sizeof(int32_t)]; - ssize_t bytes_read = read_from_socket(_RTI.federates[fed_id].socket, sizeof(uint16_t), (unsigned char*)buffer); + ssize_t bytes_read = read_from_socket(fed->socket, sizeof(uint16_t), (unsigned char*)buffer); if (bytes_read == 0) { lf_print_error_and_exit("Failed to read address query."); } uint16_t remote_fed_id = extract_uint16(buffer); - if (_RTI.tracing_enabled){ + if (_RTI.enclave_rti.tracing_enabled){ tracepoint_RTI_from_federate(receive_ADR_QR, fed_id, NULL); } @@ -782,31 +747,33 @@ void handle_address_query(uint16_t fed_id) { // from this federate. In that case, it will respond by sending -1. // Encode the port number. - encode_int32(_RTI.federates[remote_fed_id].server_port, (unsigned char*)buffer); + federate_t *remote_fed = (federate_t *)_RTI.enclave_rti.enclaves[remote_fed_id]; + encode_int32(remote_fed->server_port, (unsigned char*)buffer); // Send the port number (which could be -1). - write_to_socket_errexit(_RTI.federates[fed_id].socket, sizeof(int32_t), (unsigned char*)buffer, + write_to_socket_errexit(fed->socket, sizeof(int32_t), (unsigned char*)buffer, "Failed to write port number to socket of federate %d.", fed_id); // Send the server IP address to federate. - write_to_socket_errexit(_RTI.federates[fed_id].socket, sizeof(_RTI.federates[remote_fed_id].server_ip_addr), - (unsigned char *)&_RTI.federates[remote_fed_id].server_ip_addr, + write_to_socket_errexit(fed->socket, sizeof(remote_fed->server_ip_addr), + (unsigned char *)&remote_fed->server_ip_addr, "Failed to write ip address to socket of federate %d.", fed_id); - if (_RTI.federates[remote_fed_id].server_port != -1) { + if (remote_fed->server_port != -1) { LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", - fed_id, _RTI.federates[remote_fed_id].server_hostname, _RTI.federates[remote_fed_id].server_port); + fed_id, remote_fed->server_hostname, remote_fed->server_port); } } void handle_address_ad(uint16_t federate_id) { + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[federate_id]; // Read the port number of the federate that can be used for physical // connections to other federates int32_t server_port = -1; unsigned char buffer[sizeof(int32_t)]; - ssize_t bytes_read = read_from_socket(_RTI.federates[federate_id].socket, sizeof(int32_t), (unsigned char *)buffer); + ssize_t bytes_read = read_from_socket(fed->socket, sizeof(int32_t), (unsigned char *)buffer); if (bytes_read < (ssize_t)sizeof(int32_t)) { - LF_PRINT_DEBUG("Error reading port data from federate %d.", _RTI.federates[federate_id].enclave.id); + LF_PRINT_DEBUG("Error reading port data from federate %d.", federate_id); // Leave the server port at -1, which means "I don't know". return; } @@ -816,8 +783,8 @@ void handle_address_ad(uint16_t federate_id) { assert(server_port < 65536); lf_mutex_lock(&rti_mutex); - _RTI.federates[federate_id].server_port = server_port; - if (_RTI.tracing_enabled) { + fed->server_port = server_port; + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_ADR_AD, federate_id, NULL); } LF_PRINT_LOG("Received address advertisement from federate %d.", federate_id); @@ -833,7 +800,7 @@ void handle_timestamp(federate_t *my_fed) { } int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t *)(&buffer))); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tag_t tag = {.time = timestamp, .microstep = 0}; tracepoint_RTI_from_federate(receive_TIMESTAMP, my_fed->enclave.id, &tag); } @@ -844,13 +811,13 @@ void handle_timestamp(federate_t *my_fed) { if (timestamp > _RTI.max_start_time) { _RTI.max_start_time = timestamp; } - if (_RTI.num_feds_proposed_start == _RTI.number_of_federates) { + if (_RTI.num_feds_proposed_start == _RTI.enclave_rti.number_of_enclaves) { // All federates have proposed a start time. lf_cond_broadcast(&received_start_times); } else { // Some federates have not yet proposed a start time. // wait for a notification. - while (_RTI.num_feds_proposed_start < _RTI.number_of_federates) { + while (_RTI.num_feds_proposed_start < _RTI.enclave_rti.number_of_enclaves) { // FIXME: Should have a timeout here? lf_cond_wait(&received_start_times); } @@ -866,7 +833,7 @@ void handle_timestamp(federate_t *my_fed) { start_time = _RTI.max_start_time + DELAY_START; encode_int64(swap_bytes_if_big_endian_int64(start_time), &start_time_buffer[1]); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tag_t tag = {.time = start_time, .microstep = 0}; tracepoint_RTI_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } @@ -942,7 +909,7 @@ void* clock_synchronization_thread(void* noargs) { // Wait until all federates have been notified of the start time. // FIXME: Use lf_ version of this when merged with master. lf_mutex_lock(&rti_mutex); - while (_RTI.num_feds_proposed_start < _RTI.number_of_federates) { + while (_RTI.num_feds_proposed_start < _RTI.enclave_rti.number_of_enclaves) { lf_cond_wait(&received_start_times); } lf_mutex_unlock(&rti_mutex); @@ -967,20 +934,21 @@ void* clock_synchronization_thread(void* noargs) { // Sleep nanosleep(&sleep_time, &remaining_time); // Can be interrupted any_federates_connected = false; - for (int fed = 0; fed < _RTI.number_of_federates; fed++) { - if (_RTI.federates[fed].enclave.state == NOT_CONNECTED) { + for (int fed_id = 0; fed_id < _RTI.enclave_rti.number_of_enclaves; fed_id++) { + federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[fed_id]; + if (fed->enclave.state == NOT_CONNECTED) { // FIXME: We need better error handling here, but clock sync failure // should not stop execution. - lf_print_error("Clock sync failed with federate %d. Not connected.", _RTI.federates[fed].enclave.id); - // mark_federate_requesting_stop(&_RTI.federates[fed]); + lf_print_error("Clock sync failed with federate %d. Not connected.", fed_id); + // mark_federate_requesting_stop(&fed); continue; - } else if (!_RTI.federates[fed].clock_synchronization_enabled) { + } else if (!fed->clock_synchronization_enabled) { continue; } // Send the RTI's current physical time to the federate // Send on UDP. LF_PRINT_DEBUG("RTI sending T1 message to initiate clock sync round."); - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &_RTI.federates[fed], UDP); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &fed, UDP); // Listen for reply message, which should be T3. size_t message_size = 1 + sizeof(int32_t); @@ -995,17 +963,17 @@ void* clock_synchronization_thread(void* noargs) { // If any errors occur, either discard the message or the clock sync round. if (bytes_read == message_size) { if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { - int32_t fed_id = extract_int32(&(buffer[1])); + int32_t fed_id_2 = extract_int32(&(buffer[1])); // Check that this message came from the correct federate. - if (fed_id != _RTI.federates[fed].enclave.id) { + if (fed_id_2 != fed->enclave.id) { // Message is from the wrong federate. Discard the message. lf_print_warning("Clock sync: Received T3 message from federate %d, " "but expected one from %d. Discarding message.", - fed_id, _RTI.federates[fed].enclave.id); + fed_id_2, fed->enclave.id); continue; } - LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id); - handle_physical_clock_sync_message(&_RTI.federates[fed_id], UDP); + LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id_2); + handle_physical_clock_sync_message((federate_t *)_RTI.enclave_rti.enclaves[fed_id_2], UDP); break; } else { // The message is not a T3 message. Discard the message and @@ -1015,14 +983,14 @@ void* clock_synchronization_thread(void* noargs) { "Discarding message.", buffer[0], MSG_TYPE_CLOCK_SYNC_T3, - _RTI.federates[fed].enclave.id); + fed->enclave.id); continue; } } else { lf_print_warning("Clock sync: Read from UDP socket failed: %s. " "Skipping clock sync round for federate %d.", strerror(errno), - _RTI.federates[fed].enclave.id); + fed->enclave.id); remaining_attempts = -1; } } @@ -1037,7 +1005,7 @@ void* clock_synchronization_thread(void* noargs) { void handle_federate_resign(federate_t *my_fed) { // Nothing more to do. Close the socket and exit. lf_mutex_lock(&rti_mutex); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { // Extract the tag, for tracing purposes size_t header_size = 1 + sizeof(tag_t); unsigned char buffer[header_size]; @@ -1070,7 +1038,7 @@ void handle_federate_resign(federate_t *my_fed) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_federates, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(&my_fed->enclave, visited); free(visited); @@ -1136,7 +1104,7 @@ void* federate_thread_TCP(void* fed) { break; default: lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->enclave.id, buffer[0]); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_UNIDENTIFIED, my_fed->enclave.id, NULL); } } @@ -1168,7 +1136,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // FIXME: This should not exit with error but rather should just reject the connection. read_from_socket_errexit(socket_id, length, buffer, "RTI failed to read from accepted socket."); - uint16_t fed_id = _RTI.number_of_federates; // Initialize to an invalid value. + uint16_t fed_id = _RTI.enclave_rti.number_of_enclaves; // Initialize to an invalid value. // First byte received is the message type. if (buffer[0] != MSG_TYPE_FED_IDS) { @@ -1184,7 +1152,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } else { send_reject(socket_id, UNEXPECTED_MESSAGE); } - if (_RTI.tracing_enabled){ + if (_RTI.enclave_rti.tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); @@ -1208,7 +1176,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI received federation ID: %s.", federation_id_received); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_FED_ID, fed_id, NULL); } // Compare the received federation ID to mine. @@ -1217,24 +1185,24 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie lf_print_error("WARNING: Federate from another federation %s attempted to connect to RTI in federation %s.\n", federation_id_received, _RTI.federation_id); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH); return -1; } else { - if (fed_id >= _RTI.number_of_federates) { + if (fed_id >= _RTI.enclave_rti.number_of_enclaves) { // Federate ID is out of range. lf_print_error("RTI received federate ID %d, which is out of range.", fed_id); - if (_RTI.tracing_enabled){ + if (_RTI.enclave_rti.tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { - if (_RTI.federates[fed_id].enclave.state != NOT_CONNECTED) { + if (_RTI.enclave_rti.enclaves[fed_id]->state != NOT_CONNECTED) { lf_print_error("RTI received duplicate federate ID: %d.", fed_id); - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_IN_USE); @@ -1243,7 +1211,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } } } - + federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[fed_id]; // The MSG_TYPE_FED_IDS message has the right federation ID. // Assign the address information for federate. // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful @@ -1251,28 +1219,28 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // First, convert the sockaddr structure into a sockaddr_in that contains an internet address. struct sockaddr_in* pV4_addr = client_fd; // Then extract the internet address (which is in IPv4 format) and assign it as the federate's socket server - _RTI.federates[fed_id].server_ip_addr = pV4_addr->sin_addr; + fed->server_ip_addr = pV4_addr->sin_addr; #if LOG_LEVEL >= LOG_LEVEL_DEBUG // Create the human readable format and copy that into // the .server_hostname field of the federate. char str[INET_ADDRSTRLEN]; - inet_ntop( AF_INET, &_RTI.federates[fed_id].server_ip_addr, str, INET_ADDRSTRLEN ); - strncpy (_RTI.federates[fed_id].server_hostname, str, INET_ADDRSTRLEN); + inet_ntop( AF_INET, &fed->server_ip_addr, str, INET_ADDRSTRLEN ); + strncpy (fed->server_hostname, str, INET_ADDRSTRLEN); - LF_PRINT_DEBUG("RTI got address %s from federate %d.", _RTI.federates[fed_id].server_hostname, fed_id); + LF_PRINT_DEBUG("RTI got address %s from federate %d.", fed->server_hostname, fed_id); #endif - _RTI.federates[fed_id].socket = socket_id; + fed->socket = socket_id; // Set the federate's state as pending // because it is waiting for the start time to be // sent by the RTI before beginning its execution. - _RTI.federates[fed_id].enclave.state = PENDING; + fed->enclave.state = PENDING; LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); // Send an MSG_TYPE_ACK message. unsigned char ack_message = MSG_TYPE_ACK; - if (_RTI.tracing_enabled) { + if (_RTI.enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_ACK, fed_id, NULL); } write_to_socket_errexit(socket_id, 1, &ack_message, @@ -1298,27 +1266,28 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { + federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[fed_id]; // Read the number of upstream and downstream connections - _RTI.federates[fed_id].enclave.num_upstream = extract_int32(&(connection_info_header[1])); - _RTI.federates[fed_id].enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); + fed->enclave.num_upstream = extract_int32(&(connection_info_header[1])); + fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); LF_PRINT_DEBUG( "RTI got %d upstreams and %d downstreams from federate %d.", - _RTI.federates[fed_id].enclave.num_upstream, - _RTI.federates[fed_id].enclave.num_downstream, + fed->enclave.num_upstream, + fed->enclave.num_downstream, fed_id); // Allocate memory for the upstream and downstream pointers - _RTI.federates[fed_id].enclave.upstream = (int*)malloc(sizeof(federate_t*) * _RTI.federates[fed_id].enclave.num_upstream); - _RTI.federates[fed_id].enclave.downstream = (int*)malloc(sizeof(federate_t*) * _RTI.federates[fed_id].enclave.num_downstream); + fed->enclave.upstream = (int*)malloc(sizeof(federate_t*) * fed->enclave.num_upstream); + fed->enclave.downstream = (int*)malloc(sizeof(federate_t*) * fed->enclave.num_downstream); // Allocate memory for the upstream delay pointers - _RTI.federates[fed_id].enclave.upstream_delay = + fed->enclave.upstream_delay = (interval_t*)malloc( - sizeof(interval_t) * _RTI.federates[fed_id].enclave.num_upstream + sizeof(interval_t) * fed->enclave.num_upstream ); size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) * - _RTI.federates[fed_id].enclave.num_upstream) + (sizeof(uint16_t) * _RTI.federates[fed_id].enclave.num_downstream); + fed->enclave.num_upstream) + (sizeof(uint16_t) * fed->enclave.num_downstream); unsigned char* connections_info_body = (unsigned char*)malloc(connections_info_body_size); read_from_socket_errexit( socket_id, @@ -1331,16 +1300,16 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { // Keep track of where we are in the buffer size_t message_head = 0; // First, read the info about upstream federates - for (int i=0; i<_RTI.federates[fed_id].enclave.num_upstream; i++) { - _RTI.federates[fed_id].enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head])); + for (int i=0; ienclave.num_upstream; i++) { + fed->enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head])); message_head += sizeof(uint16_t); - _RTI.federates[fed_id].enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head])); + fed->enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head])); message_head += sizeof(int64_t); } // Next, read the info about downstream federates - for (int i=0; i<_RTI.federates[fed_id].enclave.num_downstream; i++) { - _RTI.federates[fed_id].enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head])); + for (int i=0; ienclave.num_downstream; i++) { + fed->enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head])); message_head += sizeof(uint16_t); } @@ -1363,6 +1332,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[fed_id]; if (_RTI.clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. uint16_t federate_UDP_port_number = extract_uint16(&(response[1])); @@ -1374,7 +1344,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { // Send the required number of messages for clock synchronization for (int i=0; i < _RTI.clock_sync_exchanges_per_interval; i++) { // Send the RTI's current physical time T1 to the federate. - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &_RTI.federates[fed_id], TCP); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &fed, TCP); // Listen for reply message, which should be T3. size_t message_size = 1 + sizeof(int32_t); @@ -1386,7 +1356,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { assert(fed_id > -1); assert(fed_id < 65536); LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id); - handle_physical_clock_sync_message(&_RTI.federates[fed_id], TCP); + handle_physical_clock_sync_message(&fed, TCP); } else { lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id); send_reject(socket_id, UNEXPECTED_MESSAGE); @@ -1398,19 +1368,19 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { if (_RTI.clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. if (federate_UDP_port_number > 0) { // Initialize the UDP_addr field of the federate struct - _RTI.federates[fed_id].UDP_addr.sin_family = AF_INET; - _RTI.federates[fed_id].UDP_addr.sin_port = htons(federate_UDP_port_number); - _RTI.federates[fed_id].UDP_addr.sin_addr = _RTI.federates[fed_id].server_ip_addr; + fed->UDP_addr.sin_family = AF_INET; + fed->UDP_addr.sin_port = htons(federate_UDP_port_number); + fed->UDP_addr.sin_addr = fed->server_ip_addr; } } else { // Disable clock sync after initial round. - _RTI.federates[fed_id].clock_synchronization_enabled = false; + fed->clock_synchronization_enabled = false; } } else { // No clock synchronization at all. // Clock synchronization is universally disabled via the clock-sync command-line parameter // (-c off was passed to the RTI). // Note that the federates are still going to send a MSG_TYPE_UDP_PORT message but with a payload (port) of -1. - _RTI.federates[fed_id].clock_synchronization_enabled = false; + fed->clock_synchronization_enabled = false; } } return 1; @@ -1475,7 +1445,7 @@ bool authenticate_federate(int socket) { #endif void connect_to_federates(int socket_descriptor) { - for (int i = 0; i < _RTI.number_of_federates; i++) { + for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { // Wait for an incoming connection request. struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); @@ -1517,7 +1487,8 @@ void connect_to_federates(int socket_descriptor) { // This has to be done after clock synchronization is finished // or that thread may end up attempting to handle incoming clock // synchronization messages. - lf_thread_create(&(_RTI.federates[fed_id].thread_id), federate_thread_TCP, &(_RTI.federates[fed_id])); + federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[fed_id]; + lf_thread_create(&(fed->thread_id), federate_thread_TCP, fed); } else { // Received message was rejected. Try again. @@ -1532,8 +1503,8 @@ void connect_to_federates(int socket_descriptor) { // over the UDP channel, but only if the UDP channel is open and at least one // federate is performing runtime clock synchronization. bool clock_sync_enabled = false; - for (int i = 0; i < _RTI.number_of_federates; i++) { - if (_RTI.federates[i].clock_synchronization_enabled) { + for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { + if (((federate_t *)_RTI.enclave_rti.enclaves[i])->clock_synchronization_enabled) { clock_sync_enabled = true; break; } @@ -1615,11 +1586,12 @@ void wait_for_federates(int socket_descriptor) { // Wait for federate threads to exit. void* thread_exit_status; - for (int i = 0; i < _RTI.number_of_federates; i++) { - lf_print("RTI: Waiting for thread handling federate %d.", _RTI.federates[i].enclave.id); - lf_thread_join(_RTI.federates[i].thread_id, &thread_exit_status); - free_in_transit_message_q(_RTI.federates[i].in_transit_message_tags); - lf_print("RTI: Federate %d thread exited.", _RTI.federates[i].enclave.id); + for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { + federate_t* fed = (federate_t *)_RTI.enclave_rti.enclaves[i]; + lf_print("RTI: Waiting for thread handling federate %d.", fed->enclave.id); + lf_thread_join(fed->thread_id, &thread_exit_status); + free_in_transit_message_q(fed->in_transit_message_tags); + lf_print("RTI: Federate %d thread exited.", fed->enclave.id); } _RTI.all_federates_exited = true; @@ -1795,8 +1767,8 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _RTI.number_of_federates = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines - printf("RTI: Number of federates: %d\n", _RTI.number_of_federates); + _RTI.enclave_rti.number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines + printf("RTI: Number of federates: %d\n", _RTI.enclave_rti.number_of_enclaves); } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) { if (argc < i + 2) { fprintf( @@ -1830,7 +1802,7 @@ int process_args(int argc, const char* argv[]) { } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--auth") == 0) { _RTI.authentication_enabled = true; } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { - _RTI.tracing_enabled = true; + _RTI.enclave_rti.tracing_enabled = true; } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1840,7 +1812,7 @@ int process_args(int argc, const char* argv[]) { return 0; } } - if (_RTI.number_of_federates == 0) { + if (_RTI.enclave_rti.number_of_enclaves == 0) { fprintf(stderr, "Error: --number_of_federates needs a valid positive integer argument.\n"); usage(argc, argv); return 0; diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index 0f8939a27..c223b84b9 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -78,15 +78,9 @@ typedef enum clock_sync_stat { * Structure that an RTI instance uses to keep track of its own and its * corresponding federates' state. */ -typedef struct RTI_instance_t { - // RTI's decided stop tag for federates - tag_t max_stop_tag; - - // Number of federates in the federation - int32_t number_of_federates; - - // The federates. - federate_t* federates; +typedef struct federation_RTI_t { + // Enclave RTI + enclave_RTI_t enclave_rti; // Maximum start time seen so far from the federates. int64_t max_start_time; @@ -94,9 +88,6 @@ typedef struct RTI_instance_t { // Number of federates that have proposed start times. int num_feds_proposed_start; - // Number of federates handling stop - int num_feds_handling_stop; - /** * Boolean indicating that all federates have exited. * This gets set to true exactly once before the program exits. @@ -160,12 +151,12 @@ typedef struct RTI_instance_t { * Boolean indicating that tracing is enabled. */ bool tracing_enabled; -} RTI_instance_t; +} federation_RTI_t; -/** - * The main mutex lock for the RTI. - */ -extern lf_mutex_t rti_mutex; +// /** +// * The main mutex lock for the RTI. +// */ +// extern lf_mutex_t rti_mutex; /** * Condition variable used to signal receipt of all proposed start times. @@ -206,30 +197,6 @@ extern int lf_critical_section_exit(); */ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_type); -/** - * Find the earliest tag at which the specified federate may - * experience its next event. This is the least next event tag (NET) - * of the specified federate and (transitively) upstream federates - * (with delays of the connections added). For upstream federates, - * we assume (conservatively) that federate upstream of those - * may also send an event. The result will never be less than - * the completion time of the federate (which may be NEVER, - * if the federate has not yet completed a logical time). - * - * FIXME: This could be made less conservative by building - * at code generation time a causality interface table indicating - * which outputs can be triggered by which inputs. For now, we - * assume any output can be triggered by any input. - * - * @param fed The federate. - * @param candidate A candidate tag (for the first invocation, - * this should be fed->next_event). - * @param visited An array of booleans indicating which federates - * have been visited (for the first invocation, this should be - * an array of falses of size _RTI.number_of_federates). - */ -tag_t transitive_next_event(federate_t* fed, tag_t candidate, bool visited[]); - /** * @brief Update the next event tag of federate `federate_id`. * From 800a6f63f88761fb44a4eb8ac7226faa3c96d997 Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Tue, 23 May 2023 11:26:47 -0700 Subject: [PATCH 07/25] Fix the memory allocation of the federates. --- core/federated/RTI/rti.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index b4b68af0f..0f99834c4 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -75,9 +75,10 @@ int main(int argc, const char* argv[]) { } printf("Starting RTI for %d federates in federation ID %s\n", _RTI.enclave_rti.number_of_enclaves, _RTI.federation_id); assert(_RTI.enclave_rti.number_of_enclaves < UINT16_MAX); - _RTI.enclave_rti.enclaves = (federate_t*)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(federate_t)); + _RTI.enclave_rti.enclaves = (federate_t**)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(federate_t*)); for (uint16_t i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { - initialize_federate((federate_t*)&_RTI.enclave_rti.enclaves[i], i); + _RTI.enclave_rti.enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); + initialize_federate((federate_t *)_RTI.enclave_rti.enclaves[i], i); } int socket_descriptor = start_rti_server(_RTI.user_specified_port); wait_for_federates(socket_descriptor); From 7bb3e3a42f0e0c69c106f451b1173fe68fec877d Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Tue, 23 May 2023 13:50:25 -0700 Subject: [PATCH 08/25] Fix segmentation fault, due to wrong access to enclaves using _RTI. --- core/federated/RTI/enclave.c | 10 +++++----- core/federated/RTI/enclave_impl.c | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 5283df46d..a85d3128e 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -49,7 +49,7 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { // Check downstream federates to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { // FIXME: Shouldn't use enclave_t here. - enclave_t* downstream = &_RTI.enclaves[enclave->downstream[i]]; + enclave_t* downstream = _RTI.enclaves[enclave->downstream[i]]; // Notify downstream federate if appropriate. notify_advance_grant_if_safe((enclave_t*)downstream); bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. @@ -68,7 +68,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < e->num_upstream; j++) { - enclave_t* upstream = &_RTI.enclaves[e->upstream[j]]; + enclave_t* upstream = _RTI.enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) continue; @@ -109,7 +109,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { NEVER - start_time, 0u); for (int j = 0; j < e->num_upstream; j++) { - enclave_t* upstream = &_RTI.enclaves[e->upstream[j]]; + enclave_t* upstream = _RTI.enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) continue; @@ -172,7 +172,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { visited[e->id] = true; for (int i = 0; i < e->num_downstream; i++) { - enclave_t* downstream = (enclave_t*)&_RTI.enclaves[e->downstream[i]]; + enclave_t* downstream = (enclave_t*)_RTI.enclaves[e->downstream[i]]; if (visited[downstream->id]) continue; notify_advance_grant_if_safe(downstream); notify_downstream_advance_grant_if_safe(downstream, visited); @@ -274,7 +274,7 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { // an event that would result in an earlier next event. for (int i = 0; i < e->num_upstream; i++) { tag_t upstream_result = transitive_next_event( - &_RTI.enclaves[e->upstream[i]], result, visited); + _RTI.enclaves[e->upstream[i]], result, visited); // Add the "after" delay of the connection to the result. upstream_result = lf_delay_tag(upstream_result, e->upstream_delay[i]); diff --git a/core/federated/RTI/enclave_impl.c b/core/federated/RTI/enclave_impl.c index 12d4b0cd7..42a397eb6 100644 --- a/core/federated/RTI/enclave_impl.c +++ b/core/federated/RTI/enclave_impl.c @@ -1,7 +1,6 @@ #include "enclave.h" // FIXME: This should not be here. -#include "enclave_impl.h" #include "platform.h" void notify_tag_advance_grant(enclave_t* e, tag_t tag) { @@ -17,3 +16,5 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { e->last_granted = tag; lf_cond_signal(&e->next_event_condition); } + + From 2a9e2e9a4ff207c46d3cb6c14a081be0c15d25cc Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 25 May 2023 11:36:56 -0700 Subject: [PATCH 09/25] Fix the segmentation fault. Cycles are not yet correctly handled! --- core/federated/RTI/enclave.c | 21 +-- core/federated/RTI/rti.c | 24 +-- core/federated/RTI/rti_lib.c | 304 ++++++++++++++++++----------------- core/federated/RTI/rti_lib.h | 5 +- 4 files changed, 182 insertions(+), 172 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index a85d3128e..b054bc29a 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -1,6 +1,7 @@ #include "enclave.h" -extern enclave_RTI_t _RTI; + +extern enclave_RTI_t* _RTI; // Global variables defined in tag.c: extern instant_t start_time; @@ -35,7 +36,7 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { lf_mutex_lock(&rti_mutex); enclave->completed = completed; - if (_RTI.tracing_enabled) { + if ((enclave_RTI_t*)_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_LTC, enclave->id, &(enclave->completed)); } @@ -49,10 +50,10 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { // Check downstream federates to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { // FIXME: Shouldn't use enclave_t here. - enclave_t* downstream = _RTI.enclaves[enclave->downstream[i]]; + enclave_t *downstream = (enclave_RTI_t *)_RTI->enclaves[enclave->downstream[i]]; // Notify downstream federate if appropriate. notify_advance_grant_if_safe((enclave_t*)downstream); - bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc((enclave_RTI_t *)_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Notify enclaves downstream of downstream if appropriate. notify_downstream_advance_grant_if_safe((enclave_t*)downstream, visited); free(visited); @@ -68,7 +69,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < e->num_upstream; j++) { - enclave_t* upstream = _RTI.enclaves[e->upstream[j]]; + enclave_t *upstream = (enclave_RTI_t *)_RTI->enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) continue; @@ -99,7 +100,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc((enclave_RTI_t *)_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the tag of the earliest possible incoming message from // upstream federates. @@ -109,7 +110,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { NEVER - start_time, 0u); for (int j = 0; j < e->num_upstream; j++) { - enclave_t* upstream = _RTI.enclaves[e->upstream[j]]; + enclave_t *upstream = (enclave_RTI_t *)_RTI->enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) continue; @@ -172,7 +173,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { visited[e->id] = true; for (int i = 0; i < e->num_downstream; i++) { - enclave_t* downstream = (enclave_t*)_RTI.enclaves[e->downstream[i]]; + enclave_t* downstream = (enclave_t*)_RTI->enclaves[e->downstream[i]]; if (visited[downstream->id]) continue; notify_advance_grant_if_safe(downstream); notify_downstream_advance_grant_if_safe(downstream, visited); @@ -198,7 +199,7 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc((enclave_RTI_t *)_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(e, visited); free(visited); } @@ -274,7 +275,7 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { // an event that would result in an earlier next event. for (int i = 0; i < e->num_upstream; i++) { tag_t upstream_result = transitive_next_event( - _RTI.enclaves[e->upstream[i]], result, visited); + (enclave_RTI_t *)_RTI->enclaves[e->upstream[i]], result, visited); // Add the "after" delay of the connection to the result. upstream_result = lf_delay_tag(upstream_result, e->upstream_delay[i]); diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 0f99834c4..a5f047781 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -49,7 +49,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "rti_lib.h" unsigned int _lf_number_of_workers = 0u; -extern federation_RTI_t _RTI; +extern federation_RTI_t *_RTI; extern lf_mutex_t rti_mutex; @@ -60,6 +60,8 @@ const char *rti_trace_file_name = "rti.lft"; int main(int argc, const char* argv[]) { + initialize_RTI(); + lf_mutex_init(&rti_mutex); lf_cond_init(&received_start_times, &rti_mutex); lf_cond_init(&sent_start_time, &rti_mutex); @@ -68,21 +70,21 @@ int main(int argc, const char* argv[]) { // Processing command-line arguments failed. return -1; } - if (_RTI.enclave_rti.tracing_enabled) { - _lf_number_of_workers = _RTI.enclave_rti.number_of_enclaves; + if (_RTI->enclave_rti.tracing_enabled) { + _lf_number_of_workers = _RTI->enclave_rti.number_of_enclaves; start_trace(rti_trace_file_name); printf("Tracing the RTI execution in %s file.\n", rti_trace_file_name); } - printf("Starting RTI for %d federates in federation ID %s\n", _RTI.enclave_rti.number_of_enclaves, _RTI.federation_id); - assert(_RTI.enclave_rti.number_of_enclaves < UINT16_MAX); - _RTI.enclave_rti.enclaves = (federate_t**)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(federate_t*)); - for (uint16_t i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { - _RTI.enclave_rti.enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); - initialize_federate((federate_t *)_RTI.enclave_rti.enclaves[i], i); + printf("Starting RTI for %d federates in federation ID %s\n", _RTI->enclave_rti.number_of_enclaves, _RTI->federation_id); + assert(_RTI->enclave_rti.number_of_enclaves < UINT16_MAX); + _RTI->enclave_rti.enclaves = (federate_t**)calloc(_RTI->enclave_rti.number_of_enclaves, sizeof(federate_t*)); + for (uint16_t i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { + _RTI->enclave_rti.enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); + initialize_federate((federate_t *)_RTI->enclave_rti.enclaves[i], i); } - int socket_descriptor = start_rti_server(_RTI.user_specified_port); + int socket_descriptor = start_rti_server(_RTI->user_specified_port); wait_for_federates(socket_descriptor); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { stop_trace(); } printf("RTI is exiting.\n"); diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 548d5ead5..eab69d06e 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -32,25 +32,7 @@ extern instant_t start_time; /** * The state of this RTI instance. */ -federation_RTI_t _RTI = { - .enclave_rti.max_stop_tag = NEVER_TAG, - .max_start_time = 0LL, - .enclave_rti.number_of_enclaves = 0, - .num_feds_proposed_start = 0, - .enclave_rti.num_feds_handling_stop = 0, - .all_federates_exited = false, - .federation_id = "Unidentified Federation", - .user_specified_port = 0, - .final_port_TCP = 0, - .socket_descriptor_TCP = -1, - .final_port_UDP = UINT16_MAX, - .socket_descriptor_UDP = -1, - .clock_sync_global_status = clock_sync_init, - .clock_sync_period_ns = MSEC(10), - .clock_sync_exchanges_per_interval = 10, - .authentication_enabled = false, - .tracing_enabled = false -}; +federation_RTI_t* _RTI; lf_mutex_t rti_mutex; lf_cond_t received_start_times; @@ -175,16 +157,16 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty if (socket_type == UDP) { type = "UDP"; } - lf_print("RTI using %s port %d for federation %s.", type, port, _RTI.federation_id); + lf_print("RTI using %s port %d for federation %s.", type, port, _RTI->federation_id); if (socket_type == TCP) { - _RTI.final_port_TCP = port; + _RTI->final_port_TCP = port; // Enable listening for socket connections. // The second argument is the maximum number of queued socket requests, // which according to the Mac man page is limited to 128. listen(socket_descriptor, 128); } else if (socket_type == UDP) { - _RTI.final_port_UDP = port; + _RTI->final_port_UDP = port; // No need to listen on the UDP socket } @@ -210,7 +192,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -250,7 +232,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_RTI.enclave_rti.tracing_enabled){ + if (_RTI->enclave_rti.tracing_enabled){ tracepoint_RTI_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -277,20 +259,18 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { // we have an available encoding of causality interfaces. // That might be more efficient. for (int j = 0; j < e->num_upstream; j++) { - federate_t* upstream = (federate_t*)&_RTI.enclave_rti.enclaves[e->upstream[j]]; + federate_t* upstream = (federate_t*)&_RTI->enclave_rti.enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->enclave.state == NOT_CONNECTED) continue; - // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI->enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( upstream, upstream->enclave.next_event, visited); free(visited); - // If these tags are equal, then // a TAG or PTAG should have already been granted, // in which case, another will not be sent. But it @@ -298,12 +278,13 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { if (lf_tag_compare(upstream_next_event, tag) >= 0) { notify_provisional_tag_advance_grant(&upstream->enclave, tag); } + } } } void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { - federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[federate_id]; + federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[federate_id]; tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(fed->in_transit_message_tags); if (lf_tag_compare( min_in_transit_tag, @@ -326,7 +307,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); tag_t tag = extract_tag(&(buffer[1 + 2 * sizeof(uint16_t)])); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_PORT_ABS, federate_id, &tag); } @@ -337,7 +318,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // If the destination federate is no longer connected, issue a warning // and return. - federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[federate_id]; + federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[federate_id]; if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", @@ -371,7 +352,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // Forward the message. int destination_socket = fed->socket; - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_PORT_ABS, federate_id, &tag); } write_to_socket_errexit(destination_socket, message_size + 1, buffer, @@ -417,7 +398,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // Following only works for string messages. // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_TAGGED_MSG, sending_federate->enclave.id, &intended_tag); } @@ -428,7 +409,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // If the destination federate is no longer connected, issue a warning // and return. - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[federate_id]; + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[federate_id]; if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", @@ -494,7 +475,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { lf_cond_wait(&sent_start_time); } - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_TAGGED_MSG, federate_id, &intended_tag); } @@ -555,7 +536,7 @@ void handle_next_event_tag(federate_t* fed) { tag_t intended_tag = extract_tag(buffer); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_NET, fed->enclave.id, &intended_tag); } LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) " PRINTF_TAG, @@ -582,28 +563,28 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { } // Reply with a stop granted to all federates unsigned char outgoing_buffer[MSG_TYPE_STOP_GRANTED_LENGTH]; - ENCODE_STOP_GRANTED(outgoing_buffer, _RTI.enclave_rti.max_stop_tag.time, _RTI.enclave_rti.max_stop_tag.microstep); + ENCODE_STOP_GRANTED(outgoing_buffer, _RTI->enclave_rti.max_stop_tag.time, _RTI->enclave_rti.max_stop_tag.microstep); // Iterate over federates and send each the message. - for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[i]; + for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[i]; if (fed->enclave.state == NOT_CONNECTED) { continue; } - if (lf_tag_compare(fed->enclave.next_event, _RTI.enclave_rti.max_stop_tag) >= 0) { + if (lf_tag_compare(fed->enclave.next_event, _RTI->enclave_rti.max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. - fed->enclave.next_event = _RTI.enclave_rti.max_stop_tag; + fed->enclave.next_event = _RTI->enclave_rti.max_stop_tag; } - if (_RTI.enclave_rti.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_RTI.enclave_rti.max_stop_tag); + if (_RTI->enclave_rti.tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_RTI->enclave_rti.max_stop_tag); } write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); } LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag (" PRINTF_TIME ", %u).", - _RTI.enclave_rti.max_stop_tag.time - start_time, - _RTI.enclave_rti.max_stop_tag.microstep); + _RTI->enclave_rti.max_stop_tag.time - start_time, + _RTI->enclave_rti.max_stop_tag.microstep); _lf_rti_stop_granted_already_sent_to_federates = true; } @@ -611,10 +592,10 @@ void mark_federate_requesting_stop(federate_t* fed) { if (!fed->enclave.requested_stop) { // Assume that the federate // has requested stop - _RTI.enclave_rti.num_feds_handling_stop++; + _RTI->enclave_rti.num_feds_handling_stop++; fed->enclave.requested_stop = true; } - if (_RTI.enclave_rti.num_feds_handling_stop == _RTI.enclave_rti.number_of_enclaves) { + if (_RTI->enclave_rti.num_feds_handling_stop == _RTI->enclave_rti.number_of_enclaves) { // We now have information about the stop time of all // federates. _lf_rti_broadcast_stop_time_to_federates_already_locked(); @@ -644,13 +625,13 @@ void handle_stop_request_message(federate_t* fed) { // Extract the proposed stop tag for the federate tag_t proposed_stop_tag = extract_tag(buffer); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ, fed->enclave.id, &proposed_stop_tag); } // Update the maximum stop tag received from federates - if (lf_tag_compare(proposed_stop_tag, _RTI.enclave_rti.max_stop_tag) > 0) { - _RTI.enclave_rti.max_stop_tag = proposed_stop_tag; + if (lf_tag_compare(proposed_stop_tag, _RTI->enclave_rti.max_stop_tag) > 0) { + _RTI->enclave_rti.max_stop_tag = proposed_stop_tag; } LF_PRINT_LOG("RTI received from federate %d a MSG_TYPE_STOP_REQUEST message with tag (%lld, %u).", @@ -660,7 +641,7 @@ void handle_stop_request_message(federate_t* fed) { // for a stop, add it to the tally. mark_federate_requesting_stop(fed); - if (_RTI.enclave_rti.num_feds_handling_stop == _RTI.enclave_rti.number_of_enclaves) { + if (_RTI->enclave_rti.num_feds_handling_stop == _RTI->enclave_rti.number_of_enclaves) { // We now have information about the stop time of all // federates. This is extremely unlikely, but it can occur // all federates call lf_request_stop() at the same tag. @@ -670,30 +651,30 @@ void handle_stop_request_message(federate_t* fed) { // Forward the stop request to all other federates that have not // also issued a stop request. unsigned char stop_request_buffer[MSG_TYPE_STOP_REQUEST_LENGTH]; - ENCODE_STOP_REQUEST(stop_request_buffer, _RTI.enclave_rti.max_stop_tag.time, _RTI.enclave_rti.max_stop_tag.microstep); + ENCODE_STOP_REQUEST(stop_request_buffer, _RTI->enclave_rti.max_stop_tag.time, _RTI->enclave_rti.max_stop_tag.microstep); // Iterate over federates and send each the MSG_TYPE_STOP_REQUEST message // if we do not have a stop_time already for them. - for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[i]; + for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[i]; if (fed->enclave.id != fed->enclave.id && fed->enclave.requested_stop == false) { if (fed->enclave.state == NOT_CONNECTED) { mark_federate_requesting_stop(fed); continue; } - if (_RTI.enclave_rti.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI.enclave_rti.max_stop_tag); + if (_RTI->enclave_rti.tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI->enclave_rti.max_stop_tag); } write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", fed->enclave.id); - if (_RTI.enclave_rti.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI.enclave_rti.max_stop_tag); + if (_RTI->enclave_rti.tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI->enclave_rti.max_stop_tag); } } } LF_PRINT_LOG("RTI forwarded to federates MSG_TYPE_STOP_REQUEST with tag (" PRINTF_TIME ", %u).", - _RTI.enclave_rti.max_stop_tag.time - start_time, - _RTI.enclave_rti.max_stop_tag.microstep); + _RTI->enclave_rti.max_stop_tag.time - start_time, + _RTI->enclave_rti.max_stop_tag.microstep); lf_mutex_unlock(&rti_mutex); } @@ -705,7 +686,7 @@ void handle_stop_request_reply(federate_t* fed) { tag_t federate_stop_tag = extract_tag(buffer_stop_time); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ_REP, fed->enclave.id, &federate_stop_tag); } @@ -716,8 +697,8 @@ void handle_stop_request_reply(federate_t* fed) { // Acquire the mutex lock so that we can change the state of the RTI lf_mutex_lock(&rti_mutex); // If the federate has not requested stop before, count the reply - if (lf_tag_compare(federate_stop_tag, _RTI.enclave_rti.max_stop_tag) > 0) { - _RTI.enclave_rti.max_stop_tag = federate_stop_tag; + if (lf_tag_compare(federate_stop_tag, _RTI->enclave_rti.max_stop_tag) > 0) { + _RTI->enclave_rti.max_stop_tag = federate_stop_tag; } mark_federate_requesting_stop(fed); lf_mutex_unlock(&rti_mutex); @@ -726,7 +707,7 @@ void handle_stop_request_reply(federate_t* fed) { ////////////////////////////////////////////////// void handle_address_query(uint16_t fed_id) { - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[fed_id]; + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[fed_id]; // Use buffer both for reading and constructing the reply. // The length is what is needed for the reply. unsigned char buffer[sizeof(int32_t)]; @@ -736,7 +717,7 @@ void handle_address_query(uint16_t fed_id) { } uint16_t remote_fed_id = extract_uint16(buffer); - if (_RTI.enclave_rti.tracing_enabled){ + if (_RTI->enclave_rti.tracing_enabled){ tracepoint_RTI_from_federate(receive_ADR_QR, fed_id, NULL); } @@ -747,7 +728,7 @@ void handle_address_query(uint16_t fed_id) { // from this federate. In that case, it will respond by sending -1. // Encode the port number. - federate_t *remote_fed = (federate_t *)_RTI.enclave_rti.enclaves[remote_fed_id]; + federate_t *remote_fed = (federate_t *)_RTI->enclave_rti.enclaves[remote_fed_id]; encode_int32(remote_fed->server_port, (unsigned char*)buffer); // Send the port number (which could be -1). write_to_socket_errexit(fed->socket, sizeof(int32_t), (unsigned char*)buffer, @@ -765,7 +746,7 @@ void handle_address_query(uint16_t fed_id) { } void handle_address_ad(uint16_t federate_id) { - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[federate_id]; + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[federate_id]; // Read the port number of the federate that can be used for physical // connections to other federates int32_t server_port = -1; @@ -784,7 +765,7 @@ void handle_address_ad(uint16_t federate_id) { lf_mutex_lock(&rti_mutex); fed->server_port = server_port; - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_ADR_AD, federate_id, NULL); } LF_PRINT_LOG("Received address advertisement from federate %d.", federate_id); @@ -800,24 +781,24 @@ void handle_timestamp(federate_t *my_fed) { } int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t *)(&buffer))); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tag_t tag = {.time = timestamp, .microstep = 0}; tracepoint_RTI_from_federate(receive_TIMESTAMP, my_fed->enclave.id, &tag); } LF_PRINT_LOG("RTI received timestamp message: %lld.", timestamp); lf_mutex_lock(&rti_mutex); - _RTI.num_feds_proposed_start++; - if (timestamp > _RTI.max_start_time) { - _RTI.max_start_time = timestamp; + _RTI->num_feds_proposed_start++; + if (timestamp > _RTI->max_start_time) { + _RTI->max_start_time = timestamp; } - if (_RTI.num_feds_proposed_start == _RTI.enclave_rti.number_of_enclaves) { + if (_RTI->num_feds_proposed_start == _RTI->enclave_rti.number_of_enclaves) { // All federates have proposed a start time. lf_cond_broadcast(&received_start_times); } else { // Some federates have not yet proposed a start time. // wait for a notification. - while (_RTI.num_feds_proposed_start < _RTI.enclave_rti.number_of_enclaves) { + while (_RTI->num_feds_proposed_start < _RTI->enclave_rti.number_of_enclaves) { // FIXME: Should have a timeout here? lf_cond_wait(&received_start_times); } @@ -830,10 +811,10 @@ void handle_timestamp(federate_t *my_fed) { unsigned char start_time_buffer[MSG_TYPE_TIMESTAMP_LENGTH]; start_time_buffer[0] = MSG_TYPE_TIMESTAMP; // Add an offset to this start time to get everyone starting together. - start_time = _RTI.max_start_time + DELAY_START; + start_time = _RTI->max_start_time + DELAY_START; encode_int64(swap_bytes_if_big_endian_int64(start_time), &start_time_buffer[1]); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tag_t tag = {.time = start_time, .microstep = 0}; tracepoint_RTI_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } @@ -870,7 +851,7 @@ void send_physical_clock(unsigned char message_type, federate_t* fed, socket_typ if (socket_type == UDP) { // FIXME: UDP_addr is never initialized. LF_PRINT_DEBUG("Clock sync: RTI sending UDP message type %u.", buffer[0]); - ssize_t bytes_written = sendto(_RTI.socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, + ssize_t bytes_written = sendto(_RTI->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr)); if (bytes_written < (ssize_t)sizeof(int64_t) + 1) { lf_print_warning("Clock sync: RTI failed to send physical time to federate %d: %s\n", @@ -909,7 +890,7 @@ void* clock_synchronization_thread(void* noargs) { // Wait until all federates have been notified of the start time. // FIXME: Use lf_ version of this when merged with master. lf_mutex_lock(&rti_mutex); - while (_RTI.num_feds_proposed_start < _RTI.enclave_rti.number_of_enclaves) { + while (_RTI->num_feds_proposed_start < _RTI->enclave_rti.number_of_enclaves) { lf_cond_wait(&received_start_times); } lf_mutex_unlock(&rti_mutex); @@ -924,9 +905,9 @@ void* clock_synchronization_thread(void* noargs) { nanosleep(&wait_time, &rem_time); } - // Initiate a clock synchronization every _RTI.clock_sync_period_ns - struct timespec sleep_time = {(time_t) _RTI.clock_sync_period_ns / BILLION, - _RTI.clock_sync_period_ns % BILLION}; + // Initiate a clock synchronization every _RTI->clock_sync_period_ns + struct timespec sleep_time = {(time_t) _RTI->clock_sync_period_ns / BILLION, + _RTI->clock_sync_period_ns % BILLION}; struct timespec remaining_time; bool any_federates_connected = true; @@ -934,8 +915,8 @@ void* clock_synchronization_thread(void* noargs) { // Sleep nanosleep(&sleep_time, &remaining_time); // Can be interrupted any_federates_connected = false; - for (int fed_id = 0; fed_id < _RTI.enclave_rti.number_of_enclaves; fed_id++) { - federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[fed_id]; + for (int fed_id = 0; fed_id < _RTI->enclave_rti.number_of_enclaves; fed_id++) { + federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[fed_id]; if (fed->enclave.state == NOT_CONNECTED) { // FIXME: We need better error handling here, but clock sync failure // should not stop execution. @@ -959,7 +940,7 @@ void* clock_synchronization_thread(void* noargs) { int remaining_attempts = 5; while (remaining_attempts > 0) { remaining_attempts--; - int bytes_read = read_from_socket(_RTI.socket_descriptor_UDP, message_size, buffer); + int bytes_read = read_from_socket(_RTI->socket_descriptor_UDP, message_size, buffer); // If any errors occur, either discard the message or the clock sync round. if (bytes_read == message_size) { if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { @@ -973,7 +954,7 @@ void* clock_synchronization_thread(void* noargs) { continue; } LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id_2); - handle_physical_clock_sync_message((federate_t *)_RTI.enclave_rti.enclaves[fed_id_2], UDP); + handle_physical_clock_sync_message((federate_t *)_RTI->enclave_rti.enclaves[fed_id_2], UDP); break; } else { // The message is not a T3 message. Discard the message and @@ -1005,7 +986,7 @@ void* clock_synchronization_thread(void* noargs) { void handle_federate_resign(federate_t *my_fed) { // Nothing more to do. Close the socket and exit. lf_mutex_lock(&rti_mutex); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { // Extract the tag, for tracing purposes size_t header_size = 1 + sizeof(tag_t); unsigned char buffer[header_size]; @@ -1038,7 +1019,7 @@ void handle_federate_resign(federate_t *my_fed) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI.enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_RTI->enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(&my_fed->enclave, visited); free(visited); @@ -1104,7 +1085,7 @@ void* federate_thread_TCP(void* fed) { break; default: lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->enclave.id, buffer[0]); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_UNIDENTIFIED, my_fed->enclave.id, NULL); } } @@ -1136,7 +1117,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // FIXME: This should not exit with error but rather should just reject the connection. read_from_socket_errexit(socket_id, length, buffer, "RTI failed to read from accepted socket."); - uint16_t fed_id = _RTI.enclave_rti.number_of_enclaves; // Initialize to an invalid value. + uint16_t fed_id = _RTI->enclave_rti.number_of_enclaves; // Initialize to an invalid value. // First byte received is the message type. if (buffer[0] != MSG_TYPE_FED_IDS) { @@ -1152,7 +1133,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } else { send_reject(socket_id, UNEXPECTED_MESSAGE); } - if (_RTI.enclave_rti.tracing_enabled){ + if (_RTI->enclave_rti.tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); @@ -1176,33 +1157,33 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI received federation ID: %s.", federation_id_received); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_from_federate(receive_FED_ID, fed_id, NULL); } // Compare the received federation ID to mine. - if (strncmp(_RTI.federation_id, federation_id_received, federation_id_length) != 0) { + if (strncmp(_RTI->federation_id, federation_id_received, federation_id_length) != 0) { // Federation IDs do not match. Send back a MSG_TYPE_REJECT message. lf_print_error("WARNING: Federate from another federation %s attempted to connect to RTI in federation %s.\n", federation_id_received, - _RTI.federation_id); - if (_RTI.enclave_rti.tracing_enabled) { + _RTI->federation_id); + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH); return -1; } else { - if (fed_id >= _RTI.enclave_rti.number_of_enclaves) { + if (fed_id >= _RTI->enclave_rti.number_of_enclaves) { // Federate ID is out of range. lf_print_error("RTI received federate ID %d, which is out of range.", fed_id); - if (_RTI.enclave_rti.tracing_enabled){ + if (_RTI->enclave_rti.tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { - if (_RTI.enclave_rti.enclaves[fed_id]->state != NOT_CONNECTED) { + if (_RTI->enclave_rti.enclaves[fed_id]->state != NOT_CONNECTED) { lf_print_error("RTI received duplicate federate ID: %d.", fed_id); - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_IN_USE); @@ -1211,7 +1192,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } } } - federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[fed_id]; + federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[fed_id]; // The MSG_TYPE_FED_IDS message has the right federation ID. // Assign the address information for federate. // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful @@ -1240,7 +1221,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); // Send an MSG_TYPE_ACK message. unsigned char ack_message = MSG_TYPE_ACK; - if (_RTI.enclave_rti.tracing_enabled) { + if (_RTI->enclave_rti.tracing_enabled) { tracepoint_RTI_to_federate(send_ACK, fed_id, NULL); } write_to_socket_errexit(socket_id, 1, &ack_message, @@ -1266,7 +1247,7 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { - federate_t* fed = (federate_t*) _RTI.enclave_rti.enclaves[fed_id]; + federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[fed_id]; // Read the number of upstream and downstream connections fed->enclave.num_upstream = extract_int32(&(connection_info_header[1])); fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); @@ -1332,8 +1313,8 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[fed_id]; - if (_RTI.clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[fed_id]; + if (_RTI->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. uint16_t federate_UDP_port_number = extract_uint16(&(response[1])); LF_PRINT_DEBUG("RTI got MSG_TYPE_UDP_PORT %u from federate %d.", federate_UDP_port_number, fed_id); @@ -1342,7 +1323,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { if (federate_UDP_port_number != UINT16_MAX) { // Perform the initialization clock synchronization with the federate. // Send the required number of messages for clock synchronization - for (int i=0; i < _RTI.clock_sync_exchanges_per_interval; i++) { + for (int i=0; i < _RTI->clock_sync_exchanges_per_interval; i++) { // Send the RTI's current physical time T1 to the federate. send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &fed, TCP); @@ -1365,7 +1346,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { } LF_PRINT_DEBUG("RTI finished initial clock synchronization with federate %d.", fed_id); } - if (_RTI.clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. + if (_RTI->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. if (federate_UDP_port_number > 0) { // Initialize the UDP_addr field of the federate struct fed->UDP_addr.sin_family = AF_INET; @@ -1400,7 +1381,7 @@ bool authenticate_federate(int socket) { // Check HMAC of received FED_RESPONSE message. size_t hmac_length = SHA256_HMAC_LENGTH; - size_t federation_id_length = strnlen(_RTI.federation_id, 255); + size_t federation_id_length = strnlen(_RTI->federation_id, 255); size_t fed_id_length = sizeof(uint16_t); unsigned char received[1 + NONCE_LENGTH + fed_id_length + hmac_length]; @@ -1417,7 +1398,7 @@ bool authenticate_federate(int socket) { memcpy(&buf_to_check[1], &received[1 + NONCE_LENGTH], fed_id_length); memcpy(&buf_to_check[1 + fed_id_length], rti_nonce, NONCE_LENGTH); unsigned char rti_tag[hmac_length]; - HMAC(EVP_sha256(), _RTI.federation_id, federation_id_length, buf_to_check, 1 + fed_id_length + NONCE_LENGTH, + HMAC(EVP_sha256(), _RTI->federation_id, federation_id_length, buf_to_check, 1 + fed_id_length + NONCE_LENGTH, rti_tag, &hmac_length); // Compare received tag and created tag. @@ -1436,7 +1417,7 @@ bool authenticate_federate(int socket) { // Buffer for message type and HMAC tag. unsigned char sender[1 + hmac_length]; sender[0] = MSG_TYPE_RTI_RESPONSE; - HMAC(EVP_sha256(), _RTI.federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, + HMAC(EVP_sha256(), _RTI->federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1], &hmac_length); write_to_socket(socket, 1 + hmac_length, sender); return true; @@ -1445,14 +1426,14 @@ bool authenticate_federate(int socket) { #endif void connect_to_federates(int socket_descriptor) { - for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { + for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { // Wait for an incoming connection request. struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); // The following blocks until a federate connects. int socket_id = -1; while(1) { - socket_id = accept(_RTI.socket_descriptor_TCP, &client_fd, &client_length); + socket_id = accept(_RTI->socket_descriptor_TCP, &client_fd, &client_length); if (socket_id >= 0) { // Got a socket break; @@ -1467,7 +1448,7 @@ void connect_to_federates(int socket_descriptor) { // Send RTI hello when RTI -a option is on. #ifdef __RTI_AUTH__ - if (_RTI.authentication_enabled) { + if (_RTI->authentication_enabled) { if (!authenticate_federate(socket_id)) { lf_print_warning("RTI failed to authenticate the incoming federate."); // Ignore the federate that failed authentication. @@ -1487,7 +1468,7 @@ void connect_to_federates(int socket_descriptor) { // This has to be done after clock synchronization is finished // or that thread may end up attempting to handle incoming clock // synchronization messages. - federate_t *fed = (federate_t *)_RTI.enclave_rti.enclaves[fed_id]; + federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[fed_id]; lf_thread_create(&(fed->thread_id), federate_thread_TCP, fed); } else { @@ -1498,19 +1479,19 @@ void connect_to_federates(int socket_descriptor) { // All federates have connected. LF_PRINT_DEBUG("All federates have connected to RTI."); - if (_RTI.clock_sync_global_status >= clock_sync_on) { + if (_RTI->clock_sync_global_status >= clock_sync_on) { // Create the thread that performs periodic PTP clock synchronization sessions // over the UDP channel, but only if the UDP channel is open and at least one // federate is performing runtime clock synchronization. bool clock_sync_enabled = false; - for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { - if (((federate_t *)_RTI.enclave_rti.enclaves[i])->clock_synchronization_enabled) { + for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { + if (((federate_t *)_RTI->enclave_rti.enclaves[i])->clock_synchronization_enabled) { clock_sync_enabled = true; break; } } - if (_RTI.final_port_UDP != UINT16_MAX && clock_sync_enabled) { - lf_thread_create(&_RTI.clock_thread, clock_synchronization_thread, NULL); + if (_RTI->final_port_UDP != UINT16_MAX && clock_sync_enabled) { + lf_thread_create(&_RTI->clock_thread, clock_synchronization_thread, NULL); } } } @@ -1521,11 +1502,11 @@ void* respond_to_erroneous_connections(void* nothing) { struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); // The following will block until either a federate attempts to connect - // or close(_RTI.socket_descriptor_TCP) is called. - int socket_id = accept(_RTI.socket_descriptor_TCP, &client_fd, &client_length); + // or close(_RTI->socket_descriptor_TCP) is called. + int socket_id = accept(_RTI->socket_descriptor_TCP, &client_fd, &client_length); if (socket_id < 0) return NULL; - if (_RTI.all_federates_exited) { + if (_RTI->all_federates_exited) { return NULL; } @@ -1560,14 +1541,14 @@ int32_t start_rti_server(uint16_t port) { } lf_initialize_clock(); // Create the TCP socket server - _RTI.socket_descriptor_TCP = create_server(specified_port, port, TCP); + _RTI->socket_descriptor_TCP = create_server(specified_port, port, TCP); lf_print("RTI: Listening for federates."); // Create the UDP socket server - // Try to get the _RTI.final_port_TCP + 1 port - if (_RTI.clock_sync_global_status >= clock_sync_on) { - _RTI.socket_descriptor_UDP = create_server(specified_port, _RTI.final_port_TCP + 1, UDP); + // Try to get the _RTI->final_port_TCP + 1 port + if (_RTI->clock_sync_global_status >= clock_sync_on) { + _RTI->socket_descriptor_UDP = create_server(specified_port, _RTI->final_port_TCP + 1, UDP); } - return _RTI.socket_descriptor_TCP; + return _RTI->socket_descriptor_TCP; } void wait_for_federates(int socket_descriptor) { @@ -1586,19 +1567,19 @@ void wait_for_federates(int socket_descriptor) { // Wait for federate threads to exit. void* thread_exit_status; - for (int i = 0; i < _RTI.enclave_rti.number_of_enclaves; i++) { - federate_t* fed = (federate_t *)_RTI.enclave_rti.enclaves[i]; + for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { + federate_t* fed = (federate_t *)_RTI->enclave_rti.enclaves[i]; lf_print("RTI: Waiting for thread handling federate %d.", fed->enclave.id); lf_thread_join(fed->thread_id, &thread_exit_status); free_in_transit_message_q(fed->in_transit_message_tags); lf_print("RTI: Federate %d thread exited.", fed->enclave.id); } - _RTI.all_federates_exited = true; + _RTI->all_federates_exited = true; // Shutdown and close the socket so that the accept() call in // respond_to_erroneous_connections returns. That thread should then - // check _RTI.all_federates_exited and it should exit. + // check _RTI->all_federates_exited and it should exit. if (shutdown(socket_descriptor, SHUT_RDWR)) { LF_PRINT_LOG("On shut down TCP socket, received reply: %s", strerror(errno)); } @@ -1609,7 +1590,7 @@ void wait_for_federates(int socket_descriptor) { // NOTE: Apparently, closing the socket will not necessarily // cause the respond_to_erroneous_connections accept() call to return, - // so instead, we connect here so that it can check the _RTI.all_federates_exited + // so instead, we connect here so that it can check the _RTI->all_federates_exited // variable. // Create an IPv4 socket for TCP (not UDP) communication over IP (0). @@ -1628,7 +1609,7 @@ void wait_for_federates(int socket_descriptor) { (char *)&server_fd.sin_addr.s_addr, server->h_length); // Convert the port number from host byte order to network byte order. - server_fd.sin_port = htons(_RTI.final_port_TCP); + server_fd.sin_port = htons(_RTI->final_port_TCP); connect( tmp_socket, (struct sockaddr *)&server_fd, @@ -1645,11 +1626,11 @@ void wait_for_federates(int socket_descriptor) { close(socket_descriptor); */ - if (_RTI.socket_descriptor_UDP > 0) { - if (shutdown(_RTI.socket_descriptor_UDP, SHUT_RDWR)) { + if (_RTI->socket_descriptor_UDP > 0) { + if (shutdown(_RTI->socket_descriptor_UDP, SHUT_RDWR)) { LF_PRINT_LOG("On shut down UDP socket, received reply: %s", strerror(errno)); } - close(_RTI.socket_descriptor_UDP); + close(_RTI->socket_descriptor_UDP); } } @@ -1684,16 +1665,16 @@ void usage(int argc, const char* argv[]) { int process_clock_sync_args(int argc, const char* argv[]) { for (int i = 0; i < argc; i++) { if (strcmp(argv[i], "off") == 0) { - _RTI.clock_sync_global_status = clock_sync_off; + _RTI->clock_sync_global_status = clock_sync_off; printf("RTI: Clock sync: off\n"); } else if (strcmp(argv[i], "init") == 0 || strcmp(argv[i], "initial") == 0) { - _RTI.clock_sync_global_status = clock_sync_init; + _RTI->clock_sync_global_status = clock_sync_init; printf("RTI: Clock sync: init\n"); } else if (strcmp(argv[i], "on") == 0) { - _RTI.clock_sync_global_status = clock_sync_on; + _RTI->clock_sync_global_status = clock_sync_on; printf("RTI: Clock sync: on\n"); } else if (strcmp(argv[i], "period") == 0) { - if (_RTI.clock_sync_global_status != clock_sync_on) { + if (_RTI->clock_sync_global_status != clock_sync_on) { fprintf(stderr, "Error: clock sync period can only be set if --clock-sync is set to on.\n"); usage(argc, argv); i++; @@ -1709,10 +1690,10 @@ int process_clock_sync_args(int argc, const char* argv[]) { fprintf(stderr, "Error: clock sync period value is invalid.\n"); continue; // Try to parse the rest of the arguments as clock sync args. } - _RTI.clock_sync_period_ns = (int64_t)period_ns; - printf("RTI: Clock sync period: %lld\n", (long long int)_RTI.clock_sync_period_ns); + _RTI->clock_sync_period_ns = (int64_t)period_ns; + printf("RTI: Clock sync period: %lld\n", (long long int)_RTI->clock_sync_period_ns); } else if (strcmp(argv[i], "exchanges-per-interval") == 0) { - if (_RTI.clock_sync_global_status != clock_sync_on && _RTI.clock_sync_global_status != clock_sync_init) { + if (_RTI->clock_sync_global_status != clock_sync_on && _RTI->clock_sync_global_status != clock_sync_init) { fprintf(stderr, "Error: clock sync exchanges-per-interval can only be set if\n"); fprintf(stderr, "--clock-sync is set to on or init.\n"); usage(argc, argv); @@ -1728,8 +1709,8 @@ int process_clock_sync_args(int argc, const char* argv[]) { fprintf(stderr, "Error: clock sync exchanges-per-interval value is invalid.\n"); continue; // Try to parse the rest of the arguments as clock sync args. } - _RTI.clock_sync_exchanges_per_interval = (int32_t)exchanges; // FIXME: Loses numbers on 64-bit machines - printf("RTI: Clock sync exchanges per interval: %d\n", _RTI.clock_sync_exchanges_per_interval); + _RTI->clock_sync_exchanges_per_interval = (int32_t)exchanges; // FIXME: Loses numbers on 64-bit machines + printf("RTI: Clock sync exchanges per interval: %d\n", _RTI->clock_sync_exchanges_per_interval); } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1753,7 +1734,7 @@ int process_args(int argc, const char* argv[]) { } i++; printf("RTI: Federation ID: %s\n", argv[i]); - _RTI.federation_id = argv[i]; + _RTI->federation_id = argv[i]; } else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--number_of_federates") == 0) { if (argc < i + 2) { fprintf(stderr, "Error: --number_of_federates needs an integer argument.\n"); @@ -1767,8 +1748,8 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _RTI.enclave_rti.number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines - printf("RTI: Number of federates: %d\n", _RTI.enclave_rti.number_of_enclaves); + _RTI->enclave_rti.number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines + printf("RTI: Number of federates: %d\n", _RTI->enclave_rti.number_of_enclaves); } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) { if (argc < i + 2) { fprintf( @@ -1790,7 +1771,7 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _RTI.user_specified_port = (uint16_t)RTI_port; + _RTI->user_specified_port = (uint16_t)RTI_port; } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--clock_sync") == 0) { if (argc < i + 2) { fprintf(stderr, "Error: --clock-sync needs off|init|on.\n"); @@ -1800,9 +1781,9 @@ int process_args(int argc, const char* argv[]) { i++; i += process_clock_sync_args((argc-i), &argv[i]); } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--auth") == 0) { - _RTI.authentication_enabled = true; + _RTI->authentication_enabled = true; } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { - _RTI.enclave_rti.tracing_enabled = true; + _RTI->enclave_rti.tracing_enabled = true; } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1812,10 +1793,33 @@ int process_args(int argc, const char* argv[]) { return 0; } } - if (_RTI.enclave_rti.number_of_enclaves == 0) { + if (_RTI->enclave_rti.number_of_enclaves == 0) { fprintf(stderr, "Error: --number_of_federates needs a valid positive integer argument.\n"); usage(argc, argv); return 0; } return 1; } + +void initialize_RTI(){ + _RTI = (federation_RTI_t *)malloc(sizeof(federation_RTI_t)); + // enclave_rti related initializations + _RTI->enclave_rti.max_stop_tag = NEVER_TAG, + _RTI->enclave_rti.number_of_enclaves = 0, + _RTI->enclave_rti.num_feds_handling_stop = 0, + // federation_rti related initializations + _RTI->max_start_time = 0LL, + _RTI->num_feds_proposed_start = 0, + _RTI->all_federates_exited = false, + _RTI->federation_id = "Unidentified Federation", + _RTI->user_specified_port = 0, + _RTI->final_port_TCP = 0, + _RTI->socket_descriptor_TCP = -1, + _RTI->final_port_UDP = UINT16_MAX, + _RTI->socket_descriptor_UDP = -1, + _RTI->clock_sync_global_status = clock_sync_init, + _RTI->clock_sync_period_ns = MSEC(10), + _RTI->clock_sync_exchanges_per_interval = 10, + _RTI->authentication_enabled = false, + _RTI->tracing_enabled = false; +} \ No newline at end of file diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index c223b84b9..cbf9633e3 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -513,6 +513,9 @@ int process_clock_sync_args(int argc, const char* argv[]); */ int process_args(int argc, const char* argv[]); - +/** + * Initialize the _RTI instance. + */ +void initialize_RTI(); #endif // RTI_LIB_H \ No newline at end of file From 9a7296373c417c16bfefc2535593b853dcc9479c Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 25 May 2023 17:31:15 -0700 Subject: [PATCH 10/25] Separte federates RTI (_F_RTI) from enclave RTI (_E_RTI) and get rid of all castings, except one. --- core/federated/RTI/enclave.c | 42 ++--- core/federated/RTI/rti.c | 35 ++-- core/federated/RTI/rti_lib.c | 330 +++++++++++++++++------------------ core/federated/RTI/rti_lib.h | 24 ++- 4 files changed, 229 insertions(+), 202 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index b054bc29a..201294854 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -1,7 +1,9 @@ #include "enclave.h" - -extern enclave_RTI_t* _RTI; +/** + * Reference to enclave_RTI_t instance. + */ +enclave_RTI_t* _E_RTI; // Global variables defined in tag.c: extern instant_t start_time; @@ -36,11 +38,11 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { lf_mutex_lock(&rti_mutex); enclave->completed = completed; - if ((enclave_RTI_t*)_RTI->tracing_enabled) { + if (_E_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_LTC, enclave->id, &(enclave->completed)); } - LF_PRINT_LOG("RTI received from federate %d the Logical Tag Complete (LTC) (%lld, %u).", + LF_PRINT_LOG("RTI received from federate %d the Logical Tag Complete (LTC) " PRINTF_TAG ".", enclave->id, enclave->completed.time - start_time, enclave->completed.microstep); // See if we can remove any of the recorded in-transit messages for this. @@ -50,12 +52,12 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { // Check downstream federates to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { // FIXME: Shouldn't use enclave_t here. - enclave_t *downstream = (enclave_RTI_t *)_RTI->enclaves[enclave->downstream[i]]; + enclave_t *downstream = _E_RTI->enclaves[enclave->downstream[i]]; // Notify downstream federate if appropriate. - notify_advance_grant_if_safe((enclave_t*)downstream); - bool *visited = (bool *)calloc((enclave_RTI_t *)_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + notify_advance_grant_if_safe(downstream); + bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Notify enclaves downstream of downstream if appropriate. - notify_downstream_advance_grant_if_safe((enclave_t*)downstream, visited); + notify_downstream_advance_grant_if_safe(downstream, visited); free(visited); } @@ -69,7 +71,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < e->num_upstream; j++) { - enclave_t *upstream = (enclave_RTI_t *)_RTI->enclaves[e->upstream[j]]; + enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) continue; @@ -80,7 +82,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { min_upstream_completed = candidate; } } - LF_PRINT_LOG("Minimum upstream LTC for fed %d is (%lld, %u) " + LF_PRINT_LOG("Minimum upstream LTC for fed %d is " PRINTF_TAG "(adjusted by after delay).", e->id, min_upstream_completed.time - start_time, min_upstream_completed.microstep); @@ -100,17 +102,17 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool *visited = (bool *)calloc((enclave_RTI_t *)_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the tag of the earliest possible incoming message from // upstream federates. tag_t t_d = FOREVER_TAG; - LF_PRINT_DEBUG("NOTE: FOREVER is displayed as (%lld, %u) and NEVER as (%lld, %u)", - FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, - NEVER - start_time, 0u); + LF_PRINT_DEBUG("NOTE: FOREVER is displayed as" PRINTF_TAG "and NEVER as " PRINTF_TAG, + FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, + NEVER - start_time, 0u); for (int j = 0; j < e->num_upstream; j++) { - enclave_t *upstream = (enclave_RTI_t *)_RTI->enclaves[e->upstream[j]]; + enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->state == NOT_CONNECTED) continue; @@ -119,7 +121,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_t upstream_next_event = transitive_next_event( upstream, upstream->next_event, visited); - LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag (%lld, %u).", + LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag " PRINTF_TAG ".", e->id, upstream->id, upstream_next_event.time - start_time, upstream_next_event.microstep); @@ -135,7 +137,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { } free(visited); - LF_PRINT_LOG("Earliest next event upstream has tag (%lld, %u).", + LF_PRINT_LOG("Earliest next event upstream has tag " PRINTF_TAG ".", t_d.time - start_time, t_d.microstep); if ( @@ -173,7 +175,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { visited[e->id] = true; for (int i = 0; i < e->num_downstream; i++) { - enclave_t* downstream = (enclave_t*)_RTI->enclaves[e->downstream[i]]; + enclave_t* downstream = _E_RTI->enclaves[e->downstream[i]]; if (visited[downstream->id]) continue; notify_advance_grant_if_safe(downstream); notify_downstream_advance_grant_if_safe(downstream, visited); @@ -199,7 +201,7 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool *visited = (bool *)calloc((enclave_RTI_t *)_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(e, visited); free(visited); } @@ -275,7 +277,7 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { // an event that would result in an earlier next event. for (int i = 0; i < e->num_upstream; i++) { tag_t upstream_result = transitive_next_event( - (enclave_RTI_t *)_RTI->enclaves[e->upstream[i]], result, visited); + _E_RTI->enclaves[e->upstream[i]], result, visited); // Add the "after" delay of the connection to the result. upstream_result = lf_delay_tag(upstream_result, e->upstream_delay[i]); diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index a5f047781..9a59a6baa 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -49,7 +49,13 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "rti_lib.h" unsigned int _lf_number_of_workers = 0u; -extern federation_RTI_t *_RTI; +/** + * References to the federation RTI and the enclave RTI. + * They both point to the same enclaves stuctures. In the case of federation RTI, + * however, enclaves are encapsulated in federates. + */ +extern enclave_RTI_t * _E_RTI; +extern federation_RTI_t* _F_RTI; extern lf_mutex_t rti_mutex; @@ -70,21 +76,28 @@ int main(int argc, const char* argv[]) { // Processing command-line arguments failed. return -1; } - if (_RTI->enclave_rti.tracing_enabled) { - _lf_number_of_workers = _RTI->enclave_rti.number_of_enclaves; + if (_F_RTI->tracing_enabled) { + _lf_number_of_workers = _F_RTI->number_of_enclaves; start_trace(rti_trace_file_name); printf("Tracing the RTI execution in %s file.\n", rti_trace_file_name); } - printf("Starting RTI for %d federates in federation ID %s\n", _RTI->enclave_rti.number_of_enclaves, _RTI->federation_id); - assert(_RTI->enclave_rti.number_of_enclaves < UINT16_MAX); - _RTI->enclave_rti.enclaves = (federate_t**)calloc(_RTI->enclave_rti.number_of_enclaves, sizeof(federate_t*)); - for (uint16_t i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { - _RTI->enclave_rti.enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); - initialize_federate((federate_t *)_RTI->enclave_rti.enclaves[i], i); + + printf("Starting RTI for %d federates in federation ID %s\n", _F_RTI->number_of_enclaves, _F_RTI->federation_id); + assert(_F_RTI->number_of_enclaves < UINT16_MAX); + + // Allocate memory for the federates + _F_RTI->enclaves = (federate_t**)calloc(_F_RTI->number_of_enclaves, sizeof(federate_t*)); + for (uint16_t i = 0; i < _F_RTI->number_of_enclaves; i++) { + _F_RTI->enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); + initialize_federate(_F_RTI->enclaves[i], i); } - int socket_descriptor = start_rti_server(_RTI->user_specified_port); + + // Initialize the RTI enclaves + _E_RTI = (enclave_RTI_t*)_F_RTI; + + int socket_descriptor = start_rti_server(_F_RTI->user_specified_port); wait_for_federates(socket_descriptor); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { stop_trace(); } printf("RTI is exiting.\n"); diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index eab69d06e..377347778 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -30,9 +30,9 @@ extern instant_t start_time; /** - * The state of this RTI instance. + * Reference to federate_RTI_t instance. */ -federation_RTI_t* _RTI; +federation_RTI_t *_F_RTI; lf_mutex_t rti_mutex; lf_cond_t received_start_times; @@ -157,16 +157,16 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty if (socket_type == UDP) { type = "UDP"; } - lf_print("RTI using %s port %d for federation %s.", type, port, _RTI->federation_id); + lf_print("RTI using %s port %d for federation %s.", type, port, _F_RTI->federation_id); if (socket_type == TCP) { - _RTI->final_port_TCP = port; + _F_RTI->final_port_TCP = port; // Enable listening for socket connections. // The second argument is the maximum number of queued socket requests, // which according to the Mac man page is limited to 128. listen(socket_descriptor, 128); } else if (socket_type == UDP) { - _RTI->final_port_UDP = port; + _F_RTI->final_port_UDP = port; // No need to listen on the UDP socket } @@ -192,7 +192,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -208,7 +208,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { } } else { e->last_granted = tag; - LF_PRINT_LOG("RTI sent to federate %d the tag advance grant (TAG) (%lld, %u).", + LF_PRINT_LOG("RTI sent to federate %d the tag advance grant (TAG) " PRINTF_TAG ".", e->id, tag.time - start_time, tag.microstep); } } @@ -232,7 +232,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_RTI->enclave_rti.tracing_enabled){ + if (_F_RTI->tracing_enabled){ tracepoint_RTI_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -249,8 +249,8 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { } } else { e->last_provisionally_granted = tag; - LF_PRINT_LOG("RTI sent to federate %d the Provisional Tag Advance Grant (PTAG) (%lld, %u).", - e->id, tag.time - start_time, tag.microstep); + LF_PRINT_LOG("RTI sent to federate %d the Provisional Tag Advance Grant (PTAG) " PRINTF_TAG ".", + e->id, tag.time - start_time, tag.microstep); // Send PTAG to all upstream federates, if they have not had // a later or equal PTAG or TAG sent previously and if their transitive @@ -259,24 +259,24 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { // we have an available encoding of causality interfaces. // That might be more efficient. for (int j = 0; j < e->num_upstream; j++) { - federate_t* upstream = (federate_t*)&_RTI->enclave_rti.enclaves[e->upstream[j]]; + federate_t* upstream = _F_RTI->enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->enclave.state == NOT_CONNECTED) continue; // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI->enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_F_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( - upstream, upstream->enclave.next_event, visited); + &(upstream->enclave), upstream->enclave.next_event, visited); free(visited); // If these tags are equal, then // a TAG or PTAG should have already been granted, // in which case, another will not be sent. But it // may not have been already granted. if (lf_tag_compare(upstream_next_event, tag) >= 0) { - notify_provisional_tag_advance_grant(&upstream->enclave, tag); + notify_provisional_tag_advance_grant(&(upstream->enclave), tag); } } @@ -284,7 +284,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { } void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { - federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[federate_id]; + federate_t* fed = _F_RTI->enclaves[federate_id]; tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(fed->in_transit_message_tags); if (lf_tag_compare( min_in_transit_tag, @@ -307,7 +307,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); tag_t tag = extract_tag(&(buffer[1 + 2 * sizeof(uint16_t)])); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_PORT_ABS, federate_id, &tag); } @@ -318,7 +318,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // If the destination federate is no longer connected, issue a warning // and return. - federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[federate_id]; + federate_t* fed = (federate_t*) _F_RTI->enclaves[federate_id]; if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", @@ -352,7 +352,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // Forward the message. int destination_socket = fed->socket; - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_PORT_ABS, federate_id, &tag); } write_to_socket_errexit(destination_socket, message_size + 1, buffer, @@ -398,7 +398,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // Following only works for string messages. // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_TAGGED_MSG, sending_federate->enclave.id, &intended_tag); } @@ -409,7 +409,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // If the destination federate is no longer connected, issue a warning // and return. - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[federate_id]; + federate_t *fed = _F_RTI->enclaves[federate_id]; if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", @@ -475,7 +475,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { lf_cond_wait(&sent_start_time); } - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_TAGGED_MSG, federate_id, &intended_tag); } @@ -513,7 +513,7 @@ void handle_logical_tag_complete(federate_t* fed) { read_from_socket_errexit(fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, "RTI failed to read the content of the logical tag complete from federate %d.", fed->enclave.id); tag_t completed = extract_tag(buffer); - logical_tag_complete(&fed->enclave, completed); + logical_tag_complete(&(fed->enclave), completed); // FIXME: Should this function be in the enclave version? lf_mutex_lock(&rti_mutex); @@ -536,7 +536,7 @@ void handle_next_event_tag(federate_t* fed) { tag_t intended_tag = extract_tag(buffer); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_NET, fed->enclave.id, &intended_tag); } LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) " PRINTF_TAG, @@ -563,28 +563,28 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { } // Reply with a stop granted to all federates unsigned char outgoing_buffer[MSG_TYPE_STOP_GRANTED_LENGTH]; - ENCODE_STOP_GRANTED(outgoing_buffer, _RTI->enclave_rti.max_stop_tag.time, _RTI->enclave_rti.max_stop_tag.microstep); + ENCODE_STOP_GRANTED(outgoing_buffer, _F_RTI->max_stop_tag.time, _F_RTI->max_stop_tag.microstep); // Iterate over federates and send each the message. - for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[i]; + for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { + federate_t *fed = _F_RTI->enclaves[i]; if (fed->enclave.state == NOT_CONNECTED) { continue; } - if (lf_tag_compare(fed->enclave.next_event, _RTI->enclave_rti.max_stop_tag) >= 0) { + if (lf_tag_compare(fed->enclave.next_event, _F_RTI->max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. - fed->enclave.next_event = _RTI->enclave_rti.max_stop_tag; + fed->enclave.next_event = _F_RTI->max_stop_tag; } - if (_RTI->enclave_rti.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_RTI->enclave_rti.max_stop_tag); + if (_F_RTI->tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_F_RTI->max_stop_tag); } write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); } LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag (" PRINTF_TIME ", %u).", - _RTI->enclave_rti.max_stop_tag.time - start_time, - _RTI->enclave_rti.max_stop_tag.microstep); + _F_RTI->max_stop_tag.time - start_time, + _F_RTI->max_stop_tag.microstep); _lf_rti_stop_granted_already_sent_to_federates = true; } @@ -592,10 +592,10 @@ void mark_federate_requesting_stop(federate_t* fed) { if (!fed->enclave.requested_stop) { // Assume that the federate // has requested stop - _RTI->enclave_rti.num_feds_handling_stop++; + _F_RTI->num_feds_handling_stop++; fed->enclave.requested_stop = true; } - if (_RTI->enclave_rti.num_feds_handling_stop == _RTI->enclave_rti.number_of_enclaves) { + if (_F_RTI->num_feds_handling_stop == _F_RTI->number_of_enclaves) { // We now have information about the stop time of all // federates. _lf_rti_broadcast_stop_time_to_federates_already_locked(); @@ -625,23 +625,23 @@ void handle_stop_request_message(federate_t* fed) { // Extract the proposed stop tag for the federate tag_t proposed_stop_tag = extract_tag(buffer); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ, fed->enclave.id, &proposed_stop_tag); } // Update the maximum stop tag received from federates - if (lf_tag_compare(proposed_stop_tag, _RTI->enclave_rti.max_stop_tag) > 0) { - _RTI->enclave_rti.max_stop_tag = proposed_stop_tag; + if (lf_tag_compare(proposed_stop_tag, _F_RTI->max_stop_tag) > 0) { + _F_RTI->max_stop_tag = proposed_stop_tag; } - LF_PRINT_LOG("RTI received from federate %d a MSG_TYPE_STOP_REQUEST message with tag (%lld, %u).", + LF_PRINT_LOG("RTI received from federate %d a MSG_TYPE_STOP_REQUEST message with tag " PRINTF_TAG ".", fed->enclave.id, proposed_stop_tag.time - start_time, proposed_stop_tag.microstep); // If this federate has not already asked // for a stop, add it to the tally. mark_federate_requesting_stop(fed); - if (_RTI->enclave_rti.num_feds_handling_stop == _RTI->enclave_rti.number_of_enclaves) { + if (_F_RTI->num_feds_handling_stop == _F_RTI->number_of_enclaves) { // We now have information about the stop time of all // federates. This is extremely unlikely, but it can occur // all federates call lf_request_stop() at the same tag. @@ -651,30 +651,30 @@ void handle_stop_request_message(federate_t* fed) { // Forward the stop request to all other federates that have not // also issued a stop request. unsigned char stop_request_buffer[MSG_TYPE_STOP_REQUEST_LENGTH]; - ENCODE_STOP_REQUEST(stop_request_buffer, _RTI->enclave_rti.max_stop_tag.time, _RTI->enclave_rti.max_stop_tag.microstep); + ENCODE_STOP_REQUEST(stop_request_buffer, _F_RTI->max_stop_tag.time, _F_RTI->max_stop_tag.microstep); // Iterate over federates and send each the MSG_TYPE_STOP_REQUEST message // if we do not have a stop_time already for them. - for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[i]; + for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { + federate_t *fed = _F_RTI->enclaves[i]; if (fed->enclave.id != fed->enclave.id && fed->enclave.requested_stop == false) { if (fed->enclave.state == NOT_CONNECTED) { mark_federate_requesting_stop(fed); continue; } - if (_RTI->enclave_rti.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI->enclave_rti.max_stop_tag); + if (_F_RTI->tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_F_RTI->max_stop_tag); } write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", fed->enclave.id); - if (_RTI->enclave_rti.tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_RTI->enclave_rti.max_stop_tag); + if (_F_RTI->tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_F_RTI->max_stop_tag); } } } LF_PRINT_LOG("RTI forwarded to federates MSG_TYPE_STOP_REQUEST with tag (" PRINTF_TIME ", %u).", - _RTI->enclave_rti.max_stop_tag.time - start_time, - _RTI->enclave_rti.max_stop_tag.microstep); + _F_RTI->max_stop_tag.time - start_time, + _F_RTI->max_stop_tag.microstep); lf_mutex_unlock(&rti_mutex); } @@ -686,19 +686,19 @@ void handle_stop_request_reply(federate_t* fed) { tag_t federate_stop_tag = extract_tag(buffer_stop_time); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ_REP, fed->enclave.id, &federate_stop_tag); } - LF_PRINT_LOG("RTI received from federate %d STOP reply tag (%lld, %u).", fed->enclave.id, + LF_PRINT_LOG("RTI received from federate %d STOP reply tag " PRINTF_TAG ".", fed->enclave.id, federate_stop_tag.time - start_time, federate_stop_tag.microstep); // Acquire the mutex lock so that we can change the state of the RTI lf_mutex_lock(&rti_mutex); // If the federate has not requested stop before, count the reply - if (lf_tag_compare(federate_stop_tag, _RTI->enclave_rti.max_stop_tag) > 0) { - _RTI->enclave_rti.max_stop_tag = federate_stop_tag; + if (lf_tag_compare(federate_stop_tag, _F_RTI->max_stop_tag) > 0) { + _F_RTI->max_stop_tag = federate_stop_tag; } mark_federate_requesting_stop(fed); lf_mutex_unlock(&rti_mutex); @@ -707,7 +707,7 @@ void handle_stop_request_reply(federate_t* fed) { ////////////////////////////////////////////////// void handle_address_query(uint16_t fed_id) { - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[fed_id]; + federate_t *fed = _F_RTI->enclaves[fed_id]; // Use buffer both for reading and constructing the reply. // The length is what is needed for the reply. unsigned char buffer[sizeof(int32_t)]; @@ -717,7 +717,7 @@ void handle_address_query(uint16_t fed_id) { } uint16_t remote_fed_id = extract_uint16(buffer); - if (_RTI->enclave_rti.tracing_enabled){ + if (_F_RTI->tracing_enabled){ tracepoint_RTI_from_federate(receive_ADR_QR, fed_id, NULL); } @@ -728,7 +728,7 @@ void handle_address_query(uint16_t fed_id) { // from this federate. In that case, it will respond by sending -1. // Encode the port number. - federate_t *remote_fed = (federate_t *)_RTI->enclave_rti.enclaves[remote_fed_id]; + federate_t *remote_fed = _F_RTI->enclaves[remote_fed_id]; encode_int32(remote_fed->server_port, (unsigned char*)buffer); // Send the port number (which could be -1). write_to_socket_errexit(fed->socket, sizeof(int32_t), (unsigned char*)buffer, @@ -746,7 +746,7 @@ void handle_address_query(uint16_t fed_id) { } void handle_address_ad(uint16_t federate_id) { - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[federate_id]; + federate_t *fed = _F_RTI->enclaves[federate_id]; // Read the port number of the federate that can be used for physical // connections to other federates int32_t server_port = -1; @@ -765,7 +765,7 @@ void handle_address_ad(uint16_t federate_id) { lf_mutex_lock(&rti_mutex); fed->server_port = server_port; - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_ADR_AD, federate_id, NULL); } LF_PRINT_LOG("Received address advertisement from federate %d.", federate_id); @@ -781,24 +781,24 @@ void handle_timestamp(federate_t *my_fed) { } int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t *)(&buffer))); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tag_t tag = {.time = timestamp, .microstep = 0}; tracepoint_RTI_from_federate(receive_TIMESTAMP, my_fed->enclave.id, &tag); } - LF_PRINT_LOG("RTI received timestamp message: %lld.", timestamp); + LF_PRINT_LOG("RTI received timestamp message: %ld.", timestamp); lf_mutex_lock(&rti_mutex); - _RTI->num_feds_proposed_start++; - if (timestamp > _RTI->max_start_time) { - _RTI->max_start_time = timestamp; + _F_RTI->num_feds_proposed_start++; + if (timestamp > _F_RTI->max_start_time) { + _F_RTI->max_start_time = timestamp; } - if (_RTI->num_feds_proposed_start == _RTI->enclave_rti.number_of_enclaves) { + if (_F_RTI->num_feds_proposed_start == _F_RTI->number_of_enclaves) { // All federates have proposed a start time. lf_cond_broadcast(&received_start_times); } else { // Some federates have not yet proposed a start time. // wait for a notification. - while (_RTI->num_feds_proposed_start < _RTI->enclave_rti.number_of_enclaves) { + while (_F_RTI->num_feds_proposed_start < _F_RTI->number_of_enclaves) { // FIXME: Should have a timeout here? lf_cond_wait(&received_start_times); } @@ -811,10 +811,10 @@ void handle_timestamp(federate_t *my_fed) { unsigned char start_time_buffer[MSG_TYPE_TIMESTAMP_LENGTH]; start_time_buffer[0] = MSG_TYPE_TIMESTAMP; // Add an offset to this start time to get everyone starting together. - start_time = _RTI->max_start_time + DELAY_START; + start_time = _F_RTI->max_start_time + DELAY_START; encode_int64(swap_bytes_if_big_endian_int64(start_time), &start_time_buffer[1]); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tag_t tag = {.time = start_time, .microstep = 0}; tracepoint_RTI_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } @@ -832,7 +832,7 @@ void handle_timestamp(federate_t *my_fed) { // the federate to the start time. my_fed->enclave.state = GRANTED; lf_cond_broadcast(&sent_start_time); - LF_PRINT_LOG("RTI sent start time %lld to federate %d.", start_time, my_fed->enclave.id); + LF_PRINT_LOG("RTI sent start time %ld to federate %d.", start_time, my_fed->enclave.id); lf_mutex_unlock(&rti_mutex); } @@ -851,7 +851,7 @@ void send_physical_clock(unsigned char message_type, federate_t* fed, socket_typ if (socket_type == UDP) { // FIXME: UDP_addr is never initialized. LF_PRINT_DEBUG("Clock sync: RTI sending UDP message type %u.", buffer[0]); - ssize_t bytes_written = sendto(_RTI->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, + ssize_t bytes_written = sendto(_F_RTI->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr)); if (bytes_written < (ssize_t)sizeof(int64_t) + 1) { lf_print_warning("Clock sync: RTI failed to send physical time to federate %d: %s\n", @@ -866,7 +866,7 @@ void send_physical_clock(unsigned char message_type, federate_t* fed, socket_typ fed->enclave.id, strerror(errno)); } - LF_PRINT_DEBUG("Clock sync: RTI sent PHYSICAL_TIME_SYNC_MESSAGE with timestamp %lld to federate %d.", + LF_PRINT_DEBUG("Clock sync: RTI sent PHYSICAL_TIME_SYNC_MESSAGE with timestamp %ld to federate %d.", current_physical_time, fed->enclave.id); } @@ -890,7 +890,7 @@ void* clock_synchronization_thread(void* noargs) { // Wait until all federates have been notified of the start time. // FIXME: Use lf_ version of this when merged with master. lf_mutex_lock(&rti_mutex); - while (_RTI->num_feds_proposed_start < _RTI->enclave_rti.number_of_enclaves) { + while (_F_RTI->num_feds_proposed_start < _F_RTI->number_of_enclaves) { lf_cond_wait(&received_start_times); } lf_mutex_unlock(&rti_mutex); @@ -905,9 +905,9 @@ void* clock_synchronization_thread(void* noargs) { nanosleep(&wait_time, &rem_time); } - // Initiate a clock synchronization every _RTI->clock_sync_period_ns - struct timespec sleep_time = {(time_t) _RTI->clock_sync_period_ns / BILLION, - _RTI->clock_sync_period_ns % BILLION}; + // Initiate a clock synchronization every _F_RTI->clock_sync_period_ns + struct timespec sleep_time = {(time_t) _F_RTI->clock_sync_period_ns / BILLION, + _F_RTI->clock_sync_period_ns % BILLION}; struct timespec remaining_time; bool any_federates_connected = true; @@ -915,8 +915,8 @@ void* clock_synchronization_thread(void* noargs) { // Sleep nanosleep(&sleep_time, &remaining_time); // Can be interrupted any_federates_connected = false; - for (int fed_id = 0; fed_id < _RTI->enclave_rti.number_of_enclaves; fed_id++) { - federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[fed_id]; + for (int fed_id = 0; fed_id < _F_RTI->number_of_enclaves; fed_id++) { + federate_t* fed = _F_RTI->enclaves[fed_id]; if (fed->enclave.state == NOT_CONNECTED) { // FIXME: We need better error handling here, but clock sync failure // should not stop execution. @@ -929,7 +929,7 @@ void* clock_synchronization_thread(void* noargs) { // Send the RTI's current physical time to the federate // Send on UDP. LF_PRINT_DEBUG("RTI sending T1 message to initiate clock sync round."); - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &fed, UDP); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, UDP); // Listen for reply message, which should be T3. size_t message_size = 1 + sizeof(int32_t); @@ -940,7 +940,7 @@ void* clock_synchronization_thread(void* noargs) { int remaining_attempts = 5; while (remaining_attempts > 0) { remaining_attempts--; - int bytes_read = read_from_socket(_RTI->socket_descriptor_UDP, message_size, buffer); + int bytes_read = read_from_socket(_F_RTI->socket_descriptor_UDP, message_size, buffer); // If any errors occur, either discard the message or the clock sync round. if (bytes_read == message_size) { if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { @@ -954,7 +954,7 @@ void* clock_synchronization_thread(void* noargs) { continue; } LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id_2); - handle_physical_clock_sync_message((federate_t *)_RTI->enclave_rti.enclaves[fed_id_2], UDP); + handle_physical_clock_sync_message(_F_RTI->enclaves[fed_id_2], UDP); break; } else { // The message is not a T3 message. Discard the message and @@ -986,7 +986,7 @@ void* clock_synchronization_thread(void* noargs) { void handle_federate_resign(federate_t *my_fed) { // Nothing more to do. Close the socket and exit. lf_mutex_lock(&rti_mutex); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { // Extract the tag, for tracing purposes size_t header_size = 1 + sizeof(tag_t); unsigned char buffer[header_size]; @@ -1019,8 +1019,8 @@ void handle_federate_resign(federate_t *my_fed) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_RTI->enclave_rti.number_of_enclaves, sizeof(bool)); // Initializes to 0. - notify_downstream_advance_grant_if_safe(&my_fed->enclave, visited); + bool* visited = (bool*)calloc(_F_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + notify_downstream_advance_grant_if_safe(&(my_fed->enclave), visited); free(visited); lf_mutex_unlock(&rti_mutex); @@ -1085,7 +1085,7 @@ void* federate_thread_TCP(void* fed) { break; default: lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->enclave.id, buffer[0]); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_UNIDENTIFIED, my_fed->enclave.id, NULL); } } @@ -1117,7 +1117,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // FIXME: This should not exit with error but rather should just reject the connection. read_from_socket_errexit(socket_id, length, buffer, "RTI failed to read from accepted socket."); - uint16_t fed_id = _RTI->enclave_rti.number_of_enclaves; // Initialize to an invalid value. + uint16_t fed_id = _F_RTI->number_of_enclaves; // Initialize to an invalid value. // First byte received is the message type. if (buffer[0] != MSG_TYPE_FED_IDS) { @@ -1133,7 +1133,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } else { send_reject(socket_id, UNEXPECTED_MESSAGE); } - if (_RTI->enclave_rti.tracing_enabled){ + if (_F_RTI->tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); @@ -1157,33 +1157,33 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI received federation ID: %s.", federation_id_received); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_from_federate(receive_FED_ID, fed_id, NULL); } // Compare the received federation ID to mine. - if (strncmp(_RTI->federation_id, federation_id_received, federation_id_length) != 0) { + if (strncmp(_F_RTI->federation_id, federation_id_received, federation_id_length) != 0) { // Federation IDs do not match. Send back a MSG_TYPE_REJECT message. lf_print_error("WARNING: Federate from another federation %s attempted to connect to RTI in federation %s.\n", federation_id_received, - _RTI->federation_id); - if (_RTI->enclave_rti.tracing_enabled) { + _F_RTI->federation_id); + if (_F_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH); return -1; } else { - if (fed_id >= _RTI->enclave_rti.number_of_enclaves) { + if (fed_id >= _F_RTI->number_of_enclaves) { // Federate ID is out of range. lf_print_error("RTI received federate ID %d, which is out of range.", fed_id); - if (_RTI->enclave_rti.tracing_enabled){ + if (_F_RTI->tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { - if (_RTI->enclave_rti.enclaves[fed_id]->state != NOT_CONNECTED) { + if ((_F_RTI->enclaves[fed_id])->enclave.state != NOT_CONNECTED) { lf_print_error("RTI received duplicate federate ID: %d.", fed_id); - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_IN_USE); @@ -1192,7 +1192,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } } } - federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[fed_id]; + federate_t* fed = _F_RTI->enclaves[fed_id]; // The MSG_TYPE_FED_IDS message has the right federation ID. // Assign the address information for federate. // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful @@ -1221,7 +1221,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); // Send an MSG_TYPE_ACK message. unsigned char ack_message = MSG_TYPE_ACK; - if (_RTI->enclave_rti.tracing_enabled) { + if (_F_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_ACK, fed_id, NULL); } write_to_socket_errexit(socket_id, 1, &ack_message, @@ -1247,7 +1247,7 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { - federate_t* fed = (federate_t*) _RTI->enclave_rti.enclaves[fed_id]; + federate_t* fed = _F_RTI->enclaves[fed_id]; // Read the number of upstream and downstream connections fed->enclave.num_upstream = extract_int32(&(connection_info_header[1])); fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); @@ -1313,8 +1313,8 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[fed_id]; - if (_RTI->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. + federate_t *fed = _F_RTI->enclaves[fed_id]; + if (_F_RTI->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. uint16_t federate_UDP_port_number = extract_uint16(&(response[1])); LF_PRINT_DEBUG("RTI got MSG_TYPE_UDP_PORT %u from federate %d.", federate_UDP_port_number, fed_id); @@ -1323,9 +1323,9 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { if (federate_UDP_port_number != UINT16_MAX) { // Perform the initialization clock synchronization with the federate. // Send the required number of messages for clock synchronization - for (int i=0; i < _RTI->clock_sync_exchanges_per_interval; i++) { + for (int i=0; i < _F_RTI->clock_sync_exchanges_per_interval; i++) { // Send the RTI's current physical time T1 to the federate. - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, &fed, TCP); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, TCP); // Listen for reply message, which should be T3. size_t message_size = 1 + sizeof(int32_t); @@ -1337,7 +1337,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { assert(fed_id > -1); assert(fed_id < 65536); LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id); - handle_physical_clock_sync_message(&fed, TCP); + handle_physical_clock_sync_message(fed, TCP); } else { lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id); send_reject(socket_id, UNEXPECTED_MESSAGE); @@ -1346,7 +1346,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { } LF_PRINT_DEBUG("RTI finished initial clock synchronization with federate %d.", fed_id); } - if (_RTI->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. + if (_F_RTI->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. if (federate_UDP_port_number > 0) { // Initialize the UDP_addr field of the federate struct fed->UDP_addr.sin_family = AF_INET; @@ -1381,7 +1381,7 @@ bool authenticate_federate(int socket) { // Check HMAC of received FED_RESPONSE message. size_t hmac_length = SHA256_HMAC_LENGTH; - size_t federation_id_length = strnlen(_RTI->federation_id, 255); + size_t federation_id_length = strnlen(_F_RTI->federation_id, 255); size_t fed_id_length = sizeof(uint16_t); unsigned char received[1 + NONCE_LENGTH + fed_id_length + hmac_length]; @@ -1398,7 +1398,7 @@ bool authenticate_federate(int socket) { memcpy(&buf_to_check[1], &received[1 + NONCE_LENGTH], fed_id_length); memcpy(&buf_to_check[1 + fed_id_length], rti_nonce, NONCE_LENGTH); unsigned char rti_tag[hmac_length]; - HMAC(EVP_sha256(), _RTI->federation_id, federation_id_length, buf_to_check, 1 + fed_id_length + NONCE_LENGTH, + HMAC(EVP_sha256(), _F_RTI->federation_id, federation_id_length, buf_to_check, 1 + fed_id_length + NONCE_LENGTH, rti_tag, &hmac_length); // Compare received tag and created tag. @@ -1417,7 +1417,7 @@ bool authenticate_federate(int socket) { // Buffer for message type and HMAC tag. unsigned char sender[1 + hmac_length]; sender[0] = MSG_TYPE_RTI_RESPONSE; - HMAC(EVP_sha256(), _RTI->federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, + HMAC(EVP_sha256(), _F_RTI->federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1], &hmac_length); write_to_socket(socket, 1 + hmac_length, sender); return true; @@ -1426,14 +1426,14 @@ bool authenticate_federate(int socket) { #endif void connect_to_federates(int socket_descriptor) { - for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { + for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { // Wait for an incoming connection request. struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); // The following blocks until a federate connects. int socket_id = -1; while(1) { - socket_id = accept(_RTI->socket_descriptor_TCP, &client_fd, &client_length); + socket_id = accept(_F_RTI->socket_descriptor_TCP, &client_fd, &client_length); if (socket_id >= 0) { // Got a socket break; @@ -1448,7 +1448,7 @@ void connect_to_federates(int socket_descriptor) { // Send RTI hello when RTI -a option is on. #ifdef __RTI_AUTH__ - if (_RTI->authentication_enabled) { + if (_F_RTI->authentication_enabled) { if (!authenticate_federate(socket_id)) { lf_print_warning("RTI failed to authenticate the incoming federate."); // Ignore the federate that failed authentication. @@ -1468,7 +1468,7 @@ void connect_to_federates(int socket_descriptor) { // This has to be done after clock synchronization is finished // or that thread may end up attempting to handle incoming clock // synchronization messages. - federate_t *fed = (federate_t *)_RTI->enclave_rti.enclaves[fed_id]; + federate_t *fed = _F_RTI->enclaves[fed_id]; lf_thread_create(&(fed->thread_id), federate_thread_TCP, fed); } else { @@ -1479,19 +1479,19 @@ void connect_to_federates(int socket_descriptor) { // All federates have connected. LF_PRINT_DEBUG("All federates have connected to RTI."); - if (_RTI->clock_sync_global_status >= clock_sync_on) { + if (_F_RTI->clock_sync_global_status >= clock_sync_on) { // Create the thread that performs periodic PTP clock synchronization sessions // over the UDP channel, but only if the UDP channel is open and at least one // federate is performing runtime clock synchronization. bool clock_sync_enabled = false; - for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { - if (((federate_t *)_RTI->enclave_rti.enclaves[i])->clock_synchronization_enabled) { + for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { + if ((_F_RTI->enclaves[i])->clock_synchronization_enabled) { clock_sync_enabled = true; break; } } - if (_RTI->final_port_UDP != UINT16_MAX && clock_sync_enabled) { - lf_thread_create(&_RTI->clock_thread, clock_synchronization_thread, NULL); + if (_F_RTI->final_port_UDP != UINT16_MAX && clock_sync_enabled) { + lf_thread_create(&_F_RTI->clock_thread, clock_synchronization_thread, NULL); } } } @@ -1502,11 +1502,11 @@ void* respond_to_erroneous_connections(void* nothing) { struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); // The following will block until either a federate attempts to connect - // or close(_RTI->socket_descriptor_TCP) is called. - int socket_id = accept(_RTI->socket_descriptor_TCP, &client_fd, &client_length); + // or close(_F_RTI->socket_descriptor_TCP) is called. + int socket_id = accept(_F_RTI->socket_descriptor_TCP, &client_fd, &client_length); if (socket_id < 0) return NULL; - if (_RTI->all_federates_exited) { + if (_F_RTI->all_federates_exited) { return NULL; } @@ -1524,7 +1524,7 @@ void* respond_to_erroneous_connections(void* nothing) { } void initialize_federate(federate_t* fed, uint16_t id) { - initialize_enclave(&fed->enclave, id); + initialize_enclave(&(fed->enclave), id); fed->socket = -1; // No socket. fed->clock_synchronization_enabled = true; fed->in_transit_message_tags = initialize_in_transit_message_q(); @@ -1541,14 +1541,14 @@ int32_t start_rti_server(uint16_t port) { } lf_initialize_clock(); // Create the TCP socket server - _RTI->socket_descriptor_TCP = create_server(specified_port, port, TCP); + _F_RTI->socket_descriptor_TCP = create_server(specified_port, port, TCP); lf_print("RTI: Listening for federates."); // Create the UDP socket server - // Try to get the _RTI->final_port_TCP + 1 port - if (_RTI->clock_sync_global_status >= clock_sync_on) { - _RTI->socket_descriptor_UDP = create_server(specified_port, _RTI->final_port_TCP + 1, UDP); + // Try to get the _F_RTI->final_port_TCP + 1 port + if (_F_RTI->clock_sync_global_status >= clock_sync_on) { + _F_RTI->socket_descriptor_UDP = create_server(specified_port, _F_RTI->final_port_TCP + 1, UDP); } - return _RTI->socket_descriptor_TCP; + return _F_RTI->socket_descriptor_TCP; } void wait_for_federates(int socket_descriptor) { @@ -1567,19 +1567,19 @@ void wait_for_federates(int socket_descriptor) { // Wait for federate threads to exit. void* thread_exit_status; - for (int i = 0; i < _RTI->enclave_rti.number_of_enclaves; i++) { - federate_t* fed = (federate_t *)_RTI->enclave_rti.enclaves[i]; + for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { + federate_t* fed = _F_RTI->enclaves[i]; lf_print("RTI: Waiting for thread handling federate %d.", fed->enclave.id); lf_thread_join(fed->thread_id, &thread_exit_status); free_in_transit_message_q(fed->in_transit_message_tags); lf_print("RTI: Federate %d thread exited.", fed->enclave.id); } - _RTI->all_federates_exited = true; + _F_RTI->all_federates_exited = true; // Shutdown and close the socket so that the accept() call in // respond_to_erroneous_connections returns. That thread should then - // check _RTI->all_federates_exited and it should exit. + // check _F_RTI->all_federates_exited and it should exit. if (shutdown(socket_descriptor, SHUT_RDWR)) { LF_PRINT_LOG("On shut down TCP socket, received reply: %s", strerror(errno)); } @@ -1590,7 +1590,7 @@ void wait_for_federates(int socket_descriptor) { // NOTE: Apparently, closing the socket will not necessarily // cause the respond_to_erroneous_connections accept() call to return, - // so instead, we connect here so that it can check the _RTI->all_federates_exited + // so instead, we connect here so that it can check the _F_RTI->all_federates_exited // variable. // Create an IPv4 socket for TCP (not UDP) communication over IP (0). @@ -1609,7 +1609,7 @@ void wait_for_federates(int socket_descriptor) { (char *)&server_fd.sin_addr.s_addr, server->h_length); // Convert the port number from host byte order to network byte order. - server_fd.sin_port = htons(_RTI->final_port_TCP); + server_fd.sin_port = htons(_F_RTI->final_port_TCP); connect( tmp_socket, (struct sockaddr *)&server_fd, @@ -1626,11 +1626,11 @@ void wait_for_federates(int socket_descriptor) { close(socket_descriptor); */ - if (_RTI->socket_descriptor_UDP > 0) { - if (shutdown(_RTI->socket_descriptor_UDP, SHUT_RDWR)) { + if (_F_RTI->socket_descriptor_UDP > 0) { + if (shutdown(_F_RTI->socket_descriptor_UDP, SHUT_RDWR)) { LF_PRINT_LOG("On shut down UDP socket, received reply: %s", strerror(errno)); } - close(_RTI->socket_descriptor_UDP); + close(_F_RTI->socket_descriptor_UDP); } } @@ -1665,16 +1665,16 @@ void usage(int argc, const char* argv[]) { int process_clock_sync_args(int argc, const char* argv[]) { for (int i = 0; i < argc; i++) { if (strcmp(argv[i], "off") == 0) { - _RTI->clock_sync_global_status = clock_sync_off; + _F_RTI->clock_sync_global_status = clock_sync_off; printf("RTI: Clock sync: off\n"); } else if (strcmp(argv[i], "init") == 0 || strcmp(argv[i], "initial") == 0) { - _RTI->clock_sync_global_status = clock_sync_init; + _F_RTI->clock_sync_global_status = clock_sync_init; printf("RTI: Clock sync: init\n"); } else if (strcmp(argv[i], "on") == 0) { - _RTI->clock_sync_global_status = clock_sync_on; + _F_RTI->clock_sync_global_status = clock_sync_on; printf("RTI: Clock sync: on\n"); } else if (strcmp(argv[i], "period") == 0) { - if (_RTI->clock_sync_global_status != clock_sync_on) { + if (_F_RTI->clock_sync_global_status != clock_sync_on) { fprintf(stderr, "Error: clock sync period can only be set if --clock-sync is set to on.\n"); usage(argc, argv); i++; @@ -1690,10 +1690,10 @@ int process_clock_sync_args(int argc, const char* argv[]) { fprintf(stderr, "Error: clock sync period value is invalid.\n"); continue; // Try to parse the rest of the arguments as clock sync args. } - _RTI->clock_sync_period_ns = (int64_t)period_ns; - printf("RTI: Clock sync period: %lld\n", (long long int)_RTI->clock_sync_period_ns); + _F_RTI->clock_sync_period_ns = (int64_t)period_ns; + printf("RTI: Clock sync period: %lld\n", (long long int)_F_RTI->clock_sync_period_ns); } else if (strcmp(argv[i], "exchanges-per-interval") == 0) { - if (_RTI->clock_sync_global_status != clock_sync_on && _RTI->clock_sync_global_status != clock_sync_init) { + if (_F_RTI->clock_sync_global_status != clock_sync_on && _F_RTI->clock_sync_global_status != clock_sync_init) { fprintf(stderr, "Error: clock sync exchanges-per-interval can only be set if\n"); fprintf(stderr, "--clock-sync is set to on or init.\n"); usage(argc, argv); @@ -1709,8 +1709,8 @@ int process_clock_sync_args(int argc, const char* argv[]) { fprintf(stderr, "Error: clock sync exchanges-per-interval value is invalid.\n"); continue; // Try to parse the rest of the arguments as clock sync args. } - _RTI->clock_sync_exchanges_per_interval = (int32_t)exchanges; // FIXME: Loses numbers on 64-bit machines - printf("RTI: Clock sync exchanges per interval: %d\n", _RTI->clock_sync_exchanges_per_interval); + _F_RTI->clock_sync_exchanges_per_interval = (int32_t)exchanges; // FIXME: Loses numbers on 64-bit machines + printf("RTI: Clock sync exchanges per interval: %d\n", _F_RTI->clock_sync_exchanges_per_interval); } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1734,7 +1734,7 @@ int process_args(int argc, const char* argv[]) { } i++; printf("RTI: Federation ID: %s\n", argv[i]); - _RTI->federation_id = argv[i]; + _F_RTI->federation_id = argv[i]; } else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--number_of_federates") == 0) { if (argc < i + 2) { fprintf(stderr, "Error: --number_of_federates needs an integer argument.\n"); @@ -1748,8 +1748,8 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _RTI->enclave_rti.number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines - printf("RTI: Number of federates: %d\n", _RTI->enclave_rti.number_of_enclaves); + _F_RTI->number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines + printf("RTI: Number of federates: %d\n", _F_RTI->number_of_enclaves); } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) { if (argc < i + 2) { fprintf( @@ -1771,7 +1771,7 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _RTI->user_specified_port = (uint16_t)RTI_port; + _F_RTI->user_specified_port = (uint16_t)RTI_port; } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--clock_sync") == 0) { if (argc < i + 2) { fprintf(stderr, "Error: --clock-sync needs off|init|on.\n"); @@ -1781,9 +1781,9 @@ int process_args(int argc, const char* argv[]) { i++; i += process_clock_sync_args((argc-i), &argv[i]); } else if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--auth") == 0) { - _RTI->authentication_enabled = true; + _F_RTI->authentication_enabled = true; } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { - _RTI->enclave_rti.tracing_enabled = true; + _F_RTI->tracing_enabled = true; } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1793,7 +1793,7 @@ int process_args(int argc, const char* argv[]) { return 0; } } - if (_RTI->enclave_rti.number_of_enclaves == 0) { + if (_F_RTI->number_of_enclaves == 0) { fprintf(stderr, "Error: --number_of_federates needs a valid positive integer argument.\n"); usage(argc, argv); return 0; @@ -1802,24 +1802,24 @@ int process_args(int argc, const char* argv[]) { } void initialize_RTI(){ - _RTI = (federation_RTI_t *)malloc(sizeof(federation_RTI_t)); + _F_RTI = (federation_RTI_t *)malloc(sizeof(federation_RTI_t)); // enclave_rti related initializations - _RTI->enclave_rti.max_stop_tag = NEVER_TAG, - _RTI->enclave_rti.number_of_enclaves = 0, - _RTI->enclave_rti.num_feds_handling_stop = 0, + _F_RTI->max_stop_tag = NEVER_TAG, + _F_RTI->number_of_enclaves = 0, + _F_RTI->num_feds_handling_stop = 0, // federation_rti related initializations - _RTI->max_start_time = 0LL, - _RTI->num_feds_proposed_start = 0, - _RTI->all_federates_exited = false, - _RTI->federation_id = "Unidentified Federation", - _RTI->user_specified_port = 0, - _RTI->final_port_TCP = 0, - _RTI->socket_descriptor_TCP = -1, - _RTI->final_port_UDP = UINT16_MAX, - _RTI->socket_descriptor_UDP = -1, - _RTI->clock_sync_global_status = clock_sync_init, - _RTI->clock_sync_period_ns = MSEC(10), - _RTI->clock_sync_exchanges_per_interval = 10, - _RTI->authentication_enabled = false, - _RTI->tracing_enabled = false; + _F_RTI->max_start_time = 0LL, + _F_RTI->num_feds_proposed_start = 0, + _F_RTI->all_federates_exited = false, + _F_RTI->federation_id = "Unidentified Federation", + _F_RTI->user_specified_port = 0, + _F_RTI->final_port_TCP = 0, + _F_RTI->socket_descriptor_TCP = -1, + _F_RTI->final_port_UDP = UINT16_MAX, + _F_RTI->socket_descriptor_UDP = -1, + _F_RTI->clock_sync_global_status = clock_sync_init, + _F_RTI->clock_sync_period_ns = MSEC(10), + _F_RTI->clock_sync_exchanges_per_interval = 10, + _F_RTI->authentication_enabled = false, + _F_RTI->tracing_enabled = false; } \ No newline at end of file diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index cbf9633e3..9cec37e5b 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -80,7 +80,24 @@ typedef enum clock_sync_stat { */ typedef struct federation_RTI_t { // Enclave RTI - enclave_RTI_t enclave_rti; + // enclave_RTI_t enclave_rti; + + // The federates. + federate_t **enclaves; + + // Number of enclaves + int32_t number_of_enclaves; + + // RTI's decided stop tag for federates + tag_t max_stop_tag; + + // Number of federates handling stop + int num_feds_handling_stop; + + /** + * Boolean indicating that tracing is enabled. + */ + bool tracing_enabled; // Maximum start time seen so far from the federates. int64_t max_start_time; @@ -146,11 +163,6 @@ typedef struct federation_RTI_t { * Boolean indicating that authentication is enabled. */ bool authentication_enabled; - - /** - * Boolean indicating that tracing is enabled. - */ - bool tracing_enabled; } federation_RTI_t; // /** From 241ffc314732c2eb4412a726bff0a3cf42fb851a Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 25 May 2023 18:12:13 -0700 Subject: [PATCH 11/25] Upstream and downstream attributes of a federate/enclave is and array of ids, and not an array of federates?! --- core/federated/RTI/rti_lib.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 377347778..cf32e0724 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -1258,8 +1258,8 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { fed_id); // Allocate memory for the upstream and downstream pointers - fed->enclave.upstream = (int*)malloc(sizeof(federate_t*) * fed->enclave.num_upstream); - fed->enclave.downstream = (int*)malloc(sizeof(federate_t*) * fed->enclave.num_downstream); + fed->enclave.upstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_upstream); + fed->enclave.downstream = (int*)malloc(sizeof(uint16_t) * fed->enclave.num_downstream); // Allocate memory for the upstream delay pointers fed->enclave.upstream_delay = From d918168fe50c5dc27a9f27c61979fc139c4b859d Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 25 May 2023 18:25:50 -0700 Subject: [PATCH 12/25] Fix the very wrong variable name when requesting stop! --- core/federated/RTI/rti_lib.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index cf32e0724..bba37dc1d 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -656,19 +656,19 @@ void handle_stop_request_message(federate_t* fed) { // Iterate over federates and send each the MSG_TYPE_STOP_REQUEST message // if we do not have a stop_time already for them. for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { - federate_t *fed = _F_RTI->enclaves[i]; - if (fed->enclave.id != fed->enclave.id && fed->enclave.requested_stop == false) { - if (fed->enclave.state == NOT_CONNECTED) { - mark_federate_requesting_stop(fed); + federate_t *f = _F_RTI->enclaves[i]; + if (f->enclave.id != fed->enclave.id && f->enclave.requested_stop == false) { + if (f->enclave.state == NOT_CONNECTED) { + mark_federate_requesting_stop(f); continue; } if (_F_RTI->tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_F_RTI->max_stop_tag); + tracepoint_RTI_to_federate(send_STOP_REQ, f->enclave.id, &_F_RTI->max_stop_tag); } - write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, - "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", fed->enclave.id); + write_to_socket_errexit(f->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, + "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", f->enclave.id); if (_F_RTI->tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, fed->enclave.id, &_F_RTI->max_stop_tag); + tracepoint_RTI_to_federate(send_STOP_REQ, f->enclave.id, &_F_RTI->max_stop_tag); } } } From bd3168506d9c1f8cbed1857fdd41f1dccebe619d Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Fri, 26 May 2023 11:37:18 -0700 Subject: [PATCH 13/25] Minor update and More documentation. --- core/federated/RTI/enclave.c | 65 +++++++++++++++++++----------------- core/federated/RTI/enclave.h | 22 +++++++++--- core/federated/RTI/rti.c | 10 +++--- core/federated/RTI/rti_lib.c | 11 +++--- core/federated/RTI/rti_lib.h | 39 +++++++++------------- 5 files changed, 81 insertions(+), 66 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 201294854..1986b4e04 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -12,6 +12,12 @@ extern instant_t start_time; extern lf_mutex_t rti_mutex; // FIXME: rename "federate" everywhere in this file. +// FIXME: Done, except for log and debug message. What sould be kept: 'enclave', +// 'federate', or 'enlcave/federate'? +// FIXME: Should enclaves tracing use the same mechanism as federates? +// It needs to account a federate having itself a number of enclaves. +// Currently, all calls to tracepoint_from_federate() and +// tracepoint_to_federate() are in rti_lib.c void initialize_enclave(enclave_t* e, uint16_t id) { e->id = id; @@ -38,22 +44,19 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { lf_mutex_lock(&rti_mutex); enclave->completed = completed; - if (_E_RTI->tracing_enabled) { - tracepoint_RTI_from_federate(receive_LTC, enclave->id, &(enclave->completed)); - } - LF_PRINT_LOG("RTI received from federate %d the Logical Tag Complete (LTC) " PRINTF_TAG ".", + LF_PRINT_LOG("RTI received from federate/enclave %d the Logical Tag Complete (LTC) " PRINTF_TAG ".", enclave->id, enclave->completed.time - start_time, enclave->completed.microstep); // See if we can remove any of the recorded in-transit messages for this. // FIXME: Should this be here? // clean_in_transit_message_record_up_to_tag(enclave->in_transit_message_tags, enclave->completed); - // Check downstream federates to see whether they should now be granted a TAG. + // Check downstream enclaves to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { // FIXME: Shouldn't use enclave_t here. enclave_t *downstream = _E_RTI->enclaves[enclave->downstream[i]]; - // Notify downstream federate if appropriate. + // Notify downstream enclave if appropriate. notify_advance_grant_if_safe(downstream); bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Notify enclaves downstream of downstream if appropriate. @@ -67,13 +70,13 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_advance_grant_t result = {.tag = NEVER_TAG, .is_provisional = false}; - // Find the earliest LTC of upstream federates (M). + // Find the earliest LTC of upstream enclaves (M). tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < e->num_upstream; j++) { enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; - // Ignore this federate if it has resigned. + // Ignore this enclave if it no longer connected. if (upstream->state == NOT_CONNECTED) continue; tag_t candidate = lf_delay_tag(upstream->completed, e->upstream_delay[j]); @@ -82,30 +85,30 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { min_upstream_completed = candidate; } } - LF_PRINT_LOG("Minimum upstream LTC for fed %d is " PRINTF_TAG + LF_PRINT_LOG("Minimum upstream LTC for federate/enclave %d is " PRINTF_TAG "(adjusted by after delay).", e->id, min_upstream_completed.time - start_time, min_upstream_completed.microstep); if (lf_tag_compare(min_upstream_completed, e->last_granted) > 0 - && lf_tag_compare(min_upstream_completed, e->next_event) >= 0 // The federate has to advance its tag + && lf_tag_compare(min_upstream_completed, e->next_event) >= 0 // The enclave has to advance its tag ) { result.tag = min_upstream_completed; return result; } // Can't make progress based only on upstream LTCs. - // If all (transitive) upstream federates of the federate + // If all (transitive) upstream enclaves of the enclave // have earliest event tags such that the - // federate can now advance its tag, then send it a TAG message. - // Find the earliest event time of each such upstream federate, + // enclave can now advance its tag, then send it a TAG message. + // Find the earliest event time of each such upstream enclave, // adjusted by delays on the connections. // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. + // track of which upstream enclave have been visited. bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the tag of the earliest possible incoming message from - // upstream federates. + // upstream enclaves. tag_t t_d = FOREVER_TAG; LF_PRINT_DEBUG("NOTE: FOREVER is displayed as" PRINTF_TAG "and NEVER as " PRINTF_TAG, FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, @@ -114,14 +117,14 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { for (int j = 0; j < e->num_upstream; j++) { enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; - // Ignore this federate if it has resigned. + // Ignore this enclave if it is no longer connected. if (upstream->state == NOT_CONNECTED) continue; // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( upstream, upstream->next_event, visited); - LF_PRINT_DEBUG("Earliest next event upstream of fed %d at fed %d has tag " PRINTF_TAG ".", + LF_PRINT_DEBUG("Earliest next event upstream of fed/encl %d at fed/encl %d has tag " PRINTF_TAG ".", e->id, upstream->id, upstream_next_event.time - start_time, upstream_next_event.microstep); @@ -141,14 +144,14 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { t_d.time - start_time, t_d.microstep); if ( - lf_tag_compare(t_d, e->next_event) > 0 // The federate has something to do. + lf_tag_compare(t_d, e->next_event) > 0 // The enclave has something to do. && lf_tag_compare(t_d, e->last_provisionally_granted) >= 0 // The grant is not redundant // (equal is important to override any previous // PTAGs). && lf_tag_compare(t_d, e->last_granted) > 0 // The grant is not redundant. ) { - // All upstream federates have events with a larger tag than fed, so it is safe to send a TAG. - LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG + // All upstream enclaves have events with a larger tag than fed, so it is safe to send a TAG. + LF_PRINT_LOG("Earliest upstream message time for fed/encl %d is " PRINTF_TAG "(adjusted by after delay). Granting tag advance for " PRINTF_TAG, e->id, t_d.time - lf_time_start(), t_d.microstep, @@ -156,13 +159,13 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { e->next_event.microstep); result.tag = e->next_event; } else if ( - lf_tag_compare(t_d, e->next_event) == 0 // The federate has something to do. + lf_tag_compare(t_d, e->next_event) == 0 // The enclave has something to do. && lf_tag_compare(t_d, e->last_provisionally_granted) > 0 // The grant is not redundant. && lf_tag_compare(t_d, e->last_granted) > 0 // The grant is not redundant. ) { - // Some upstream federate has an event that has the same tag as fed's next event, so we can only provisionally + // Some upstream enclaves has an event that has the same tag as fed's next event, so we can only provisionally // grant a TAG (via a PTAG). - LF_PRINT_LOG("Earliest upstream message time for fed %d is " PRINTF_TAG + LF_PRINT_LOG("Earliest upstream message time for fed/encl %d is " PRINTF_TAG " (adjusted by after delay). Granting provisional tag advance.", e->id, t_d.time - start_time, t_d.microstep); @@ -186,21 +189,21 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag) { e->next_event = next_event_tag; LF_PRINT_DEBUG( - "RTI: Updated the recorded next event tag for enclave %d to " PRINTF_TAG, + "RTI: Updated the recorded next event tag for federate/enclave %d to " PRINTF_TAG, e->id, next_event_tag.time - lf_time_start(), next_event_tag.microstep ); // Check to see whether we can reply now with a tag advance grant. - // If the federate has no upstream federates, then it does not wait for + // If the enclave has no upstream enclaves, then it does not wait for // nor expect a reply. It just proceeds to advance time. if (e->num_upstream > 0) { notify_advance_grant_if_safe(e); } - // Check downstream federates to see whether they should now be granted a TAG. + // Check downstream enclaves to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep - // track of which upstream federates have been visited. + // track of which upstream enclaves have been visited. bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(e, visited); free(visited); @@ -254,15 +257,15 @@ void notify_advance_grant_if_safe(enclave_t* e) { tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { if (visited[e->id] || e->state == NOT_CONNECTED) { - // Federate has stopped executing or we have visited it before. - // No point in checking upstream federates. + // Enclave has stopped executing or we have visited it before. + // No point in checking upstream enclaves. return candidate; } visited[e->id] = true; tag_t result = e->next_event; - // If the candidate is less than this federate's next_event, use the candidate. + // If the candidate is less than this enclave's next_event, use the candidate. if (lf_tag_compare(candidate, result) < 0) { result = candidate; } @@ -273,7 +276,7 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { result = (tag_t){.time = start_time, .microstep = 0u}; } - // Check upstream federates to see whether any of them might send + // Check upstream enclaves to see whether any of them might send // an event that would result in an earlier next event. for (int i = 0; i < e->num_upstream; i++) { tag_t upstream_result = transitive_next_event( diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index c51f8a25c..1f4855fb2 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -19,13 +19,21 @@ typedef enum execution_mode_t { REALTIME } execution_mode_t; -/** State of a federate during execution. */ +/** State of a enclave during execution. */ typedef enum fed_state_t { NOT_CONNECTED, // The federate has not connected. GRANTED, // Most recent MSG_TYPE_NEXT_EVENT_TAG has been granted. PENDING // Waiting for upstream federates. } fed_state_t; +/** + * Information about enclave known to the RTI, including its runtime state, + * mode of execution, and connectivity with other enclaves. + * The list of upstream and downstream enclaves does not include + * those that are connected via a "physical" connection (one + * denoted with ~>) because those connections do not impose + * any scheduling constraints. + */ typedef struct enclave_t { uint16_t id; // ID of this enclave. tag_t completed; // The largest logical tag completed by the federate (or NEVER if no LTC has been received). @@ -50,7 +58,13 @@ typedef struct enclave_t { /** * Structure that an enclave RTI instance uses to keep track of its own and its * corresponding enclaves'state. + * // **************** IMPORTANT!!! ******************** + * // ** If you make any change to this struct, ** + * // ** you MUST also change federation_RTI_t in ** + * // ** (rti_lib.h)! The change must exactly match. ** + * // ************************************************** */ + typedef struct enclave_RTI_t { // The enclaves. enclave_t **enclaves; @@ -58,11 +72,11 @@ typedef struct enclave_RTI_t { // Number of enclaves int32_t number_of_enclaves; - // RTI's decided stop tag for federates + // RTI's decided stop tag for enclaves tag_t max_stop_tag; - // Number of federates handling stop - int num_feds_handling_stop; + // Number of enclaves handling stop + int num_enclaves_handling_stop; /** * Boolean indicating that tracing is enabled. diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 9a59a6baa..12ac722e5 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -58,11 +58,13 @@ extern enclave_RTI_t * _E_RTI; extern federation_RTI_t* _F_RTI; extern lf_mutex_t rti_mutex; +extern lf_cond_t received_start_times; +extern lf_cond_t sent_start_time; -/** - * RTI trace file name - */ -const char *rti_trace_file_name = "rti.lft"; + /** + * RTI trace file name + */ + const char *rti_trace_file_name = "rti.lft"; int main(int argc, const char* argv[]) { diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index bba37dc1d..065b29d7b 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -513,6 +513,9 @@ void handle_logical_tag_complete(federate_t* fed) { read_from_socket_errexit(fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, "RTI failed to read the content of the logical tag complete from federate %d.", fed->enclave.id); tag_t completed = extract_tag(buffer); + if (_F_RTI->tracing_enabled) { + tracepoint_RTI_from_federate(receive_LTC, fed->enclave.id, &completed); + } logical_tag_complete(&(fed->enclave), completed); // FIXME: Should this function be in the enclave version? @@ -592,10 +595,10 @@ void mark_federate_requesting_stop(federate_t* fed) { if (!fed->enclave.requested_stop) { // Assume that the federate // has requested stop - _F_RTI->num_feds_handling_stop++; + _F_RTI->num_enclaves_handling_stop++; fed->enclave.requested_stop = true; } - if (_F_RTI->num_feds_handling_stop == _F_RTI->number_of_enclaves) { + if (_F_RTI->num_enclaves_handling_stop == _F_RTI->number_of_enclaves) { // We now have information about the stop time of all // federates. _lf_rti_broadcast_stop_time_to_federates_already_locked(); @@ -641,7 +644,7 @@ void handle_stop_request_message(federate_t* fed) { // for a stop, add it to the tally. mark_federate_requesting_stop(fed); - if (_F_RTI->num_feds_handling_stop == _F_RTI->number_of_enclaves) { + if (_F_RTI->num_enclaves_handling_stop == _F_RTI->number_of_enclaves) { // We now have information about the stop time of all // federates. This is extremely unlikely, but it can occur // all federates call lf_request_stop() at the same tag. @@ -1806,7 +1809,7 @@ void initialize_RTI(){ // enclave_rti related initializations _F_RTI->max_stop_tag = NEVER_TAG, _F_RTI->number_of_enclaves = 0, - _F_RTI->num_feds_handling_stop = 0, + _F_RTI->num_enclaves_handling_stop = 0, // federation_rti related initializations _F_RTI->max_start_time = 0LL, _F_RTI->num_feds_proposed_start = 0, diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index 9cec37e5b..0f76b7d21 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -77,10 +77,18 @@ typedef enum clock_sync_stat { /** * Structure that an RTI instance uses to keep track of its own and its * corresponding federates' state. + * It is a special case of `enclave_RTI_t` (declared in enclave.h). Inheritence + * is mimicked by having the first attributes to be the same as of enclave_RTI_t, + * except that enclaves attribute here is of type `federate_t**`, while it + * is of type `enclave_t**` in `enclave_RTI_t`. + * // **************** IMPORTANT!!! ******************** + * // ** If you make any change to this struct, ** + * // ** you MUST also change enclave_RTI_t in ** + * // ** (enclave.h)! The change must exactly match. ** + * // ************************************************** */ typedef struct federation_RTI_t { - // Enclave RTI - // enclave_RTI_t enclave_rti; + ////////////////// Enclave specific attributes ////////////////// // The federates. federate_t **enclaves; @@ -88,17 +96,17 @@ typedef struct federation_RTI_t { // Number of enclaves int32_t number_of_enclaves; - // RTI's decided stop tag for federates + // RTI's decided stop tag for enclaves tag_t max_stop_tag; - // Number of federates handling stop - int num_feds_handling_stop; + // Number of enclaves handling stop + int num_enclaves_handling_stop; - /** - * Boolean indicating that tracing is enabled. - */ + // Boolean indicating that tracing is enabled. bool tracing_enabled; + ////////////// Federation only specific attributes ////////////// + // Maximum start time seen so far from the federates. int64_t max_start_time; @@ -165,21 +173,6 @@ typedef struct federation_RTI_t { bool authentication_enabled; } federation_RTI_t; -// /** -// * The main mutex lock for the RTI. -// */ -// extern lf_mutex_t rti_mutex; - -/** - * Condition variable used to signal receipt of all proposed start times. - */ -extern lf_cond_t received_start_times; - -/** - * Condition variable used to signal that a start time has been sent to a federate. - */ -extern lf_cond_t sent_start_time; - /** * Enter a critical section where logical time and the event queue are guaranteed * to not change unless they are changed within the critical section. From 1b47b3239d958714a6af9c2ebabb86e8cf09b1fb Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Fri, 26 May 2023 12:00:27 -0700 Subject: [PATCH 14/25] Minor fix --- core/federated/RTI/rti_lib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 54f1f0648..1f1affefd 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -1384,7 +1384,7 @@ bool authenticate_federate(int socket) { // Check HMAC of received FED_RESPONSE message. unsigned int hmac_length = SHA256_HMAC_LENGTH; - size_t federation_id_length = strnlen(_RTI.federation_id, 255); + size_t federation_id_length = strnlen(_F_RTI.federation_id, 255); size_t fed_id_length = sizeof(uint16_t); unsigned char received[1 + NONCE_LENGTH + fed_id_length + hmac_length]; From 0343ab6dc6a41559870fd92abd3855a523c4f65e Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Fri, 26 May 2023 12:03:25 -0700 Subject: [PATCH 15/25] Re-minor fix --- core/federated/RTI/rti_lib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 1f1affefd..d843efcec 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -1384,7 +1384,7 @@ bool authenticate_federate(int socket) { // Check HMAC of received FED_RESPONSE message. unsigned int hmac_length = SHA256_HMAC_LENGTH; - size_t federation_id_length = strnlen(_F_RTI.federation_id, 255); + size_t federation_id_length = strnlen(_F_RTI->federation_id, 255); size_t fed_id_length = sizeof(uint16_t); unsigned char received[1 + NONCE_LENGTH + fed_id_length + hmac_length]; From 5c6c346cb492fc743222928d5e88283a30d6866d Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Wed, 31 May 2023 12:06:15 -0700 Subject: [PATCH 16/25] Minor fixes. --- core/federated/RTI/enclave.c | 4 ++-- core/federated/RTI/rti_lib.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 1986b4e04..67e6a4d43 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -110,9 +110,9 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { // Find the tag of the earliest possible incoming message from // upstream enclaves. tag_t t_d = FOREVER_TAG; - LF_PRINT_DEBUG("NOTE: FOREVER is displayed as" PRINTF_TAG "and NEVER as " PRINTF_TAG, + LF_PRINT_DEBUG("NOTE: FOREVER is displayed as " PRINTF_TAG " and NEVER as " PRINTF_TAG, FOREVER_TAG.time - start_time, FOREVER_TAG.microstep, - NEVER - start_time, 0u); + NEVER_TAG.time - start_time, 0); for (int j = 0; j < e->num_upstream; j++) { enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index d095f0f45..7d4e23b95 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -836,7 +836,7 @@ void handle_timestamp(federate_t *my_fed) { // the federate to the start time. my_fed->enclave.state = GRANTED; lf_cond_broadcast(&sent_start_time); - LF_PRINT_LOG("RTI sent start time " PRINTF_TIME " to federate %d.", start_time, my_fed->id); + LF_PRINT_LOG("RTI sent start time " PRINTF_TIME " to federate %d.", start_time, my_fed->enclave.id); lf_mutex_unlock(&rti_mutex); } @@ -907,7 +907,7 @@ void* clock_synchronization_thread(void* noargs) { lf_sleep(ns_to_wait); } - // Initiate a clock synchronization every _RTI.clock_sync_period_ns + // Initiate a clock synchronization every _F_RTI.clock_sync_period_ns // Initiate a clock synchronization every _F_RTI->clock_sync_period_ns struct timespec sleep_time = {(time_t) _F_RTI->clock_sync_period_ns / BILLION, _F_RTI->clock_sync_period_ns % BILLION}; @@ -916,7 +916,7 @@ void* clock_synchronization_thread(void* noargs) { bool any_federates_connected = true; while (any_federates_connected) { // Sleep - lf_sleep(_RTI.clock_sync_period_ns); // Can be interrupted + lf_sleep(_F_RTI->clock_sync_period_ns); // Can be interrupted any_federates_connected = false; for (int fed_id = 0; fed_id < _F_RTI->number_of_enclaves; fed_id++) { federate_t* fed = _F_RTI->enclaves[fed_id]; From a11c1b03a135056550eb64c3a110ee56f8fa0c42 Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Tue, 6 Jun 2023 17:59:44 -0700 Subject: [PATCH 17/25] Address comments from code review. --- core/federated/RTI/enclave.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 67e6a4d43..3beb06568 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -11,9 +11,8 @@ extern instant_t start_time; // RTI mutex, which is the main lock extern lf_mutex_t rti_mutex; -// FIXME: rename "federate" everywhere in this file. -// FIXME: Done, except for log and debug message. What sould be kept: 'enclave', -// 'federate', or 'enlcave/federate'? +// FIXME: For log and debug message in this file, what sould be kept: 'enclave', +// 'federate', or 'enlcave/federate'? Currently its is 'enclave/federate'. // FIXME: Should enclaves tracing use the same mechanism as federates? // It needs to account a federate having itself a number of enclaves. // Currently, all calls to tracepoint_from_federate() and @@ -48,13 +47,8 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { LF_PRINT_LOG("RTI received from federate/enclave %d the Logical Tag Complete (LTC) " PRINTF_TAG ".", enclave->id, enclave->completed.time - start_time, enclave->completed.microstep); - // See if we can remove any of the recorded in-transit messages for this. - // FIXME: Should this be here? - // clean_in_transit_message_record_up_to_tag(enclave->in_transit_message_tags, enclave->completed); - // Check downstream enclaves to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { - // FIXME: Shouldn't use enclave_t here. enclave_t *downstream = _E_RTI->enclaves[enclave->downstream[i]]; // Notify downstream enclave if appropriate. notify_advance_grant_if_safe(downstream); From d7335230ac13447783133071972f651e361ad3a6 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 8 Jun 2023 09:49:35 -0700 Subject: [PATCH 18/25] Update .gitignore --- .gitignore | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitignore b/.gitignore index df202f15d..34bd6ac3d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,10 +2,6 @@ /docs/_build /docs/api **/.vscode -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json /build/ **/.DS_Store /core/federated/RTI/build/ From 3c7ee1026648607c19a38d0e9ed758b468556044 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 8 Jun 2023 09:50:18 -0700 Subject: [PATCH 19/25] Update .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 34bd6ac3d..2feca864c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /.idea/ /docs/_build /docs/api -**/.vscode /build/ **/.DS_Store /core/federated/RTI/build/ From 4c31572c12004ad252de19832b275969122d9b63 Mon Sep 17 00:00:00 2001 From: Marten Lohstroh Date: Thu, 8 Jun 2023 09:52:28 -0700 Subject: [PATCH 20/25] Fix .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2feca864c..658a260bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.idea/ /docs/_build /docs/api +**/.vscode/ /build/ **/.DS_Store /core/federated/RTI/build/ From 027c65ba1b5efe1e84d85f98d7b4c0dcfea8e9c3 Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Wed, 7 Jun 2023 10:47:28 -0700 Subject: [PATCH 21/25] Remove redundant declaration. --- core/federated/RTI/rti.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 38e38d788..0ce019dcb 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -63,14 +63,6 @@ extern federation_RTI_t* _F_RTI; */ unsigned int _lf_number_of_workers = 0u; -/** - * References to the federation RTI and the enclave RTI. - * They both point to the same enclaves stuctures. In the case of federation RTI, - * however, enclaves are encapsulated in federates. - */ -extern enclave_RTI_t * _E_RTI; -extern federation_RTI_t* _F_RTI; - extern lf_mutex_t rti_mutex; extern lf_cond_t received_start_times; extern lf_cond_t sent_start_time; From 42a08bfb89e3bf171a00ca0ffb20bc370ca090fe Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 8 Jun 2023 10:46:29 -0700 Subject: [PATCH 22/25] Code review: Add logical_tag_complete documentation. --- core/federated/RTI/enclave.h | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index 4bcc7b5a8..5637306ba 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -76,14 +76,19 @@ typedef struct enclave_RTI_t { // Number of enclaves handling stop int num_enclaves_handling_stop; - /** - * Boolean indicating that tracing is enabled. - */ + // Boolean indicating that tracing is enabled. bool tracing_enabled; } enclave_RTI_t; -// FIXME: Docs +/** + * An enclave calls this function after it completed a tag. + * The function updates the completed tag and check if the downstream enclaves + * are eligible for receiving TAGs. + * + * @param enclave The enclave + * @param completed The completed tag of the enclave + */ void logical_tag_complete(enclave_t* enclave, tag_t completed); typedef struct { @@ -93,6 +98,8 @@ typedef struct { /** * Initialize the enclave with the specified ID. + * + * @param e The enclave * @param id The enclave ID. */ void initialize_enclave(enclave_t* e, uint16_t id); @@ -179,7 +186,9 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag); * * This function assumes that the caller holds the mutex lock. * - * @return True if the TAG message is sent and false otherwise. + * @param e The enclave + * @return If granted, return the tag value and whether it is provisional. + * Otherwise, return the NEVER_TAG. */ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e); @@ -196,6 +205,8 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e); * * @param e The enclave. * @param next_event_tag The next event tag for e. + * @return If granted, return the TAG and whether it is provisional or not. + * Otherwise, return the NEVER_TAG. */ tag_advance_grant_t next_event_tag(enclave_t* e, tag_t next_event_tag); @@ -226,12 +237,13 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag); * which outputs can be triggered by which inputs. For now, we * assume any output can be triggered by any input. * - * @param fed The federate. + * @param e The enclave. * @param candidate A candidate tag (for the first invocation, * this should be fed->next_event). * @param visited An array of booleans indicating which federates * have been visited (for the first invocation, this should be * an array of falses of size _RTI.number_of_federates). + * @return The earliest next event tag of the enclave e. */ tag_t transitive_next_event(enclave_t *e, tag_t candidate, bool visited[]); From ea1071c05170f596bf772f6a71f4297f03be9704 Mon Sep 17 00:00:00 2001 From: Chadlia Jerad <37504116+ChadliaJerad@users.noreply.github.com> Date: Thu, 8 Jun 2023 10:52:33 -0700 Subject: [PATCH 23/25] Update core/federated/RTI/rti_lib.h Co-authored-by: Marten Lohstroh --- core/federated/RTI/rti_lib.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index 9ec058cda..8b3e5ab56 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -521,4 +521,4 @@ int process_args(int argc, const char* argv[]); */ void initialize_RTI(); -#endif // RTI_LIB_H \ No newline at end of file +#endif // RTI_LIB_H From 8aaeaafc6f9595771e2c48a505779ccbff6dfa4a Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 8 Jun 2023 10:51:07 -0700 Subject: [PATCH 24/25] Code review of enclave_impl.c --- core/federated/RTI/enclave_impl.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/federated/RTI/enclave_impl.c b/core/federated/RTI/enclave_impl.c index 42a397eb6..525c0442f 100644 --- a/core/federated/RTI/enclave_impl.c +++ b/core/federated/RTI/enclave_impl.c @@ -1,7 +1,7 @@ #include "enclave.h" -// FIXME: This should not be here. -#include "platform.h" +// References to the enclave RTI. +extern enclave_RTI_t * _E_RTI; void notify_tag_advance_grant(enclave_t* e, tag_t tag) { if (e->state == NOT_CONNECTED @@ -10,7 +10,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { ) { return; } - if (_RTI.tracing_enabled) { + if (_E_RTI->tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } e->last_granted = tag; From 287a521f365f33ab6f3bfc68dbad168c334e8ab9 Mon Sep 17 00:00:00 2001 From: ChadliaJerad Date: Thu, 8 Jun 2023 12:02:02 -0700 Subject: [PATCH 25/25] Code review: lower case for RTI variables and types. --- core/federated/RTI/enclave.c | 20 +- core/federated/RTI/enclave.h | 6 +- core/federated/RTI/enclave_impl.c | 4 +- core/federated/RTI/rti.c | 26 +-- core/federated/RTI/rti_lib.c | 304 +++++++++++++++--------------- core/federated/RTI/rti_lib.h | 12 +- 6 files changed, 186 insertions(+), 186 deletions(-) diff --git a/core/federated/RTI/enclave.c b/core/federated/RTI/enclave.c index 3beb06568..4aebe608a 100644 --- a/core/federated/RTI/enclave.c +++ b/core/federated/RTI/enclave.c @@ -1,9 +1,9 @@ #include "enclave.h" /** - * Reference to enclave_RTI_t instance. + * Reference to enclave_rti_t instance. */ -enclave_RTI_t* _E_RTI; +enclave_rti_t* _e_rti; // Global variables defined in tag.c: extern instant_t start_time; @@ -49,10 +49,10 @@ void logical_tag_complete(enclave_t* enclave, tag_t completed) { // Check downstream enclaves to see whether they should now be granted a TAG. for (int i = 0; i < enclave->num_downstream; i++) { - enclave_t *downstream = _E_RTI->enclaves[enclave->downstream[i]]; + enclave_t *downstream = _e_rti->enclaves[enclave->downstream[i]]; // Notify downstream enclave if appropriate. notify_advance_grant_if_safe(downstream); - bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc(_e_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Notify enclaves downstream of downstream if appropriate. notify_downstream_advance_grant_if_safe(downstream, visited); free(visited); @@ -68,7 +68,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { tag_t min_upstream_completed = FOREVER_TAG; for (int j = 0; j < e->num_upstream; j++) { - enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; + enclave_t *upstream = _e_rti->enclaves[e->upstream[j]]; // Ignore this enclave if it no longer connected. if (upstream->state == NOT_CONNECTED) continue; @@ -99,7 +99,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { // To handle cycles, need to create a boolean array to keep // track of which upstream enclave have been visited. - bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc(_e_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the tag of the earliest possible incoming message from // upstream enclaves. @@ -109,7 +109,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { NEVER_TAG.time - start_time, 0); for (int j = 0; j < e->num_upstream; j++) { - enclave_t *upstream = _E_RTI->enclaves[e->upstream[j]]; + enclave_t *upstream = _e_rti->enclaves[e->upstream[j]]; // Ignore this enclave if it is no longer connected. if (upstream->state == NOT_CONNECTED) continue; @@ -172,7 +172,7 @@ tag_advance_grant_t tag_advance_grant_if_safe(enclave_t* e) { void notify_downstream_advance_grant_if_safe(enclave_t* e, bool visited[]) { visited[e->id] = true; for (int i = 0; i < e->num_downstream; i++) { - enclave_t* downstream = _E_RTI->enclaves[e->downstream[i]]; + enclave_t* downstream = _e_rti->enclaves[e->downstream[i]]; if (visited[downstream->id]) continue; notify_advance_grant_if_safe(downstream); notify_downstream_advance_grant_if_safe(downstream, visited); @@ -198,7 +198,7 @@ void update_enclave_next_event_tag_locked(enclave_t* e, tag_t next_event_tag) { // Check downstream enclaves to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream enclaves have been visited. - bool *visited = (bool *)calloc(_E_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool *visited = (bool *)calloc(_e_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(e, visited); free(visited); } @@ -274,7 +274,7 @@ tag_t transitive_next_event(enclave_t* e, tag_t candidate, bool visited[]) { // an event that would result in an earlier next event. for (int i = 0; i < e->num_upstream; i++) { tag_t upstream_result = transitive_next_event( - _E_RTI->enclaves[e->upstream[i]], result, visited); + _e_rti->enclaves[e->upstream[i]], result, visited); // Add the "after" delay of the connection to the result. upstream_result = lf_delay_tag(upstream_result, e->upstream_delay[i]); diff --git a/core/federated/RTI/enclave.h b/core/federated/RTI/enclave.h index 5637306ba..4dc74ed8b 100644 --- a/core/federated/RTI/enclave.h +++ b/core/federated/RTI/enclave.h @@ -58,12 +58,12 @@ typedef struct enclave_t { * corresponding enclaves'state. * // **************** IMPORTANT!!! ******************** * // ** If you make any change to this struct, ** - * // ** you MUST also change federation_RTI_t in ** + * // ** you MUST also change federation_rti_t in ** * // ** (rti_lib.h)! The change must exactly match. ** * // ************************************************** */ -typedef struct enclave_RTI_t { +typedef struct enclave_rti_t { // The enclaves. enclave_t **enclaves; @@ -78,7 +78,7 @@ typedef struct enclave_RTI_t { // Boolean indicating that tracing is enabled. bool tracing_enabled; -} enclave_RTI_t; +} enclave_rti_t; /** diff --git a/core/federated/RTI/enclave_impl.c b/core/federated/RTI/enclave_impl.c index 525c0442f..367a8aed0 100644 --- a/core/federated/RTI/enclave_impl.c +++ b/core/federated/RTI/enclave_impl.c @@ -1,7 +1,7 @@ #include "enclave.h" // References to the enclave RTI. -extern enclave_RTI_t * _E_RTI; +extern enclave_rti_t * _e_rti; void notify_tag_advance_grant(enclave_t* e, tag_t tag) { if (e->state == NOT_CONNECTED @@ -10,7 +10,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { ) { return; } - if (_E_RTI->tracing_enabled) { + if (_e_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } e->last_granted = tag; diff --git a/core/federated/RTI/rti.c b/core/federated/RTI/rti.c index 0ce019dcb..237711b4a 100644 --- a/core/federated/RTI/rti.c +++ b/core/federated/RTI/rti.c @@ -54,8 +54,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * They both point to the same enclaves stuctures. In the case of federation RTI, * however, enclaves are encapsulated in federates. */ -extern enclave_RTI_t * _E_RTI; -extern federation_RTI_t* _F_RTI; +extern enclave_rti_t * _e_rti; +extern federation_rti_t* _f_rti; /** * The tracing mechanism uses the number of workers variable `_lf_number_of_workers`. @@ -77,7 +77,7 @@ const char *rti_trace_file_name = "rti.lft"; * enabled, before exiting. */ void termination() { - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { stop_trace(); lf_print("RTI trace file saved."); } @@ -102,26 +102,26 @@ int main(int argc, const char* argv[]) { // Processing command-line arguments failed. return -1; } - if (_F_RTI->tracing_enabled) { - _lf_number_of_workers = _F_RTI->number_of_enclaves; + if (_f_rti->tracing_enabled) { + _lf_number_of_workers = _f_rti->number_of_enclaves; start_trace(rti_trace_file_name); lf_print("Tracing the RTI execution in %s file.", rti_trace_file_name); } - lf_print("Starting RTI for %d federates in federation ID %s.", _F_RTI->number_of_enclaves, _F_RTI->federation_id); - assert(_F_RTI->number_of_enclaves < UINT16_MAX); + lf_print("Starting RTI for %d federates in federation ID %s.", _f_rti->number_of_enclaves, _f_rti->federation_id); + assert(_f_rti->number_of_enclaves < UINT16_MAX); // Allocate memory for the federates - _F_RTI->enclaves = (federate_t**)calloc(_F_RTI->number_of_enclaves, sizeof(federate_t*)); - for (uint16_t i = 0; i < _F_RTI->number_of_enclaves; i++) { - _F_RTI->enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); - initialize_federate(_F_RTI->enclaves[i], i); + _f_rti->enclaves = (federate_t**)calloc(_f_rti->number_of_enclaves, sizeof(federate_t*)); + for (uint16_t i = 0; i < _f_rti->number_of_enclaves; i++) { + _f_rti->enclaves[i] = (federate_t *)malloc(sizeof(federate_t)); + initialize_federate(_f_rti->enclaves[i], i); } // Initialize the RTI enclaves - _E_RTI = (enclave_RTI_t*)_F_RTI; + _e_rti = (enclave_rti_t*)_f_rti; - int socket_descriptor = start_rti_server(_F_RTI->user_specified_port); + int socket_descriptor = start_rti_server(_f_rti->user_specified_port); wait_for_federates(socket_descriptor); return 0; diff --git a/core/federated/RTI/rti_lib.c b/core/federated/RTI/rti_lib.c index 0e22a1a71..5b01c3ac3 100644 --- a/core/federated/RTI/rti_lib.c +++ b/core/federated/RTI/rti_lib.c @@ -30,9 +30,9 @@ extern instant_t start_time; /** - * Reference to federate_RTI_t instance. + * Reference to federate_rti_t instance. */ -federation_RTI_t *_F_RTI; +federation_rti_t *_f_rti; lf_mutex_t rti_mutex; lf_cond_t received_start_times; @@ -157,16 +157,16 @@ int create_server(int32_t specified_port, uint16_t port, socket_type_t socket_ty if (socket_type == UDP) { type = "UDP"; } - lf_print("RTI using %s port %d for federation %s.", type, port, _F_RTI->federation_id); + lf_print("RTI using %s port %d for federation %s.", type, port, _f_rti->federation_id); if (socket_type == TCP) { - _F_RTI->final_port_TCP = port; + _f_rti->final_port_TCP = port; // Enable listening for socket connections. // The second argument is the maximum number of queued socket requests, // which according to the Mac man page is limited to 128. listen(socket_descriptor, 128); } else if (socket_type == UDP) { - _F_RTI->final_port_UDP = port; + _f_rti->final_port_UDP = port; // No need to listen on the UDP socket } @@ -192,7 +192,7 @@ void notify_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -232,7 +232,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { encode_int64(tag.time, &(buffer[1])); encode_int32((int32_t)tag.microstep, &(buffer[1 + sizeof(int64_t)])); - if (_F_RTI->tracing_enabled){ + if (_f_rti->tracing_enabled){ tracepoint_RTI_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long @@ -259,13 +259,13 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { // we have an available encoding of causality interfaces. // That might be more efficient. for (int j = 0; j < e->num_upstream; j++) { - federate_t* upstream = _F_RTI->enclaves[e->upstream[j]]; + federate_t* upstream = _f_rti->enclaves[e->upstream[j]]; // Ignore this federate if it has resigned. if (upstream->enclave.state == NOT_CONNECTED) continue; // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_F_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_f_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0. // Find the (transitive) next event tag upstream. tag_t upstream_next_event = transitive_next_event( @@ -284,7 +284,7 @@ void notify_provisional_tag_advance_grant(enclave_t* e, tag_t tag) { } void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_event_tag) { - federate_t* fed = _F_RTI->enclaves[federate_id]; + federate_t* fed = _f_rti->enclaves[federate_id]; tag_t min_in_transit_tag = get_minimum_in_transit_message_tag(fed->in_transit_message_tags); if (lf_tag_compare( min_in_transit_tag, @@ -307,7 +307,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); tag_t tag = extract_tag(&(buffer[1 + 2 * sizeof(uint16_t)])); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_PORT_ABS, federate_id, &tag); } @@ -318,7 +318,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // If the destination federate is no longer connected, issue a warning // and return. - federate_t* fed = (federate_t*) _F_RTI->enclaves[federate_id]; + federate_t* fed = (federate_t*) _f_rti->enclaves[federate_id]; if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", @@ -352,7 +352,7 @@ void handle_port_absent_message(federate_t* sending_federate, unsigned char* buf // Forward the message. int destination_socket = fed->socket; - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_PORT_ABS, federate_id, &tag); } write_to_socket_errexit(destination_socket, message_size + 1, buffer, @@ -398,7 +398,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // Following only works for string messages. // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_TAGGED_MSG, sending_federate->enclave.id, &intended_tag); } @@ -409,7 +409,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { // If the destination federate is no longer connected, issue a warning // and return. - federate_t *fed = _F_RTI->enclaves[federate_id]; + federate_t *fed = _f_rti->enclaves[federate_id]; if (fed->enclave.state == NOT_CONNECTED) { lf_mutex_unlock(&rti_mutex); lf_print_warning("RTI: Destination federate %d is no longer connected. Dropping message.", @@ -475,7 +475,7 @@ void handle_timed_message(federate_t* sending_federate, unsigned char* buffer) { lf_cond_wait(&sent_start_time); } - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_TAGGED_MSG, federate_id, &intended_tag); } @@ -513,7 +513,7 @@ void handle_logical_tag_complete(federate_t* fed) { read_from_socket_errexit(fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, "RTI failed to read the content of the logical tag complete from federate %d.", fed->enclave.id); tag_t completed = extract_tag(buffer); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_LTC, fed->enclave.id, &completed); } logical_tag_complete(&(fed->enclave), completed); @@ -539,7 +539,7 @@ void handle_next_event_tag(federate_t* fed) { tag_t intended_tag = extract_tag(buffer); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_NET, fed->enclave.id, &intended_tag); } LF_PRINT_LOG("RTI received from federate %d the Next Event Tag (NET) " PRINTF_TAG, @@ -566,28 +566,28 @@ void _lf_rti_broadcast_stop_time_to_federates_already_locked() { } // Reply with a stop granted to all federates unsigned char outgoing_buffer[MSG_TYPE_STOP_GRANTED_LENGTH]; - ENCODE_STOP_GRANTED(outgoing_buffer, _F_RTI->max_stop_tag.time, _F_RTI->max_stop_tag.microstep); + ENCODE_STOP_GRANTED(outgoing_buffer, _f_rti->max_stop_tag.time, _f_rti->max_stop_tag.microstep); // Iterate over federates and send each the message. - for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { - federate_t *fed = _F_RTI->enclaves[i]; + for (int i = 0; i < _f_rti->number_of_enclaves; i++) { + federate_t *fed = _f_rti->enclaves[i]; if (fed->enclave.state == NOT_CONNECTED) { continue; } - if (lf_tag_compare(fed->enclave.next_event, _F_RTI->max_stop_tag) >= 0) { + if (lf_tag_compare(fed->enclave.next_event, _f_rti->max_stop_tag) >= 0) { // Need the next_event to be no greater than the stop tag. - fed->enclave.next_event = _F_RTI->max_stop_tag; + fed->enclave.next_event = _f_rti->max_stop_tag; } - if (_F_RTI->tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_F_RTI->max_stop_tag); + if (_f_rti->tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_GRN, fed->enclave.id, &_f_rti->max_stop_tag); } write_to_socket_errexit(fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); } LF_PRINT_LOG("RTI sent to federates MSG_TYPE_STOP_GRANTED with tag (" PRINTF_TIME ", %u).", - _F_RTI->max_stop_tag.time - start_time, - _F_RTI->max_stop_tag.microstep); + _f_rti->max_stop_tag.time - start_time, + _f_rti->max_stop_tag.microstep); _lf_rti_stop_granted_already_sent_to_federates = true; } @@ -595,10 +595,10 @@ void mark_federate_requesting_stop(federate_t* fed) { if (!fed->enclave.requested_stop) { // Assume that the federate // has requested stop - _F_RTI->num_enclaves_handling_stop++; + _f_rti->num_enclaves_handling_stop++; fed->enclave.requested_stop = true; } - if (_F_RTI->num_enclaves_handling_stop == _F_RTI->number_of_enclaves) { + if (_f_rti->num_enclaves_handling_stop == _f_rti->number_of_enclaves) { // We now have information about the stop time of all // federates. _lf_rti_broadcast_stop_time_to_federates_already_locked(); @@ -628,13 +628,13 @@ void handle_stop_request_message(federate_t* fed) { // Extract the proposed stop tag for the federate tag_t proposed_stop_tag = extract_tag(buffer); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ, fed->enclave.id, &proposed_stop_tag); } // Update the maximum stop tag received from federates - if (lf_tag_compare(proposed_stop_tag, _F_RTI->max_stop_tag) > 0) { - _F_RTI->max_stop_tag = proposed_stop_tag; + if (lf_tag_compare(proposed_stop_tag, _f_rti->max_stop_tag) > 0) { + _f_rti->max_stop_tag = proposed_stop_tag; } LF_PRINT_LOG("RTI received from federate %d a MSG_TYPE_STOP_REQUEST message with tag " PRINTF_TAG ".", @@ -644,7 +644,7 @@ void handle_stop_request_message(federate_t* fed) { // for a stop, add it to the tally. mark_federate_requesting_stop(fed); - if (_F_RTI->num_enclaves_handling_stop == _F_RTI->number_of_enclaves) { + if (_f_rti->num_enclaves_handling_stop == _f_rti->number_of_enclaves) { // We now have information about the stop time of all // federates. This is extremely unlikely, but it can occur // all federates call lf_request_stop() at the same tag. @@ -654,30 +654,30 @@ void handle_stop_request_message(federate_t* fed) { // Forward the stop request to all other federates that have not // also issued a stop request. unsigned char stop_request_buffer[MSG_TYPE_STOP_REQUEST_LENGTH]; - ENCODE_STOP_REQUEST(stop_request_buffer, _F_RTI->max_stop_tag.time, _F_RTI->max_stop_tag.microstep); + ENCODE_STOP_REQUEST(stop_request_buffer, _f_rti->max_stop_tag.time, _f_rti->max_stop_tag.microstep); // Iterate over federates and send each the MSG_TYPE_STOP_REQUEST message // if we do not have a stop_time already for them. - for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { - federate_t *f = _F_RTI->enclaves[i]; + for (int i = 0; i < _f_rti->number_of_enclaves; i++) { + federate_t *f = _f_rti->enclaves[i]; if (f->enclave.id != fed->enclave.id && f->enclave.requested_stop == false) { if (f->enclave.state == NOT_CONNECTED) { mark_federate_requesting_stop(f); continue; } - if (_F_RTI->tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, f->enclave.id, &_F_RTI->max_stop_tag); + if (_f_rti->tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, f->enclave.id, &_f_rti->max_stop_tag); } write_to_socket_errexit(f->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", f->enclave.id); - if (_F_RTI->tracing_enabled) { - tracepoint_RTI_to_federate(send_STOP_REQ, f->enclave.id, &_F_RTI->max_stop_tag); + if (_f_rti->tracing_enabled) { + tracepoint_RTI_to_federate(send_STOP_REQ, f->enclave.id, &_f_rti->max_stop_tag); } } } LF_PRINT_LOG("RTI forwarded to federates MSG_TYPE_STOP_REQUEST with tag (" PRINTF_TIME ", %u).", - _F_RTI->max_stop_tag.time - start_time, - _F_RTI->max_stop_tag.microstep); + _f_rti->max_stop_tag.time - start_time, + _f_rti->max_stop_tag.microstep); lf_mutex_unlock(&rti_mutex); } @@ -689,7 +689,7 @@ void handle_stop_request_reply(federate_t* fed) { tag_t federate_stop_tag = extract_tag(buffer_stop_time); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_STOP_REQ_REP, fed->enclave.id, &federate_stop_tag); } @@ -700,8 +700,8 @@ void handle_stop_request_reply(federate_t* fed) { // Acquire the mutex lock so that we can change the state of the RTI lf_mutex_lock(&rti_mutex); // If the federate has not requested stop before, count the reply - if (lf_tag_compare(federate_stop_tag, _F_RTI->max_stop_tag) > 0) { - _F_RTI->max_stop_tag = federate_stop_tag; + if (lf_tag_compare(federate_stop_tag, _f_rti->max_stop_tag) > 0) { + _f_rti->max_stop_tag = federate_stop_tag; } mark_federate_requesting_stop(fed); lf_mutex_unlock(&rti_mutex); @@ -710,7 +710,7 @@ void handle_stop_request_reply(federate_t* fed) { ////////////////////////////////////////////////// void handle_address_query(uint16_t fed_id) { - federate_t *fed = _F_RTI->enclaves[fed_id]; + federate_t *fed = _f_rti->enclaves[fed_id]; // Use buffer both for reading and constructing the reply. // The length is what is needed for the reply. unsigned char buffer[sizeof(int32_t)]; @@ -720,7 +720,7 @@ void handle_address_query(uint16_t fed_id) { } uint16_t remote_fed_id = extract_uint16(buffer); - if (_F_RTI->tracing_enabled){ + if (_f_rti->tracing_enabled){ tracepoint_RTI_from_federate(receive_ADR_QR, fed_id, NULL); } @@ -731,7 +731,7 @@ void handle_address_query(uint16_t fed_id) { // from this federate. In that case, it will respond by sending -1. // Encode the port number. - federate_t *remote_fed = _F_RTI->enclaves[remote_fed_id]; + federate_t *remote_fed = _f_rti->enclaves[remote_fed_id]; encode_int32(remote_fed->server_port, (unsigned char*)buffer); // Send the port number (which could be -1). write_to_socket_errexit(fed->socket, sizeof(int32_t), (unsigned char*)buffer, @@ -749,7 +749,7 @@ void handle_address_query(uint16_t fed_id) { } void handle_address_ad(uint16_t federate_id) { - federate_t *fed = _F_RTI->enclaves[federate_id]; + federate_t *fed = _f_rti->enclaves[federate_id]; // Read the port number of the federate that can be used for physical // connections to other federates int32_t server_port = -1; @@ -768,7 +768,7 @@ void handle_address_ad(uint16_t federate_id) { lf_mutex_lock(&rti_mutex); fed->server_port = server_port; - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_ADR_AD, federate_id, NULL); } LF_PRINT_LOG("Received address advertisement from federate %d.", federate_id); @@ -784,7 +784,7 @@ void handle_timestamp(federate_t *my_fed) { } int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t *)(&buffer))); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tag_t tag = {.time = timestamp, .microstep = 0}; tracepoint_RTI_from_federate(receive_TIMESTAMP, my_fed->enclave.id, &tag); } @@ -792,17 +792,17 @@ void handle_timestamp(federate_t *my_fed) { LF_PRINT_LOG("RTI received timestamp message: " PRINTF_TIME ".", timestamp); lf_mutex_lock(&rti_mutex); - _F_RTI->num_feds_proposed_start++; - if (timestamp > _F_RTI->max_start_time) { - _F_RTI->max_start_time = timestamp; + _f_rti->num_feds_proposed_start++; + if (timestamp > _f_rti->max_start_time) { + _f_rti->max_start_time = timestamp; } - if (_F_RTI->num_feds_proposed_start == _F_RTI->number_of_enclaves) { + if (_f_rti->num_feds_proposed_start == _f_rti->number_of_enclaves) { // All federates have proposed a start time. lf_cond_broadcast(&received_start_times); } else { // Some federates have not yet proposed a start time. // wait for a notification. - while (_F_RTI->num_feds_proposed_start < _F_RTI->number_of_enclaves) { + while (_f_rti->num_feds_proposed_start < _f_rti->number_of_enclaves) { // FIXME: Should have a timeout here? lf_cond_wait(&received_start_times); } @@ -815,10 +815,10 @@ void handle_timestamp(federate_t *my_fed) { unsigned char start_time_buffer[MSG_TYPE_TIMESTAMP_LENGTH]; start_time_buffer[0] = MSG_TYPE_TIMESTAMP; // Add an offset to this start time to get everyone starting together. - start_time = _F_RTI->max_start_time + DELAY_START; + start_time = _f_rti->max_start_time + DELAY_START; encode_int64(swap_bytes_if_big_endian_int64(start_time), &start_time_buffer[1]); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tag_t tag = {.time = start_time, .microstep = 0}; tracepoint_RTI_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } @@ -855,7 +855,7 @@ void send_physical_clock(unsigned char message_type, federate_t* fed, socket_typ if (socket_type == UDP) { // FIXME: UDP_addr is never initialized. LF_PRINT_DEBUG("Clock sync: RTI sending UDP message type %u.", buffer[0]); - ssize_t bytes_written = sendto(_F_RTI->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, + ssize_t bytes_written = sendto(_f_rti->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr)); if (bytes_written < (ssize_t)sizeof(int64_t) + 1) { lf_print_warning("Clock sync: RTI failed to send physical time to federate %d: %s\n", @@ -894,7 +894,7 @@ void* clock_synchronization_thread(void* noargs) { // Wait until all federates have been notified of the start time. // FIXME: Use lf_ version of this when merged with master. lf_mutex_lock(&rti_mutex); - while (_F_RTI->num_feds_proposed_start < _F_RTI->number_of_enclaves) { + while (_f_rti->num_feds_proposed_start < _f_rti->number_of_enclaves) { lf_cond_wait(&received_start_times); } lf_mutex_unlock(&rti_mutex); @@ -907,19 +907,19 @@ void* clock_synchronization_thread(void* noargs) { lf_sleep(ns_to_wait); } - // Initiate a clock synchronization every _F_RTI->clock_sync_period_ns - // Initiate a clock synchronization every _F_RTI->clock_sync_period_ns - struct timespec sleep_time = {(time_t) _F_RTI->clock_sync_period_ns / BILLION, - _F_RTI->clock_sync_period_ns % BILLION}; + // Initiate a clock synchronization every _f_rti->clock_sync_period_ns + // Initiate a clock synchronization every _f_rti->clock_sync_period_ns + struct timespec sleep_time = {(time_t) _f_rti->clock_sync_period_ns / BILLION, + _f_rti->clock_sync_period_ns % BILLION}; struct timespec remaining_time; bool any_federates_connected = true; while (any_federates_connected) { // Sleep - lf_sleep(_F_RTI->clock_sync_period_ns); // Can be interrupted + lf_sleep(_f_rti->clock_sync_period_ns); // Can be interrupted any_federates_connected = false; - for (int fed_id = 0; fed_id < _F_RTI->number_of_enclaves; fed_id++) { - federate_t* fed = _F_RTI->enclaves[fed_id]; + for (int fed_id = 0; fed_id < _f_rti->number_of_enclaves; fed_id++) { + federate_t* fed = _f_rti->enclaves[fed_id]; if (fed->enclave.state == NOT_CONNECTED) { // FIXME: We need better error handling here, but clock sync failure // should not stop execution. @@ -943,7 +943,7 @@ void* clock_synchronization_thread(void* noargs) { int remaining_attempts = 5; while (remaining_attempts > 0) { remaining_attempts--; - int bytes_read = read_from_socket(_F_RTI->socket_descriptor_UDP, message_size, buffer); + int bytes_read = read_from_socket(_f_rti->socket_descriptor_UDP, message_size, buffer); // If any errors occur, either discard the message or the clock sync round. if (bytes_read == message_size) { if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { @@ -957,7 +957,7 @@ void* clock_synchronization_thread(void* noargs) { continue; } LF_PRINT_DEBUG("Clock sync: RTI received T3 message from federate %d.", fed_id_2); - handle_physical_clock_sync_message(_F_RTI->enclaves[fed_id_2], UDP); + handle_physical_clock_sync_message(_f_rti->enclaves[fed_id_2], UDP); break; } else { // The message is not a T3 message. Discard the message and @@ -989,7 +989,7 @@ void* clock_synchronization_thread(void* noargs) { void handle_federate_resign(federate_t *my_fed) { // Nothing more to do. Close the socket and exit. lf_mutex_lock(&rti_mutex); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { // Extract the tag, for tracing purposes size_t header_size = 1 + sizeof(tag_t); unsigned char buffer[header_size]; @@ -1022,7 +1022,7 @@ void handle_federate_resign(federate_t *my_fed) { // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep // track of which upstream federates have been visited. - bool* visited = (bool*)calloc(_F_RTI->number_of_enclaves, sizeof(bool)); // Initializes to 0. + bool* visited = (bool*)calloc(_f_rti->number_of_enclaves, sizeof(bool)); // Initializes to 0. notify_downstream_advance_grant_if_safe(&(my_fed->enclave), visited); free(visited); @@ -1088,7 +1088,7 @@ void* federate_thread_TCP(void* fed) { break; default: lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->enclave.id, buffer[0]); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_UNIDENTIFIED, my_fed->enclave.id, NULL); } } @@ -1120,7 +1120,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie // FIXME: This should not exit with error but rather should just reject the connection. read_from_socket_errexit(socket_id, length, buffer, "RTI failed to read from accepted socket."); - uint16_t fed_id = _F_RTI->number_of_enclaves; // Initialize to an invalid value. + uint16_t fed_id = _f_rti->number_of_enclaves; // Initialize to an invalid value. // First byte received is the message type. if (buffer[0] != MSG_TYPE_FED_IDS) { @@ -1136,7 +1136,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } else { send_reject(socket_id, UNEXPECTED_MESSAGE); } - if (_F_RTI->tracing_enabled){ + if (_f_rti->tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); @@ -1160,33 +1160,33 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI received federation ID: %s.", federation_id_received); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_from_federate(receive_FED_ID, fed_id, NULL); } // Compare the received federation ID to mine. - if (strncmp(_F_RTI->federation_id, federation_id_received, federation_id_length) != 0) { + if (strncmp(_f_rti->federation_id, federation_id_received, federation_id_length) != 0) { // Federation IDs do not match. Send back a MSG_TYPE_REJECT message. lf_print_error("WARNING: Federate from another federation %s attempted to connect to RTI in federation %s.\n", federation_id_received, - _F_RTI->federation_id); - if (_F_RTI->tracing_enabled) { + _f_rti->federation_id); + if (_f_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH); return -1; } else { - if (fed_id >= _F_RTI->number_of_enclaves) { + if (fed_id >= _f_rti->number_of_enclaves) { // Federate ID is out of range. lf_print_error("RTI received federate ID %d, which is out of range.", fed_id); - if (_F_RTI->tracing_enabled){ + if (_f_rti->tracing_enabled){ tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { - if ((_F_RTI->enclaves[fed_id])->enclave.state != NOT_CONNECTED) { + if ((_f_rti->enclaves[fed_id])->enclave.state != NOT_CONNECTED) { lf_print_error("RTI received duplicate federate ID: %d.", fed_id); - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_REJECT, fed_id, NULL); } send_reject(socket_id, FEDERATE_ID_IN_USE); @@ -1195,7 +1195,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie } } } - federate_t* fed = _F_RTI->enclaves[fed_id]; + federate_t* fed = _f_rti->enclaves[fed_id]; // The MSG_TYPE_FED_IDS message has the right federation ID. // Assign the address information for federate. // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful @@ -1224,7 +1224,7 @@ int32_t receive_and_check_fed_id_message(int socket_id, struct sockaddr_in* clie LF_PRINT_DEBUG("RTI responding with MSG_TYPE_ACK to federate %d.", fed_id); // Send an MSG_TYPE_ACK message. unsigned char ack_message = MSG_TYPE_ACK; - if (_F_RTI->tracing_enabled) { + if (_f_rti->tracing_enabled) { tracepoint_RTI_to_federate(send_ACK, fed_id, NULL); } write_to_socket_errexit(socket_id, 1, &ack_message, @@ -1250,7 +1250,7 @@ int receive_connection_information(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { - federate_t* fed = _F_RTI->enclaves[fed_id]; + federate_t* fed = _f_rti->enclaves[fed_id]; // Read the number of upstream and downstream connections fed->enclave.num_upstream = extract_int32(&(connection_info_header[1])); fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); @@ -1316,8 +1316,8 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { send_reject(socket_id, UNEXPECTED_MESSAGE); return 0; } else { - federate_t *fed = _F_RTI->enclaves[fed_id]; - if (_F_RTI->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. + federate_t *fed = _f_rti->enclaves[fed_id]; + if (_f_rti->clock_sync_global_status >= clock_sync_init) {// If no initial clock sync, no need perform initial clock sync. uint16_t federate_UDP_port_number = extract_uint16(&(response[1])); LF_PRINT_DEBUG("RTI got MSG_TYPE_UDP_PORT %u from federate %d.", federate_UDP_port_number, fed_id); @@ -1326,7 +1326,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { if (federate_UDP_port_number != UINT16_MAX) { // Perform the initialization clock synchronization with the federate. // Send the required number of messages for clock synchronization - for (int i=0; i < _F_RTI->clock_sync_exchanges_per_interval; i++) { + for (int i=0; i < _f_rti->clock_sync_exchanges_per_interval; i++) { // Send the RTI's current physical time T1 to the federate. send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, TCP); @@ -1349,7 +1349,7 @@ int receive_udp_message_and_set_up_clock_sync(int socket_id, uint16_t fed_id) { } LF_PRINT_DEBUG("RTI finished initial clock synchronization with federate %d.", fed_id); } - if (_F_RTI->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. + if (_f_rti->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. if (federate_UDP_port_number > 0) { // Initialize the UDP_addr field of the federate struct fed->UDP_addr.sin_family = AF_INET; @@ -1384,7 +1384,7 @@ bool authenticate_federate(int socket) { // Check HMAC of received FED_RESPONSE message. unsigned int hmac_length = SHA256_HMAC_LENGTH; - size_t federation_id_length = strnlen(_F_RTI->federation_id, 255); + size_t federation_id_length = strnlen(_f_rti->federation_id, 255); size_t fed_id_length = sizeof(uint16_t); unsigned char received[1 + NONCE_LENGTH + fed_id_length + hmac_length]; @@ -1401,7 +1401,7 @@ bool authenticate_federate(int socket) { memcpy(&buf_to_check[1], &received[1 + NONCE_LENGTH], fed_id_length); memcpy(&buf_to_check[1 + fed_id_length], rti_nonce, NONCE_LENGTH); unsigned char rti_tag[hmac_length]; - HMAC(EVP_sha256(), _F_RTI->federation_id, federation_id_length, buf_to_check, 1 + fed_id_length + NONCE_LENGTH, + HMAC(EVP_sha256(), _f_rti->federation_id, federation_id_length, buf_to_check, 1 + fed_id_length + NONCE_LENGTH, rti_tag, &hmac_length); // Compare received tag and created tag. @@ -1420,7 +1420,7 @@ bool authenticate_federate(int socket) { // Buffer for message type and HMAC tag. unsigned char sender[1 + hmac_length]; sender[0] = MSG_TYPE_RTI_RESPONSE; - HMAC(EVP_sha256(), _F_RTI->federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, + HMAC(EVP_sha256(), _f_rti->federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1], &hmac_length); write_to_socket(socket, 1 + hmac_length, sender); return true; @@ -1429,14 +1429,14 @@ bool authenticate_federate(int socket) { #endif void connect_to_federates(int socket_descriptor) { - for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { + for (int i = 0; i < _f_rti->number_of_enclaves; i++) { // Wait for an incoming connection request. struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); // The following blocks until a federate connects. int socket_id = -1; while(1) { - socket_id = accept(_F_RTI->socket_descriptor_TCP, &client_fd, &client_length); + socket_id = accept(_f_rti->socket_descriptor_TCP, &client_fd, &client_length); if (socket_id >= 0) { // Got a socket break; @@ -1451,7 +1451,7 @@ void connect_to_federates(int socket_descriptor) { // Send RTI hello when RTI -a option is on. #ifdef __RTI_AUTH__ - if (_F_RTI->authentication_enabled) { + if (_f_rti->authentication_enabled) { if (!authenticate_federate(socket_id)) { lf_print_warning("RTI failed to authenticate the incoming federate."); // Ignore the federate that failed authentication. @@ -1471,7 +1471,7 @@ void connect_to_federates(int socket_descriptor) { // This has to be done after clock synchronization is finished // or that thread may end up attempting to handle incoming clock // synchronization messages. - federate_t *fed = _F_RTI->enclaves[fed_id]; + federate_t *fed = _f_rti->enclaves[fed_id]; lf_thread_create(&(fed->thread_id), federate_thread_TCP, fed); } else { @@ -1482,19 +1482,19 @@ void connect_to_federates(int socket_descriptor) { // All federates have connected. LF_PRINT_DEBUG("All federates have connected to RTI."); - if (_F_RTI->clock_sync_global_status >= clock_sync_on) { + if (_f_rti->clock_sync_global_status >= clock_sync_on) { // Create the thread that performs periodic PTP clock synchronization sessions // over the UDP channel, but only if the UDP channel is open and at least one // federate is performing runtime clock synchronization. bool clock_sync_enabled = false; - for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { - if ((_F_RTI->enclaves[i])->clock_synchronization_enabled) { + for (int i = 0; i < _f_rti->number_of_enclaves; i++) { + if ((_f_rti->enclaves[i])->clock_synchronization_enabled) { clock_sync_enabled = true; break; } } - if (_F_RTI->final_port_UDP != UINT16_MAX && clock_sync_enabled) { - lf_thread_create(&_F_RTI->clock_thread, clock_synchronization_thread, NULL); + if (_f_rti->final_port_UDP != UINT16_MAX && clock_sync_enabled) { + lf_thread_create(&_f_rti->clock_thread, clock_synchronization_thread, NULL); } } } @@ -1505,11 +1505,11 @@ void* respond_to_erroneous_connections(void* nothing) { struct sockaddr client_fd; uint32_t client_length = sizeof(client_fd); // The following will block until either a federate attempts to connect - // or close(_F_RTI->socket_descriptor_TCP) is called. - int socket_id = accept(_F_RTI->socket_descriptor_TCP, &client_fd, &client_length); + // or close(_f_rti->socket_descriptor_TCP) is called. + int socket_id = accept(_f_rti->socket_descriptor_TCP, &client_fd, &client_length); if (socket_id < 0) return NULL; - if (_F_RTI->all_federates_exited) { + if (_f_rti->all_federates_exited) { return NULL; } @@ -1544,14 +1544,14 @@ int32_t start_rti_server(uint16_t port) { } lf_initialize_clock(); // Create the TCP socket server - _F_RTI->socket_descriptor_TCP = create_server(specified_port, port, TCP); + _f_rti->socket_descriptor_TCP = create_server(specified_port, port, TCP); lf_print("RTI: Listening for federates."); // Create the UDP socket server - // Try to get the _F_RTI->final_port_TCP + 1 port - if (_F_RTI->clock_sync_global_status >= clock_sync_on) { - _F_RTI->socket_descriptor_UDP = create_server(specified_port, _F_RTI->final_port_TCP + 1, UDP); + // Try to get the _f_rti->final_port_TCP + 1 port + if (_f_rti->clock_sync_global_status >= clock_sync_on) { + _f_rti->socket_descriptor_UDP = create_server(specified_port, _f_rti->final_port_TCP + 1, UDP); } - return _F_RTI->socket_descriptor_TCP; + return _f_rti->socket_descriptor_TCP; } void wait_for_federates(int socket_descriptor) { @@ -1570,19 +1570,19 @@ void wait_for_federates(int socket_descriptor) { // Wait for federate threads to exit. void* thread_exit_status; - for (int i = 0; i < _F_RTI->number_of_enclaves; i++) { - federate_t* fed = _F_RTI->enclaves[i]; + for (int i = 0; i < _f_rti->number_of_enclaves; i++) { + federate_t* fed = _f_rti->enclaves[i]; lf_print("RTI: Waiting for thread handling federate %d.", fed->enclave.id); lf_thread_join(fed->thread_id, &thread_exit_status); free_in_transit_message_q(fed->in_transit_message_tags); lf_print("RTI: Federate %d thread exited.", fed->enclave.id); } - _F_RTI->all_federates_exited = true; + _f_rti->all_federates_exited = true; // Shutdown and close the socket so that the accept() call in // respond_to_erroneous_connections returns. That thread should then - // check _F_RTI->all_federates_exited and it should exit. + // check _f_rti->all_federates_exited and it should exit. if (shutdown(socket_descriptor, SHUT_RDWR)) { LF_PRINT_LOG("On shut down TCP socket, received reply: %s", strerror(errno)); } @@ -1593,7 +1593,7 @@ void wait_for_federates(int socket_descriptor) { // NOTE: Apparently, closing the socket will not necessarily // cause the respond_to_erroneous_connections accept() call to return, - // so instead, we connect here so that it can check the _F_RTI->all_federates_exited + // so instead, we connect here so that it can check the _f_rti->all_federates_exited // variable. // Create an IPv4 socket for TCP (not UDP) communication over IP (0). @@ -1612,7 +1612,7 @@ void wait_for_federates(int socket_descriptor) { (char *)&server_fd.sin_addr.s_addr, server->h_length); // Convert the port number from host byte order to network byte order. - server_fd.sin_port = htons(_F_RTI->final_port_TCP); + server_fd.sin_port = htons(_f_rti->final_port_TCP); connect( tmp_socket, (struct sockaddr *)&server_fd, @@ -1629,11 +1629,11 @@ void wait_for_federates(int socket_descriptor) { close(socket_descriptor); */ - if (_F_RTI->socket_descriptor_UDP > 0) { - if (shutdown(_F_RTI->socket_descriptor_UDP, SHUT_RDWR)) { + if (_f_rti->socket_descriptor_UDP > 0) { + if (shutdown(_f_rti->socket_descriptor_UDP, SHUT_RDWR)) { LF_PRINT_LOG("On shut down UDP socket, received reply: %s", strerror(errno)); } - close(_F_RTI->socket_descriptor_UDP); + close(_f_rti->socket_descriptor_UDP); } } @@ -1668,16 +1668,16 @@ void usage(int argc, const char* argv[]) { int process_clock_sync_args(int argc, const char* argv[]) { for (int i = 0; i < argc; i++) { if (strcmp(argv[i], "off") == 0) { - _F_RTI->clock_sync_global_status = clock_sync_off; + _f_rti->clock_sync_global_status = clock_sync_off; lf_print("RTI: Clock sync: off"); } else if (strcmp(argv[i], "init") == 0 || strcmp(argv[i], "initial") == 0) { - _F_RTI->clock_sync_global_status = clock_sync_init; + _f_rti->clock_sync_global_status = clock_sync_init; lf_print("RTI: Clock sync: init"); } else if (strcmp(argv[i], "on") == 0) { - _F_RTI->clock_sync_global_status = clock_sync_on; + _f_rti->clock_sync_global_status = clock_sync_on; lf_print("RTI: Clock sync: on"); } else if (strcmp(argv[i], "period") == 0) { - if (_F_RTI->clock_sync_global_status != clock_sync_on) { + if (_f_rti->clock_sync_global_status != clock_sync_on) { lf_print_error("clock sync period can only be set if --clock-sync is set to on."); usage(argc, argv); i++; @@ -1693,10 +1693,10 @@ int process_clock_sync_args(int argc, const char* argv[]) { lf_print_error("clock sync period value is invalid."); continue; // Try to parse the rest of the arguments as clock sync args. } - _F_RTI->clock_sync_period_ns = (int64_t)period_ns; - lf_print("RTI: Clock sync period: " PRINTF_TIME, (long long int)_F_RTI->clock_sync_period_ns); + _f_rti->clock_sync_period_ns = (int64_t)period_ns; + lf_print("RTI: Clock sync period: " PRINTF_TIME, (long long int)_f_rti->clock_sync_period_ns); } else if (strcmp(argv[i], "exchanges-per-interval") == 0) { - if (_F_RTI->clock_sync_global_status != clock_sync_on && _F_RTI->clock_sync_global_status != clock_sync_init) { + if (_f_rti->clock_sync_global_status != clock_sync_on && _f_rti->clock_sync_global_status != clock_sync_init) { lf_print_error("clock sync exchanges-per-interval can only be set if\n" "--clock-sync is set to on or init."); usage(argc, argv); @@ -1712,8 +1712,8 @@ int process_clock_sync_args(int argc, const char* argv[]) { lf_print_error("clock sync exchanges-per-interval value is invalid."); continue; // Try to parse the rest of the arguments as clock sync args. } - _F_RTI->clock_sync_exchanges_per_interval = (int32_t)exchanges; // FIXME: Loses numbers on 64-bit machines - lf_print("RTI: Clock sync exchanges per interval: %d", _F_RTI->clock_sync_exchanges_per_interval); + _f_rti->clock_sync_exchanges_per_interval = (int32_t)exchanges; // FIXME: Loses numbers on 64-bit machines + lf_print("RTI: Clock sync exchanges per interval: %d", _f_rti->clock_sync_exchanges_per_interval); } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1737,7 +1737,7 @@ int process_args(int argc, const char* argv[]) { } i++; lf_print("RTI: Federation ID: %s", argv[i]); - _F_RTI->federation_id = argv[i]; + _f_rti->federation_id = argv[i]; } else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--number_of_federates") == 0) { if (argc < i + 2) { lf_print_error("--number_of_federates needs an integer argument."); @@ -1751,8 +1751,8 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _F_RTI->number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines - lf_print("RTI: Number of federates: %d\n", _F_RTI->number_of_enclaves); + _f_rti->number_of_enclaves = (int32_t)num_federates; // FIXME: Loses numbers on 64-bit machines + lf_print("RTI: Number of federates: %d\n", _f_rti->number_of_enclaves); } else if (strcmp(argv[i], "-p") == 0 || strcmp(argv[i], "--port") == 0) { if (argc < i + 2) { lf_print_error( @@ -1772,7 +1772,7 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; } - _F_RTI->user_specified_port = (uint16_t)RTI_port; + _f_rti->user_specified_port = (uint16_t)RTI_port; } else if (strcmp(argv[i], "-c") == 0 || strcmp(argv[i], "--clock_sync") == 0) { if (argc < i + 2) { lf_print_error("--clock-sync needs off|init|on."); @@ -1787,9 +1787,9 @@ int process_args(int argc, const char* argv[]) { usage(argc, argv); return 0; #endif - _F_RTI->authentication_enabled = true; + _f_rti->authentication_enabled = true; } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { - _F_RTI->tracing_enabled = true; + _f_rti->tracing_enabled = true; } else if (strcmp(argv[i], " ") == 0) { // Tolerate spaces continue; @@ -1799,7 +1799,7 @@ int process_args(int argc, const char* argv[]) { return 0; } } - if (_F_RTI->number_of_enclaves == 0) { + if (_f_rti->number_of_enclaves == 0) { lf_print_error("--number_of_federates needs a valid positive integer argument."); usage(argc, argv); return 0; @@ -1808,24 +1808,24 @@ int process_args(int argc, const char* argv[]) { } void initialize_RTI(){ - _F_RTI = (federation_RTI_t *)malloc(sizeof(federation_RTI_t)); + _f_rti = (federation_rti_t *)malloc(sizeof(federation_rti_t)); // enclave_rti related initializations - _F_RTI->max_stop_tag = NEVER_TAG, - _F_RTI->number_of_enclaves = 0, - _F_RTI->num_enclaves_handling_stop = 0, + _f_rti->max_stop_tag = NEVER_TAG, + _f_rti->number_of_enclaves = 0, + _f_rti->num_enclaves_handling_stop = 0, // federation_rti related initializations - _F_RTI->max_start_time = 0LL, - _F_RTI->num_feds_proposed_start = 0, - _F_RTI->all_federates_exited = false, - _F_RTI->federation_id = "Unidentified Federation", - _F_RTI->user_specified_port = 0, - _F_RTI->final_port_TCP = 0, - _F_RTI->socket_descriptor_TCP = -1, - _F_RTI->final_port_UDP = UINT16_MAX, - _F_RTI->socket_descriptor_UDP = -1, - _F_RTI->clock_sync_global_status = clock_sync_init, - _F_RTI->clock_sync_period_ns = MSEC(10), - _F_RTI->clock_sync_exchanges_per_interval = 10, - _F_RTI->authentication_enabled = false, - _F_RTI->tracing_enabled = false; + _f_rti->max_start_time = 0LL, + _f_rti->num_feds_proposed_start = 0, + _f_rti->all_federates_exited = false, + _f_rti->federation_id = "Unidentified Federation", + _f_rti->user_specified_port = 0, + _f_rti->final_port_TCP = 0, + _f_rti->socket_descriptor_TCP = -1, + _f_rti->final_port_UDP = UINT16_MAX, + _f_rti->socket_descriptor_UDP = -1, + _f_rti->clock_sync_global_status = clock_sync_init, + _f_rti->clock_sync_period_ns = MSEC(10), + _f_rti->clock_sync_exchanges_per_interval = 10, + _f_rti->authentication_enabled = false, + _f_rti->tracing_enabled = false; } \ No newline at end of file diff --git a/core/federated/RTI/rti_lib.h b/core/federated/RTI/rti_lib.h index 8b3e5ab56..964defb99 100644 --- a/core/federated/RTI/rti_lib.h +++ b/core/federated/RTI/rti_lib.h @@ -75,17 +75,17 @@ typedef enum clock_sync_stat { /** * Structure that an RTI instance uses to keep track of its own and its * corresponding federates' state. - * It is a special case of `enclave_RTI_t` (declared in enclave.h). Inheritence - * is mimicked by having the first attributes to be the same as of enclave_RTI_t, + * It is a special case of `enclave_rti_t` (declared in enclave.h). Inheritence + * is mimicked by having the first attributes to be the same as of enclave_rti_t, * except that enclaves attribute here is of type `federate_t**`, while it - * is of type `enclave_t**` in `enclave_RTI_t`. + * is of type `enclave_t**` in `enclave_rti_t`. * // **************** IMPORTANT!!! ******************** * // ** If you make any change to this struct, ** - * // ** you MUST also change enclave_RTI_t in ** + * // ** you MUST also change enclave_rti_t in ** * // ** (enclave.h)! The change must exactly match. ** * // ************************************************** */ -typedef struct federation_RTI_t { +typedef struct federation_rti_t { ////////////////// Enclave specific attributes ////////////////// // The federates. @@ -169,7 +169,7 @@ typedef struct federation_RTI_t { * Boolean indicating that authentication is enabled. */ bool authentication_enabled; -} federation_RTI_t; +} federation_rti_t; /** * Enter a critical section where logical time and the event queue are guaranteed