diff --git a/CMakeLists.txt b/CMakeLists.txt index 186e5b670..4127b2b65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ include_directories(${CMAKE_SOURCE_DIR}/include/core/platform) include_directories(${CMAKE_SOURCE_DIR}/include/core/threaded) include_directories(${CMAKE_SOURCE_DIR}/include/core/utils) include_directories(${CMAKE_SOURCE_DIR}/include/api) +include_directories(/usr/local/include) enable_testing() add_subdirectory(${Lib}) diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index ca51fd50c..ec6ba8647 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -18,7 +18,6 @@ list(APPEND REACTORC_SOURCES ${GENERAL_SOURCES}) # Add sources for either threaded or single-threaded runtime if (DEFINED FEDERATED) include(federated/CMakeLists.txt) - include(federated/network/CMakeLists.txt) endif() # Add sources for either threaded or single-threaded runtime @@ -96,6 +95,21 @@ include(${LF_ROOT}/platform/impl/CMakeLists.txt) target_link_libraries(reactor-c PUBLIC lf::platform-api) target_link_libraries(reactor-c PRIVATE lf::platform-impl) +# message("reactor-c/core/CMakeLists.txt First") +option(COMM_TYPE "Communication type between RTI and federate(s)." ON) +IF(COMM_TYPE MATCHES ON) + set(COMM_TYPE TCP) +ENDIF() + +include(${LF_ROOT}/include/core/federated/network/CMakeLists.txt) +target_link_libraries(reactor-c PUBLIC lf::network-api) + +include(${LF_ROOT}/core/federated/network/CMakeLists.txt) +target_link_libraries(reactor-c PRIVATE lf::network-impl) + +# Apply compile definitions to the reactor-c library. +target_compile_definitions(reactor-c PUBLIC ${REACTORC_COMPILE_DEFS}) + target_include_directories(reactor-c PUBLIC ../include) target_include_directories(reactor-c PUBLIC ../include/core) target_include_directories(reactor-c PUBLIC ../include/core/federated) @@ -105,6 +119,7 @@ target_include_directories(reactor-c PUBLIC ../include/core/modal_models) target_include_directories(reactor-c PUBLIC ../include/core/threaded) target_include_directories(reactor-c PUBLIC ../include/core/utils) target_include_directories(reactor-c PUBLIC federated/RTI/) +target_include_directories(reactor-c PUBLIC /usr/local/include) if (APPLE) SET(CMAKE_C_ARCHIVE_CREATE " Scr ") @@ -122,6 +137,12 @@ if(DEFINED FEDERATED_AUTHENTICATED) target_link_libraries(reactor-c PUBLIC OpenSSL::SSL) endif() +if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + set(OPENSSL_ROOT_DIR "/usr/local/lib64") +endif() + +set(OPENSSL_ROOT_DIR "/usr/local/lib64") + if(DEFINED FEDERATED) find_library(MATH_LIBRARY m) if(MATH_LIBRARY) diff --git a/core/federated/RTI/CMakeLists.txt b/core/federated/RTI/CMakeLists.txt index 5bfbf0196..a010d253e 100644 --- a/core/federated/RTI/CMakeLists.txt +++ b/core/federated/RTI/CMakeLists.txt @@ -1,6 +1,10 @@ cmake_minimum_required(VERSION 3.12) project(RTI VERSION 1.0.0 LANGUAGES C) +if(${CMAKE_SYSTEM_NAME} STREQUAL "Linux") + set(OPENSSL_ROOT_DIR "/usr/local/lib64") +endif() + set(CoreLib ../../../core) set(LF_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../..) set(IncludeDir ../../../include/core) @@ -16,12 +20,16 @@ add_library(${RTI_LIB} STATIC ${CoreLib}/utils/util.c ${CoreLib}/tag.c ${CoreLib}/clock.c - ${CoreLib}/federated/network/net_util.c ${CoreLib}/utils/pqueue_base.c ${CoreLib}/utils/pqueue_tag.c ${CoreLib}/utils/pqueue.c ) +option(COMM_TYPE "Communication type between RTI and federate(s)." ON) +IF(COMM_TYPE MATCHES ON) + set(COMM_TYPE TCP) +ENDIF() + # Add the main target which will link with the library. add_executable(${RTI_MAIN} main.c) @@ -70,6 +78,12 @@ target_link_libraries(${RTI_LIB} PUBLIC lf::low-level-platform-impl) include(${LF_ROOT}/low_level_platform/api/CMakeLists.txt) target_link_libraries(${RTI_LIB} PUBLIC lf::low-level-platform-api) +include(${IncludeDir}/federated/network/CMakeLists.txt) +target_link_libraries(${RTI_LIB} PUBLIC lf::network-api) + +include(${CoreLib}/federated/network/CMakeLists.txt) +target_link_libraries(${RTI_LIB} PUBLIC lf::network-impl) + # Set the STANDALONE_RTI flag to include the rti_remote and rti_common. target_compile_definitions(${RTI_LIB} PUBLIC STANDALONE_RTI=1) @@ -77,6 +91,8 @@ target_compile_definitions(${RTI_LIB} PUBLIC STANDALONE_RTI=1) target_compile_definitions(${RTI_LIB} PUBLIC FEDERATED=1) target_compile_definitions(${RTI_LIB} PUBLIC PLATFORM_${CMAKE_SYSTEM_NAME}) +target_compile_definitions(RTI PUBLIC COMM_TYPE_${COMM_TYPE}) + # Set RTI Tracing target_compile_definitions(${RTI_LIB} PUBLIC RTI_TRACE) diff --git a/core/federated/RTI/RTI.config b/core/federated/RTI/RTI.config new file mode 100644 index 000000000..5b601544c --- /dev/null +++ b/core/federated/RTI/RTI.config @@ -0,0 +1,10 @@ +entityInfo.name=net1.rti +entityInfo.purpose={"group":"Servers"} +entityInfo.number_key=1 +authInfo.pubkey.path=/home/dongha/project/iotauth/entity/auth_certs/Auth101EntityCert.pem +entityInfo.privkey.path=/home/dongha/project/iotauth/entity/credentials/keys/net1/Net1.rtiKey.pem +auth.ip.address=127.0.0.1 +auth.port.number=21900 +entity.server.ip.address=127.0.0.1 +entity.server.port.number=21100 +network.protocol=TCP \ No newline at end of file diff --git a/core/federated/RTI/main.c b/core/federated/RTI/main.c index 17d73e93e..b7e6b4c29 100644 --- a/core/federated/RTI/main.c +++ b/core/federated/RTI/main.c @@ -48,7 +48,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "rti_remote.h" -#include "net_util.h" #include // To trap ctrl-c and invoke a clean stop to save the trace file, if needed. #include @@ -80,11 +79,11 @@ static void send_failed_signal(federate_info_t* fed) { if (rti.base.tracing_enabled) { tracepoint_rti_to_federate(send_FAILED, fed->enclave.id, NULL); } - int failed = write_to_socket(fed->socket, bytes_to_write, &(buffer[0])); - if (failed == 0) { + int bytes_written = write_to_netdrv(fed->fed_netdrv, bytes_to_write, &(buffer[0])); + if (bytes_written > 0) { LF_PRINT_LOG("RTI has sent failed signal to federate %d due to abnormal termination.", fed->enclave.id); } else { - lf_print_error("RTI failed to send failed signal to federate %d on socket ID %d.", fed->enclave.id, fed->socket); + // lf_print_error("RTI failed to send failed signal to federate %d on socket ID %d.", fed->enclave.id, fed->socket); } } @@ -123,7 +122,7 @@ void usage(int argc, const char* argv[]) { lf_print(" The number of federates in the federation that this RTI will control.\n"); lf_print(" -p, --port "); lf_print(" The port number to use for the RTI. Must be larger than 0 and smaller than %d. Default is %d.\n", - UINT16_MAX, DEFAULT_PORT); + UINT16_MAX, RTI_DEFAULT_PORT); lf_print(" -c, --clock_sync [off|init|on] [period ] [exchanges-per-interval ]"); lf_print(" The status of clock synchronization for this federate."); lf_print(" - off: Clock synchronization is off."); @@ -136,6 +135,7 @@ void usage(int argc, const char* argv[]) { lf_print(" clock sync attempt (default is 10). Applies to 'init' and 'on'.\n"); lf_print(" -a, --auth Turn on HMAC authentication options.\n"); lf_print(" -t, --tracing Turn on tracing.\n"); + lf_print(" -sst, --sst SST config path for RTI.\n"); lf_print("Command given:"); for (int i = 0; i < argc; i++) { @@ -261,6 +261,15 @@ int process_args(int argc, const char* argv[]) { return 0; #endif rti.authentication_enabled = true; + } else if (strcmp(argv[i], "-sst") == 0 || strcmp(argv[i], "--sst") == 0) { +#ifndef COMM_TYPE_SST + lf_print_error("--sst requires the RTI to be built with the --DCOMM_TYPE=SST option."); + usage(argc, argv); + return 0; +#else + i++; + lf_set_rti_sst_config_path(argv[i]); +#endif } else if (strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--tracing") == 0) { rti.base.tracing_enabled = true; } else if (strcmp(argv[i], " ") == 0) { @@ -324,9 +333,8 @@ int main(int argc, const char* argv[]) { rti.base.scheduling_nodes[i] = (scheduling_node_t*)fed_info; } - int socket_descriptor = start_rti_server(rti.user_specified_port); - if (socket_descriptor >= 0) { - wait_for_federates(socket_descriptor); + if (start_rti_server(rti.user_specified_port)) { + wait_for_federates(rti.rti_netdrv); normal_termination = true; if (rti.base.tracing_enabled) { // No need for a mutex lock because all threads have exited. diff --git a/core/federated/RTI/rti_remote.c b/core/federated/RTI/rti_remote.c index cc0252843..be1d2326a 100644 --- a/core/federated/RTI/rti_remote.c +++ b/core/federated/RTI/rti_remote.c @@ -52,132 +52,6 @@ extern int lf_critical_section_enter(environment_t* env) { return lf_mutex_lock( extern int lf_critical_section_exit(environment_t* env) { return lf_mutex_unlock(&rti_mutex); } -/** - * Create a server and enable listening for socket connections. - * If the specified port if it is non-zero, it will attempt to acquire that port. - * If it fails, it will repeatedly attempt up to PORT_BIND_RETRY_LIMIT times with - * a delay of PORT_BIND_RETRY_INTERVAL in between. If the specified port is - * zero, then it will attempt to acquire DEFAULT_PORT first. If this fails, then it - * will repeatedly attempt up to PORT_BIND_RETRY_LIMIT times, incrementing the port - * number between attempts, with no delay between attempts. Once it has incremented - * the port number MAX_NUM_PORT_ADDRESSES times, it will cycle around and begin again - * with DEFAULT_PORT. - * - * @param port The port number to use or 0 to start trying at DEFAULT_PORT. - * @param socket_type The type of the socket for the server (TCP or UDP). - * @return The socket descriptor on which to accept connections. - */ -static int create_rti_server(uint16_t port, socket_type_t socket_type) { - // Timeout time for the communications of the server - struct timeval timeout_time = {.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000}; - // Create an IPv4 socket for TCP (not UDP) communication over IP (0). - int socket_descriptor = -1; - if (socket_type == TCP) { - socket_descriptor = create_real_time_tcp_socket_errexit(); - } else if (socket_type == UDP) { - socket_descriptor = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - // Set the appropriate timeout time - timeout_time = - (struct timeval){.tv_sec = UDP_TIMEOUT_TIME / BILLION, .tv_usec = (UDP_TIMEOUT_TIME % BILLION) / 1000}; - } - if (socket_descriptor < 0) { - lf_print_error_system_failure("Failed to create RTI socket."); - } - - // Set the option for this socket to reuse the same address - int true_variable = 1; // setsockopt() requires a reference to the value assigned to an option - if (setsockopt(socket_descriptor, SOL_SOCKET, SO_REUSEADDR, &true_variable, sizeof(int32_t)) < 0) { - lf_print_error("RTI failed to set SO_REUSEADDR option on the socket: %s.", strerror(errno)); - } - // Set the timeout on the socket so that read and write operations don't block for too long - if (setsockopt(socket_descriptor, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout_time, sizeof(timeout_time)) < 0) { - lf_print_error("RTI failed to set SO_RCVTIMEO option on the socket: %s.", strerror(errno)); - } - if (setsockopt(socket_descriptor, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout_time, sizeof(timeout_time)) < 0) { - lf_print_error("RTI failed to set SO_SNDTIMEO option on the socket: %s.", strerror(errno)); - } - - /* - * The following used to permit reuse of a port that an RTI has previously - * used that has not been released. We no longer do this, and instead retry - * some number of times after waiting. - - // SO_REUSEPORT (since Linux 3.9) - // Permits multiple AF_INET or AF_INET6 sockets to be bound to an - // identical socket address. This option must be set on each - // socket (including the first socket) prior to calling bind(2) - // on the socket. To prevent port hijacking, all of the - // processes binding to the same address must have the same - // effective UID. This option can be employed with both TCP and - // UDP sockets. - - int reuse = 1; - #ifdef SO_REUSEPORT - if (setsockopt(socket_descriptor, SOL_SOCKET, SO_REUSEPORT, - (const char*)&reuse, sizeof(reuse)) < 0) { - perror("setsockopt(SO_REUSEPORT) failed"); - } - #endif - */ - - // Server file descriptor. - struct sockaddr_in server_fd; - // Zero out the server address structure. - bzero((char*)&server_fd, sizeof(server_fd)); - - uint16_t specified_port = port; - if (specified_port == 0) - port = DEFAULT_PORT; - - server_fd.sin_family = AF_INET; // IPv4 - server_fd.sin_addr.s_addr = INADDR_ANY; // All interfaces, 0.0.0.0. - // Convert the port number from host byte order to network byte order. - server_fd.sin_port = htons(port); - - int result = bind(socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); - - // Try repeatedly to bind to a port. If no specific port is specified, then - // increment the port number each time. - - int count = 1; - while (result != 0 && count++ < PORT_BIND_RETRY_LIMIT) { - if (specified_port == 0) { - lf_print_warning("RTI failed to get port %d.", port); - port++; - if (port >= DEFAULT_PORT + MAX_NUM_PORT_ADDRESSES) - port = DEFAULT_PORT; - lf_print_warning("RTI will try again with port %d.", port); - server_fd.sin_port = htons(port); - // Do not sleep. - } else { - lf_print("RTI failed to get port %d. Will try again.", port); - lf_sleep(PORT_BIND_RETRY_INTERVAL); - } - result = bind(socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); - } - if (result != 0) { - lf_print_error_and_exit("Failed to bind the RTI socket. Port %d is not available. ", port); - } - char* type = "TCP"; - if (socket_type == UDP) { - type = "UDP"; - } - lf_print("RTI using %s port %d for federation %s.", type, port, rti_remote->federation_id); - - if (socket_type == TCP) { - rti_remote->final_port_TCP = port; - // Enable listening for socket connections. - // The second argument is the maximum number of queued socket requests, - // which according to the Mac man page is limited to 128. - listen(socket_descriptor, 128); - } else if (socket_type == UDP) { - rti_remote->final_port_UDP = port; - // No need to listen on the UDP socket - } - - return socket_descriptor; -} - void notify_tag_advance_grant(scheduling_node_t* e, tag_t tag) { if (e->state == NOT_CONNECTED || lf_tag_compare(tag, e->last_granted) <= 0 || lf_tag_compare(tag, e->last_provisionally_granted) < 0) { @@ -199,9 +73,9 @@ void notify_tag_advance_grant(scheduling_node_t* e, tag_t tag) { tracepoint_rti_to_federate(send_TAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long - // function. During this call, the socket might close, causing the following write_to_socket + // function. During this call, the netdriver might close, causing the following write_to_netdrv // to fail. Consider a failure here a soft failure and update the federate's status. - if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) { + if (write_to_netdrv(((federate_info_t*)e)->fed_netdrv, message_length, buffer) <= 0) { lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id); e->state = NOT_CONNECTED; } else { @@ -232,9 +106,9 @@ void notify_provisional_tag_advance_grant(scheduling_node_t* e, tag_t tag) { tracepoint_rti_to_federate(send_PTAG, e->id, &tag); } // This function is called in notify_advance_grant_if_safe(), which is a long - // function. During this call, the socket might close, causing the following write_to_socket + // function. During this call, the netdriver might close, causing the following write_to_netdrv // to fail. Consider a failure here a soft failure and update the federate's status. - if (write_to_socket(((federate_info_t*)e)->socket, message_length, buffer)) { + if (write_to_netdrv(((federate_info_t*)e)->fed_netdrv, message_length, buffer) <= 0) { lf_print_error("RTI failed to send tag advance grant to federate %d.", e->id); e->state = NOT_CONNECTED; } else { @@ -282,22 +156,16 @@ void update_federate_next_event_tag_locked(uint16_t federate_id, tag_t next_even } void handle_port_absent_message(federate_info_t* sending_federate, unsigned char* buffer) { - size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); - - read_from_socket_fail_on_error(&sending_federate->socket, message_size, &(buffer[1]), NULL, - " RTI failed to read port absent message from federate %u.", - sending_federate->enclave.id); - - uint16_t reactor_port_id = extract_uint16(&(buffer[1])); - uint16_t federate_id = extract_uint16(&(buffer[1 + sizeof(uint16_t)])); - tag_t tag = extract_tag(&(buffer[1 + 2 * sizeof(uint16_t)])); + uint16_t reactor_port_id = extract_uint16(buffer + 1); + uint16_t federate_id = extract_uint16(buffer + 1 + sizeof(uint16_t)); + tag_t tag = extract_tag(buffer + 1 + 2 * sizeof(uint16_t)); if (rti_remote->base.tracing_enabled) { tracepoint_rti_from_federate(receive_PORT_ABS, sending_federate->enclave.id, &tag); } // Need to acquire the mutex lock to ensure that the thread handling - // messages coming from the socket connected to the destination does not + // messages coming from the netdriver connected to the destination does not // issue a TAG before this message has been forwarded. LF_MUTEX_LOCK(&rti_mutex); @@ -333,47 +201,43 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char } // Forward the message. - write_to_socket_fail_on_error(&fed->socket, message_size + 1, buffer, &rti_mutex, + size_t message_size = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); + write_to_netdrv_fail_on_error(fed->fed_netdrv, message_size + 1, buffer, &rti_mutex, "RTI failed to forward message to federate %d.", federate_id); LF_MUTEX_UNLOCK(&rti_mutex); } -void handle_timed_message(federate_info_t* sending_federate, unsigned char* buffer) { +void handle_timed_message(federate_info_t* sending_federate, unsigned char* buffer, ssize_t bytes_read) { size_t header_size = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(int64_t) + sizeof(uint32_t); - // Read the header, minus the first byte which has already been read. - read_from_socket_fail_on_error(&sending_federate->socket, header_size - 1, &(buffer[1]), NULL, - "RTI failed to read the timed message header from remote federate."); // Extract the header information. of the sender uint16_t reactor_port_id; uint16_t federate_id; size_t length; tag_t intended_tag; // Extract information from the header. - extract_timed_header(&(buffer[1]), &reactor_port_id, &federate_id, &length, &intended_tag); + extract_timed_header(buffer + 1, &reactor_port_id, &federate_id, &length, &intended_tag); - size_t total_bytes_to_read = length + header_size; - size_t bytes_to_read = length; + // size_t total_bytes_to_read = length + header_size; + // size_t bytes_to_read = length; - if (FED_COM_BUFFER_SIZE < header_size + 1) { - lf_print_error_and_exit("Buffer size (%d) is not large enough to " - "read the header plus one byte.", - FED_COM_BUFFER_SIZE); - } + // if (FED_COM_BUFFER_SIZE < header_size + 1) { + // lf_print_error_and_exit("Buffer size (%d) is not large enough to " + // "read the header plus one byte.", + // FED_COM_BUFFER_SIZE); + // } - // Cut up the payload in chunks. - if (bytes_to_read > FED_COM_BUFFER_SIZE - header_size) { - bytes_to_read = FED_COM_BUFFER_SIZE - header_size; - } + // // Cut up the payload in chunks. + // if (bytes_to_read > FED_COM_BUFFER_SIZE - header_size) { + // bytes_to_read = FED_COM_BUFFER_SIZE - header_size; + // } LF_PRINT_LOG("RTI received message from federate %d for federate %u port %u with intended tag " PRINTF_TAG ". Forwarding.", sending_federate->enclave.id, federate_id, reactor_port_id, intended_tag.time - lf_time_start(), intended_tag.microstep); - read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, &(buffer[header_size]), NULL, - "RTI failed to read timed message from federate %d.", federate_id); - size_t bytes_read = bytes_to_read + header_size; + // size_t bytes_read = bytes_to_read + header_size; // Following only works for string messages. // LF_PRINT_DEBUG("Message received by RTI: %s.", buffer + header_size); @@ -382,7 +246,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff } // Need to acquire the mutex lock to ensure that the thread handling - // messages coming from the socket connected to the destination does not + // messages coming from the netdriver connected to the destination does not // issue a TAG before this message has been forwarded. LF_MUTEX_LOCK(&rti_mutex); @@ -401,15 +265,9 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff fed->enclave.last_provisionally_granted.time - start_time, fed->enclave.last_provisionally_granted.microstep); // If the message was larger than the buffer, we must empty out the remainder also. - size_t total_bytes_read = bytes_read; - while (total_bytes_read < total_bytes_to_read) { - bytes_to_read = total_bytes_to_read - total_bytes_read; - if (bytes_to_read > FED_COM_BUFFER_SIZE) { - bytes_to_read = FED_COM_BUFFER_SIZE; - } - read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, NULL, + while (sending_federate->fed_netdrv->read_remaining_bytes > 0) { + read_from_netdrv_fail_on_error(sending_federate->fed_netdrv, buffer, FED_COM_BUFFER_SIZE, NULL, "RTI failed to clear message chunks."); - total_bytes_read += bytes_to_read; } LF_MUTEX_UNLOCK(&rti_mutex); return; @@ -429,29 +287,38 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff tracepoint_rti_to_federate(send_TAGGED_MSG, federate_id, &intended_tag); } - write_to_socket_fail_on_error(&fed->socket, bytes_read, buffer, &rti_mutex, + write_to_netdrv_fail_on_error(fed->fed_netdrv, bytes_read, buffer, &rti_mutex, "RTI failed to forward message to federate %d.", federate_id); - // The message length may be longer than the buffer, - // in which case we have to handle it in chunks. - size_t total_bytes_read = bytes_read; - while (total_bytes_read < total_bytes_to_read) { - LF_PRINT_DEBUG("Forwarding message in chunks."); - bytes_to_read = total_bytes_to_read - total_bytes_read; - if (bytes_to_read > FED_COM_BUFFER_SIZE) { - bytes_to_read = FED_COM_BUFFER_SIZE; + while (sending_federate->fed_netdrv->read_remaining_bytes > 0) { + ssize_t bytes_read_again = read_from_netdrv(sending_federate->fed_netdrv, buffer, FED_COM_BUFFER_SIZE); + if (bytes_read_again <= 0) { + lf_print_error_and_exit("RTI failed to read message chunks."); } - read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, NULL, - "RTI failed to read message chunks."); - total_bytes_read += bytes_to_read; - - // FIXME: a mutex needs to be held for this so that other threads - // do not write to destination_socket and cause interleaving. However, - // holding the rti_mutex might be very expensive. Instead, each outgoing - // socket should probably have its own mutex. - write_to_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, &rti_mutex, - "RTI failed to send message chunks."); - } + write_to_netdrv_fail_on_error(fed->fed_netdrv, bytes_read_again, buffer, &rti_mutex, + "RTI failed to forward message to federate %d.", federate_id); + } + + // // The message length may be longer than the buffer, + // // in which case we have to handle it in chunks. + // size_t total_bytes_read = bytes_read; + // while (total_bytes_read < total_bytes_to_read) { + // LF_PRINT_DEBUG("Forwarding message in chunks."); + // bytes_to_read = total_bytes_to_read - total_bytes_read; + // if (bytes_to_read > FED_COM_BUFFER_SIZE) { + // bytes_to_read = FED_COM_BUFFER_SIZE; + // } + // read_from_socket_fail_on_error(&sending_federate->socket, bytes_to_read, buffer, NULL, + // "RTI failed to read message chunks."); + // total_bytes_read += bytes_to_read; + + // // FIXME: a mutex needs to be held for this so that other threads + // // do not write to destination_socket and cause interleaving. However, + // // holding the rti_mutex might be very expensive. Instead, each outgoing + // // socket should probably have its own mutex. + // write_to_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, &rti_mutex, + // "RTI failed to send message chunks."); + // } // Record this in-transit message in federate's in-transit message queue. if (lf_tag_compare(fed->enclave.completed, intended_tag) < 0) { @@ -477,11 +344,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff LF_MUTEX_UNLOCK(&rti_mutex); } -void handle_latest_tag_complete(federate_info_t* fed) { - unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, NULL, - "RTI failed to read the content of the logical tag complete from federate %d.", - fed->enclave.id); +void handle_latest_tag_complete(federate_info_t* fed, unsigned char* buffer) { tag_t completed = extract_tag(buffer); if (rti_remote->base.tracing_enabled) { tracepoint_rti_from_federate(receive_LTC, fed->enclave.id, &completed); @@ -495,12 +358,7 @@ void handle_latest_tag_complete(federate_info_t* fed) { LF_MUTEX_UNLOCK(&rti_mutex); } -void handle_next_event_tag(federate_info_t* fed) { - unsigned char buffer[sizeof(int64_t) + sizeof(uint32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(int64_t) + sizeof(uint32_t), buffer, NULL, - "RTI failed to read the content of the next event tag from federate %d.", - fed->enclave.id); - +void handle_next_event_tag(federate_info_t* fed, unsigned char* buffer) { // Acquire a mutex lock to ensure that this state does not change while a // message is in transport or being used to determine a TAG. LF_MUTEX_LOCK(&rti_mutex); // FIXME: Instead of using a mutex, it might be more efficient to use a @@ -555,7 +413,7 @@ static void broadcast_stop_time_to_federates_locked() { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_STOP_GRN, fed->enclave.id, &rti_remote->base.max_stop_tag); } - write_to_socket_fail_on_error(&fed->socket, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, &rti_mutex, + write_to_netdrv_fail_on_error(fed->fed_netdrv, MSG_TYPE_STOP_GRANTED_LENGTH, outgoing_buffer, &rti_mutex, "RTI failed to send MSG_TYPE_STOP_GRANTED message to federate %d.", fed->enclave.id); } @@ -603,15 +461,9 @@ static void* wait_for_stop_request_reply(void* args) { return NULL; } -void handle_stop_request_message(federate_info_t* fed) { +void handle_stop_request_message(federate_info_t* fed, unsigned char* buffer) { LF_PRINT_DEBUG("RTI handling stop_request from federate %d.", fed->enclave.id); - size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; - unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&fed->socket, bytes_to_read, buffer, NULL, - "RTI failed to read the MSG_TYPE_STOP_REQUEST payload from federate %d.", - fed->enclave.id); - // Extract the proposed stop tag for the federate tag_t proposed_stop_tag = extract_tag(buffer); @@ -675,7 +527,7 @@ void handle_stop_request_message(federate_info_t* fed) { if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_STOP_REQ, f->enclave.id, &rti_remote->base.max_stop_tag); } - write_to_socket_fail_on_error(&f->socket, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, &rti_mutex, + write_to_netdrv_fail_on_error(f->fed_netdrv, MSG_TYPE_STOP_REQUEST_LENGTH, stop_request_buffer, &rti_mutex, "RTI failed to forward MSG_TYPE_STOP_REQUEST message to federate %d.", f->enclave.id); } @@ -685,14 +537,8 @@ void handle_stop_request_message(federate_info_t* fed) { LF_MUTEX_UNLOCK(&rti_mutex); } -void handle_stop_request_reply(federate_info_t* fed) { - size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_REPLY_LENGTH - 1; - unsigned char buffer_stop_time[bytes_to_read]; - read_from_socket_fail_on_error(&fed->socket, bytes_to_read, buffer_stop_time, NULL, - "RTI failed to read the reply to MSG_TYPE_STOP_REQUEST message from federate %d.", - fed->enclave.id); - - tag_t federate_stop_tag = extract_tag(buffer_stop_time); +void handle_stop_request_reply(federate_info_t* fed, unsigned char* buffer) { + tag_t federate_stop_tag = extract_tag(buffer); if (rti_remote->base.tracing_enabled) { tracepoint_rti_from_federate(receive_STOP_REQ_REP, fed->enclave.id, &federate_stop_tag); @@ -713,13 +559,9 @@ void handle_stop_request_reply(federate_info_t* fed) { ////////////////////////////////////////////////// -void handle_address_query(uint16_t fed_id) { +void handle_address_query(uint16_t fed_id, unsigned char* buffer) { federate_info_t* fed = GET_FED_INFO(fed_id); - // Use buffer both for reading and constructing the reply. - // The length is what is needed for the reply. - unsigned char buffer[1 + sizeof(int32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(uint16_t), (unsigned char*)buffer, NULL, - "Failed to read address query."); + uint16_t remote_fed_id = extract_uint16(buffer); if (rti_remote->base.tracing_enabled) { @@ -732,43 +574,50 @@ void handle_address_query(uint16_t fed_id) { // the port number because it has not yet received an MSG_TYPE_ADDRESS_ADVERTISEMENT message // from this federate. In that case, it will respond by sending -1. + // The length is what is needed for the reply. + unsigned char buf[1 + sizeof(int32_t) + sizeof(struct in_addr)]; // Response message is MSG_TYPE_ADDRESS_QUERY_REPLY. - buffer[0] = MSG_TYPE_ADDRESS_QUERY_REPLY; + buf[0] = MSG_TYPE_ADDRESS_QUERY_REPLY; // Encode the port number. federate_info_t* remote_fed = GET_FED_INFO(remote_fed_id); - // Send the port number (which could be -1). + // Send the port number (which could be -1) and server IP address to federate. LF_MUTEX_LOCK(&rti_mutex); - encode_int32(remote_fed->server_port, (unsigned char*)&buffer[1]); - write_to_socket_fail_on_error(&fed->socket, sizeof(int32_t) + 1, (unsigned char*)buffer, &rti_mutex, - "Failed to write port number to socket of federate %d.", fed_id); - - // Send the server IP address to federate. - write_to_socket_fail_on_error(&fed->socket, sizeof(remote_fed->server_ip_addr), - (unsigned char*)&remote_fed->server_ip_addr, &rti_mutex, - "Failed to write ip address to socket of federate %d.", fed_id); + int32_t server_port = get_port(remote_fed->fed_netdrv); + char* server_hostname; + encode_int32(server_port, (unsigned char*)&buf[1]); + if (server_port == -1) { + // RTI does not know ip_address and port yet. + server_hostname = "localhost"; + buf[1 + sizeof(int32_t)] = -1; + } else { + server_hostname = get_host_name(remote_fed->fed_netdrv); + memcpy(buf + 1 + sizeof(int32_t), (unsigned char*)get_ip_addr(remote_fed->fed_netdrv), + sizeof(*get_ip_addr(remote_fed->fed_netdrv))); + } + write_to_netdrv_fail_on_error(fed->fed_netdrv, sizeof(int32_t) + 1 + sizeof(*get_ip_addr(remote_fed->fed_netdrv)), + (unsigned char*)buf, &rti_mutex, + "Failed to write port number to netdrv of federate %d.", fed_id); + LF_MUTEX_UNLOCK(&rti_mutex); - LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id, remote_fed->server_hostname, - remote_fed->server_port); + LF_PRINT_DEBUG("Replied to address query from federate %d with address %s:%d.", fed_id, server_hostname, server_port); } -void handle_address_ad(uint16_t federate_id) { +// TODO: NEED to be fixed. +void handle_address_ad(uint16_t federate_id, unsigned char* buffer) { federate_info_t* fed = GET_FED_INFO(federate_id); // Read the port number of the federate that can be used for physical // connections to other federates int32_t server_port = -1; - unsigned char buffer[sizeof(int32_t)]; - read_from_socket_fail_on_error(&fed->socket, sizeof(int32_t), (unsigned char*)buffer, NULL, - "Error reading port data from federate %d.", federate_id); server_port = extract_int32(buffer); assert(server_port < 65536); LF_MUTEX_LOCK(&rti_mutex); - fed->server_port = server_port; + set_port(fed->fed_netdrv, server_port); LF_MUTEX_UNLOCK(&rti_mutex); LF_PRINT_LOG("Received address advertisement with port %d from federate %d.", server_port, federate_id); @@ -777,13 +626,8 @@ void handle_address_ad(uint16_t federate_id) { } } -void handle_timestamp(federate_info_t* my_fed) { - unsigned char buffer[sizeof(int64_t)]; - // Read bytes from the socket. We need 8 bytes. - read_from_socket_fail_on_error(&my_fed->socket, sizeof(int64_t), (unsigned char*)&buffer, NULL, - "ERROR reading timestamp from federate %d.\n", my_fed->enclave.id); - - int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t*)(&buffer))); +void handle_timestamp(federate_info_t* my_fed, unsigned char* buffer) { + int64_t timestamp = swap_bytes_if_big_endian_int64(*((int64_t*)(buffer))); if (rti_remote->base.tracing_enabled) { tag_t tag = {.time = timestamp, .microstep = 0}; tracepoint_rti_from_federate(receive_TIMESTAMP, my_fed->enclave.id, &tag); @@ -822,7 +666,7 @@ void handle_timestamp(federate_info_t* my_fed) { tag_t tag = {.time = start_time, .microstep = 0}; tracepoint_rti_to_federate(send_TIMESTAMP, my_fed->enclave.id, &tag); } - if (write_to_socket(my_fed->socket, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer)) { + if (write_to_netdrv(my_fed->fed_netdrv, MSG_TYPE_TIMESTAMP_LENGTH, start_time_buffer) <= 0) { lf_print_error("Failed to send the starting time to federate %d.", my_fed->enclave.id); } @@ -835,10 +679,9 @@ void handle_timestamp(federate_info_t* my_fed) { LF_PRINT_LOG("RTI sent start time " PRINTF_TIME " to federate %d.", start_time, my_fed->enclave.id); LF_MUTEX_UNLOCK(&rti_mutex); } - -void send_physical_clock(unsigned char message_type, federate_info_t* fed, socket_type_t socket_type) { +void send_physical_clock(unsigned char message_type, federate_info_t* fed, netdrv_type_t netdrv_type) { if (fed->enclave.state == NOT_CONNECTED) { - lf_print_warning("Clock sync: RTI failed to send physical time to federate %d. Socket not connected.\n", + lf_print_warning("Clock sync: RTI failed to send physical time to federate %d. Netdrv not connected.\n", fed->enclave.id); return; } @@ -848,20 +691,19 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke encode_int64(current_physical_time, &(buffer[1])); // Send the message - if (socket_type == UDP) { - // FIXME: UDP_addr is never initialized. + if (netdrv_type == UDP) { LF_PRINT_DEBUG("Clock sync: RTI sending UDP message type %u.", buffer[0]); - ssize_t bytes_written = sendto(rti_remote->socket_descriptor_UDP, buffer, 1 + sizeof(int64_t), 0, + ssize_t bytes_written = sendto(rti_remote->clock_sync_socket, buffer, 1 + sizeof(int64_t), 0, (struct sockaddr*)&fed->UDP_addr, sizeof(fed->UDP_addr)); if (bytes_written < (ssize_t)sizeof(int64_t) + 1) { lf_print_warning("Clock sync: RTI failed to send physical time to federate %d: %s\n", fed->enclave.id, strerror(errno)); return; } - } else if (socket_type == TCP) { + } else if (netdrv_type == NETDRV) { LF_PRINT_DEBUG("Clock sync: RTI sending TCP message type %u.", buffer[0]); LF_MUTEX_LOCK(&rti_mutex); - write_to_socket_fail_on_error(&fed->socket, 1 + sizeof(int64_t), buffer, &rti_mutex, + write_to_netdrv_fail_on_error(fed->fed_netdrv, 1 + sizeof(int64_t), buffer, &rti_mutex, "Clock sync: RTI failed to send physical time to federate %d.", fed->enclave.id); LF_MUTEX_UNLOCK(&rti_mutex); } @@ -869,16 +711,16 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke current_physical_time, fed->enclave.id); } -void handle_physical_clock_sync_message(federate_info_t* my_fed, socket_type_t socket_type) { +void handle_physical_clock_sync_message(federate_info_t* my_fed, netdrv_type_t netdrv_type) { // Lock the mutex to prevent interference between sending the two // coded probe messages. LF_MUTEX_LOCK(&rti_mutex); // Reply with a T4 type message - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed, socket_type); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T4, my_fed, netdrv_type); // Send the corresponding coded probe immediately after, // but only if this is a UDP channel. - if (socket_type == UDP) { - send_physical_clock(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed, socket_type); + if (netdrv_type == UDP) { + send_physical_clock(MSG_TYPE_CLOCK_SYNC_CODED_PROBE, my_fed, netdrv_type); } LF_MUTEX_UNLOCK(&rti_mutex); } @@ -931,9 +773,16 @@ void* clock_synchronization_thread(void* noargs) { int remaining_attempts = 5; while (remaining_attempts > 0) { remaining_attempts--; - int read_failed = read_from_socket(rti_remote->socket_descriptor_UDP, message_size, buffer); + // Read from the UDP socket + ssize_t bytes_read = 0; + do { + ssize_t bytes = recv(rti_remote->clock_sync_socket, buffer, message_size, MSG_WAITALL); + if (bytes > 0) { + bytes_read += bytes; + } + } while ((errno == EAGAIN || errno == EWOULDBLOCK) && bytes_read < (ssize_t)message_size); // If any errors occur, either discard the message or the clock sync round. - if (!read_failed) { + if (!bytes_read) { if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { int32_t fed_id_2 = extract_int32(&(buffer[1])); // Check that this message came from the correct federate. @@ -981,7 +830,7 @@ void* clock_synchronization_thread(void* noargs) { * @param my_fed The federate sending a MSG_TYPE_FAILED message. */ static void handle_federate_failed(federate_info_t* my_fed) { - // Nothing more to do. Close the socket and exit. + // Nothing more to do. Close the netdrv and exit. LF_MUTEX_LOCK(&rti_mutex); if (rti_remote->base.tracing_enabled) { @@ -997,14 +846,8 @@ static void handle_federate_failed(federate_info_t* my_fed) { // Indicate that there will no further events from this federate. my_fed->enclave.next_event = FOREVER_TAG; - // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket, - // the close should happen when receiving a 0 length message from the other end. - // Here, we just signal the other side that no further writes to the socket are - // forthcoming, which should result in the other end getting a zero-length reception. - shutdown(my_fed->socket, SHUT_RDWR); - - // We can now safely close the socket. - close(my_fed->socket); // from unistd.h + // Shutdown and close netdriver. + close_netdrv(my_fed->fed_netdrv); // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep @@ -1022,6 +865,7 @@ static void handle_federate_failed(federate_info_t* my_fed) { * * This function assumes the caller does not hold the mutex. * + * //TODO: Change comments. * @note At this point, the RTI might have outgoing messages to the federate. This * function thus first performs a shutdown on the socket, which sends an EOF. It then * waits for the remote socket to be closed before closing the socket itself. @@ -1043,21 +887,7 @@ static void handle_federate_resign(federate_info_t* my_fed) { // Indicate that there will no further events from this federate. my_fed->enclave.next_event = FOREVER_TAG; - // According to this: https://stackoverflow.com/questions/4160347/close-vs-shutdown-socket, - // the close should happen when receiving a 0 length message from the other end. - // Here, we just signal the other side that no further writes to the socket are - // forthcoming, which should result in the other end getting a zero-length reception. - shutdown(my_fed->socket, SHUT_WR); - - // Wait for the federate to send an EOF or a socket error to occur. - // Discard any incoming bytes. Normally, this read should return 0 because - // the federate is resigning and should itself invoke shutdown. - unsigned char buffer[10]; - while (read(my_fed->socket, buffer, 10) > 0) - ; - - // We can now safely close the socket. - close(my_fed->socket); // from unistd.h + close_netdrv(my_fed->fed_netdrv); // Check downstream federates to see whether they should now be granted a TAG. // To handle cycles, need to create a boolean array to keep @@ -1081,46 +911,41 @@ void* federate_info_thread_TCP(void* fed) { // Listen for messages from the federate. while (my_fed->enclave.state != NOT_CONNECTED) { // Read no more than one byte to get the message type. - int read_failed = read_from_socket(my_fed->socket, 1, buffer); - if (read_failed) { - // Socket is closed - lf_print_error("RTI: Socket to federate %d is closed. Exiting the thread.", my_fed->enclave.id); + ssize_t bytes_read = read_from_netdrv(my_fed->fed_netdrv, buffer, FED_COM_BUFFER_SIZE); + if (bytes_read <= 0) { + // Netdriver is closed + lf_print_error("RTI: Netdriver to federate %d is closed. Exiting the thread.", my_fed->enclave.id); my_fed->enclave.state = NOT_CONNECTED; - my_fed->socket = -1; // FIXME: We need better error handling here, but do not stop execution here. break; } LF_PRINT_DEBUG("RTI: Received message type %u from federate %d.", buffer[0], my_fed->enclave.id); switch (buffer[0]) { case MSG_TYPE_TIMESTAMP: - handle_timestamp(my_fed); + handle_timestamp(my_fed, buffer + 1); break; case MSG_TYPE_ADDRESS_QUERY: - handle_address_query(my_fed->enclave.id); + handle_address_query(my_fed->enclave.id, buffer + 1); break; case MSG_TYPE_ADDRESS_ADVERTISEMENT: - handle_address_ad(my_fed->enclave.id); + handle_address_ad(my_fed->enclave.id, buffer + 1); break; case MSG_TYPE_TAGGED_MESSAGE: - handle_timed_message(my_fed, buffer); + handle_timed_message(my_fed, buffer, bytes_read); break; case MSG_TYPE_RESIGN: handle_federate_resign(my_fed); return NULL; case MSG_TYPE_NEXT_EVENT_TAG: - handle_next_event_tag(my_fed); + handle_next_event_tag(my_fed, buffer + 1); break; case MSG_TYPE_LATEST_TAG_COMPLETE: - handle_latest_tag_complete(my_fed); + handle_latest_tag_complete(my_fed, buffer + 1); break; case MSG_TYPE_STOP_REQUEST: - handle_stop_request_message(my_fed); // FIXME: Reviewed until here. - // Need to also look at - // notify_advance_grant_if_safe() - // and notify_downstream_advance_grant_if_safe() - break; + handle_stop_request_message(my_fed, buffer + 1); case MSG_TYPE_STOP_REQUEST_REPLY: - handle_stop_request_reply(my_fed); + handle_stop_request_reply(my_fed, buffer + 1); break; case MSG_TYPE_PORT_ABSENT: handle_port_absent_message(my_fed, buffer); @@ -1129,36 +954,33 @@ void* federate_info_thread_TCP(void* fed) { handle_federate_failed(my_fed); return NULL; default: - lf_print_error("RTI received from federate %d an unrecognized TCP message type: %u.", my_fed->enclave.id, - buffer[0]); + lf_print_error("RTI received from federate %d an unrecognized message type: %u.", my_fed->enclave.id, buffer[0]); if (rti_remote->base.tracing_enabled) { tracepoint_rti_from_federate(receive_UNIDENTIFIED, my_fed->enclave.id, NULL); } } } - // Nothing more to do. Close the socket and exit. - // Prevent multiple threads from closing the same socket at the same time. + // Nothing more to do. Close the netdriver and exit. + // Prevent multiple threads from closing the same netdriver at the same time. LF_MUTEX_LOCK(&rti_mutex); - close(my_fed->socket); // from unistd.h + close_netdrv(my_fed->fed_netdrv); LF_MUTEX_UNLOCK(&rti_mutex); return NULL; } -void send_reject(int* socket_id, unsigned char error_code) { +void send_reject(netdrv_t* netdrv, unsigned char error_code) { LF_PRINT_DEBUG("RTI sending MSG_TYPE_REJECT."); unsigned char response[2]; response[0] = MSG_TYPE_REJECT; response[1] = error_code; LF_MUTEX_LOCK(&rti_mutex); // NOTE: Ignore errors on this response. - if (write_to_socket(*socket_id, 2, response)) { - lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the socket."); + if (write_to_netdrv(netdrv, 2, response) <= 0) { + lf_print_warning("RTI failed to write MSG_TYPE_REJECT message on the netdriver."); } - // Close the socket. - shutdown(*socket_id, SHUT_RDWR); - close(*socket_id); - *socket_id = -1; + // Shutdown and close the netdrv. + close_netdrv(netdrv); LF_MUTEX_UNLOCK(&rti_mutex); } @@ -1167,18 +989,17 @@ void send_reject(int* socket_id, unsigned char error_code) { * a federate ID and a federation ID. If the federation ID * matches this federation, send an MSG_TYPE_ACK and otherwise send * a MSG_TYPE_REJECT message. - * @param socket_id Pointer to the socket on which to listen. - * @param client_fd The socket address. + * @param netdrv_t Pointer to the netdriver on which to listen. * @return The federate ID for success or -1 for failure. */ -static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_in* client_fd) { - // Buffer for message ID, federate ID, and federation ID length. - size_t length = 1 + sizeof(uint16_t) + 1; // Message ID, federate ID, length of fedration ID. - unsigned char buffer[length]; - +// TODO: UPDATE comments. +static int32_t receive_and_check_fed_id_message(netdrv_t* netdrv) { + // Buffer for message ID, federate ID, federation ID length, and federation ID. + unsigned char buffer[256]; // TODO: NEED TO CHECK. // Read bytes from the socket. We need 4 bytes. - if (read_from_socket_close_on_error(socket_id, length, buffer)) { - lf_print_error("RTI failed to read from accepted socket."); + + if (read_from_netdrv_close_on_error(netdrv, buffer, 256) <= 0) { // TODO: check length. + lf_print_error("RTI failed to read from accepted netdriver."); return -1; } @@ -1197,9 +1018,12 @@ static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_ // of the peer they want to connect to from the RTI. // If the connection is a peer-to-peer connection between two // federates, reject the connection with the WRONG_SERVER error. - send_reject(socket_id, WRONG_SERVER); + send_reject(netdrv, WRONG_SERVER); } else { - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(netdrv, UNEXPECTED_MESSAGE); + } + if (rti_remote->base.tracing_enabled) { + tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } lf_print_error("RTI expected a MSG_TYPE_FED_IDS message. Got %u (see net_common.h).", buffer[0]); return -1; @@ -1210,30 +1034,24 @@ static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_ // Read the federation ID. First read the length, which is one byte. size_t federation_id_length = (size_t)buffer[sizeof(uint16_t) + 1]; - char federation_id_received[federation_id_length + 1]; // One extra for null terminator. - // Next read the actual federation ID. - if (read_from_socket_close_on_error(socket_id, federation_id_length, (unsigned char*)federation_id_received)) { - lf_print_error("RTI failed to read federation id from federate %d.", fed_id); - return -1; - } // Terminate the string with a null. - federation_id_received[federation_id_length] = 0; + buffer[2 + sizeof(uint16_t) + federation_id_length] = 0; - LF_PRINT_DEBUG("RTI received federation ID: %s.", federation_id_received); + LF_PRINT_DEBUG("RTI received federation ID: %s.", buffer + 2 + sizeof(uint16_t)); if (rti_remote->base.tracing_enabled) { tracepoint_rti_from_federate(receive_FED_ID, fed_id, NULL); } // Compare the received federation ID to mine. - if (strncmp(rti_remote->federation_id, federation_id_received, federation_id_length) != 0) { + if (strncmp(rti_remote->federation_id, (const char*)buffer + 2 + sizeof(uint16_t), federation_id_length) != 0) { // Federation IDs do not match. Send back a MSG_TYPE_REJECT message. lf_print_warning("Federate from another federation %s attempted to connect to RTI in federation %s.", - federation_id_received, rti_remote->federation_id); + buffer + 2 + sizeof(uint16_t), rti_remote->federation_id); if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } - send_reject(socket_id, FEDERATION_ID_DOES_NOT_MATCH); + send_reject(netdrv, FEDERATION_ID_DOES_NOT_MATCH); return -1; } else { if (fed_id >= rti_remote->base.number_of_scheduling_nodes) { @@ -1242,7 +1060,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_ if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } - send_reject(socket_id, FEDERATE_ID_OUT_OF_RANGE); + send_reject(netdrv, FEDERATE_ID_OUT_OF_RANGE); return -1; } else { if ((rti_remote->base.scheduling_nodes[fed_id])->state != NOT_CONNECTED) { @@ -1250,32 +1068,30 @@ static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_ if (rti_remote->base.tracing_enabled) { tracepoint_rti_to_federate(send_REJECT, fed_id, NULL); } - send_reject(socket_id, FEDERATE_ID_IN_USE); + send_reject(netdrv, FEDERATE_ID_IN_USE); return -1; } } } } federate_info_t* fed = GET_FED_INFO(fed_id); - // The MSG_TYPE_FED_IDS message has the right federation ID. - // Assign the address information for federate. - // The IP address is stored here as an in_addr struct (in .server_ip_addr) that can be useful - // to create sockets and can be efficiently sent over the network. - // First, convert the sockaddr structure into a sockaddr_in that contains an internet address. - struct sockaddr_in* pV4_addr = client_fd; - // Then extract the internet address (which is in IPv4 format) and assign it as the federate's socket server - fed->server_ip_addr = pV4_addr->sin_addr; + netdrv->my_federate_id = (uint16_t)fed_id; + fed->fed_netdrv = netdrv; +// TODO: Make this work for only TCP. #if LOG_LEVEL >= LOG_LEVEL_DEBUG // Create the human readable format and copy that into // the .server_hostname field of the federate. char str[INET_ADDRSTRLEN + 1]; - inet_ntop(AF_INET, &fed->server_ip_addr, str, INET_ADDRSTRLEN); - strncpy(fed->server_hostname, str, INET_ADDRSTRLEN); - LF_PRINT_DEBUG("RTI got address %s from federate %d.", fed->server_hostname, fed_id); +// get_ip_addr(fed->fed_netdrv); +// // TODO: NEED TO FIX HERE! +// inet_ntop(AF_INET, get_ip_addr(fed->fed_netdrv), str, INET_ADDRSTRLEN); +// // strncpy(get_host_name(fed->fed_netdrv), str, INET_ADDRSTRLEN); +// set_host_name(fed->fed_netdrv, str); + +// LF_PRINT_DEBUG("RTI got address %s from federate %d.", get_host_name(fed->fed_netdrv), fed_id); #endif - fed->socket = *socket_id; // Set the federate's state as pending // because it is waiting for the start time to be @@ -1289,7 +1105,7 @@ static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_ tracepoint_rti_to_federate(send_ACK, fed_id, NULL); } LF_MUTEX_LOCK(&rti_mutex); - if (write_to_socket_close_on_error(&fed->socket, 1, &ack_message)) { + if (write_to_netdrv_close_on_error(fed->fed_netdrv, 1, &ack_message) <= 0) { LF_MUTEX_UNLOCK(&rti_mutex); lf_print_error("RTI failed to write MSG_TYPE_ACK message to federate %d.", fed_id); return -1; @@ -1306,24 +1122,24 @@ static int32_t receive_and_check_fed_id_message(int* socket_id, struct sockaddr_ * out the relevant information in the federate's struct. * @return 1 on success and 0 on failure. */ -static int receive_connection_information(int* socket_id, uint16_t fed_id) { +static int receive_connection_information(netdrv_t* netdrv, uint16_t fed_id) { + unsigned char connection_info_buffer[1024]; LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_NEIGHBOR_STRUCTURE from federate %d.", fed_id); - unsigned char connection_info_header[MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE]; - read_from_socket_fail_on_error(socket_id, MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE, connection_info_header, NULL, + read_from_netdrv_fail_on_error(netdrv, connection_info_buffer, 1024, NULL, "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message header from federate %d.", fed_id); - if (connection_info_header[0] != MSG_TYPE_NEIGHBOR_STRUCTURE) { - lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. " + if (connection_info_buffer[0] != MSG_TYPE_NEIGHBOR_STRUCTURE) { + lf_print_error("RTI was expecting a MSG_TYPE_NEIGHBOR_STRUCTURE message from federate %d. Got %u instead. " "Rejecting federate.", - fed_id, connection_info_header[0]); - send_reject(socket_id, UNEXPECTED_MESSAGE); + fed_id, connection_info_buffer[0]); + send_reject(netdrv, UNEXPECTED_MESSAGE); return 0; } else { federate_info_t* fed = GET_FED_INFO(fed_id); // Read the number of upstream and downstream connections - fed->enclave.num_upstream = extract_int32(&(connection_info_header[1])); - fed->enclave.num_downstream = extract_int32(&(connection_info_header[1 + sizeof(int32_t)])); + fed->enclave.num_upstream = extract_int32(&(connection_info_buffer[1])); + fed->enclave.num_downstream = extract_int32(&(connection_info_buffer[1 + sizeof(int32_t)])); LF_PRINT_DEBUG("RTI got %d upstreams and %d downstreams from federate %d.", fed->enclave.num_upstream, fed->enclave.num_downstream, fed_id); @@ -1347,30 +1163,22 @@ static int receive_connection_information(int* socket_id, uint16_t fed_id) { size_t connections_info_body_size = ((sizeof(uint16_t) + sizeof(int64_t)) * fed->enclave.num_upstream) + (sizeof(uint16_t) * fed->enclave.num_downstream); - unsigned char* connections_info_body = NULL; if (connections_info_body_size > 0) { - connections_info_body = (unsigned char*)malloc(connections_info_body_size); - LF_ASSERT_NON_NULL(connections_info_body); - read_from_socket_fail_on_error(socket_id, connections_info_body_size, connections_info_body, NULL, - "RTI failed to read MSG_TYPE_NEIGHBOR_STRUCTURE message body from federate %d.", - fed_id); // Keep track of where we are in the buffer - size_t message_head = 0; + size_t message_head = MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE; // First, read the info about upstream federates for (int i = 0; i < fed->enclave.num_upstream; i++) { - fed->enclave.upstream[i] = extract_uint16(&(connections_info_body[message_head])); + fed->enclave.upstream[i] = extract_uint16(&(connection_info_buffer[message_head])); message_head += sizeof(uint16_t); - fed->enclave.upstream_delay[i] = extract_int64(&(connections_info_body[message_head])); + fed->enclave.upstream_delay[i] = extract_int64(&(connection_info_buffer[message_head])); message_head += sizeof(int64_t); } // Next, read the info about downstream federates for (int i = 0; i < fed->enclave.num_downstream; i++) { - fed->enclave.downstream[i] = extract_uint16(&(connections_info_body[message_head])); + fed->enclave.downstream[i] = extract_uint16(&(connection_info_buffer[message_head])); message_head += sizeof(uint16_t); } - - free(connections_info_body); } } LF_PRINT_DEBUG("RTI received neighbor structure from federate %d.", fed_id); @@ -1385,23 +1193,29 @@ static int receive_connection_information(int* socket_id, uint16_t fed_id) { * up to perform runtime clock synchronization using the UDP port number * specified in the payload to communicate with the federate's clock * synchronization logic. - * @param socket_id The socket on which to listen. + * @param socket_id The netdriver on which to listen. * @param fed_id The federate ID. * @return 1 for success, 0 for failure. */ -static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fed_id) { +static int receive_udp_message_and_set_up_clock_sync(netdrv_t* netdrv, uint16_t fed_id) { // Read the MSG_TYPE_UDP_PORT message from the federate regardless of the status of // clock synchronization. This message will tell the RTI whether the federate // is doing clock synchronization, and if it is, what port to use for UDP. LF_PRINT_DEBUG("RTI waiting for MSG_TYPE_UDP_PORT from federate %d.", fed_id); - unsigned char response[1 + sizeof(uint16_t)]; - read_from_socket_fail_on_error(socket_id, 1 + sizeof(uint16_t), response, NULL, + size_t buffer_size; +#if defined(COMM_TYPE_TCP) || defined(COMM_TYPE_SST) + buffer_size = 1 + sizeof(uint16_t); +#elif defined(COMM_TYPE_MQTT) + buffer_size = 1 + sizeof(uint16_t) + INET_ADDRSTRLEN; +#endif + unsigned char response[buffer_size]; + read_from_netdrv_fail_on_error(netdrv, response, buffer_size, NULL, "RTI failed to read MSG_TYPE_UDP_PORT message from federate %d.", fed_id); if (response[0] != MSG_TYPE_UDP_PORT) { lf_print_error("RTI was expecting a MSG_TYPE_UDP_PORT message from federate %d. Got %u instead. " "Rejecting federate.", fed_id, response[0]); - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(netdrv, UNEXPECTED_MESSAGE); return 0; } else { federate_info_t* fed = GET_FED_INFO(fed_id); @@ -1417,22 +1231,22 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe // Send the required number of messages for clock synchronization for (int i = 0; i < rti_remote->clock_sync_exchanges_per_interval; i++) { // Send the RTI's current physical time T1 to the federate. - send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, TCP); + send_physical_clock(MSG_TYPE_CLOCK_SYNC_T1, fed, NETDRV); // Listen for reply message, which should be T3. size_t message_size = 1 + sizeof(int32_t); unsigned char buffer[message_size]; - read_from_socket_fail_on_error(socket_id, message_size, buffer, NULL, - "Socket to federate %d unexpectedly closed.", fed_id); + read_from_netdrv_fail_on_error(netdrv, buffer, message_size, NULL, + "Netdriver to federate %d unexpectedly closed.", fed_id); if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T3) { int32_t fed_id = extract_int32(&(buffer[1])); assert(fed_id > -1); assert(fed_id < 65536); LF_PRINT_DEBUG("RTI received T3 clock sync message from federate %d.", fed_id); - handle_physical_clock_sync_message(fed, TCP); + handle_physical_clock_sync_message(fed, NETDRV); } else { lf_print_error("Unexpected message %u from federate %d.", buffer[0], fed_id); - send_reject(socket_id, UNEXPECTED_MESSAGE); + send_reject(netdrv, UNEXPECTED_MESSAGE); return 0; } } @@ -1440,12 +1254,16 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe } if (rti_remote->clock_sync_global_status >= clock_sync_on) { // If no runtime clock sync, no need to set up the UDP port. - if (federate_UDP_port_number > 0) { - // Initialize the UDP_addr field of the federate struct - fed->UDP_addr.sin_family = AF_INET; - fed->UDP_addr.sin_port = htons(federate_UDP_port_number); - fed->UDP_addr.sin_addr = fed->server_ip_addr; - } + // Initialize the UDP_addr field of the federate struct + // TODO: DONGHA - Need to know the ip_address of the federate. + fed->UDP_addr.sin_family = AF_INET; + fed->UDP_addr.sin_port = htons(federate_UDP_port_number); + +#if defined(COMM_TYPE_TCP) || defined(COMM_TYPE_SST) + fed->UDP_addr.sin_addr = *get_ip_addr(netdrv); +#elif defined(COMM_TYPE_MQTT) + inet_pton(AF_INET, &(response[1 + sizeof(uint16_t)]), &(fed->UDP_addr.sin_addr)); +#endif } else { // Disable clock sync after initial round. fed->clock_synchronization_enabled = false; @@ -1467,17 +1285,17 @@ static int receive_udp_message_and_set_up_clock_sync(int* socket_id, uint16_t fe /** * Authenticate incoming federate by performing HMAC-based authentication. * - * @param socket Socket for the incoming federate tryting to authenticate. + * @param fed_netdrv Netdriver for the incoming federate tryting to authenticate. * @return True if authentication is successful and false otherwise. */ -static bool authenticate_federate(int* socket) { +static bool authenticate_federate(netdrv_t* fed_netdrv) { // Wait for MSG_TYPE_FED_NONCE from federate. size_t fed_id_length = sizeof(uint16_t); unsigned char buffer[1 + fed_id_length + NONCE_LENGTH]; - read_from_socket_fail_on_error(socket, 1 + fed_id_length + NONCE_LENGTH, buffer, NULL, - "Failed to read MSG_TYPE_FED_NONCE"); + read_from_netdrv_fail_on_error(fed_netdrv, buffer, 1 + fed_id_length + NONCE_LENGTH, NULL, + "Failed to read MSG_TYPE_FED_NONCE."); if (buffer[0] != MSG_TYPE_FED_NONCE) { - lf_print_error_and_exit("Received unexpected response %u from the FED (see net_common.h).", buffer[0]); + lf_print_error_and_exit("Received unexpected response %u from the federate (see net_common.h).", buffer[0]); } unsigned int hmac_length = SHA256_HMAC_LENGTH; size_t federation_id_length = strnlen(rti_remote->federation_id, 255); @@ -1499,13 +1317,13 @@ static bool authenticate_federate(int* socket) { RAND_bytes(rti_nonce, NONCE_LENGTH); memcpy(&sender[1], rti_nonce, NONCE_LENGTH); memcpy(&sender[1 + NONCE_LENGTH], hmac_tag, hmac_length); - if (write_to_socket(*socket, 1 + NONCE_LENGTH + hmac_length, sender)) { + if (write_to_netdrv(fed_netdrv, 1 + NONCE_LENGTH + hmac_length, sender) <= 0) { lf_print_error("Failed to send nonce to federate."); } // Wait for MSG_TYPE_FED_RESPONSE unsigned char received[1 + hmac_length]; - read_from_socket_fail_on_error(socket, 1 + hmac_length, received, NULL, "Failed to read federate response."); + read_from_netdrv_fail_on_error(fed_netdrv, received, 1 + hmac_length, NULL, "Failed to read federate response."); if (received[0] != MSG_TYPE_FED_RESPONSE) { lf_print_error_and_exit("Received unexpected response %u from the federate (see net_common.h).", received[0]); return false; @@ -1524,7 +1342,7 @@ static bool authenticate_federate(int* socket) { if (memcmp(&received[1], rti_tag, hmac_length) != 0) { // Federation IDs do not match. Send back a HMAC_DOES_NOT_MATCH message. lf_print_warning("HMAC authentication failed. Rejecting the federate."); - send_reject(socket, HMAC_DOES_NOT_MATCH); + send_reject(fed_netdrv, HMAC_DOES_NOT_MATCH); return false; } else { LF_PRINT_LOG("Federate's HMAC verified."); @@ -1533,36 +1351,20 @@ static bool authenticate_federate(int* socket) { } #endif -void lf_connect_to_federates(int socket_descriptor) { +void lf_connect_to_federates(netdrv_t* rti_netdrv) { + netdrv_t* netdrv_array[rti_remote->base.number_of_scheduling_nodes]; + // Connect with the federates first, before receiving fed_id, connection_info, and udp port. for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) { - // Wait for an incoming connection request. - struct sockaddr client_fd; - uint32_t client_length = sizeof(client_fd); - // The following blocks until a federate connects. - int socket_id = -1; - while (1) { - socket_id = accept(rti_remote->socket_descriptor_TCP, &client_fd, &client_length); - if (socket_id >= 0) { - // Got a socket - break; - } else if (socket_id < 0 && (errno != EAGAIN || errno != EWOULDBLOCK)) { - lf_print_error_system_failure("RTI failed to accept the socket."); - } else { - // Try again - lf_print_warning("RTI failed to accept the socket. %s. Trying again.", strerror(errno)); - continue; - } - } - + netdrv_array[i] = establish_communication_session(rti_netdrv); + } + for (int i = 0; i < rti_remote->base.number_of_scheduling_nodes; i++) { + netdrv_t* fed_netdrv = netdrv_array[i]; // Wait for the first message from the federate when RTI -a option is on. #ifdef __RTI_AUTH__ if (rti_remote->authentication_enabled) { - if (!authenticate_federate(&socket_id)) { + if (!authenticate_federate(fed_netdrv)) { lf_print_warning("RTI failed to authenticate the incoming federate."); - // Close the socket. - shutdown(socket_id, SHUT_RDWR); - close(socket_id); - socket_id = -1; + close_netdrv(fed_netdrv); // Ignore the federate that failed authentication. i--; continue; @@ -1571,9 +1373,9 @@ void lf_connect_to_federates(int socket_descriptor) { #endif // The first message from the federate should contain its ID and the federation ID. - int32_t fed_id = receive_and_check_fed_id_message(&socket_id, (struct sockaddr_in*)&client_fd); - if (fed_id >= 0 && socket_id >= 0 && receive_connection_information(&socket_id, (uint16_t)fed_id) && - receive_udp_message_and_set_up_clock_sync(&socket_id, (uint16_t)fed_id)) { + int32_t fed_id = receive_and_check_fed_id_message(fed_netdrv); + if (fed_id >= 0 && receive_connection_information(fed_netdrv, (uint16_t)fed_id) && + receive_udp_message_and_set_up_clock_sync(fed_netdrv, (uint16_t)fed_id)) { // Create a thread to communicate with the federate. // This has to be done after clock synchronization is finished @@ -1601,7 +1403,7 @@ void lf_connect_to_federates(int socket_descriptor) { break; } } - if (rti_remote->final_port_UDP != UINT16_MAX && clock_sync_enabled) { + if (rti_remote->clock_sync_port != UINT16_MAX && clock_sync_enabled) { lf_thread_create(&rti_remote->clock_thread, clock_synchronization_thread, NULL); } } @@ -1610,14 +1412,10 @@ void lf_connect_to_federates(int socket_descriptor) { void* respond_to_erroneous_connections(void* nothing) { initialize_lf_thread_id(); while (true) { - // Wait for an incoming connection request. - struct sockaddr client_fd; - uint32_t client_length = sizeof(client_fd); - // The following will block until either a federate attempts to connect - // or close(rti->socket_descriptor_TCP) is called. - int socket_id = accept(rti_remote->socket_descriptor_TCP, &client_fd, &client_length); - if (socket_id < 0) - return NULL; + netdrv_t* fed_netdrv = establish_communication_session(rti_remote->rti_netdrv); + + if (fed_netdrv == NULL) + continue; if (rti_remote->all_federates_exited) { return NULL; @@ -1628,12 +1426,11 @@ void* respond_to_erroneous_connections(void* nothing) { response[0] = MSG_TYPE_REJECT; response[1] = FEDERATION_ID_DOES_NOT_MATCH; // Ignore errors on this response. - if (write_to_socket(socket_id, 2, response)) { + if (write_to_netdrv(fed_netdrv, 2, response) <= 0) { lf_print_warning("RTI failed to write FEDERATION_ID_DOES_NOT_MATCH to erroneous incoming connection."); } - // Close the socket. - shutdown(socket_id, SHUT_RDWR); - close(socket_id); + // Close the netdriver. + close_netdrv(fed_netdrv); } return NULL; } @@ -1641,35 +1438,32 @@ void* respond_to_erroneous_connections(void* nothing) { void initialize_federate(federate_info_t* fed, uint16_t id) { initialize_scheduling_node(&(fed->enclave), id); fed->requested_stop = false; - fed->socket = -1; // No socket. fed->clock_synchronization_enabled = true; fed->in_transit_message_tags = pqueue_tag_init(10); - strncpy(fed->server_hostname, "localhost", INET_ADDRSTRLEN); - fed->server_ip_addr.s_addr = 0; - fed->server_port = -1; } -int32_t start_rti_server(uint16_t port) { +int start_rti_server(uint16_t user_specified_port) { _lf_initialize_clock(); - // Create the TCP socket server - rti_remote->socket_descriptor_TCP = create_rti_server(port, TCP); + // Initialize RTI's netdriver. The ID is -1 for the RTI. + rti_remote->rti_netdrv = initialize_netdrv(-1, rti_remote->federation_id); + // Create the RTI's netdriver. The user_specified_port will be 0 when not specified. + create_listener(rti_remote->rti_netdrv, RTI, user_specified_port); // 0 for RTI lf_print("RTI: Listening for federates."); - // Create the UDP socket server - // Try to get the rti_remote->final_port_TCP + 1 port + // Create the clocksync's netdriver. if (rti_remote->clock_sync_global_status >= clock_sync_on) { - rti_remote->socket_descriptor_UDP = create_rti_server(rti_remote->final_port_TCP + 1, UDP); + rti_remote->clock_sync_socket = create_clock_sync_server(&rti_remote->clock_sync_port); } - return rti_remote->socket_descriptor_TCP; + return 1; } -void wait_for_federates(int socket_descriptor) { +void wait_for_federates(netdrv_t* netdrv) { // Wait for connections from federates and create a thread for each. - lf_connect_to_federates(socket_descriptor); + lf_connect_to_federates(netdrv); // All federates have connected. lf_print("RTI: All expected federates have connected. Starting execution."); - // The socket server will not continue to accept connections after all the federates + // The netdriver server will not continue to accept connections after all the federates // have joined. // In case some other federation's federates are trying to join the wrong // federation, need to respond. Start a separate thread to do that. @@ -1688,25 +1482,7 @@ void wait_for_federates(int socket_descriptor) { rti_remote->all_federates_exited = true; - // Shutdown and close the socket that is listening for incoming connections - // so that the accept() call in respond_to_erroneous_connections returns. - // That thread should then check rti->all_federates_exited and it should exit. - if (shutdown(socket_descriptor, SHUT_RDWR)) { - LF_PRINT_LOG("On shut down TCP socket, received reply: %s", strerror(errno)); - } - // NOTE: In all common TCP/IP stacks, there is a time period, - // typically between 30 and 120 seconds, called the TIME_WAIT period, - // before the port is released after this close. This is because - // the OS is preventing another program from accidentally receiving - // duplicated packets intended for this program. - close(socket_descriptor); - - if (rti_remote->socket_descriptor_UDP > 0) { - if (shutdown(rti_remote->socket_descriptor_UDP, SHUT_RDWR)) { - LF_PRINT_LOG("On shut down UDP socket, received reply: %s", strerror(errno)); - } - close(rti_remote->socket_descriptor_UDP); - } + close_netdrv(netdrv); } void initialize_RTI(rti_remote_t* rti) { @@ -1717,6 +1493,8 @@ void initialize_RTI(rti_remote_t* rti) { LF_COND_INIT(&received_start_times, &rti_mutex); LF_COND_INIT(&sent_start_time, &rti_mutex); + LF_MUTEX_INIT(&netdrv_mutex); + initialize_rti_common(&rti_remote->base); rti_remote->base.mutex = &rti_mutex; @@ -1726,16 +1504,18 @@ void initialize_RTI(rti_remote_t* rti) { rti_remote->all_federates_exited = false; rti_remote->federation_id = "Unidentified Federation"; rti_remote->user_specified_port = 0; - rti_remote->final_port_TCP = 0; - rti_remote->socket_descriptor_TCP = -1; - rti_remote->final_port_UDP = UINT16_MAX; - rti_remote->socket_descriptor_UDP = -1; + rti_remote->clock_sync_port = UINT16_MAX; + rti_remote->clock_sync_socket = -1; rti_remote->clock_sync_global_status = clock_sync_init; rti_remote->clock_sync_period_ns = MSEC(10); rti_remote->clock_sync_exchanges_per_interval = 10; rti_remote->authentication_enabled = false; rti_remote->base.tracing_enabled = false; rti_remote->stop_in_progress = false; + + // #ifdef OPENSSL_REQUIRED + // OPENSSL_init_crypto(OPENSSL_INIT_NO_ATEXIT, NULL); + // #endif } // The RTI includes clock.c, which requires the following functions that are defined diff --git a/core/federated/RTI/rti_remote.h b/core/federated/RTI/rti_remote.h index a83179f62..45559f4ee 100644 --- a/core/federated/RTI/rti_remote.h +++ b/core/federated/RTI/rti_remote.h @@ -22,8 +22,11 @@ #include // Defines read(), write(), and close() #include // Defines bzero(). +#include "net_util.h" #include "rti_common.h" +#include "netdriver.h" + #ifdef __RTI_AUTH__ #include // For secure random number generation. #include // For HMAC authentication. @@ -38,8 +41,6 @@ ///////////////////////////////////////////// //// Data structures -typedef enum socket_type_t { TCP, UDP } socket_type_t; - /** * Information about a federate known to the RTI, including its runtime state, * mode of execution, and connectivity with other federates. @@ -54,20 +55,13 @@ typedef struct federate_info_t { // to a request for stop from the RTI. Used to prevent double-counting // a federate when handling lf_request_stop(). lf_thread_t thread_id; // The ID of the thread handling communication with this federate. - int socket; // The TCP socket descriptor for communicating with this federate. struct sockaddr_in UDP_addr; // The UDP address for the federate. bool clock_synchronization_enabled; // Indicates the status of clock synchronization // for this federate. Enabled by default. pqueue_tag_t* in_transit_message_tags; // Record of in-transit messages to this federate that are not // yet processed. This record is ordered based on the time // value of each message for a more efficient access. - char server_hostname[INET_ADDRSTRLEN]; // Human-readable IP address and - int32_t server_port; // port number of the socket server of the federate - // if it has any incoming direct connections from other federates. - // The port number will be -1 if there is no server or if the - // RTI has not been informed of the port number. - struct in_addr server_ip_addr; // Information about the IP address of the socket - // server of the federate. + netdrv_t* fed_netdrv; // The netdriver that the RTI handling each federate. } federate_info_t; /** @@ -101,7 +95,7 @@ typedef struct rti_remote_t { * This gets set to true exactly once before the program exits. * It is marked volatile because the write is not guarded by a mutex. * The main thread makes this true, then calls shutdown and close on - * the socket, which will cause accept() to return with an error code + * the netdriver, which will cause accept() to return with an error code * in respond_to_erroneous_connections(). */ volatile bool all_federates_exited; @@ -113,24 +107,19 @@ typedef struct rti_remote_t { */ const char* federation_id; + netdrv_t* rti_netdrv; + /************* TCP server information *************/ - /** The desired port specified by the user on the command line. */ + /** The desired port specified by the user on the command line. This only works for TCP connections.*/ uint16_t user_specified_port; - /** The final port number that the TCP socket server ends up using. */ - uint16_t final_port_TCP; - - /** The TCP socket descriptor for the socket server. */ - int socket_descriptor_TCP; - - /************* UDP server information *************/ + /************* Clock synchronization server information *************/ /** The final port number that the UDP socket server ends up using. */ - uint16_t final_port_UDP; + uint16_t clock_sync_port; /** The UDP socket descriptor for the socket server. */ - int socket_descriptor_UDP; + int clock_sync_socket; - /************* Clock synchronization information *************/ /* Thread performing PTP clock sync sessions periodically. */ lf_thread_t clock_thread; @@ -214,7 +203,7 @@ void handle_port_absent_message(federate_info_t* sending_federate, unsigned char * @param sending_federate The sending federate. * @param buffer The buffer to read into (the first byte is already there). */ -void handle_timed_message(federate_info_t* sending_federate, unsigned char* buffer); +void handle_timed_message(federate_info_t* sending_federate, unsigned char* buffer, ssize_t bytes_read); /** * Handle a latest tag complete (LTC) message. @see @@ -224,7 +213,7 @@ void handle_timed_message(federate_info_t* sending_federate, unsigned char* buff * * @param fed The federate that has completed a logical tag. */ -void handle_latest_tag_complete(federate_info_t* fed); +void handle_latest_tag_complete(federate_info_t* fed, unsigned char* buffer); /** * Handle a next event tag (NET) message. @see MSG_TYPE_NEXT_EVENT_TAG in rti.h. @@ -233,7 +222,7 @@ void handle_latest_tag_complete(federate_info_t* fed); * * @param fed The federate sending a NET message. */ -void handle_next_event_tag(federate_info_t* fed); +void handle_next_event_tag(federate_info_t* fed, unsigned char* buffer); /////////////////// STOP functions //////////////////// @@ -244,7 +233,7 @@ void handle_next_event_tag(federate_info_t* fed); * * @param fed The federate sending a MSG_TYPE_STOP_REQUEST message. */ -void handle_stop_request_message(federate_info_t* fed); +void handle_stop_request_message(federate_info_t* fed, unsigned char* buffer); /** * Handle a MSG_TYPE_STOP_REQUEST_REPLY message. @@ -253,7 +242,7 @@ void handle_stop_request_message(federate_info_t* fed); * * @param fed The federate replying the MSG_TYPE_STOP_REQUEST */ -void handle_stop_request_reply(federate_info_t* fed); +void handle_stop_request_reply(federate_info_t* fed, unsigned char* buffer); ////////////////////////////////////////////////// @@ -261,14 +250,14 @@ void handle_stop_request_reply(federate_info_t* fed); * Handle address query messages. * This function reads the body of a MSG_TYPE_ADDRESS_QUERY (@see net_common.h) message * which is the requested destination federate ID and replies with the stored - * port value for the socket server of that federate. The port values + * port value for the netdriver server of that federate. The port values * are initialized to -1. If no MSG_TYPE_ADDRESS_ADVERTISEMENT message has been received from * the destination federate, the RTI will simply reply with -1 for the port. * The sending federate is responsible for checking back with the RTI after a * period of time. * @param fed_id The federate sending a MSG_TYPE_ADDRESS_QUERY message. */ -void handle_address_query(uint16_t fed_id); +void handle_address_query(uint16_t fed_id, unsigned char* buffer); /** * Handle address advertisement messages (@see MSG_TYPE_ADDRESS_ADVERTISEMENT in net_common.h). @@ -277,7 +266,7 @@ void handle_address_query(uint16_t fed_id); * field of the _RTI.federates[federate_id] array of structs. * * The server_hostname and server_ip_addr fields are assigned - * in lf_connect_to_federates() upon accepting the socket + * in lf_connect_to_federates() upon accepting the netdriver * from the remote federate. * * This function assumes the caller does not hold the mutex. @@ -285,13 +274,13 @@ void handle_address_query(uint16_t fed_id); * @param federate_id The id of the remote federate that is * sending the address advertisement. */ -void handle_address_ad(uint16_t federate_id); +void handle_address_ad(uint16_t federate_id, unsigned char* buffer); /** * A function to handle timestamp messages. * This function assumes the caller does not hold the mutex. */ -void handle_timestamp(federate_info_t* my_fed); +void handle_timestamp(federate_info_t* my_fed, unsigned char* buffer); /** * Take a snapshot of the physical clock time and send @@ -301,9 +290,9 @@ void handle_timestamp(federate_info_t* my_fed); * * @param message_type The type of the clock sync message (see net_common.h). * @param fed The federate to send the physical time to. - * @param socket_type The socket type (TCP or UDP). + * @param netdrv_type The netdrv_type (NETDRV or UDP). */ -void send_physical_clock(unsigned char message_type, federate_info_t* fed, socket_type_t socket_type); +void send_physical_clock(unsigned char message_type, federate_info_t* fed, netdrv_type_t netdrv_type); /** * Handle clock synchronization T3 messages from federates. @@ -316,9 +305,9 @@ void send_physical_clock(unsigned char message_type, federate_info_t* fed, socke * clock synchronization round. * * @param my_fed The sending federate. - * @param socket_type The RTI's socket type used for the communication (TCP or UDP) + * @param netdrv_type The RTI's netdrv_type used for the communication (NETDRV or UDP) */ -void handle_physical_clock_sync_message(federate_info_t* my_fed, socket_type_t socket_type); +void handle_physical_clock_sync_message(federate_info_t* my_fed, netdrv_type_t netdrv_type); /** * A (quasi-)periodic thread that performs clock synchronization with each @@ -336,24 +325,26 @@ void* clock_synchronization_thread(void* noargs); /** * Thread handling TCP communication with a federate. * @param fed A pointer to the federate's struct that has the - * socket descriptor for the federate. + * netdriver for the federate. */ void* federate_info_thread_TCP(void* fed); /** - * Send a MSG_TYPE_REJECT message to the specified socket and close the socket. - * @param socket_id Pointer to the socket ID. + * Send a MSG_TYPE_REJECT message to the specified netdriver and close the netdriver. + * @param netdrv Pointer to the netdriver. * @param error_code An error code. */ -void send_reject(int* socket_id, unsigned char error_code); +// TODO: Update comments. +void send_reject(netdrv_t* netdrv, unsigned char error_code); /** * Wait for one incoming connection request from each federate, * and upon receiving it, create a thread to communicate with * that federate. Return when all federates have connected. - * @param socket_descriptor The socket on which to accept connections. + * @param rti_netdrv The netdriver on which to accept connections. */ -void lf_connect_to_federates(int socket_descriptor); +// TODO: Update comments. +void lf_connect_to_federates(netdrv_t* rti_netdrv); /** * Thread to respond to new connections, which could be federates of other @@ -369,20 +360,18 @@ void* respond_to_erroneous_connections(void* nothing); void initialize_federate(federate_info_t* fed, uint16_t id); /** - * Start the socket server for the runtime infrastructure (RTI) and - * return the socket descriptor. - * @param num_feds Number of federates. - * @param port The port on which to listen for socket connections, or + * Start the netdriver server for the runtime infrastructure (RTI). + * @param user_specified_port The port on which to listen for netdriver connections, or * 0 to use the default port range. */ -int32_t start_rti_server(uint16_t port); +int start_rti_server(uint16_t user_specified_port); /** * Start the runtime infrastructure (RTI) interaction with the federates * and wait for the federates to exit. - * @param socket_descriptor The socket descriptor returned by start_rti_server(). + * @param netdrv The netdrv returned by start_rti_server(). */ -void wait_for_federates(int socket_descriptor); +void wait_for_federates(netdrv_t* netdrv); /** * Print a usage message. diff --git a/core/federated/clock-sync.c b/core/federated/clock-sync.c index b18efb650..9d1ce73f3 100644 --- a/core/federated/clock-sync.c +++ b/core/federated/clock-sync.c @@ -37,6 +37,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include +#include #include "low_level_platform.h" #include "clock-sync.h" @@ -160,33 +161,31 @@ void reset_socket_stat(struct socket_stat_t* socket_stat) { * If clock synchronization is set to on, a reserved UDP port number * will be sent. */ -uint16_t setup_clock_synchronization_with_rti() { +uint16_t setup_clock_synchronization_with_rti(struct sockaddr_in* federate_UDP_addr) { uint16_t port_to_return = UINT16_MAX; // Default if clock sync is off. #if (LF_CLOCK_SYNC >= LF_CLOCK_SYNC_ON) // Initialize the UDP socket _lf_rti_socket_UDP = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); - // Initialize the necessary information for the UDP address - struct sockaddr_in federate_UDP_addr; - federate_UDP_addr.sin_family = AF_INET; - federate_UDP_addr.sin_port = htons(0u); // Port 0 indicates to bind that + federate_UDP_addr->sin_family = AF_INET; + federate_UDP_addr->sin_port = htons(0u); // Port 0 indicates to bind that // it can assign any port to this // socket. This is okay because // the port number is then sent // to the RTI. - federate_UDP_addr.sin_addr.s_addr = INADDR_ANY; - if (bind(_lf_rti_socket_UDP, (struct sockaddr*)&federate_UDP_addr, sizeof(federate_UDP_addr)) < 0) { + federate_UDP_addr->sin_addr.s_addr = INADDR_ANY; + if (bind(_lf_rti_socket_UDP, (struct sockaddr*)federate_UDP_addr, sizeof(*federate_UDP_addr)) < 0) { lf_print_error_system_failure("Failed to bind its UDP socket."); } // Retrieve the port number that was assigned by the operating system - socklen_t addr_length = sizeof(federate_UDP_addr); - if (getsockname(_lf_rti_socket_UDP, (struct sockaddr*)&federate_UDP_addr, &addr_length) == -1) { + socklen_t addr_length = sizeof(*federate_UDP_addr); + if (getsockname(_lf_rti_socket_UDP, (struct sockaddr*)federate_UDP_addr, &addr_length) == -1) { // FIXME: Send 0 UDP_PORT message instead of exiting. // That will disable clock synchronization. lf_print_error_system_failure("Failed to retrieve UDP port."); } - LF_PRINT_DEBUG("Assigned UDP port number %u to its socket.", ntohs(federate_UDP_addr.sin_port)); + LF_PRINT_DEBUG("Assigned UDP port number %u to its socket.", ntohs(federate_UDP_addr->sin_port)); - port_to_return = ntohs(federate_UDP_addr.sin_port); + port_to_return = ntohs(federate_UDP_addr->sin_port); // Set the option for this socket to reuse the same address int option_value = 1; @@ -204,10 +203,11 @@ uint16_t setup_clock_synchronization_with_rti() { #elif (LF_CLOCK_SYNC == LF_CLOCK_SYNC_INIT) port_to_return = 0u; #endif // (LF_CLOCK_SYNC >= LF_CLOCK_SYNC_ON) + (void) federate_UDP_addr; // To pass unused-value warning. return port_to_return; } -void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { +void synchronize_initial_physical_clock_with_rti(netdrv_t* netdrv_to_rti) { LF_PRINT_DEBUG("Waiting for initial clock synchronization messages from the RTI."); size_t message_size = 1 + sizeof(instant_t); @@ -215,7 +215,7 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { for (int i = 0; i < _LF_CLOCK_SYNC_EXCHANGES_PER_INTERVAL; i++) { // The first message expected from the RTI is MSG_TYPE_CLOCK_SYNC_T1 - read_from_socket_fail_on_error(rti_socket_TCP, message_size, buffer, NULL, + read_from_netdrv_fail_on_error(netdrv_to_rti, buffer, message_size, NULL, "Federate %d did not get the initial clock synchronization message T1 from the RTI.", _lf_my_fed_id); @@ -229,12 +229,12 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { // Handle the message and send a reply T3 message. // NOTE: No need to acquire the mutex lock during initialization because only // one thread is running. - if (handle_T1_clock_sync_message(buffer, *rti_socket_TCP, receive_time) != 0) { + if (handle_T1_clock_sync_message(buffer, netdrv_to_rti, NETDRV, receive_time) != 0) { lf_print_error_and_exit("Initial clock sync: Failed to send T3 reply to RTI."); } // Next message from the RTI is required to be MSG_TYPE_CLOCK_SYNC_T4 - read_from_socket_fail_on_error(rti_socket_TCP, message_size, buffer, NULL, + read_from_netdrv_fail_on_error(netdrv_to_rti, buffer, message_size, NULL, "Federate %d did not get the clock synchronization message T4 from the RTI.", _lf_my_fed_id); @@ -244,7 +244,7 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { } // Handle the message. - handle_T4_clock_sync_message(buffer, *rti_socket_TCP, receive_time); + handle_T4_clock_sync_message(buffer, netdrv_to_rti, NETDRV, receive_time); } LF_PRINT_LOG("Finished initial clock synchronization with the RTI."); @@ -261,7 +261,7 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP) { * @param t2 The physical time at which the T1 message was received. * @return 0 if T3 reply is successfully sent, -1 otherwise. */ -int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2) { +int handle_T1_clock_sync_message(unsigned char* buffer, void* netdrv_or_sock, netdrv_type_t netdrv_type, instant_t t2) { // Extract the payload instant_t t1 = extract_int64(&(buffer[1])); @@ -281,7 +281,13 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2 // Write the reply to the socket. LF_PRINT_DEBUG("Sending T3 message to RTI."); - if (write_to_socket(socket, 1 + sizeof(int), reply_buffer)) { + int ret = -1; + if (netdrv_type == NETDRV) { + ret = write_to_netdrv((netdrv_t*)netdrv_or_sock, 1 + sizeof(int), reply_buffer); + } else if (netdrv_type == UDP) { + ret = write(*(int*)netdrv_or_sock, reply_buffer, 1 + sizeof(int)); + } + if (ret <= 0) { lf_print_error("Clock sync: Failed to send T3 message to RTI."); return -1; } @@ -309,7 +315,10 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2 * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP). * @param r4 The physical time at which this T4 message was received. */ -void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r4) { + +// TODO: DONGHA: Need to check here... +void handle_T4_clock_sync_message(unsigned char* buffer, void* netdrv_or_sock, netdrv_type_t netdrv_type, + instant_t r4) { // Increment the number of received T4 messages _lf_rti_socket_stat.received_T4_messages_in_current_sync_window++; @@ -338,17 +347,19 @@ void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r // The adjustment to the clock offset (to be calculated) interval_t adjustment = 0; + + // TODO: DONGHA: CHECK here. // If the socket is _lf_rti_socket_UDP, then // after sending T4, the RTI sends a "coded probe" message, // which can be used to filter out noise. - if (socket == _lf_rti_socket_UDP) { + if (netdrv_type == UDP) { // Read the coded probe message. // We can reuse the same buffer. - int read_failed = read_from_socket(socket, 1 + sizeof(instant_t), buffer); + int bytes_read = read(*(int*)netdrv_or_sock, buffer, 1 + sizeof(instant_t)); instant_t r5 = lf_time_physical(); - if (read_failed || buffer[0] != MSG_TYPE_CLOCK_SYNC_CODED_PROBE) { + if (bytes_read <= 0 || buffer[0] != MSG_TYPE_CLOCK_SYNC_CODED_PROBE) { lf_print_warning("Clock sync: Did not get the expected coded probe message from the RTI. " "Skipping clock synchronization round."); return; @@ -500,7 +511,7 @@ void* listen_to_rti_UDP_thread(void* args) { break; } connected = true; - if (handle_T1_clock_sync_message(buffer, _lf_rti_socket_UDP, receive_time) != 0) { + if (handle_T1_clock_sync_message(buffer, &_lf_rti_socket_UDP, UDP, receive_time) != 0) { // Failed to send T3 reply. Wait for the next T1. waiting_for_T1 = true; continue; @@ -513,7 +524,7 @@ void* listen_to_rti_UDP_thread(void* args) { continue; } } else if (buffer[0] == MSG_TYPE_CLOCK_SYNC_T4) { - handle_T4_clock_sync_message(buffer, _lf_rti_socket_UDP, receive_time); + handle_T4_clock_sync_message(buffer, &_lf_rti_socket_UDP, UDP, receive_time); waiting_for_T1 = true; } else { lf_print_warning("Clock sync: Received from RTI an unexpected UDP message type: %u. " diff --git a/core/federated/federate.c b/core/federated/federate.c index 498687d07..ab9c418ab 100644 --- a/core/federated/federate.c +++ b/core/federated/federate.c @@ -49,7 +49,7 @@ extern instant_t start_time; extern bool _lf_termination_executed; // Global variables references in federate.h -lf_mutex_t lf_outbound_socket_mutex; +lf_mutex_t lf_outbound_netdrv_mutex; lf_cond_t lf_port_status_changed; lf_cond_t lf_current_tag_changed; @@ -76,13 +76,11 @@ int max_level_allowed_to_advance; * The state of this federate instance. Each executable has exactly one federate instance, * and the _fed global variable refers to that instance. */ -federate_instance_t _fed = {.socket_TCP_RTI = -1, - .number_of_inbound_p2p_connections = 0, - .inbound_socket_listeners = NULL, + +federate_instance_t _fed = {.number_of_inbound_p2p_connections = 0, + .inbound_netdriv_listeners = NULL, .number_of_outbound_p2p_connections = 0, .inbound_p2p_handling_thread_id = 0, - .server_socket = -1, - .server_port = -1, .last_TAG = {.time = NEVER, .microstep = 0u}, .is_last_TAG_provisional = false, .has_upstream = false, @@ -92,6 +90,7 @@ federate_instance_t _fed = {.socket_TCP_RTI = -1, .last_sent_NET = {.time = NEVER, .microstep = 0u}, .min_delay_from_physical_action_to_federate_output = NEVER}; +// TODO: DONGHA: Need to change host and port. federation_metadata_t federation_metadata = { .federation_id = "Unidentified Federation", .rti_host = NULL, .rti_port = -1, .rti_user = NULL}; @@ -100,7 +99,7 @@ federation_metadata_t federation_metadata = { // Static functions (used only internally) /** - * Send a time to the RTI. This acquires the lf_outbound_socket_mutex. + * Send a time to the RTI. This acquires the lf_outbound_netdrv_mutex. * @param type The message type (MSG_TYPE_TIMESTAMP). * @param time The time. */ @@ -115,15 +114,15 @@ static void send_time(unsigned char type, instant_t time) { tag_t tag = {.time = time, .microstep = 0}; tracepoint_federate_to_rti(send_TIMESTAMP, _lf_my_fed_id, &tag); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_socket_mutex, + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, bytes_to_write, buffer, &lf_outbound_netdrv_mutex, "Failed to send time " PRINTF_TIME " to the RTI.", time - start_time); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); } /** * Send a tag to the RTI. - * This function acquires the lf_outbound_socket_mutex. + * This function acquires the lf_outbound_netdrv_mutex. * @param type The message type (MSG_TYPE_NEXT_EVENT_TAG or MSG_TYPE_LATEST_TAG_COMPLETE). * @param tag The tag. */ @@ -134,27 +133,28 @@ static void send_tag(unsigned char type, tag_t tag) { buffer[0] = type; encode_tag(&(buffer[1]), tag); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + if (_fed.netdrv_to_rti == NULL) { + lf_print_warning("Netdriver is no longer connected. Dropping message."); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); return; } trace_event_t event_type = (type == MSG_TYPE_NEXT_EVENT_TAG) ? send_NET : send_LTC; // Trace the event when tracing is enabled + tracepoint_federate_to_rti(event_type, _lf_my_fed_id, &tag); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, buffer, &lf_outbound_socket_mutex, + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, bytes_to_write, buffer, &lf_outbound_netdrv_mutex, "Failed to send tag " PRINTF_TAG " to the RTI.", tag.time - start_time, tag.microstep); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); } /** - * Return true if either the socket to the RTI is broken or the socket is - * alive and the first unread byte on the socket's queue is MSG_TYPE_FAILED. + * Return true if either the netdriver to the RTI is broken or the netdriver is + * alive and the first unread byte on the netdriver's queue is MSG_TYPE_FAILED. */ static bool rti_failed() { unsigned char first_byte; - ssize_t bytes = peek_from_socket(_fed.socket_TCP_RTI, &first_byte); + ssize_t bytes = peek_from_netdrv(_fed.netdrv_to_rti, &first_byte); if (bytes < 0 || (bytes == 1 && first_byte == MSG_TYPE_FAILED)) return true; else @@ -372,31 +372,34 @@ static trigger_handle_t schedule_message_received_from_network_locked(environmen } /** - * Close the socket that receives incoming messages from the + * Close the netdriver that receives incoming messages from the * specified federate ID. This function should be called when a read - * of incoming socket fails or when an EOF is received. + * of incoming netdriver fails or when an EOF is received. * It can also be called when the receiving end wants to stop communication, * in which case, flag should be 1. * * @param fed_id The ID of the peer federate sending messages to this * federate. - * @param flag 0 if an EOF was received, -1 if a socket error occurred, 1 otherwise. + * @param flag 0 if an EOF was received, -1 if a netdriver error occurred, 1 otherwise. */ -static void close_inbound_socket(int fed_id, int flag) { - LF_MUTEX_LOCK(&socket_mutex); - if (_fed.sockets_for_inbound_p2p_connections[fed_id] >= 0) { +// TODO: need discussion here. +static void close_inbound_netdrv(int fed_id, int flag) { + LF_MUTEX_LOCK(&netdrv_mutex); + if (_fed.netdrv_for_inbound_p2p_connections[fed_id] != NULL) { if (flag >= 0) { - if (flag > 0) { - shutdown(_fed.sockets_for_inbound_p2p_connections[fed_id], SHUT_RDWR); - } else { - // Have received EOF from the other end. Send EOF to the other end. - shutdown(_fed.sockets_for_inbound_p2p_connections[fed_id], SHUT_WR); - } + // if (flag > 0) { + // shutdown(_fed.sockets_for_inbound_p2p_connections[fed_id], SHUT_RDWR); + // } else { + // // Have received EOF from the other end. Send EOF to the other end. + // shutdown(_fed.sockets_for_inbound_p2p_connections[fed_id], SHUT_WR); + // } } - close(_fed.sockets_for_inbound_p2p_connections[fed_id]); - _fed.sockets_for_inbound_p2p_connections[fed_id] = -1; + // close(_fed.sockets_for_inbound_p2p_connections[fed_id]); + // _fed.sockets_for_inbound_p2p_connections[fed_id] = -1; + close_netdrv(_fed.netdrv_for_inbound_p2p_connections[fed_id]); + _fed.netdrv_for_inbound_p2p_connections[fed_id] = NULL; } - LF_MUTEX_UNLOCK(&socket_mutex); + LF_MUTEX_UNLOCK(&netdrv_mutex); } /** @@ -446,26 +449,21 @@ static bool handle_message_now(environment_t* env, trigger_t* trigger, tag_t int * Handle a message being received from a remote federate. * * This function assumes the caller does not hold the mutex lock. - * @param socket Pointer to the socket to read the message from. + * @param netdrv Pointer to the netdrv to read the message from. * @param fed_id The sending federate ID or -1 if the centralized coordination. * @return 0 for success, -1 for failure. */ -static int handle_message(int* socket, int fed_id) { - (void)fed_id; - // Read the header. - size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); - unsigned char buffer[bytes_to_read]; - if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) { - // Read failed, which means the socket has been closed between reading the - // message ID byte and here. - return -1; - } +static int handle_message(netdrv_t* netdrv, int fed_id, unsigned char* buffer, ssize_t bytes_read) { // Extract the header information. unsigned short port_id; unsigned short federate_id; size_t length; + // TODO: JUST FOR COMPILER. + if (fed_id == 0) { + } extract_header(buffer, &port_id, &federate_id, &length); + size_t header_length = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t); // Check if the message is intended for this federate assert(_lf_my_fed_id == federate_id); LF_PRINT_DEBUG("Receiving message to port %d of length %zu.", port_id, length); @@ -476,8 +474,15 @@ static int handle_message(int* socket, int fed_id) { // Read the payload. // Allocate memory for the message contents. unsigned char* message_contents = (unsigned char*)malloc(length); - if (read_from_socket_close_on_error(socket, length, message_contents)) { - return -1; + memcpy(message_contents, buffer + header_length, bytes_read); + int buf_count = bytes_read; + while (netdrv->read_remaining_bytes > 0) { + ssize_t bytes_read_again = + read_from_netdrv_close_on_error(netdrv, message_contents + buf_count, length - buf_count); + if (bytes_read_again <= 0) { + return -1; + } + buf_count += bytes_read_again; } // Trace the event when tracing is enabled tracepoint_federate_from_federate(receive_P2P_MSG, _lf_my_fed_id, federate_id, NULL); @@ -500,29 +505,22 @@ static int handle_message(int* socket, int fed_id) { * will not advance to the tag of the message if it is in the future, or * the tag will not advance at all if the tag of the message is * now or in the past. - * @param socket Pointer to the socket to read the message from. + * @param netdrv Pointer to the netdrv to read the message from. * @param fed_id The sending federate ID or -1 if the centralized coordination. - * @return 0 on successfully reading the message, -1 on failure (e.g. due to socket closed). + * @return 0 on successfully reading the message, -1 on failure (e.g. due to netdrv closed). */ -static int handle_tagged_message(int* socket, int fed_id) { +static int handle_tagged_message(netdrv_t* netdrv, int fed_id, unsigned char* buffer, ssize_t bytes_read) { // Environment is always the one corresponding to the top-level scheduling enclave. environment_t* env; _lf_get_environments(&env); - - // Read the header which contains the timestamp. - size_t bytes_to_read = - sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t) + sizeof(instant_t) + sizeof(microstep_t); - unsigned char buffer[bytes_to_read]; - if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) { - return -1; // Read failed. - } - // Extract the header information. unsigned short port_id; unsigned short federate_id; size_t length; tag_t intended_tag; extract_timed_header(buffer, &port_id, &federate_id, &length, &intended_tag); + size_t header_length = + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(instant_t) + sizeof(microstep_t); // Trace the event when tracing is enabled if (fed_id == -1) { tracepoint_federate_from_rti(receive_TAGGED_MSG, _lf_my_fed_id, &intended_tag); @@ -562,11 +560,18 @@ static int handle_tagged_message(int* socket, int fed_id) { // Read the payload. // Allocate memory for the message contents. unsigned char* message_contents = (unsigned char*)malloc(length); - if (read_from_socket_close_on_error(socket, length, message_contents)) { + memcpy(message_contents, buffer + header_length, bytes_read - header_length); + int buf_count = bytes_read - header_length; + while (netdrv->read_remaining_bytes > 0) { + ssize_t bytes_read_again = + read_from_netdrv_close_on_error(netdrv, message_contents + buf_count, length - buf_count); + if (bytes_read_again <= 0) { #ifdef FEDERATED_DECENTRALIZED - _lf_decrement_tag_barrier_locked(env); + _lf_decrement_tag_barrier_locked(env); #endif - return -1; // Read failed. + return -1; + } + buf_count += bytes_read_again; } // The following is only valid for string messages. @@ -631,11 +636,11 @@ static int handle_tagged_message(int* socket, int fed_id) { if (lf_tag_compare(env->current_tag, env->stop_tag) >= 0 && env->execution_started) { lf_print_error("Received message too late. Already at stop tag.\n" " Current tag is " PRINTF_TAG " and intended tag is " PRINTF_TAG ".\n" - " Discarding message and closing the socket.", + " Discarding message and closing the netdrv.", env->current_tag.time - start_time, env->current_tag.microstep, intended_tag.time - start_time, intended_tag.microstep); - // Close socket, reading any incoming data and discarding it. - close_inbound_socket(fed_id, 1); + // Close netdrv, reading any incoming data and discarding it. + close_inbound_netdrv(fed_id, 1); } else { // Need to use intended_tag here, not actual_tag, so that STP violations are detected. // It will become actual_tag (that is when the reactions will be invoked). @@ -664,17 +669,11 @@ static int handle_tagged_message(int* socket, int fed_id) { * This just sets the last known status tag of the port specified * in the message. * - * @param socket Pointer to the socket to read the message from + * @param buffer * @param fed_id The sending federate ID or -1 if the centralized coordination. * @return 0 for success, -1 for failure to complete the read. */ -static int handle_port_absent_message(int* socket, int fed_id) { - size_t bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(instant_t) + sizeof(microstep_t); - unsigned char buffer[bytes_to_read]; - if (read_from_socket_close_on_error(socket, bytes_to_read, buffer)) { - return -1; - } - +static int handle_port_absent_message(unsigned char* buffer, int fed_id) { // Extract the header information. unsigned short port_id = extract_uint16(buffer); // The next part of the message is the federate_id, but we don't need it. @@ -708,10 +707,9 @@ static int handle_port_absent_message(int* socket, int fed_id) { * peer federate and calls the appropriate handling function for * each message type. If an error occurs or an EOF is received * from the peer, then this procedure sets the corresponding - * socket in _fed.sockets_for_inbound_p2p_connections + * netdrv in _fed.netdrv_for_inbound_p2p_connections * to -1 and returns, terminating the thread. * @param _args The remote federate ID (cast to void*). - * @param fed_id_ptr A pointer to a uint16_t containing federate ID being listened to. * This procedure frees the memory pointed to before returning. */ static void* listen_to_federates(void* _args) { @@ -720,7 +718,8 @@ static void* listen_to_federates(void* _args) { LF_PRINT_LOG("Listening to federate %d.", fed_id); - int* socket_id = &_fed.sockets_for_inbound_p2p_connections[fed_id]; + // int* socket_id = &_fed.sockets_for_inbound_p2p_connections[fed_id]; + netdrv_t* netdrv = _fed.netdrv_for_inbound_p2p_connections[fed_id]; // Buffer for incoming messages. // This does not constrain the message size @@ -729,45 +728,46 @@ static void* listen_to_federates(void* _args) { // Listen for messages from the federate. while (1) { - bool socket_closed = false; + bool netdrv_closed = false; // Read one byte to get the message type. - LF_PRINT_DEBUG("Waiting for a P2P message on socket %d.", *socket_id); - if (read_from_socket_close_on_error(socket_id, 1, buffer)) { - // Socket has been closed. - lf_print("Socket from federate %d is closed.", fed_id); + LF_PRINT_DEBUG("Waiting for a P2P message on netdrv"); + ssize_t bytes_read = read_from_netdrv_close_on_error(netdrv, buffer, FED_COM_BUFFER_SIZE); + if (bytes_read <= 0) { + // netdrv has been closed. + lf_print("Netdriver from federate %d is closed.", fed_id); // Stop listening to this federate. - socket_closed = true; + netdrv_closed = true; break; } - LF_PRINT_DEBUG("Received a P2P message on socket %d of type %d.", *socket_id, buffer[0]); + LF_PRINT_DEBUG("Received a P2P message on netdrv of type %d.", buffer[0]); bool bad_message = false; switch (buffer[0]) { case MSG_TYPE_P2P_MESSAGE: LF_PRINT_LOG("Received untimed message from federate %d.", fed_id); - if (handle_message(socket_id, fed_id)) { + if (handle_message(netdrv, fed_id, buffer + 1, bytes_read - 1)) { // Failed to complete the reading of a message on a physical connection. lf_print_warning("Failed to complete reading of message on physical connection."); - socket_closed = true; + netdrv_closed = true; } break; case MSG_TYPE_P2P_TAGGED_MESSAGE: LF_PRINT_LOG("Received tagged message from federate %d.", fed_id); - if (handle_tagged_message(socket_id, fed_id)) { + if (handle_tagged_message(netdrv, fed_id, buffer + 1, bytes_read - 1)) { // P2P tagged messages are only used in decentralized coordination, and - // it is not a fatal error if the socket is closed before the whole message is read. + // it is not a fatal error if the netdriver is closed before the whole message is read. // But this thread should exit. lf_print_warning("Failed to complete reading of tagged message."); - socket_closed = true; + netdrv_closed = true; } break; case MSG_TYPE_PORT_ABSENT: LF_PRINT_LOG("Received port absent message from federate %d.", fed_id); - if (handle_port_absent_message(socket_id, fed_id)) { + if (handle_port_absent_message(buffer + 1, fed_id)) { // P2P tagged messages are only used in decentralized coordination, and - // it is not a fatal error if the socket is closed before the whole message is read. + // it is not a fatal error if the netdriver is closed before the whole message is read. // But this thread should exit. lf_print_warning("Failed to complete reading of tagged message."); - socket_closed = true; + netdrv_closed = true; } break; default: @@ -779,8 +779,8 @@ static void* listen_to_federates(void* _args) { tracepoint_federate_from_federate(receive_UNIDENTIFIED, _lf_my_fed_id, fed_id, NULL); break; // while loop } - if (socket_closed) { - // NOTE: For decentralized execution, once this socket is closed, we could + if (netdrv_closed) { + // NOTE: For decentralized execution, once this netdriver is closed, we could // update last known tags of all ports connected to the specified federate to FOREVER_TAG, // which would eliminate the need to wait for STAA to assume an input is absent. // However, at this time, we don't know which ports correspond to which upstream federates. @@ -794,38 +794,40 @@ static void* listen_to_federates(void* _args) { } /** - * Close the socket that sends outgoing messages to the - * specified federate ID. This function acquires the lf_outbound_socket_mutex mutex lock + * Close the netdriver that sends outgoing messages to the + * specified federate ID. This function acquires the lf_outbound_netdrv_mutex mutex lock * if _lf_normal_termination is true and otherwise proceeds without the lock. * @param fed_id The ID of the peer federate receiving messages from this * federate, or -1 if the RTI (centralized coordination). - * @param flag 0 if the socket has received EOF, 1 if not, -1 if abnormal termination. + * @param flag 1 if normal termination, -1 if abnormal termination. */ -static void close_outbound_socket(int fed_id, int flag) { +// TODO: DONGHA: Need discussion here. +static void close_outbound_netdrv(int fed_id, int flag) { assert(fed_id >= 0 && fed_id < NUMBER_OF_FEDERATES); - if (_lf_normal_termination) { - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + if (flag) { + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); } - if (_fed.sockets_for_outbound_p2p_connections[fed_id] >= 0) { - // Close the socket by sending a FIN packet indicating that no further writes - // are expected. Then read until we get an EOF indication. + if (_fed.netdrv_for_outbound_p2p_connections[fed_id] != NULL) { + // // Close the socket by sending a FIN packet indicating that no further writes + // // are expected. Then read until we get an EOF indication. if (flag >= 0) { - // SHUT_WR indicates no further outgoing messages. - shutdown(_fed.sockets_for_outbound_p2p_connections[fed_id], SHUT_WR); - if (flag > 0) { - // Have not received EOF yet. read until we get an EOF or error indication. - // This compensates for delayed ACKs and disabling of Nagles algorithm - // by delaying exiting until the shutdown is complete. - unsigned char message[32]; - while (read(_fed.sockets_for_outbound_p2p_connections[fed_id], &message, 32) > 0) - ; - } + // // SHUT_WR indicates no further outgoing messages. + // shutdown(_fed.sockets_for_outbound_p2p_connections[fed_id], SHUT_WR); + // if (flag > 0) { + // // Have not received EOF yet. read until we get an EOF or error indication. + // // This compensates for delayed ACKs and disabling of Nagles algorithm + // // by delaying exiting until the shutdown is complete. + // unsigned char message[32]; + // while (read(_fed.sockets_for_outbound_p2p_connections[fed_id], &message, 32) > 0); + // } } - close(_fed.sockets_for_outbound_p2p_connections[fed_id]); - _fed.sockets_for_outbound_p2p_connections[fed_id] = -1; + // close(_fed.sockets_for_outbound_p2p_connections[fed_id]); + // _fed.sockets_for_outbound_p2p_connections[fed_id] = -1; + close_netdrv(_fed.netdrv_for_outbound_p2p_connections[fed_id]); + _fed.netdrv_for_outbound_p2p_connections[fed_id] = NULL; } - if (_lf_normal_termination) { - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + if (flag) { + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); } } @@ -843,18 +845,19 @@ static int perform_hmac_authentication() { unsigned char fed_hello_buf[message_length]; fed_hello_buf[0] = MSG_TYPE_FED_NONCE; encode_uint16((uint16_t)_lf_my_fed_id, &fed_hello_buf[1]); + // Must save fed_nonce for handshake comparison. unsigned char fed_nonce[NONCE_LENGTH]; RAND_bytes(fed_nonce, NONCE_LENGTH); memcpy(&fed_hello_buf[1 + fed_id_length], fed_nonce, NONCE_LENGTH); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, message_length, fed_hello_buf, NULL, "Failed to write nonce."); + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, message_length, fed_hello_buf, NULL, "Failed to write nonce."); // Check HMAC of received FED_RESPONSE message. unsigned int hmac_length = SHA256_HMAC_LENGTH; size_t federation_id_length = strnlen(federation_metadata.federation_id, 255); unsigned char received[1 + NONCE_LENGTH + hmac_length]; - if (read_from_socket_close_on_error(&_fed.socket_TCP_RTI, 1 + NONCE_LENGTH + hmac_length, received)) { + if (read_from_netdrv_close_on_error(_fed.netdrv_to_rti, received, 1 + NONCE_LENGTH + hmac_length) <= 0) { lf_print_warning("Failed to read RTI response."); return -1; } @@ -885,7 +888,7 @@ static int perform_hmac_authentication() { response[1] = HMAC_DOES_NOT_MATCH; // Ignore errors on writing back. - write_to_socket(_fed.socket_TCP_RTI, 2, response); + write_to_netdrv(_fed.netdrv_to_rti, 2, response); return -1; } else { LF_PRINT_LOG("HMAC verified."); @@ -899,48 +902,12 @@ static int perform_hmac_authentication() { HMAC(EVP_sha256(), federation_metadata.federation_id, federation_id_length, mac_buf, 1 + NONCE_LENGTH, &sender[1], &hmac_length); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, 1 + hmac_length, sender, NULL, "Failed to write fed response."); + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, 1 + hmac_length, sender, NULL, "Failed to write fed response."); } return 0; } #endif -static void close_rti_socket() { - shutdown(_fed.socket_TCP_RTI, SHUT_RDWR); - close(_fed.socket_TCP_RTI); - _fed.socket_TCP_RTI = -1; -} - -/** - * Return in the result a struct with the address info for the specified hostname and port. - * The memory for the result is dynamically allocated and must be freed using freeaddrinfo. - * @param hostname The host name. - * @param port The port number. - * @param result The struct into which to write. - */ -static void rti_address(const char* hostname, uint16_t port, struct addrinfo** result) { - struct addrinfo hints; - - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_INET; /* Allow IPv4 */ - hints.ai_socktype = SOCK_STREAM; /* Stream socket */ - hints.ai_protocol = IPPROTO_TCP; /* TCP protocol */ - hints.ai_addr = NULL; - hints.ai_next = NULL; - hints.ai_flags = AI_NUMERICSERV; /* Allow only numeric port numbers */ - - // Convert port number to string. - char str[6]; - sprintf(str, "%u", port); - - // Get address structure matching hostname and hints criteria, and - // set port to the port number provided in str. There should only - // ever be one matching address structure, and we connect to that. - if (getaddrinfo(hostname, (const char*)&str, &hints, result)) { - lf_print_error_and_exit("No host for RTI matching given hostname: %s", hostname); - } -} - /** * Send the specified timestamp to the RTI and wait for a response. * The specified timestamp should be current physical time of the @@ -954,12 +921,12 @@ static instant_t get_start_time_from_rti(instant_t my_physical_time) { // Send the timestamp marker first. send_time(MSG_TYPE_TIMESTAMP, my_physical_time); - // Read bytes from the socket. We need 9 bytes. + // Read bytes from the netdriver. We need 9 bytes. // Buffer for message ID plus timestamp. size_t buffer_length = 1 + sizeof(instant_t); unsigned char buffer[buffer_length]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, buffer_length, buffer, NULL, + read_from_netdrv_fail_on_error(_fed.netdrv_to_rti, buffer, buffer_length, NULL, "Failed to read MSG_TYPE_TIMESTAMP message from RTI."); LF_PRINT_DEBUG("Read 9 bytes."); @@ -996,15 +963,11 @@ static instant_t get_start_time_from_rti(instant_t my_physical_time) { * @note This function is very similar to handle_provisinal_tag_advance_grant() except that * it sets last_TAG_was_provisional to false. */ -static void handle_tag_advance_grant(void) { +static void handle_tag_advance_grant(unsigned char* buffer) { // Environment is always the one corresponding to the top-level scheduling enclave. environment_t* env; _lf_get_environments(&env); - size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); - unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL, - "Failed to read tag advance grant from RTI."); tag_t TAG = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1209,15 +1172,10 @@ static void* update_ports_from_staa_offsets(void* args) { * it sets last_TAG_was_provisional to true and also it does not update the * last known tag for input ports. */ -static void handle_provisional_tag_advance_grant() { +static void handle_provisional_tag_advance_grant(unsigned char* buffer) { // Environment is always the one corresponding to the top-level scheduling enclave. environment_t* env; _lf_get_environments(&env); - - size_t bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); - unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL, - "Failed to read provisional tag advance grant from RTI."); tag_t PTAG = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1302,13 +1260,7 @@ static void handle_provisional_tag_advance_grant() { * logical time raised when lf_request_stop() was * called in the environment for each enclave. */ -static void handle_stop_granted_message() { - - size_t bytes_to_read = MSG_TYPE_STOP_GRANTED_LENGTH - 1; - unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL, - "Failed to read stop granted from RTI."); - +static void handle_stop_granted_message(unsigned char* buffer) { tag_t received_stop_tag = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1347,11 +1299,7 @@ static void handle_stop_granted_message() { /** * Handle a MSG_TYPE_STOP_REQUEST message from the RTI. */ -static void handle_stop_request_message() { - size_t bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; - unsigned char buffer[bytes_to_read]; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_read, buffer, NULL, - "Failed to read stop request from RTI."); +static void handle_stop_request_message(unsigned char* buffer) { tag_t tag_to_stop = extract_tag(buffer); // Trace the event when tracing is enabled @@ -1376,10 +1324,10 @@ static void handle_stop_request_message() { // or we have previously sent a stop request to the RTI, // then we have already blocked tag advance in enclaves. // Do not do this twice. The record of whether the first has occurred - // is guarded by the outbound socket mutex. + // is guarded by the outbound netdriver mutex. // The second is guarded by the global mutex. // Note that the RTI should not send stop requests more than once to federates. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); if (_fed.received_stop_request_from_rti) { LF_PRINT_LOG("Redundant MSG_TYPE_STOP_REQUEST from RTI. Ignoring it."); already_blocked = true; @@ -1388,7 +1336,7 @@ static void handle_stop_request_message() { // prevent lf_request_stop from sending. _fed.received_stop_request_from_rti = true; } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); if (already_blocked) { // Either we have sent a stop request to the RTI ourselves, @@ -1422,11 +1370,11 @@ static void handle_stop_request_message() { tracepoint_federate_to_rti(send_STOP_REQ_REP, _lf_my_fed_id, &tag_to_stop); // Send the current logical time to the RTI. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer, - &lf_outbound_socket_mutex, + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, MSG_TYPE_STOP_REQUEST_REPLY_LENGTH, outgoing_buffer, + &lf_outbound_netdrv_mutex, "Failed to send the answer to MSG_TYPE_STOP_REQUEST to RTI."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); LF_PRINT_DEBUG("Sent MSG_TYPE_STOP_REQUEST_REPLY to RTI with tag " PRINTF_TAG, tag_to_stop.time, tag_to_stop.microstep); @@ -1439,10 +1387,10 @@ static void send_resign_signal() { size_t bytes_to_write = 1; unsigned char buffer[bytes_to_write]; buffer[0] = MSG_TYPE_RESIGN; - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), &lf_outbound_socket_mutex, + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, bytes_to_write, &(buffer[0]), &lf_outbound_netdrv_mutex, "Failed to send MSG_TYPE_RESIGN."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); LF_PRINT_LOG("Resigned."); } @@ -1453,7 +1401,7 @@ static void send_failed_signal() { size_t bytes_to_write = 1; unsigned char buffer[bytes_to_write]; buffer[0] = MSG_TYPE_FAILED; - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, bytes_to_write, &(buffer[0]), NULL, + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, bytes_to_write, &(buffer[0]), NULL, "Failed to send MSG_TYPE_FAILED."); LF_PRINT_LOG("Failed."); } @@ -1471,7 +1419,7 @@ static void handle_rti_failed_message(void) { exit(1); } * When messages arrive, this calls the appropriate handler. * @param args Ignored */ -static void* listen_to_rti_TCP(void* args) { +static void* listen_to_rti(void* args) { (void)args; initialize_lf_thread_id(); // Buffer for incoming messages. @@ -1481,57 +1429,54 @@ static void* listen_to_rti_TCP(void* args) { // Listen for messages from the federate. while (1) { - // Check whether the RTI socket is still valid - if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket to the RTI unexpectedly closed."); - return NULL; - } // Read one byte to get the message type. // This will exit if the read fails. - int read_failed = read_from_socket(_fed.socket_TCP_RTI, 1, buffer); - if (read_failed < 0) { - if (errno == ECONNRESET) { - lf_print_error("Socket connection to the RTI was closed by the RTI without" - " properly sending an EOF first. Considering this a soft error."); - // FIXME: If this happens, possibly a new RTI must be elected. - _fed.socket_TCP_RTI = -1; - return NULL; - } else { - lf_print_error("Socket connection to the RTI has been broken with error %d: %s." - " The RTI should close connections with an EOF first." - " Considering this a soft error.", - errno, strerror(errno)); - // FIXME: If this happens, possibly a new RTI must be elected. - _fed.socket_TCP_RTI = -1; - return NULL; - } - } else if (read_failed > 0) { - // EOF received. - lf_print("Connection to the RTI closed with an EOF."); - _fed.socket_TCP_RTI = -1; + int bytes_read = read_from_netdrv(_fed.netdrv_to_rti, buffer, FED_COM_BUFFER_SIZE); + // if (read_failed < 0) { + // if (errno == ECONNRESET) { + // lf_print_error("Socket connection to the RTI was closed by the RTI without" + // " properly sending an EOF first. Considering this a soft error."); + // // FIXME: If this happens, possibly a new RTI must be elected. + // _fed.socket_TCP_RTI = -1; + // return NULL; + // } else { + // lf_print_error("Socket connection to the RTI has been broken with error %d: %s." + // " The RTI should close connections with an EOF first." + // " Considering this a soft error.", + // errno, + // strerror(errno)); + // // FIXME: If this happens, possibly a new RTI must be elected. + // _fed.socket_TCP_RTI = -1; + // return NULL; + // } + // } else if (read_failed > 0) { + // // EOF received. + // lf_print("Connection to the RTI closed with an EOF."); + // _fed.socket_TCP_RTI = -1; + if (bytes_read <= 0) { return NULL; } switch (buffer[0]) { case MSG_TYPE_TAGGED_MESSAGE: - if (handle_tagged_message(&_fed.socket_TCP_RTI, -1)) { + if (handle_tagged_message(_fed.netdrv_to_rti, -1, buffer + 1, bytes_read - 1)) { // Failures to complete the read of messages from the RTI are fatal. lf_print_error_and_exit("Failed to complete the reading of a message from the RTI."); } break; case MSG_TYPE_TAG_ADVANCE_GRANT: - handle_tag_advance_grant(); + handle_tag_advance_grant(buffer + 1); break; case MSG_TYPE_PROVISIONAL_TAG_ADVANCE_GRANT: - handle_provisional_tag_advance_grant(); + handle_provisional_tag_advance_grant(buffer + 1); break; case MSG_TYPE_STOP_REQUEST: - handle_stop_request_message(); + handle_stop_request_message(buffer + 1); break; case MSG_TYPE_STOP_GRANTED: - handle_stop_granted_message(); + handle_stop_granted_message(buffer + 1); break; case MSG_TYPE_PORT_ABSENT: - if (handle_port_absent_message(&_fed.socket_TCP_RTI, -1)) { + if (handle_port_absent_message(buffer + 1, -1)) { // Failures to complete the read of absent messages from the RTI are fatal. lf_print_error_and_exit("Failed to complete the reading of an absent message from the RTI."); } @@ -1541,7 +1486,7 @@ static void* listen_to_rti_TCP(void* args) { break; case MSG_TYPE_CLOCK_SYNC_T1: case MSG_TYPE_CLOCK_SYNC_T4: - lf_print_error("Federate %d received unexpected clock sync message from RTI on TCP socket.", _lf_my_fed_id); + lf_print_error("Federate %d received unexpected clock sync message from RTI on netdriver.", _lf_my_fed_id); break; default: lf_print_error_and_exit("Received from RTI an unrecognized TCP message type: %hhx.", buffer[0]); @@ -1606,7 +1551,7 @@ static bool bounded_NET(tag_t* tag) { // An empty version of this function is code generated for unfederated execution. /** - * Close sockets used to communicate with other federates, if they are open, + * Close netdrivers used to communicate with other federates, if they are open, * and send a MSG_TYPE_RESIGN message to the RTI. This implements the function * defined in reactor.h. For unfederated execution, the code generator * generates an empty implementation. @@ -1617,7 +1562,7 @@ void lf_terminate_execution(environment_t* env) { // For an abnormal termination (e.g. a SIGINT), we need to send a // MSG_TYPE_FAILED message to the RTI, but we should not acquire a mutex. - if (_fed.socket_TCP_RTI >= 0) { + if (_fed.netdrv_to_rti != NULL) { if (_lf_normal_termination) { tracepoint_federate_to_rti(send_RESIGN, _lf_my_fed_id, &env->current_tag); send_resign_signal(); @@ -1627,64 +1572,89 @@ void lf_terminate_execution(environment_t* env) { } } - LF_PRINT_DEBUG("Closing incoming P2P sockets."); - // Close any incoming P2P sockets that are still open. + LF_PRINT_DEBUG("Closing incoming P2P netdrivers."); + // Close any incoming P2P netdrivers that are still open. for (int i = 0; i < NUMBER_OF_FEDERATES; i++) { - close_inbound_socket(i, 1); - // Ignore errors. Mark the socket closed. - _fed.sockets_for_inbound_p2p_connections[i] = -1; + close_inbound_netdrv(i, 1); + // Ignore errors. Mark the netdriver closed. + _fed.netdrv_for_inbound_p2p_connections[i] = NULL; } // Check for all outgoing physical connections in - // _fed.sockets_for_outbound_p2p_connections and + // _fed.netdrv_for_outbound_p2p_connections and // if the socket ID is not -1, the connection is still open. - // Send an EOF by closing the socket here. + // Send an EOF by closing the netdriver here. for (int i = 0; i < NUMBER_OF_FEDERATES; i++) { // Close outbound connections, in case they have not closed themselves. // This will result in EOF being sent to the remote federate, except for - // abnormal termination, in which case it will just close the socket. + // abnormal termination, in which case it will just close the netdriver. int flag = _lf_normal_termination ? 1 : -1; - close_outbound_socket(i, flag); + close_outbound_netdrv(i, flag); } - LF_PRINT_DEBUG("Waiting for inbound p2p socket listener threads."); - // Wait for each inbound socket listener thread to close. - if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_socket_listeners != NULL) { + LF_PRINT_DEBUG("Waiting for inbound p2p netdrviers listener threads."); + // Wait for each inbound netdriver listener thread to close. + if (_fed.number_of_inbound_p2p_connections > 0 && _fed.inbound_netdriv_listeners != NULL) { LF_PRINT_LOG("Waiting for %zu threads listening for incoming messages to exit.", _fed.number_of_inbound_p2p_connections); for (size_t i = 0; i < _fed.number_of_inbound_p2p_connections; i++) { // Ignoring errors here. - lf_thread_join(_fed.inbound_socket_listeners[i], NULL); + lf_thread_join(_fed.inbound_netdriv_listeners[i], NULL); } } - LF_PRINT_DEBUG("Waiting for RTI's socket listener threads."); + LF_PRINT_DEBUG("Waiting for RTI's netdriver listener threads."); // Wait for the thread listening for messages from the RTI to close. - lf_thread_join(_fed.RTI_socket_listener, NULL); + lf_thread_join(_fed.RTI_netdrv_listener, NULL); // For abnormal termination, there is no need to free memory. if (_lf_normal_termination) { LF_PRINT_DEBUG("Freeing memory occupied by the federate."); - free(_fed.inbound_socket_listeners); + free(_fed.inbound_netdriv_listeners); free(federation_metadata.rti_host); free(federation_metadata.rti_user); } } -////////////////////////////////////////////////////////////////////////////////// -////////////////////////////////////////////////////////////////////////////////// -// Public functions (declared in federate.h, in alphabetical order) +/** + * @brief Send the server port number to the RTI on an MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h). + * + * @param drv + */ +void send_address_advertisement_to_RTI(netdrv_t* fed_netdrv, netdrv_t* rti_netdrv) { +#if defined(COMM_TYPE_TCP) || defined(COMM_TYPE_SST) + unsigned char buffer[sizeof(int32_t) + 1]; + buffer[0] = MSG_TYPE_ADDRESS_ADVERTISEMENT; + encode_int32(get_my_port(fed_netdrv), &(buffer[1])); -void lf_connect_to_federate(uint16_t remote_federate_id) { - int result = -1; + // No need for a mutex because we have the only handle on this netdriver. + write_to_netdrv_fail_on_error(rti_netdrv, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL, + "Failed to send address advertisement."); + + LF_PRINT_DEBUG("Sent port %d to the RTI.", get_my_port(fed_netdrv)); +#elif defined(COMM_TYPE_MQTT) + // Do not send port. + if (fed_netdrv == NULL) { + } // JUST TO PASS COMPILER. + if (rti_netdrv == NULL) { + } // JUST TO PASS COMPILER. +#endif +} - // Ask the RTI for port number of the remote federate. +/** + * @brief Ask the RTI for port number of the remote federate. + * + * @param remote_federate_id + * @return int + */ +static void get_remote_federate_info_from_RTI(uint16_t remote_federate_id, netdrv_t* rti_netdrv, netdrv_t* fed_netdrv) { +// Do not send port for MQTT. It only needs to know the target federate's ID. +#if defined(COMM_TYPE_TCP) || defined(COMM_TYPE_SST) // The buffer is used for both sending and receiving replies. // The size is what is needed for receiving replies. - unsigned char buffer[sizeof(int32_t) + INET_ADDRSTRLEN + 1]; + unsigned char buffer[sizeof(int32_t) + sizeof(struct in_addr) + 1]; int port = -1; - struct in_addr host_ip_addr; instant_t start_connect = lf_time_physical(); while (port == -1 && !_lf_termination_executed) { buffer[0] = MSG_TYPE_ADDRESS_QUERY; @@ -1695,14 +1665,14 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_ADR_QR, _lf_my_fed_id, NULL); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(uint16_t) + 1, buffer, &lf_outbound_socket_mutex, + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + write_to_netdrv_fail_on_error(rti_netdrv, sizeof(uint16_t) + 1, buffer, &lf_outbound_netdrv_mutex, "Failed to send address query for federate %d to RTI.", remote_federate_id); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); // Read RTI's response. - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, buffer, NULL, - "Failed to read the requested port number for federate %d from RTI.", + read_from_netdrv_fail_on_error(rti_netdrv, buffer, sizeof(int32_t) + sizeof(struct in_addr) + 1, NULL, + "Failed to read the requested port number and IP address for federate %d from RTI.", remote_federate_id); if (buffer[0] != MSG_TYPE_ADDRESS_QUERY_REPLY) { @@ -1715,9 +1685,6 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { } port = extract_int32(&buffer[1]); - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(host_ip_addr), (unsigned char*)&host_ip_addr, NULL, - "Failed to read the IP address for federate %d from RTI.", remote_federate_id); - // A reply of -1 for the port means that the RTI does not know // the port number of the remote federate, presumably because the // remote federate has not yet sent an MSG_TYPE_ADDRESS_ADVERTISEMENT message to the RTI. @@ -1732,108 +1699,85 @@ void lf_connect_to_federate(uint16_t remote_federate_id) { } assert(port < 65536); assert(port > 0); - uint16_t uport = (uint16_t)port; - -#if LOG_LEVEL > 3 - // Print the received IP address in a human readable format - // Create the human readable format of the received address. - // This is avoided unless LOG_LEVEL is high enough to - // subdue the overhead caused by inet_ntop(). char hostname[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &host_ip_addr, hostname, INET_ADDRSTRLEN); - LF_PRINT_LOG("Received address %s port %d for federate %d from RTI.", hostname, uport, remote_federate_id); + inet_ntop(AF_INET, buffer + 1 + sizeof(int32_t), hostname, INET_ADDRSTRLEN); + LF_PRINT_LOG("Received address %s port %d for federate %d from RTI.", hostname, port, remote_federate_id); + // Set the target federate's hostname and port to the netdriver. + set_host_name(fed_netdrv, hostname); + // Must set as specified port. Or else, the port will be increased when connecting to the other federate. + set_specified_port(fed_netdrv, port); +#elif defined(COMM_TYPE_MQTT) + + // Do not send port for MQTT. It only needs to know the target federate's ID. + (void) remote_federate_id; + (void) rti_netdrv; + (void) fed_netdrv; #endif +} - // Iterate until we either successfully connect or we exceed the CONNECT_TIMEOUT - start_connect = lf_time_physical(); - int socket_id = -1; - while (result < 0 && !_lf_termination_executed) { - // Create an IPv4 socket for TCP (not UDP) communication over IP (0). - socket_id = create_real_time_tcp_socket_errexit(); - - // Server file descriptor. - struct sockaddr_in server_fd; - // Zero out the server_fd struct. - bzero((char*)&server_fd, sizeof(server_fd)); - - // Set up the server_fd fields. - server_fd.sin_family = AF_INET; // IPv4 - server_fd.sin_addr = host_ip_addr; // Received from the RTI +////////////////////////////////////////////////////////////////////////////////// +////////////////////////////////////////////////////////////////////////////////// +// Public functions (declared in federate.h, in alphabetical order) - // Convert the port number from host byte order to network byte order. - server_fd.sin_port = htons(uport); - result = connect(socket_id, (struct sockaddr*)&server_fd, sizeof(server_fd)); +// TODO: DONGHA: Handling address queries must be changed. Just leaving port and ip_addr now. +void lf_connect_to_federate(uint16_t remote_federate_id) { + int result = -1; - if (result != 0) { - lf_print_error("Failed to connect to federate %d on port %d.", remote_federate_id, uport); + // Initialize the netdriver to connect the remote federate. + netdrv_t* netdrv = initialize_netdrv(_lf_my_fed_id, federation_metadata.federation_id); +#if defined(COMM_TYPE_MQTT) + set_target_id(netdrv, remote_federate_id); +#endif + create_connector(netdrv); + get_remote_federate_info_from_RTI(remote_federate_id, _fed.netdrv_to_rti, netdrv); + result = connect_to_netdrv(netdrv); + if (result != 0) { + lf_print_error("Failed to connect to federate %d.", remote_federate_id); + } - // Try again after some time if the connection failed. - // Note that this should not really happen since the remote federate should be - // accepting socket connections. But possibly it will be busy (in process of accepting - // another socket connection?). Hence, we retry. - if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { - // If the remote federate is not accepting the connection after CONNECT_TIMEOUT - // treat it as a soft error condition and return. - lf_print_error("Failed to connect to federate %d with timeout: " PRINTF_TIME ". Giving up.", remote_federate_id, - CONNECT_TIMEOUT); - return; - } - lf_print_warning("Could not connect to federate %d. Will try again every" PRINTF_TIME "nanoseconds.\n", - remote_federate_id, ADDRESS_QUERY_RETRY_INTERVAL); + // Connect was successful. - // Check whether the RTI is still there. - if (rti_failed()) - break; + // Send P2P message to target federate. + size_t federation_id_length = strnlen(federation_metadata.federation_id, 255); + unsigned char* buf = (unsigned char*)malloc(1 + sizeof(uint16_t) + 1 + federation_id_length); + buf[0] = MSG_TYPE_P2P_SENDING_FED_ID; + if (_lf_my_fed_id > UINT16_MAX) { + // This error is very unlikely to occur. + lf_print_error_and_exit("Too many federates! More than %d.", UINT16_MAX); + } + encode_uint16((uint16_t)_lf_my_fed_id, (unsigned char*)&(buf[1])); + buf[1 + sizeof(uint16_t)] = (unsigned char)(federation_id_length & 0xff); - // Wait ADDRESS_QUERY_RETRY_INTERVAL nanoseconds. - lf_sleep(ADDRESS_QUERY_RETRY_INTERVAL); - } else { - // Connect was successful. - size_t buffer_length = 1 + sizeof(uint16_t) + 1; - unsigned char buffer[buffer_length]; - buffer[0] = MSG_TYPE_P2P_SENDING_FED_ID; - if (_lf_my_fed_id > UINT16_MAX) { - // This error is very unlikely to occur. - lf_print_error_and_exit("Too many federates! More than %d.", UINT16_MAX); - } - encode_uint16((uint16_t)_lf_my_fed_id, (unsigned char*)&(buffer[1])); - unsigned char federation_id_length = (unsigned char)strnlen(federation_metadata.federation_id, 255); - buffer[sizeof(uint16_t) + 1] = federation_id_length; - // Trace the event when tracing is enabled - tracepoint_federate_to_federate(send_FED_ID, _lf_my_fed_id, remote_federate_id, NULL); - - // No need for a mutex because we have the only handle on the socket. - write_to_socket_fail_on_error(&socket_id, buffer_length, buffer, NULL, "Failed to send fed_id to federate %d.", - remote_federate_id); - write_to_socket_fail_on_error(&socket_id, federation_id_length, (unsigned char*)federation_metadata.federation_id, - NULL, "Failed to send federation id to federate %d.", remote_federate_id); - - read_from_socket_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL, - "Failed to read MSG_TYPE_ACK from federate %d in response to sending fed_id.", - remote_federate_id); - if (buffer[0] != MSG_TYPE_ACK) { - // Get the error code. - read_from_socket_fail_on_error(&socket_id, 1, (unsigned char*)buffer, NULL, - "Failed to read error code from federate %d in response to sending fed_id.", - remote_federate_id); - lf_print_error("Received MSG_TYPE_REJECT message from remote federate (%d).", buffer[0]); - result = -1; - continue; - } else { - lf_print("Connected to federate %d, port %d.", remote_federate_id, port); - // Trace the event when tracing is enabled - tracepoint_federate_to_federate(receive_ACK, _lf_my_fed_id, remote_federate_id, NULL); - } - } + memcpy(buf + 2 + sizeof(uint16_t), (unsigned char*)federation_metadata.federation_id, federation_id_length); + // Trace the event when tracing is enabled + tracepoint_federate_to_federate(send_FED_ID, _lf_my_fed_id, remote_federate_id, NULL); + + // No need for a mutex because we have the only handle on the socket. + write_to_netdrv_fail_on_error(netdrv, 2 + sizeof(uint16_t) + federation_id_length, buf, NULL, + "Failed to send fed_id and federation id to federate %d.", remote_federate_id); + + read_from_netdrv_fail_on_error(netdrv, (unsigned char*)buf, 2, NULL, + "Failed to read MSG_TYPE_ACK from federate %d in response to sending fed_id.", + remote_federate_id); + if (buf[0] != MSG_TYPE_ACK) { + // Get the error code. + lf_print_error_and_exit("Received MSG_TYPE_REJECT message from remote federate (%d).", buf[1]); + result = -1; + } else { + lf_print("Connected to federate %d.", remote_federate_id); + // Trace the event when tracing is enabled + tracepoint_federate_to_federate(receive_ACK, _lf_my_fed_id, remote_federate_id, NULL); } + free(buf); // Once we set this variable, then all future calls to close() on this // socket ID should reset it to -1 within a critical section. - _fed.sockets_for_outbound_p2p_connections[remote_federate_id] = socket_id; + _fed.netdrv_for_outbound_p2p_connections[remote_federate_id] = netdrv; } void lf_connect_to_rti(const char* hostname, int port) { LF_PRINT_LOG("Connecting to the RTI."); + // TODO: DONGHA: Need to change these? // Override passed hostname and port if passed as runtime arguments. hostname = federation_metadata.rti_host ? federation_metadata.rti_host : hostname; port = federation_metadata.rti_port >= 0 ? federation_metadata.rti_port : port; @@ -1843,58 +1787,33 @@ void lf_connect_to_rti(const char* hostname, int port) { if (port < 0 || port > INT16_MAX) { lf_print_error("lf_connect_to_rti(): Specified port (%d) is out of range," " using the default port %d instead.", - port, DEFAULT_PORT); - uport = DEFAULT_PORT; + port, RTI_DEFAULT_PORT); + uport = RTI_DEFAULT_PORT; port = 0; // Mark so that increments occur between tries. } else { uport = (uint16_t)port; } if (uport == 0) { - uport = DEFAULT_PORT; + uport = RTI_DEFAULT_PORT; } - // Create a socket - _fed.socket_TCP_RTI = create_real_time_tcp_socket_errexit(); - - int result = -1; - struct addrinfo* res = NULL; - - instant_t start_connect = lf_time_physical(); - while (!CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT) && !_lf_termination_executed) { - if (res != NULL) { - // This is a repeated attempt. - if (_fed.socket_TCP_RTI >= 0) - close_rti_socket(); - - lf_sleep(CONNECT_RETRY_INTERVAL); - - // Create a new socket. - _fed.socket_TCP_RTI = create_real_time_tcp_socket_errexit(); - - if (port == 0) { - // Free previously allocated address info. - freeaddrinfo(res); - // Increment the port number. - uport++; - if (uport >= DEFAULT_PORT + MAX_NUM_PORT_ADDRESSES) - uport = DEFAULT_PORT; - - // Reconstruct the address info. - rti_address(hostname, uport, &res); - } - lf_print("Trying RTI again on port %d.", uport); - } else { - // This is the first attempt. - rti_address(hostname, uport, &res); - } + // Initialize netdriver to rti. + _fed.netdrv_to_rti = initialize_netdrv(_lf_my_fed_id, federation_metadata.federation_id); // set memory. + set_host_name(_fed.netdrv_to_rti, hostname); + set_port(_fed.netdrv_to_rti, uport); + set_specified_port(_fed.netdrv_to_rti, port); + set_target_id(_fed.netdrv_to_rti, -1); + create_connector(_fed.netdrv_to_rti); - result = connect(_fed.socket_TCP_RTI, res->ai_addr, res->ai_addrlen); - if (result < 0) - continue; // Connect failed. + if (connect_to_netdrv(_fed.netdrv_to_rti) < 0) { + lf_print_error_and_exit("Failed to connect() to RTI after %d tries.", CONNECT_MAX_RETRIES); + } - // Have connected to an RTI, but not sure it's the right RTI. - // Send a MSG_TYPE_FED_IDS message and wait for a reply. - // Notify the RTI of the ID of this federate and its federation. + int count_retries = 0; + while (count_retries++ < CONNECT_MAX_RETRIES && !_lf_termination_executed) { + // Have connected to an RTI, but not sure it's the right RTI. + // Send a MSG_TYPE_FED_IDS message and wait for a reply. + // Notify the RTI of the ID of this federate and its federation. #ifdef FEDERATED_AUTHENTICATED LF_PRINT_LOG("Connected to an RTI. Performing HMAC-based authentication using federation ID."); @@ -1903,16 +1822,16 @@ void lf_connect_to_rti(const char* hostname, int port) { continue; // Try again with a new port. } else { // No point in trying again because it will be the same port. - close_rti_socket(); + close_netdrv(_fed.netdrv_to_rti); lf_print_error_and_exit("Authentication failed."); } } #else LF_PRINT_LOG("Connected to an RTI. Sending federation ID for authentication."); #endif - - // Send the message type first. - unsigned char buffer[4]; + // Send message_type + federate_ID + federation_id_length + federation_id + size_t federation_id_length = strnlen(federation_metadata.federation_id, 255); + unsigned char* buffer = (unsigned char*)malloc(1 + sizeof(uint16_t) + 1 + federation_id_length); buffer[0] = MSG_TYPE_FED_IDS; // Next send the federate ID. if (_lf_my_fed_id > UINT16_MAX) { @@ -1920,146 +1839,98 @@ void lf_connect_to_rti(const char* hostname, int port) { } encode_uint16((uint16_t)_lf_my_fed_id, &buffer[1]); // Next send the federation ID length. - // The federation ID is limited to 255 bytes. - size_t federation_id_length = strnlen(federation_metadata.federation_id, 255); + // The federation ID is limited to 255 bytes. The overhead of memcpy() is acceptable. buffer[1 + sizeof(uint16_t)] = (unsigned char)(federation_id_length & 0xff); - + memcpy(buffer + 2 + sizeof(uint16_t), (unsigned char*)federation_metadata.federation_id, federation_id_length); // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_FED_ID, _lf_my_fed_id, NULL); - // No need for a mutex here because no other threads are writing to this socket. - if (write_to_socket(_fed.socket_TCP_RTI, 2 + sizeof(uint16_t), buffer)) { + // No need for a mutex here because no other threads are writing to this netdriver. + if (write_to_netdrv(_fed.netdrv_to_rti, 2 + sizeof(uint16_t) + federation_id_length, buffer) <= 0) { continue; // Try again, possibly on a new port. } - - // Next send the federation ID itself. - if (write_to_socket(_fed.socket_TCP_RTI, federation_id_length, (unsigned char*)federation_metadata.federation_id)) { - continue; // Try again. - } + free(buffer); // Wait for a response. // The response will be MSG_TYPE_REJECT if the federation ID doesn't match. // Otherwise, it will be either MSG_TYPE_ACK or MSG_TYPE_UDP_PORT, where the latter // is used if clock synchronization will be performed. - unsigned char response; + unsigned char response[2]; LF_PRINT_DEBUG("Waiting for response to federation ID from the RTI."); - if (read_from_socket(_fed.socket_TCP_RTI, 1, &response)) { + if (read_from_netdrv(_fed.netdrv_to_rti, response, 2) <= 0) { continue; // Try again. } - if (response == MSG_TYPE_REJECT) { + if (response[0] == MSG_TYPE_REJECT) { // Trace the event when tracing is enabled tracepoint_federate_from_rti(receive_REJECT, _lf_my_fed_id, NULL); // Read one more byte to determine the cause of rejection. - unsigned char cause; - read_from_socket_fail_on_error(&_fed.socket_TCP_RTI, 1, &cause, NULL, - "Failed to read the cause of rejection by the RTI."); + unsigned char cause = response[1]; if (cause == FEDERATION_ID_DOES_NOT_MATCH || cause == WRONG_SERVER) { lf_print_warning("Connected to the wrong RTI on port %d. Will try again", uport); continue; } - } else if (response == MSG_TYPE_ACK) { + } else if (response[0] == MSG_TYPE_ACK) { // Trace the event when tracing is enabled tracepoint_federate_from_rti(receive_ACK, _lf_my_fed_id, NULL); LF_PRINT_LOG("Received acknowledgment from the RTI."); break; - } else if (response == MSG_TYPE_RESIGN) { + } else if (response[0] == MSG_TYPE_RESIGN) { lf_print_warning("RTI on port %d resigned. Will try again", uport); continue; } else { - lf_print_warning("RTI on port %d gave unexpect response %u. Will try again", uport, response); + lf_print_warning("RTI on port %d gave unexpect response %u. Will try again", uport, response[0]); continue; } } - if (result < 0) { - lf_print_error_and_exit("Failed to connect to RTI with timeout: " PRINTF_TIME, CONNECT_TIMEOUT); - } - - freeaddrinfo(res); /* No longer needed */ // Call a generated (external) function that sends information // about connections between this federate and other federates // where messages are routed through the RTI. // @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h - lf_send_neighbor_structure_to_RTI(_fed.socket_TCP_RTI); + lf_send_neighbor_structure_to_RTI(_fed.netdrv_to_rti); - uint16_t udp_port = setup_clock_synchronization_with_rti(); + // The IP address of the federate is known to the RTI when using TCP connections, while + // establish_communication_session(). However, when using MQTT, the federate has to send it's IP address to the RTI to + // do UDP clock synchronization. Thus, it sends the IPv4 address after the UDP port number. + struct sockaddr_in federate_UDP_addr; + uint16_t udp_port = setup_clock_synchronization_with_rti(&federate_UDP_addr); - // Write the returned port number to the RTI - unsigned char UDP_port_number[1 + sizeof(uint16_t)]; + size_t buffer_size = 1 + sizeof(uint16_t) + INET_ADDRSTRLEN; + unsigned char UDP_port_number[buffer_size]; UDP_port_number[0] = MSG_TYPE_UDP_PORT; encode_uint16(udp_port, &(UDP_port_number[1])); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, 1 + sizeof(uint16_t), UDP_port_number, NULL, +#if defined(COMM_TYPE_TCP) || defined(COMM_TYPE_SST) + // Write the returned port number to the RTI + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, 1 + sizeof(uint16_t), UDP_port_number, NULL, "Failed to send the UDP port number to the RTI."); - +#elif defined(COMM_TYPE_MQTT) + // Write the returned port number and IPv4 address to the RTI. + // If the clock-sync is init or off, it will fill it to 0.0.0.0. + inet_ntop(AF_INET, &federate_UDP_addr, (char*)&(UDP_port_number[1 + sizeof(uint16_t)]), INET_ADDRSTRLEN); + + // Write the buffer to the network driver + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, buffer_size, UDP_port_number, NULL, + "Failed to send the UDP port number and IPv4 address to the RTI."); +#endif lf_print("Connected to RTI at %s:%d.", hostname, uport); } +// specified_port is 0 on default. void lf_create_server(int specified_port) { - assert(specified_port <= UINT16_MAX && specified_port >= 0); - uint16_t port = (uint16_t)specified_port; - LF_PRINT_LOG("Creating a socket server on port %d.", port); - // Create an IPv4 socket for TCP (not UDP) communication over IP (0). - int socket_descriptor = create_real_time_tcp_socket_errexit(); - - // Server file descriptor. - struct sockaddr_in server_fd; - // Zero out the server address structure. - bzero((char*)&server_fd, sizeof(server_fd)); - - server_fd.sin_family = AF_INET; // IPv4 - server_fd.sin_addr.s_addr = INADDR_ANY; // All interfaces, 0.0.0.0. - // Convert the port number from host byte order to network byte order. - server_fd.sin_port = htons(port); - - int result = bind(socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); - int count = 0; - while (result < 0 && count++ < PORT_BIND_RETRY_LIMIT) { - lf_sleep(PORT_BIND_RETRY_INTERVAL); - result = bind(socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); - } - if (result < 0) { - lf_print_error_and_exit("Failed to bind socket on port %d.", port); - } - - // Set the global server port. - if (specified_port == 0) { - // Need to retrieve the port number assigned by the OS. - struct sockaddr_in assigned; - socklen_t addr_len = sizeof(assigned); - if (getsockname(socket_descriptor, (struct sockaddr*)&assigned, &addr_len) < 0) { - lf_print_error_and_exit("Failed to retrieve assigned port number."); - } - _fed.server_port = ntohs(assigned.sin_port); - } else { - _fed.server_port = port; - } - - // Enable listening for socket connections. - // The second argument is the maximum number of queued socket requests, - // which according to the Mac man page is limited to 128. - listen(socket_descriptor, 128); - - LF_PRINT_LOG("Server for communicating with other federates started using port %d.", _fed.server_port); + netdrv_t* my_netdrv = initialize_netdrv(_lf_my_fed_id, federation_metadata.federation_id); + create_listener(my_netdrv, FED, specified_port); // 1 for FED - // Send the server port number to the RTI - // on an MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h). - unsigned char buffer[sizeof(int32_t) + 1]; - buffer[0] = MSG_TYPE_ADDRESS_ADVERTISEMENT; - encode_int32(_fed.server_port, &(buffer[1])); + // Set the global server netdriver. + _fed.my_netdrv = my_netdrv; - // Trace the event when tracing is enabled + // Trace the event when tracing is enabled. This will not be sent in MQTT. tracepoint_federate_to_rti(send_ADR_AD, _lf_my_fed_id, NULL); - // No need for a mutex because we have the only handle on this socket. - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, sizeof(int32_t) + 1, (unsigned char*)buffer, NULL, - "Failed to send address advertisement."); - - LF_PRINT_DEBUG("Sent port %d to the RTI.", _fed.server_port); - - // Set the global server socket - _fed.server_socket = socket_descriptor; + // Send the federate's information to the RTI, for inbound connection. For TCP, it will send the port, and for MQTT it + // will do nothing. + send_address_advertisement_to_RTI(my_netdrv, _fed.netdrv_to_rti); } void lf_enqueue_port_absent_reactions(environment_t* env) { @@ -2090,14 +1961,12 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { LF_ASSERT_NON_NULL(env_arg); size_t received_federates = 0; // Allocate memory to store thread IDs. - _fed.inbound_socket_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t)); + _fed.inbound_netdriv_listeners = (lf_thread_t*)calloc(_fed.number_of_inbound_p2p_connections, sizeof(lf_thread_t)); while (received_federates < _fed.number_of_inbound_p2p_connections && !_lf_termination_executed) { // Wait for an incoming connection request. - struct sockaddr client_fd; - uint32_t client_length = sizeof(client_fd); - int socket_id = accept(_fed.server_socket, &client_fd, &client_length); + netdrv_t* client_fed_netdrv = establish_communication_session(_fed.my_netdrv); - if (socket_id < 0) { + if (client_fed_netdrv == NULL) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { if (rti_failed()) break; @@ -2106,17 +1975,18 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { } else if (errno == EPERM) { lf_print_error_system_failure("Firewall permissions prohibit connection."); } else { - lf_print_error_system_failure("A fatal error occurred while accepting a new socket."); + lf_print_error_system_failure("A fatal error occurred while accepting a new netdriver."); } } LF_PRINT_LOG("Accepted new connection from remote federate."); size_t header_length = 1 + sizeof(uint16_t) + 1; - unsigned char buffer[header_length]; - int read_failed = read_from_socket(socket_id, header_length, (unsigned char*)&buffer); - if (read_failed || buffer[0] != MSG_TYPE_P2P_SENDING_FED_ID) { - lf_print_warning("Federate received invalid first message on P2P socket. Closing socket."); - if (read_failed == 0) { + // unsigned char buffer[header_length]; + unsigned char buffer[256]; // TODO: NEED TO CHECK. Doesn't allow super long federation IDs. + ssize_t bytes_read = read_from_netdrv(client_fed_netdrv, (unsigned char*)&buffer, header_length); + if (bytes_read <= 0 || buffer[0] != MSG_TYPE_P2P_SENDING_FED_ID) { + lf_print_warning("Federate received invalid first message on P2P netdriver. Closing netdriver."); + if (bytes_read > 0) { // Wrong message received. unsigned char response[2]; response[0] = MSG_TYPE_REJECT; @@ -2124,29 +1994,29 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL); // Ignore errors on this response. - write_to_socket(socket_id, 2, response); + write_to_netdrv(client_fed_netdrv, 2, response); } - close(socket_id); + close_netdrv(client_fed_netdrv); continue; } // Get the federation ID and check it. - unsigned char federation_id_length = buffer[header_length - 1]; - char remote_federation_id[federation_id_length]; - read_failed = read_from_socket(socket_id, federation_id_length, (unsigned char*)remote_federation_id); - if (read_failed || (strncmp(federation_metadata.federation_id, remote_federation_id, - strnlen(federation_metadata.federation_id, 255)) != 0)) { - lf_print_warning("Received invalid federation ID. Closing socket."); - if (read_failed == 0) { + // unsigned char federation_id_length = buffer[header_length - 1]; + // char remote_federation_id[federation_id_length]; + // bytes_read = read_from_netdrv(client_fed_netdrv, (unsigned char*)remote_federation_id, federation_id_length); + if (bytes_read <= 0 || (strncmp(federation_metadata.federation_id, (const char*)buffer + header_length, + strnlen(federation_metadata.federation_id, 255)) != 0)) { + lf_print_warning("Received invalid federation ID. Closing netdriver."); + if (bytes_read > 0) { unsigned char response[2]; response[0] = MSG_TYPE_REJECT; response[1] = FEDERATION_ID_DOES_NOT_MATCH; // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_REJECT, _lf_my_fed_id, -3, NULL); // Ignore errors on this response. - write_to_socket(socket_id, 2, response); + write_to_netdrv(client_fed_netdrv, 2, response); } - close(socket_id); + close_netdrv(client_fed_netdrv); continue; } @@ -2158,11 +2028,11 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { tracepoint_federate_to_federate(receive_FED_ID, _lf_my_fed_id, remote_fed_id, NULL); // Once we record the socket_id here, all future calls to close() on - // the socket should be done while holding the socket_mutex, and this array + // the netdriver should be done while holding the netdrv_mutex, and this array // element should be reset to -1 during that critical section. // Otherwise, there can be race condition where, during termination, - // two threads attempt to simultaneously close the socket. - _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = socket_id; + // two threads attempt to simultaneously close the netdriver. + _fed.netdrv_for_inbound_p2p_connections[remote_fed_id] = client_fed_netdrv; // Send an MSG_TYPE_ACK message. unsigned char response = MSG_TYPE_ACK; @@ -2170,28 +2040,27 @@ void* lf_handle_p2p_connections_from_federates(void* env_arg) { // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_ACK, _lf_my_fed_id, remote_fed_id, NULL); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - write_to_socket_fail_on_error(&_fed.sockets_for_inbound_p2p_connections[remote_fed_id], 1, - (unsigned char*)&response, &lf_outbound_socket_mutex, - "Failed to write MSG_TYPE_ACK in response to federate %d.", remote_fed_id); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + write_to_netdrv_fail_on_error(_fed.netdrv_for_inbound_p2p_connections[remote_fed_id], 1, (unsigned char*)&response, + &lf_outbound_netdrv_mutex, "Failed to write MSG_TYPE_ACK in response to federate %d.", + remote_fed_id); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); // Start a thread to listen for incoming messages from other federates. // The fed_id is a uint16_t, which we assume can be safely cast to and from void*. void* fed_id_arg = (void*)(uintptr_t)remote_fed_id; - int result = lf_thread_create(&_fed.inbound_socket_listeners[received_federates], listen_to_federates, fed_id_arg); + int result = lf_thread_create(&_fed.inbound_netdriv_listeners[received_federates], listen_to_federates, fed_id_arg); if (result != 0) { // Failed to create a listening thread. - LF_MUTEX_LOCK(&socket_mutex); - if (_fed.sockets_for_inbound_p2p_connections[remote_fed_id] != -1) { - close(socket_id); - _fed.sockets_for_inbound_p2p_connections[remote_fed_id] = -1; + LF_MUTEX_LOCK(&netdrv_mutex); + if (_fed.netdrv_for_inbound_p2p_connections[remote_fed_id] != NULL) { + close_netdrv(_fed.netdrv_for_inbound_p2p_connections[remote_fed_id]); + _fed.netdrv_for_inbound_p2p_connections[remote_fed_id] = NULL; } - LF_MUTEX_UNLOCK(&socket_mutex); + LF_MUTEX_UNLOCK(&netdrv_mutex); lf_print_error_and_exit("Failed to create a thread to listen for incoming physical connection. Error code: %d.", result); } - received_federates++; } @@ -2285,24 +2154,26 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa const int header_length = 1 + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint32_t); // Use a mutex lock to prevent multiple threads from simultaneously sending. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); - int* socket = &_fed.sockets_for_outbound_p2p_connections[federate]; + netdrv_t* netdrv = _fed.netdrv_for_outbound_p2p_connections[federate]; // Trace the event when tracing is enabled tracepoint_federate_to_federate(send_P2P_MSG, _lf_my_fed_id, federate, NULL); - int result = write_to_socket_close_on_error(socket, header_length, header_buffer); - if (result == 0) { + // TODO: DONGHA: Need to fix in future. + int bytes_written = write_to_netdrv_close_on_error(netdrv, header_length, header_buffer); + if (bytes_written > 0) { // Header sent successfully. Send the body. - result = write_to_socket_close_on_error(socket, length, message); + bytes_written = write_to_netdrv_close_on_error(netdrv, length, message); } - if (result != 0) { + if (bytes_written <= 0) { // Message did not send. Since this is used for physical connections, this is not critical. lf_print_warning("Failed to send message to %s. Dropping the message.", next_destination_str); } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); - return result; + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); + // TODO: DONGHA: Check result! + return bytes_written; } tag_t lf_send_next_event_tag(environment_t* env, tag_t tag, bool wait_for_reply) { @@ -2464,25 +2335,25 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d #ifdef FEDERATED_CENTRALIZED // Send the absent message through the RTI - int* socket = &_fed.socket_TCP_RTI; + netdrv_t* netdrv = _fed.netdrv_to_rti; #else // Send the absent message directly to the federate - int* socket = &_fed.sockets_for_outbound_p2p_connections[fed_ID]; + netdrv_t* netdrv = _fed.netdrv_for_outbound_p2p_connections[fed_ID]; #endif - if (socket == &_fed.socket_TCP_RTI) { + if (netdrv == _fed.netdrv_to_rti) { tracepoint_federate_to_rti(send_PORT_ABS, _lf_my_fed_id, ¤t_message_intended_tag); } else { tracepoint_federate_to_federate(send_PORT_ABS, _lf_my_fed_id, fed_ID, ¤t_message_intended_tag); } - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); - int result = write_to_socket_close_on_error(socket, message_length, buffer); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); + int bytes_written = write_to_netdrv_close_on_error(netdrv, message_length, buffer); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); - if (result != 0) { + if (bytes_written <= 0) { // Write failed. Response depends on whether coordination is centralized. - if (socket == &_fed.socket_TCP_RTI) { + if (netdrv == _fed.netdrv_to_rti) { // Centralized coordination. This is a critical error. lf_print_error_system_failure("Failed to send port absent message for port %hu to federate %hu.", port_ID, fed_ID); @@ -2501,29 +2372,29 @@ int lf_send_stop_request_to_rti(tag_t stop_tag) { stop_tag.microstep++; ENCODE_STOP_REQUEST(buffer, stop_tag.time, stop_tag.microstep); - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); // Do not send a stop request if a stop request has been previously received from the RTI. if (!_fed.received_stop_request_from_rti) { LF_PRINT_LOG("Sending to RTI a MSG_TYPE_STOP_REQUEST message with tag " PRINTF_TAG ".", stop_tag.time - start_time, stop_tag.microstep); - if (_fed.socket_TCP_RTI < 0) { - lf_print_warning("Socket is no longer connected. Dropping message."); - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + if (_fed.netdrv_to_rti == NULL) { + lf_print_warning("Netdriver is no longer connected. Dropping message."); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); return -1; } // Trace the event when tracing is enabled tracepoint_federate_to_rti(send_STOP_REQ, _lf_my_fed_id, &stop_tag); - write_to_socket_fail_on_error(&_fed.socket_TCP_RTI, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_socket_mutex, + write_to_netdrv_fail_on_error(_fed.netdrv_to_rti, MSG_TYPE_STOP_REQUEST_LENGTH, buffer, &lf_outbound_netdrv_mutex, "Failed to send stop time " PRINTF_TIME " to the RTI.", stop_tag.time - start_time); // Treat this sending as equivalent to having received a stop request from the RTI. _fed.received_stop_request_from_rti = true; - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); return 0; } else { - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); return 1; } } @@ -2576,23 +2447,29 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int current_message_intended_tag.microstep, next_destination_str); // Use a mutex lock to prevent multiple threads from simultaneously sending. - LF_MUTEX_LOCK(&lf_outbound_socket_mutex); + LF_MUTEX_LOCK(&lf_outbound_netdrv_mutex); - int* socket; + netdrv_t* netdrv; if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) { - socket = &_fed.sockets_for_outbound_p2p_connections[federate]; + netdrv = _fed.netdrv_for_outbound_p2p_connections[federate]; tracepoint_federate_to_federate(send_P2P_TAGGED_MSG, _lf_my_fed_id, federate, ¤t_message_intended_tag); } else { - socket = &_fed.socket_TCP_RTI; + netdrv = _fed.netdrv_to_rti; tracepoint_federate_to_rti(send_TAGGED_MSG, _lf_my_fed_id, ¤t_message_intended_tag); } - int result = write_to_socket_close_on_error(socket, header_length, header_buffer); - if (result == 0) { - // Header sent successfully. Send the body. - result = write_to_socket_close_on_error(socket, length, message); - } - if (result != 0) { + // TODO: THIS IS ONLY TEMPORARY... NEED TO FIX!! + size_t sender_length = length + header_length; + unsigned char* sender = malloc(sender_length); + memcpy(sender, header_buffer, header_length); + memcpy(sender + header_length, message, length); + // int bytes_written = write_to_netdrv_close_on_error(netdrv, header_length, header_buffer); + // if (bytes_written > 0) { + // // Header sent successfully. Send the body. + // bytes_written = write_to_netdrv_close_on_error(netdrv, length, message); + // } + int bytes_written = write_to_netdrv_close_on_error(netdrv, sender_length, sender); + if (bytes_written <= 0) { // Message did not send. Handling depends on message type. if (message_type == MSG_TYPE_P2P_TAGGED_MESSAGE) { lf_print_warning("Failed to send message to %s. Dropping the message.", next_destination_str); @@ -2601,8 +2478,9 @@ int lf_send_tagged_message(environment_t* env, interval_t additional_delay, int next_destination_str, errno, strerror(errno)); } } - LF_MUTEX_UNLOCK(&lf_outbound_socket_mutex); - return result; + LF_MUTEX_UNLOCK(&lf_outbound_netdrv_mutex); + // TODO: Check result + return bytes_written; } void lf_set_federation_id(const char* fid) { federation_metadata.federation_id = fid; } @@ -2639,7 +2517,7 @@ void lf_synchronize_with_other_federates(void) { // @note Up until this point, the federate has been listening for messages // from the RTI in a sequential manner in the main thread. From now on, a // separate thread is created to allow for asynchronous communication. - lf_thread_create(&_fed.RTI_socket_listener, listen_to_rti_TCP, NULL); + lf_thread_create(&_fed.RTI_netdrv_listener, listen_to_rti, NULL); lf_thread_t thread_id; if (create_clock_sync_thread(&thread_id)) { lf_print_warning("Failed to create thread to handle clock synchronization."); diff --git a/core/federated/network/CMakeLists.txt b/core/federated/network/CMakeLists.txt index 5306eae02..c858ccdae 100644 --- a/core/federated/network/CMakeLists.txt +++ b/core/federated/network/CMakeLists.txt @@ -1,4 +1,38 @@ -set(LF_NETWORK_FILES net_util.c) +add_library(lf-network-impl STATIC) +add_library(lf::network-impl ALIAS lf-network-impl) +target_link_libraries(lf-network-impl PUBLIC lf-logging-api) +if(COMM_TYPE MATCHES MQTT) + # $ git clone https://github.com/eclipse/paho.mqtt.c.git + # $ cd paho.mqtt.c.git + # $ mkdir build.paho; cd build.paho + # $ cmake ../ + # $ make + # $ sudo make install + # It will be installed in /usr/local/lib + find_package(eclipse-paho-mqtt-c REQUIRED) + target_link_libraries(lf-network-impl PRIVATE lf::network-api eclipse-paho-mqtt-c::paho-mqtt3a eclipse-paho-mqtt-c::paho-mqtt3c) + # $ apt-get install libssl-dev + # find_package(OpenSSL REQUIRED) + # target_link_libraries(lf-network-impl PUBLIC OpenSSL::SSL) +else() + target_link_libraries(lf-network-impl PRIVATE lf::network-api) +endif() -list(TRANSFORM LF_NETWORK_FILES PREPEND federated/network/) -list(APPEND REACTORC_SOURCES ${LF_NETWORK_FILES}) +include_directories(/usr/local/include) + +target_sources(lf-network-impl PUBLIC + ${CMAKE_CURRENT_LIST_DIR}/net_util.c + ${CMAKE_CURRENT_LIST_DIR}/netdriver.c + ${CMAKE_CURRENT_LIST_DIR}/socket_common.c +) +IF(COMM_TYPE MATCHES TCP) + target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/lf_socket_support.c) +ELSEIF(COMM_TYPE MATCHES MQTT) + target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/lf_mqtt_support.c) +ELSEIF(COMM_TYPE MATCHES SST) + target_link_libraries(lf-network-impl PUBLIC SSTLIB) + target_sources(lf-network-impl PUBLIC ${CMAKE_CURRENT_LIST_DIR}/lf_sst_support.c) + # add_compile_definitions(OPENSSL_REQUIRED) +ENDIF() + +target_compile_definitions(lf-network-impl PUBLIC COMM_TYPE_${COMM_TYPE}) diff --git a/core/federated/network/lf_mqtt_support.c b/core/federated/network/lf_mqtt_support.c new file mode 100644 index 000000000..217961f64 --- /dev/null +++ b/core/federated/network/lf_mqtt_support.c @@ -0,0 +1,697 @@ +#include +#include + +#include "util.h" +#include "net_common.h" +#include "net_util.h" +#include "netdriver.h" +#include "lf_mqtt_support.h" +#include "MQTTClientPersistence.h" // For logging +// #include "reactor_common.h" + +// #include + +#define ADDRESS "tcp://127.0.0.1:1883" +#define QOS 2 +#define TIMEOUT 10000L + +#define MAX_RETRIES 5 // Number of retry attempts +#define RETRY_DELAY 2 // Delay between attempts in seconds + +extern bool _lf_termination_executed; + +static MQTT_priv_t* MQTT_priv_init(); +static char* create_topic_federation_id_listener_id(const char* federation_id, int listener_id); +static char* create_topic_federation_id_A_to_B(const char* federation_id, int A, int B, int flag); +static void set_MQTTClient_id(MQTT_priv_t* MQTT_priv, int my_id, int target_id, int flag); +int MQTT_connect_with_retry(MQTTClient client, MQTTClient_connectOptions* conn_opts); +// int MQTT_subscribe_with_retry(MQTTClient client, const char* topic, int qos); +int MQTT_subscribe_with_retry(MQTTClient client, const char* topic, int qos, int flag); + +void log_callback(enum MQTTCLIENT_TRACE_LEVELS level, char* message) { + (void)level; // Explicitly mark the parameter as unused + lf_print("MQTT_LOG: %s\n", message); +} + +/** + * @brief + * Initializes structure of netdrv, and priv inside. + * Allocate memory. + * Check lf_socket_support.c for example. + * + * @return netdrv_t* + */ +netdrv_t* initialize_netdrv(int my_federate_id, const char* federation_id) { + netdrv_t* drv = initialize_common_netdrv(my_federate_id, federation_id); + + // Initialize priv. + MQTT_priv_t* priv = MQTT_priv_init(); + + // Set drv->priv pointer to point the malloc'd priv. + drv->priv = (void*)priv; + return drv; +} + +void close_netdrv(netdrv_t* drv) { + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + // If target_id is available (not -2, which means the listener netdriver), it sends a MQTT_RESIGNED messaged. + if (MQTT_priv->target_id != -2) { + unsigned char buffer[1]; + buffer[0] = MQTT_RESIGNED; + write_to_netdrv_fail_on_error(drv, 1, buffer, NULL, "Failed to send MQTT_RESIGNED message."); + LF_PRINT_DEBUG("Sending MQTT_RESIGNED message."); + } + int rc; + if ((rc = MQTTClient_disconnect(MQTT_priv->client, 10000)) != MQTTCLIENT_SUCCESS) { + lf_print("Failed to disconnect, return code %d.", rc); + } + MQTTClient_destroy(&MQTT_priv->client); + if (MQTT_priv->topic_name_to_send) { + free(MQTT_priv->topic_name_to_send); + } + free(MQTT_priv); + drv->priv = NULL; + free(drv); + drv = NULL; +} + +/** + * @brief Create a server object + * Initializes MQTT client, and connects to broker. + * RTI subscribes “{Federation_ID}_RTI” topic. This should be done here, because the establish_communication_session + * loops. Check socket_common.c for example. It is a common function because also lf_sst_support.c will also use it. + * @param drv + * @param server_type + * @param port The port is NULL here. + * @return int + */ +int create_listener(netdrv_t* drv, server_type_t server_type, uint16_t port) { + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + if (server_type == RTI) { + } // JUST TO PASS COMPILER. + if (port == 0) { + } // JUST TO PASS COMPILER. + + // Target is not available for listeners. We set it to -2 if it is uninitialized or unavailable. This is used when + // close_netdrv() is called. + MQTT_priv->target_id = -2; + // If RTI calls this, it will be -1. If federate server calls, it will be it's federate ID. + set_MQTTClient_id(MQTT_priv, drv->my_federate_id, drv->my_federate_id, -1); + + int rc; + + if ((rc = MQTTClient_create(&MQTT_priv->client, ADDRESS, MQTT_priv->client_id, MQTTCLIENT_PERSISTENCE_NONE, NULL)) != + MQTTCLIENT_SUCCESS) { + lf_print_error_and_exit("Failed to create client during create_listener(), return code %d.", rc); + } + LF_PRINT_DEBUG("Connecting MQTTClient %s to broker.", MQTT_priv->client_id); + LF_MUTEX_LOCK(&netdrv_mutex); + if ((rc = MQTT_connect_with_retry(MQTT_priv->client, &MQTT_priv->conn_opts)) != MQTTCLIENT_SUCCESS) { + MQTTClient_destroy(&MQTT_priv->client); + lf_print_error_and_exit( + "Failed to connect during create_listener(), return code %d. Check if MQTT broker is available.", rc); + } + LF_PRINT_DEBUG("Connected MQTTClient %s to broker, return code %d.", MQTT_priv->client_id, rc); + MQTT_priv->topic_name_to_send = create_topic_federation_id_listener_id(drv->federation_id, drv->my_federate_id); + LF_PRINT_DEBUG("create_listener(): MQTTClient %s Subscribing on topic %s.", MQTT_priv->client_id, + MQTT_priv->topic_name_to_send); + if ((rc = MQTT_subscribe_with_retry(MQTT_priv->client, MQTT_priv->topic_name_to_send, QOS, 1)) != + MQTTCLIENT_SUCCESS) { + LF_PRINT_DEBUG("create_listener(): Disconnecting MQTTClient %s.", MQTT_priv->client_id); + MQTTClient_disconnect(MQTT_priv->client, TIMEOUT); + MQTTClient_destroy(&MQTT_priv->client); + lf_print_error_and_exit("Failed to subscribe during create_listener(), return code %d.", rc); + } + LF_PRINT_DEBUG("Subscribed on topic %s, return code %d.", MQTT_priv->topic_name_to_send, rc); + LF_MUTEX_UNLOCK(&netdrv_mutex); + return 1; +} + +/** + * @brief + * This function returns a connector netdriver using the listener netdriver. + * 1. Each federate publishes fed_id to {Federation_ID}_{listenerID} + * 2. fed_{n} subscribes to “{Federation_Id}_{listenerID}_to_fed_{n}”. + * 3. Listener subscribes to “{Federation_Id}_fed_{n}_to_{listenerID}”. + * Check lf_socket_support.c for example. + + * @param netdrv + * @return netdrv_t* + */ +netdrv_t* establish_communication_session(netdrv_t* listener_netdrv) { + // Create connector netdriver which will communicate with the connector. + netdrv_t* connector_nedrv = initialize_netdrv(-2, listener_netdrv->federation_id); + MQTT_priv_t* connector_priv = (MQTT_priv_t*)connector_nedrv->priv; + connector_nedrv->my_federate_id = listener_netdrv->my_federate_id; + + // // Set the trace level to maximum verbosity + // MQTTClient_setTraceLevel(MQTTCLIENT_TRACE_MAXIMUM); + // // Set the trace callback function + // MQTTClient_setTraceCallback(log_callback); + + int rc; + unsigned char buffer[1 + sizeof(uint16_t)]; + char* topic_to_subscribe = NULL; + // Step1: The listener first waits for a MSG_TYPE_MQTT_JOIN message, through the topic federationID_listenerID. + + read_from_netdrv_fail_on_error(listener_netdrv, buffer, 1 + sizeof(uint16_t), NULL, "MQTT receive failed."); + if (buffer[0] != MSG_TYPE_MQTT_JOIN) { + lf_print_error_and_exit("Wrong message type... Expected MSG_TYPE_MQTT_JOIN."); + } + uint16_t target_fed_id = extract_uint16(buffer + 1); + LF_PRINT_LOG("Received MSG_TYPE_MQTT_JOIN message from federate %d.", target_fed_id); + + // The connector netdriver connects to the broker. + connector_priv->target_id = (int)target_fed_id; + LF_PRINT_DEBUG("Setting up MQTTClient_id to target federate %d.", connector_priv->target_id); + // If RTI calls this, it will be RTI_targetfedID. If federate calls this, it will be myfedID_targetfedID + set_MQTTClient_id(connector_priv, connector_nedrv->my_federate_id, connector_priv->target_id, -1); + LF_PRINT_DEBUG("Setup MQTTClient_id to target federate %d as %s.", connector_priv->target_id, + connector_priv->client_id); + + LF_PRINT_DEBUG("Creating topic to target federate %d.", connector_priv->target_id); + // Subscribe to topic: federationID_fedID_to_listenerID + // When centralized, this will be federationID_fedID_to_RTI + // When decentralized, this will be federationID_CONN_{targetfedID}_to_LIST_{myfedID} + // This is the channel where the federate sends messages to the listener. + topic_to_subscribe = create_topic_federation_id_A_to_B(connector_nedrv->federation_id, connector_priv->target_id, + connector_nedrv->my_federate_id, 1); + + LF_PRINT_DEBUG("Creating MQTTClient to target federate %d.", connector_priv->target_id); + if ((rc = MQTTClient_create(&connector_priv->client, ADDRESS, connector_priv->client_id, MQTTCLIENT_PERSISTENCE_NONE, + NULL)) != MQTTCLIENT_SUCCESS) { + lf_print_error_and_exit("Failed to create client during establish_communication_session(), return code %d.", rc); + } + + LF_MUTEX_LOCK(&netdrv_mutex); + + instant_t start_connect = lf_time_physical(); + while (1) { + if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { + lf_print_error_and_exit("Failed to connect and subscribe to topic %s with timeout: " PRINTF_TIME ". Giving up.", + topic_to_subscribe, CONNECT_TIMEOUT); + break; + } + LF_PRINT_DEBUG("Connecting MQTTClient %s to broker.", connector_priv->client_id); + if ((rc = MQTTClient_connect(connector_priv->client, &connector_priv->conn_opts)) != MQTTCLIENT_SUCCESS) { + // MQTTClient_destroy(&connector_priv->client); + lf_print_warning("Failed to connect during establish_communication_session(), return code %d.", rc); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } + LF_PRINT_DEBUG("Connected MQTTClient %s to broker, return code %d.", connector_priv->client_id, rc); + LF_PRINT_DEBUG("establish_communication_session(): MQTTClient %s Subscribing on topic %s.", + connector_priv->client_id, topic_to_subscribe); + + // The MQTTClient_subscribe() internally disconnects the MQTTClient to the broker. So, it should retry after + // re-connecting the MQTTClient first. + if ((rc = MQTTClient_subscribe(connector_priv->client, (const char*)topic_to_subscribe, QOS)) != + MQTTCLIENT_SUCCESS) { + // LF_PRINT_DEBUG("establish_communication_session(): Disconnecting MQTTClient %s.", connector_priv->client_id); + // MQTTClient_disconnect(connector_priv->client, TIMEOUT); + // MQTTClient_destroy(&connector_priv->client); + lf_print_warning("Failed to subscribe during establish_communication_session(), return code %d.", rc); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } + LF_PRINT_DEBUG("Subscribed on topic %s, return code %d.", topic_to_subscribe, rc); + break; + } + + LF_MUTEX_UNLOCK(&netdrv_mutex); + // Step2: The listener sends a MSG_TYPE_MQTT_ACCEPT message to the federate. + // Publish to topic: federationID_listenerID_to_fedID + // When centralized, this will be federationID_RTI_to_fedID + // When decentralized, this will be federateionID_LIST_{myfedID}_to_CONN_{targetfedID} + connector_priv->topic_name_to_send = create_topic_federation_id_A_to_B( + connector_nedrv->federation_id, connector_nedrv->my_federate_id, connector_priv->target_id, -1); + buffer[0] = MSG_TYPE_MQTT_ACCEPT; + encode_uint16((uint16_t)connector_priv->target_id, buffer + 1); + LF_PRINT_LOG("Publishing MSG_TYPE_MQTT_ACCEPT message on topic %s.", connector_priv->topic_name_to_send); + write_to_netdrv_fail_on_error(connector_nedrv, 1 + sizeof(uint16_t), buffer, NULL, + "Failed to send MSG_TYPE_MQTT_ACCEPT to federate %d", connector_priv->target_id); + + // Step3: The listner receives the MSG_TYPE_MQTT_ACCEPT_ACK message from the federate. + read_from_netdrv_fail_on_error(connector_nedrv, buffer, 1, NULL, "MQTT receive failed."); + if (buffer[0] != MSG_TYPE_MQTT_ACCEPT_ACK) { + lf_print_error_and_exit("Wrong message type... Expected MSG_TYPE_MQTT_ACCEPT_ACK."); + } + LF_PRINT_LOG("Receiving MSG_TYPE_MQTT_ACCEPT_ACK message on topic %s.", topic_to_subscribe); + free(topic_to_subscribe); + return connector_nedrv; +} + +/** + * @brief Federate connects to broker. + * + * @param drv + */ +void create_connector(netdrv_t* drv) { + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + // Only federates call this. It will be myfedID_RTI or myfedID_targetfedID. + set_MQTTClient_id(MQTT_priv, drv->my_federate_id, MQTT_priv->target_id, 1); + int rc; + if ((rc = MQTTClient_create(&MQTT_priv->client, ADDRESS, MQTT_priv->client_id, MQTTCLIENT_PERSISTENCE_NONE, NULL)) != + MQTTCLIENT_SUCCESS) { + lf_print_error_and_exit("Failed to create client, return code %d.", rc); + } + // LF_MUTEX_LOCK(&netdrv_mutex); + // LF_PRINT_DEBUG("Connecting MQTTClient %s to broker.", MQTT_priv->client_id); + // if ((rc = MQTT_connect_with_retry(MQTT_priv->client, &MQTT_priv->conn_opts)) != MQTTCLIENT_SUCCESS) { + // MQTTClient_destroy(&MQTT_priv->client); + // lf_print_error_and_exit( + // "Failed to connect during create_connector(), return code %d. Check if MQTT broker is available.", rc); + // } + // LF_PRINT_DEBUG("Connected MQTTClient %s to broker, return code %d.", MQTT_priv->client_id, rc); + // LF_MUTEX_UNLOCK(&netdrv_mutex); +} +/** + * @brief Federate publishes it's federate ID to "{Federation_ID}_RTI", then subscribes to + * "{Federation_ID}_RTI_to_fed{fed_id}" + * + * @param drv + * @return int + */ +int connect_to_netdrv(netdrv_t* drv) { + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + int rc; + + // Subscribe to topic: federationID_listenerID_to_fedID + // This should be done before sending the MSG_TYPE_MQTT_JOIN message, because the listener publishing the + // MSG_TYPE_MQTT_ACCEPT message at topic federationID_listenerID_to_fedID can be faster than the connector subscribing + // to the topic. + // When centralized, this will be federationID_RTI_to_fedID + // When decentralized, this will be federateionID_LIST_{targetfedID}_to_CONN_{myfedID} + char* topic_to_subscribe = + create_topic_federation_id_A_to_B(drv->federation_id, MQTT_priv->target_id, drv->my_federate_id, -1); + + LF_MUTEX_LOCK(&netdrv_mutex); + instant_t start_connect = lf_time_physical(); + while (1) { + if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { + lf_print_error_and_exit("Failed to connect and subscribe to topic %s with timeout: " PRINTF_TIME ". Giving up.", + topic_to_subscribe, CONNECT_TIMEOUT); + break; + } + LF_PRINT_DEBUG("Connecting MQTTClient %s to broker.", MQTT_priv->client_id); + if ((rc = MQTTClient_connect(MQTT_priv->client, &MQTT_priv->conn_opts)) != MQTTCLIENT_SUCCESS) { + // MQTTClient_destroy(&MQTT_priv->client); + lf_print_warning( + "Failed to connect during create_connector(), return code %d. Check if MQTT broker is available.", rc); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } + LF_PRINT_DEBUG("Connected MQTTClient %s to broker, return code %d.", MQTT_priv->client_id, rc); + LF_PRINT_DEBUG("connect_to_netdrv(): MQTTClient %s Subscribing on topic %s.", MQTT_priv->client_id, + topic_to_subscribe); + if ((rc = MQTTClient_subscribe(MQTT_priv->client, (const char*)topic_to_subscribe, QOS)) != MQTTCLIENT_SUCCESS) { + // LF_PRINT_DEBUG("connect_to_netdrv(): Disconnecting MQTTClient %s.", MQTT_priv->client_id); + // MQTTClient_disconnect(MQTT_priv->client, TIMEOUT); + // MQTTClient_destroy(&MQTT_priv->client); + // free(topic_to_subscribe); + lf_print_warning("Failed to subscribe during connect_to_netdrv(), return code %d.", rc); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } + LF_PRINT_DEBUG("Subscribed on topic %s, return code %d.", topic_to_subscribe, rc); + break; + } + LF_MUTEX_UNLOCK(&netdrv_mutex); + // Step1: The federate sends a MSG_TYPE_MQTT_JOIN message including it's federateID to the listener. + // Publish to topic: federationID_listenerID + MQTT_priv->topic_name_to_send = create_topic_federation_id_listener_id(drv->federation_id, MQTT_priv->target_id); + unsigned char buffer[1 + sizeof(uint16_t)]; + buffer[0] = MSG_TYPE_MQTT_JOIN; + encode_uint16((uint16_t)drv->my_federate_id, buffer + 1); + + // The connect_to_netdrv() can be called in the federates before the establish_communication_session() is called in + // the RTI. The MQTT QOS2 ensures the message to arrive to the subscribed client, but this can be called even before + // the netdriver was initialized in the RTI side. Thus, the federate must retry sending messages to the RTI until it + // replies the MSG_TYPE_MQTT_ACCEPT message. The connect retry interval (500 msecs) should be shorter than the read + // timeout time (which is 10 secs), thus it does not use read_from_netdrv(). + start_connect = lf_time_physical(); + while (1) { + if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { + lf_print_error("Failed to handshake with target with timeout: " PRINTF_TIME ". Giving up.", CONNECT_TIMEOUT); + return -1; + } + LF_PRINT_LOG("Publishing MSG_TYPE_MQTT_JOIN message with federateID %d on topic %s.", drv->my_federate_id, + MQTT_priv->topic_name_to_send); + write_to_netdrv_fail_on_error(drv, 1 + sizeof(uint16_t), buffer, NULL, + "Failed to write my_federate_id to listener for connection through MQTT."); + + // Step2: Receive MSG_TYPE_MQTT_ACCEPT from listener, which sends the connector's federateID. + // Receive from topic: federationID_listenerID_to_fedID + char* topicName = NULL; + int topicLen; + int temp_rc; + MQTTClient_message* message = NULL; + temp_rc = MQTTClient_receive(MQTT_priv->client, &topicName, &topicLen, &message, 500); + if (temp_rc != MQTTCLIENT_SUCCESS) { + lf_print_error("Failed to receive MSG_TYPE_MQTT_ACCEPT message, return code %d.", temp_rc); + if (topicName) { + MQTTClient_free(topicName); + } + if (message) { + MQTTClient_freeMessage(&message); + } + lf_sleep(MSEC(2000)); + continue; + } else if (message == NULL) { + // This means the call succeeded but no message was received within the timeout + lf_print_log("No message received within the timeout period."); + if (topicName) { + MQTTClient_free(topicName); + } + lf_sleep(MSEC(2000)); + continue; + } else { + // Successfully received a message + lf_print_log("Successfully received MSG_TYPE_MQTT_ACCEPT message, return code %d.", temp_rc); + memcpy(buffer, (unsigned char*)message->payload, message->payloadlen); + if (topicName) { + MQTTClient_free(topicName); + } + if (message) { + MQTTClient_freeMessage(&message); + } + break; + } + } + if (buffer[0] != MSG_TYPE_MQTT_ACCEPT) { + lf_print_error_and_exit("Wrong message type... Expected MSG_TYPE_MQTT_ACCEPT."); + } + LF_PRINT_LOG("Receiving MSG_TYPE_MQTT_ACCEPT message on topic %s.", topic_to_subscribe); + free((char*)MQTT_priv->topic_name_to_send); + free(topic_to_subscribe); + + // Compare the received federateID with my federateID. + uint16_t temp_fed_id = extract_uint16(buffer + 1); + if (drv->my_federate_id != temp_fed_id) { + lf_print_error_and_exit("Wrong federate ID. Received %d", temp_fed_id); + } + + // Step3: Send MSG_TYPE_MQTT_ACCEPT_ACK message to the listener. + // Publish to topic: federationID_fedID_to_listenorID + // When centralized, this will be federationID_fedID_to_RTI + // When decentralized, this will be federationID_CONN_{myfedID}_to_LIST_{targetfedID} + MQTT_priv->topic_name_to_send = + create_topic_federation_id_A_to_B(drv->federation_id, drv->my_federate_id, MQTT_priv->target_id, 1); + buffer[0] = MSG_TYPE_MQTT_ACCEPT_ACK; + write_to_netdrv_fail_on_error(drv, 1, buffer, NULL, + "Failed to write MSG_TYPE_MQTT_ACCEPT_ACK_to RTI for connection through MQTT."); + LF_PRINT_LOG("Publishing MSG_TYPE_MQTT_ACCEPT_ACK_to message on topic %s.", MQTT_priv->topic_name_to_send); + return 0; +} + +/** + * @brief Publish message. + * + * @param drv + * @param num_bytes + * @param buffer + * @return int + */ +int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) { + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + MQTTClient_message pubmsg = MQTTClient_message_initializer; + MQTTClient_deliveryToken token; + int rc; + pubmsg.payload = (void*)buffer; + pubmsg.payloadlen = num_bytes; + pubmsg.qos = QOS; + pubmsg.retained = 0; + + if ((rc = MQTTClient_publishMessage(MQTT_priv->client, MQTT_priv->topic_name_to_send, &pubmsg, &token)) != + MQTTCLIENT_SUCCESS) { + lf_print_error("Failed to publish message, return code %d.", rc); + return rc; + } + // LF_PRINT_DEBUG("Message publishing on topic %s is %.*s", MQTT_priv->topic_name_to_send, pubmsg.payloadlen, + // (char*)(pubmsg.payload)); + if ((rc = MQTTClient_waitForCompletion(MQTT_priv->client, token, TIMEOUT)) != MQTTCLIENT_SUCCESS) { + lf_print_error("Failed to complete publish message, return code %d.", rc); + return rc; + } + int bytes_written = pubmsg.payloadlen; + return bytes_written; +} + +ssize_t read_from_netdrv(netdrv_t* drv, unsigned char* buffer, size_t buffer_length) { + if (drv == NULL) { + lf_print_warning("Netdriver is closed, returning -1."); + return -1; + } + if (buffer_length == 0) { + } // JUST TO PASS COMPILER. + + char* topicName = NULL; + int topicLen = 0; + MQTTClient_message* message = NULL; + int rc; + int bytes_read = 0; + // LF_PRINT_LOG("RECEIVING message from federateID %d", MQTT_priv->target_id); + instant_t start_receive = lf_time_physical(); + // If the netdrv was closed from the outside during termination, segmentation faults happen. When federate executes + // and calls lf_terminate_execution(), closing inbound and outbound netdrivers are first before waiting for the + // threads to join. In this case, the federates may not receive the MQTT_resign message and just get closed. The + // close_outbound_netdrv() and close_inbound_netdrv() functions hold a mutex lock, so this just needs to check the + // _lf_termination_executed status befor restarting the loop. + while (!_lf_termination_executed) { + if (CHECK_TIMEOUT(start_receive, CONNECT_TIMEOUT)) { + lf_print_error("Failed to receive with timeout: " PRINTF_TIME ". Giving up.", CONNECT_TIMEOUT); + // bytes_read = -1; + break; + } + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + rc = MQTTClient_receive(MQTT_priv->client, &topicName, &topicLen, &message, 5000); + if (rc != MQTTCLIENT_SUCCESS) { + lf_print_warning("Failed to receive message, return code %d.", rc); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } else if (message == NULL) { + // This means the call succeeded but no message was received within the timeout + lf_print_log("No message received within the MQTTClient_receive() timeout period."); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } else { + // Successfully received a message + lf_print_log("Successfully received message, return code %d.", rc); + // TODO: NEED to add compare with buffer_length. Also actions. + memcpy(buffer, (unsigned char*)message->payload, message->payloadlen); + bytes_read = message->payloadlen; + // LF_PRINT_LOG("RECEIVED message from federateID %d", MQTT_priv->target_id); + if (buffer[0] == MQTT_RESIGNED) { + LF_PRINT_LOG("Received MQTT_RESIGNED message from federateID %d", MQTT_priv->target_id); + bytes_read = 0; + } + break; + } + } + if (topicName) { + MQTTClient_free(topicName); + } + if (message) { + MQTTClient_freeMessage(&message); + } + return bytes_read; +} + +char* get_host_name(netdrv_t* drv) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + return NULL; +} +int32_t get_my_port(netdrv_t* drv) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + return 0; +} +int32_t get_port(netdrv_t* drv) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + return 0; +} +struct in_addr* get_ip_addr(netdrv_t* drv) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + return NULL; +} +void set_host_name(netdrv_t* drv, const char* hostname) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (hostname == NULL) { + } // JUST TO PASS COMPILER. +} +void set_port(netdrv_t* drv, int port) { + if (drv == NULL) { + } // JUST TO PASS COMPILER + if (port == 0) { + } // JUST TO PASS COMPILER. +} +void set_specified_port(netdrv_t* drv, int port) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (port == 0) { + } // JUST TO PASS COMPILER. +} +void set_ip_addr(netdrv_t* drv, struct in_addr ip_addr) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (ip_addr.s_addr == 0) { + } // JUST TO PASS COMPILER. +} +ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (result == NULL) { + } // JUST TO PASS COMPILER. + return 0; +} + +void set_target_id(netdrv_t* drv, int federate_id) { + MQTT_priv_t* MQTT_priv = (MQTT_priv_t*)drv->priv; + MQTT_priv->target_id = federate_id; +} + +// ------------------Helper Functions------------------ // + +static MQTT_priv_t* MQTT_priv_init() { + MQTT_priv_t* MQTT_priv = malloc(sizeof(MQTT_priv_t)); + if (!MQTT_priv) { + lf_print_error_and_exit("Falied to malloc MQTT_priv_t."); + } + memset(MQTT_priv, 0, sizeof(MQTT_priv_t)); + MQTTClient_connectOptions conn_opts = MQTTClient_connectOptions_initializer; + memcpy(&MQTT_priv->conn_opts, &conn_opts, sizeof(MQTTClient_connectOptions)); + MQTT_priv->conn_opts.keepAliveInterval = MQTTkeepAliveInterval; + MQTT_priv->conn_opts.cleansession = MQTTcleansession; + MQTT_priv->conn_opts.connectTimeout = MQTTconnectTimeout; + return MQTT_priv; +} + +static char* create_topic_federation_id_listener_id(const char* federation_id, int listener_id) { + int max_length; + // Determine the maximum length of the resulting string + if (listener_id == -2) { + lf_print_error_and_exit("The ID used for the MQTT topic is not initalized."); + return NULL; + } + if (listener_id == -1) { + max_length = snprintf(NULL, 0, "%s_RTI", federation_id) + 1; // +1 for null terminator + } else { + max_length = snprintf(NULL, 0, "%s_fed_%d", federation_id, listener_id) + 1; // +1 for null terminator + } + // Allocate memory for the resulting string + char* result = (char*)malloc(max_length); + if (result == NULL) { + lf_print_error_and_exit("Falied to malloc."); + return NULL; + } + + // Format the string using snprintf + if (listener_id == -1) { + snprintf(result, max_length, "%s_RTI", federation_id); + } else { + snprintf(result, max_length, "%s_fed_%d", federation_id, listener_id); + } + return result; +} + +static char* create_topic_federation_id_A_to_B(const char* federation_id, int A, int B, int flag) { + int max_length = 0; + char* result; + if (A == -2 || B == -2) { + lf_print_error_and_exit("The ID used for the MQTT topic is not initalized."); + return NULL; + } + // Determine the maximum length of the resulting string + // If either the ID includes -1, then the flag does not matter. + if (A == -1) { + max_length = snprintf(NULL, 0, "%s_RTI_to_fed_%d", federation_id, B) + 1; // +1 for null terminator + } else if (B == -1) { + max_length = snprintf(NULL, 0, "%s_fed_%d_to_RTI", federation_id, A) + 1; // +1 for null terminator + } + + else if (flag == 1) { + max_length = snprintf(NULL, 0, "%s_CONN_fed_%d_to_LIST_fed_%d", federation_id, A, B) + 1; // +1 for null terminator + } else if (flag == -1) { + max_length = snprintf(NULL, 0, "%s_LIST_fed_%d_to_CONN_fed_%d", federation_id, A, B) + 1; // +1 for null terminator + } + + // Allocate memory for the resulting string + result = (char*)malloc(max_length); + if (result == NULL) { + lf_print_error_and_exit("Failed to malloc."); + return NULL; + } + + // Format the string using snprintf + if (A == -1) { + snprintf(result, max_length, "%s_RTI_to_fed_%d", federation_id, B); + } else if (B == -1) { + snprintf(result, max_length, "%s_fed_%d_to_RTI", federation_id, A); + } else if (flag == 1) { + snprintf(result, max_length, "%s_CONN_fed_%d_to_LIST_fed_%d", federation_id, A, B); + } else if (flag == -1) { + snprintf(result, max_length, "%s_LIST_fed_%d_to_CONN_fed_%d", federation_id, A, B); + } + return result; +} + +static void set_MQTTClient_id(MQTT_priv_t* MQTT_priv, int my_id, int target_id, int flag) { + // When the flag is 0, it is for centralized communication. + + if (my_id == -1 && target_id == -1) { + strcat(MQTT_priv->client_id, "RTI_RTI"); + } else if (my_id == -1) { + sprintf(MQTT_priv->client_id, "RTI_fed_%d", target_id); + } else if (target_id == -1) { + sprintf(MQTT_priv->client_id, "fed_%d_RTI", my_id); + } + + // When the flag is 1, it means + else if (flag == 1) { + sprintf(MQTT_priv->client_id, "CONN_fed_%d_LIST_fed_%d", my_id, target_id); + } else if (flag == -1) { + sprintf(MQTT_priv->client_id, "LIST_fed_%d_CONN_fed_%d", my_id, target_id); + } +} + +int MQTT_connect_with_retry(MQTTClient client, MQTTClient_connectOptions* conn_opts) { + int rc = -1; + instant_t start_connect = lf_time_physical(); + while (1) { + if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { + lf_print_warning("Failed to connect with timeout: " PRINTF_TIME ". Giving up.", CONNECT_TIMEOUT); + break; + } + if ((rc = MQTTClient_connect(client, conn_opts)) != MQTTCLIENT_SUCCESS) { + lf_print_warning("Failed to connect, return code %d. Retrying to connect.", rc); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } + break; + } + return rc; +} + +// int MQTT_subscribe_with_retry(MQTTClient client, const char* topic, int qos) { +int MQTT_subscribe_with_retry(MQTTClient client, const char* topic, int qos, int flag) { + int rc = -1; + instant_t start_connect = lf_time_physical(); + while (1) { + if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { + lf_print_warning("Failed to subscribe with timeout: " PRINTF_TIME ". Giving up.", CONNECT_TIMEOUT); + break; + } + if ((rc = MQTTClient_subscribe(client, topic, qos)) != MQTTCLIENT_SUCCESS) { + // lf_print_warning("Failed to subscribe, return code %d. Retrying to subscribe.", rc); + lf_print_warning("Failed to subscribe, return code %d. Retrying to subscribe. Flag: %d", rc, flag); + lf_sleep(CONNECT_RETRY_INTERVAL); + continue; + } + break; + } + return rc; +} diff --git a/core/federated/network/lf_socket_support.c b/core/federated/network/lf_socket_support.c new file mode 100644 index 000000000..424d08c37 --- /dev/null +++ b/core/federated/network/lf_socket_support.c @@ -0,0 +1,423 @@ +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "net_common.h" +#include "net_util.h" +#include "netdriver.h" +#include "lf_socket_support.h" + +static void handle_header_read(unsigned char* buffer, size_t* bytes_to_read, int* state); + +netdrv_t* initialize_netdrv(int my_federate_id, const char* federation_id) { + netdrv_t* drv = initialize_common_netdrv(my_federate_id, federation_id); + + // Initialize priv. + socket_priv_t* priv = TCP_socket_priv_init(); + + // Set drv->priv pointer to point the malloc'd priv. + drv->priv = (void*)priv; + return drv; +} + +void close_netdrv(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + TCP_socket_close(priv); + free(priv); + free(drv); +} + +// This only creates TCP servers not UDP. +int create_listener(netdrv_t* drv, server_type_t server_type, uint16_t port) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + return create_TCP_server(priv, (int)server_type, port); +} + +/** + * 1. initializes other side's netdrv. + * 2. Establishes communication session. + +**/ +netdrv_t* establish_communication_session(netdrv_t* listener_netdrv) { + // -2 is for uninitialized value. + netdrv_t* connector_nedrv = initialize_netdrv(-2, listener_netdrv->federation_id); + socket_priv_t* listener_priv = (socket_priv_t*)listener_netdrv->priv; + socket_priv_t* connector_priv = (socket_priv_t*)connector_nedrv->priv; + // Wait for an incoming connection request. + struct sockaddr client_fd; + uint32_t client_length = sizeof(client_fd); + // The following blocks until a client connects. + while (1) { + connector_priv->socket_descriptor = accept(listener_priv->socket_descriptor, &client_fd, &client_length); + if (connector_priv->socket_descriptor >= 0) { + // Got a socket + break; + } else if (connector_priv->socket_descriptor < 0 && (errno != EAGAIN || errno != EWOULDBLOCK)) { + lf_print_error_and_exit("Failed to accept the socket. %s. connector_priv->socket_descriptor = %d", + strerror(errno), connector_priv->socket_descriptor); + } else { + // Try again + lf_print_warning("Failed to accept the socket. %s. Trying again.", strerror(errno)); + continue; + } + } + + // TODO: DONGHA + // Get the IP address of the other accepting client. This is used in two cases. + // 1) Decentralized coordination - handle_address_query() - Sends the port number and address of the federate. + // 2) Clock synchronization - send_physical_clock - Send through UDP. + struct sockaddr_in* pV4_addr = (struct sockaddr_in*)&client_fd; + connector_priv->server_ip_addr = pV4_addr->sin_addr; + return connector_nedrv; +} + +void create_connector(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + TCP_socket_open(priv); +} + +int connect_to_netdrv(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + int ret = + connect_to_socket(priv->socket_descriptor, priv->server_hostname, priv->server_port, priv->user_specified_port); + return ret; +} + +/** + * Write the specified number of bytes to the specified socket from the + * specified buffer. If an error occurs, return -1 and set errno to indicate + * the cause of the error. If the write succeeds, return 0. + * This function repeats the attempt until the specified number of bytes + * have been written or an error occurs. Specifically, errors EAGAIN, + * EWOULDBLOCK, and EINTR are not considered errors and instead trigger + * another attempt. A delay between attempts is given by + * DELAY_BETWEEN_SOCKET_RETRIES. + * @param socket The socket ID. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @return The number of bytes written. + */ + +int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + if (priv->socket_descriptor < 0) { + // Socket is not open. + errno = EBADF; + return -1; + } + ssize_t bytes_written = 0; + while (bytes_written < (ssize_t)num_bytes) { + ssize_t more = write(priv->socket_descriptor, buffer + bytes_written, num_bytes - (size_t)bytes_written); + if (more <= 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { + // The error codes EAGAIN or EWOULDBLOCK indicate + // that we should try again (@see man errno). + // The error code EINTR means the system call was interrupted before completing. + LF_PRINT_DEBUG("Writing to socket was blocked. Will try again."); + lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES); + continue; + } else if (more < 0) { + // A more serious error occurred. + return -1; + } + bytes_written += more; + } + return bytes_written; +} + +/** + * Read the specified number of bytes from the specified socket into the specified buffer. + * If an error occurs during this reading, return -1 and set errno to indicate + * the cause of the error. If the read succeeds in reading the specified number of bytes, + * return 0. If an EOF occurs before reading the specified number of bytes, return 1. + * This function repeats the read attempt until the specified number of bytes + * have been read, an EOF is read, or an error occurs. Specifically, errors EAGAIN, + * EWOULDBLOCK, and EINTR are not considered errors and instead trigger + * another attempt. A delay between attempts is given by DELAY_BETWEEN_SOCKET_RETRIES. + * @param socket The socket ID. + * @param num_bytes The number of bytes to read. + * @param buffer The buffer into which to put the bytes. + * @return 0 for success, 1 for EOF, and -1 for an rerror. + */ +ssize_t read_from_netdrv(netdrv_t* drv, unsigned char* buffer, size_t buffer_length) { + if (drv == NULL) { + lf_print_warning("Netdriver is closed, returning -1."); + return -1; + } + socket_priv_t* priv = (socket_priv_t*)drv->priv; + + size_t bytes_to_read; // The bytes to read in future. + ssize_t bytes_read = 0; // The bytes that was read by a single read() function. + size_t total_bytes_read = 0; // The total bytes that have been read, and will be the return of the read_from drv. + int retry_count; + int state; + // Check if socket_descriptor is open. + if (priv->socket_descriptor < 0) { + // Socket is not open. + errno = EBADF; + return -1; + } + // First, check if there are remaining bytes. + // If there are remaining bytes, it reads as long as it can (buffer_length). + // Then it becomes KEEP_READING state. + if (drv->read_remaining_bytes > 0) { + bytes_to_read = (drv->read_remaining_bytes > buffer_length) ? buffer_length : drv->read_remaining_bytes; + state = KEEP_READING; + } else { + // If there are no left bytes to read, it reads the header byte. + bytes_to_read = 1; // read header + state = HEADER_READ; + } + + for (;;) { + retry_count = 0; + while (bytes_to_read > 0 && bytes_to_read <= buffer_length) { + bytes_read = read(priv->socket_descriptor, buffer + total_bytes_read, bytes_to_read); + if (bytes_read < 0 && // If) Error has occurred, + retry_count++ < NUM_SOCKET_RETRIES && // there are left retry counts, + (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { // and the error code was these three, + // Print warning, sleep for a short time, and retry. + lf_print_warning("Reading from socket failed. Will try again."); + lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES); + continue; + } else if (bytes_read < 0) { + return -1; + } else if (bytes_read == 0) { + // An error occurred without those three error codes. + // https://stackoverflow.com/questions/42188128/does-reading-from-a-socket-wait-or-get-eof + // bytes_read == 0 means disconnected. + return 0; + } + bytes_to_read -= bytes_read; + total_bytes_read += bytes_read; + } + + switch (state) { + case HEADER_READ: + handle_header_read(buffer, &bytes_to_read, &state); + break; + + case READ_MSG_TYPE_FED_IDS:; + size_t federation_id_length = (size_t)buffer[1 + sizeof(uint16_t)]; + bytes_to_read = federation_id_length; + state = FINISH_READ; + break; + case READ_MSG_TYPE_NEIGHBOR_STRUCTURE:; + int num_upstream = extract_int32(buffer + 1); + int num_downstream = extract_int32(buffer + 1 + sizeof(int32_t)); + bytes_to_read = ((sizeof(uint16_t) + sizeof(int64_t)) * num_upstream) + (sizeof(uint16_t) * num_downstream); + state = FINISH_READ; + break; + case READ_MSG_TYPE_TAGGED_MESSAGE:; + size_t length = (size_t)extract_uint32(buffer + 1 + sizeof(uint16_t) + sizeof(uint16_t)); + if (length > buffer_length - total_bytes_read) { + bytes_to_read = buffer_length - total_bytes_read; + drv->read_remaining_bytes = length - bytes_to_read; + } else { + bytes_to_read = length; + } + state = FINISH_READ; + break; + case KEEP_READING: + drv->read_remaining_bytes -= total_bytes_read; + return total_bytes_read; + case FINISH_READ: + return total_bytes_read; + } + } +} + +char* get_host_name(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + return priv->server_hostname; +} +int32_t get_my_port(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + return priv->port; +} +int32_t get_port(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + return (priv == NULL) ? -1 : priv->server_port; +} +// +struct in_addr* get_ip_addr(netdrv_t* drv) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + return &priv->server_ip_addr; +} +void set_host_name(netdrv_t* drv, const char* hostname) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + memcpy(priv->server_hostname, hostname, INET_ADDRSTRLEN); +} +void set_port(netdrv_t* drv, int port) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + priv->server_port = port; +} +void set_specified_port(netdrv_t* drv, int port) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + priv->user_specified_port = port; +} + +// Unused. +void set_ip_addr(netdrv_t* drv, struct in_addr ip_addr) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + priv->server_ip_addr = ip_addr; +} + +void set_clock_netdrv(netdrv_t* clock_drv, netdrv_t* rti_drv, uint16_t port_num) { + socket_priv_t* priv_clock = (socket_priv_t*)(clock_drv); + socket_priv_t* priv_rti = (socket_priv_t*)(rti_drv); + priv_clock->UDP_addr.sin_family = AF_INET; + priv_clock->UDP_addr.sin_port = htons(port_num); + priv_clock->UDP_addr.sin_addr = priv_rti->server_ip_addr; +} + +ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result) { + socket_priv_t* priv = (socket_priv_t*)drv->priv; + ssize_t bytes_read = recv(priv->socket_descriptor, result, 1, MSG_DONTWAIT | MSG_PEEK); + if (bytes_read < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) + return 0; + else + return bytes_read; +} + +void set_target_id(netdrv_t* drv, int federate_id) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (federate_id == 0) { + } // JUST TO PASS COMPILER. +} + +// ------------------Helper Functions------------------ // + +static void handle_header_read(unsigned char* buffer, size_t* bytes_to_read, int* state) { + switch (buffer[0]) { + case MSG_TYPE_REJECT: // 1 +1 + *bytes_to_read = 1; + *state = FINISH_READ; + break; + case MSG_TYPE_ACK: // 1 + *bytes_to_read = 0; + *state = FINISH_READ; + break; + case MSG_TYPE_UDP_PORT: // 1 + sizeof(uint16_t) = 3 + *bytes_to_read = sizeof(uint16_t); + *state = FINISH_READ; + break; + case MSG_TYPE_FED_IDS: // 1 + sizeof(uint16_t) + 1 + federation_id + *bytes_to_read = sizeof(uint16_t) + 1; + *state = READ_MSG_TYPE_FED_IDS; + break; + case MSG_TYPE_FED_NONCE: // 1 + sizeof(uint16_t) + NONCE_LENGTH(8) + *bytes_to_read = sizeof(uint16_t) + NONCE_LENGTH; + *state = FINISH_READ; + break; + case MSG_TYPE_RTI_RESPONSE: // 1 + NONCE_LENGTH(8) + SHA256_HMAC_LENGTH(32) + *bytes_to_read = NONCE_LENGTH + SHA256_HMAC_LENGTH; + *state = FINISH_READ; + break; + case MSG_TYPE_FED_RESPONSE: // 1 + SHA256_HMAC_LENGTH(32) + *bytes_to_read = SHA256_HMAC_LENGTH; + *state = FINISH_READ; + break; + case MSG_TYPE_TIMESTAMP: // 1+sizeof(int64_t) + *bytes_to_read = sizeof(int64_t); + *state = FINISH_READ; + break; + case MSG_TYPE_RESIGN: + *bytes_to_read = 0; + *state = FINISH_READ; + break; + case MSG_TYPE_TAGGED_MESSAGE: + *bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(int64_t) + sizeof(uint32_t); + *state = READ_MSG_TYPE_TAGGED_MESSAGE; + break; + case MSG_TYPE_NEXT_EVENT_TAG: + *bytes_to_read = sizeof(int64_t) + sizeof(uint32_t); + *state = FINISH_READ; + break; + case MSG_TYPE_TAG_ADVANCE_GRANT: + *bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); + *state = FINISH_READ; + break; + case MSG_TYPE_PROVISIONAL_TAG_ADVANCE_GRANT: + *bytes_to_read = sizeof(instant_t) + sizeof(microstep_t); + *state = FINISH_READ; + break; + case MSG_TYPE_LATEST_TAG_COMPLETE: + *bytes_to_read = sizeof(int64_t) + sizeof(uint32_t); + *state = FINISH_READ; + break; + case MSG_TYPE_STOP_REQUEST: + *bytes_to_read = MSG_TYPE_STOP_REQUEST_LENGTH - 1; + *state = FINISH_READ; + break; + case MSG_TYPE_STOP_REQUEST_REPLY: + *bytes_to_read = MSG_TYPE_STOP_REQUEST_REPLY_LENGTH - 1; + *state = FINISH_READ; + break; + case MSG_TYPE_STOP_GRANTED: + *bytes_to_read = MSG_TYPE_STOP_GRANTED_LENGTH - 1; + *state = FINISH_READ; + break; + case MSG_TYPE_ADDRESS_QUERY: + *bytes_to_read = sizeof(uint16_t); + *state = FINISH_READ; + break; + case MSG_TYPE_ADDRESS_QUERY_REPLY: + *bytes_to_read = sizeof(int32_t) + sizeof(struct in_addr); + *state = FINISH_READ; + break; + case MSG_TYPE_ADDRESS_ADVERTISEMENT: + *bytes_to_read = sizeof(int32_t); + *state = FINISH_READ; + break; + case MSG_TYPE_P2P_SENDING_FED_ID: // 1 /////////TODO: CHECK!!!!!!! + *bytes_to_read = sizeof(uint16_t) + 1; + *state = READ_MSG_TYPE_FED_IDS; + break; + case MSG_TYPE_P2P_MESSAGE: + *bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t); + *state = READ_MSG_TYPE_TAGGED_MESSAGE; + break; + case MSG_TYPE_P2P_TAGGED_MESSAGE: + *bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int32_t) + sizeof(instant_t) + sizeof(microstep_t); + *state = READ_MSG_TYPE_TAGGED_MESSAGE; + break; + case MSG_TYPE_CLOCK_SYNC_T1: + *bytes_to_read = sizeof(instant_t); + *state = FINISH_READ; + break; + case MSG_TYPE_CLOCK_SYNC_T3: + *bytes_to_read = sizeof(int32_t); + *state = FINISH_READ; + break; + case MSG_TYPE_CLOCK_SYNC_T4: + *bytes_to_read = sizeof(instant_t); + *state = FINISH_READ; + break; + case MSG_TYPE_CLOCK_SYNC_CODED_PROBE: + *bytes_to_read = sizeof(int64_t); + *state = FINISH_READ; + break; + case MSG_TYPE_PORT_ABSENT: + *bytes_to_read = sizeof(uint16_t) + sizeof(uint16_t) + sizeof(int64_t) + sizeof(uint32_t); + *state = FINISH_READ; + break; + case MSG_TYPE_NEIGHBOR_STRUCTURE: + *bytes_to_read = MSG_TYPE_NEIGHBOR_STRUCTURE_HEADER_SIZE - 1; + *state = READ_MSG_TYPE_NEIGHBOR_STRUCTURE; + break; + case MSG_TYPE_FAILED: + *bytes_to_read = 0; + *state = FINISH_READ; + break; + default: + *bytes_to_read = 0; + // Error handling? + *state = FINISH_READ; + lf_print_error_system_failure("Undefined message header. Terminating system."); + } +} diff --git a/core/federated/network/lf_sst_support.c b/core/federated/network/lf_sst_support.c new file mode 100644 index 000000000..a998593be --- /dev/null +++ b/core/federated/network/lf_sst_support.c @@ -0,0 +1,258 @@ +#include +#include +#include +#include +#include + +#include "util.h" +#include "net_common.h" +#include "net_util.h" +#include "netdriver.h" +#include "lf_sst_support.h" + +const char* sst_config_path; +const char* RTI_config_path; + +static sst_priv_t* sst_priv_init(); +static void var_length_int_to_num(unsigned char* buf, unsigned int buf_length, unsigned int* num, + unsigned int* var_len_int_buf_size); + +netdrv_t* initialize_netdrv(int my_federate_id, const char* federation_id) { + netdrv_t* drv = initialize_common_netdrv(my_federate_id, federation_id); + + // Initialize priv. + sst_priv_t* sst_priv = sst_priv_init(); + + // Set drv->priv pointer to point the malloc'd priv. + drv->priv = (void*)sst_priv; + return drv; +} +void close_netdrv(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + if (sst_priv->socket_priv != NULL) { + TCP_socket_close(sst_priv->socket_priv); + } else { + lf_print_error("Trying to close TCP socket not existing."); + } +} + +// Port will be NULL on MQTT. +int create_listener(netdrv_t* drv, server_type_t server_type, uint16_t port) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + SST_ctx_t* ctx = init_SST(RTI_config_path); + sst_priv->sst_ctx = ctx; + return create_TCP_server(sst_priv->socket_priv, server_type, port); +} + +netdrv_t* establish_communication_session(netdrv_t* listener_netdrv) { + netdrv_t* connector_nedrv = initialize_netdrv(-2, listener_netdrv->federation_id); + sst_priv_t* listener_priv = (sst_priv_t*)listener_netdrv->priv; + sst_priv_t* connector_priv = (sst_priv_t*)connector_nedrv->priv; + + // Wait for an incoming connection request. + struct sockaddr client_fd; + uint32_t client_length = sizeof(client_fd); + // The following blocks until a client connects. + while (1) { + connector_priv->socket_priv->socket_descriptor = + accept(listener_priv->socket_priv->socket_descriptor, &client_fd, &client_length); + if (connector_priv->socket_priv->socket_descriptor >= 0) { + // Got a socket + break; + } else if (connector_priv->socket_priv->socket_descriptor < 0 && (errno != EAGAIN || errno != EWOULDBLOCK)) { + lf_print_error_and_exit("Failed to accept the socket. %s. connector_priv->socket_priv->socket_descriptor = %d", + strerror(errno), connector_priv->socket_priv->socket_descriptor); + } else { + // Try again + lf_print_warning("Failed to accept the socket. %s. Trying again.", strerror(errno)); + continue; + } + } + + session_key_list_t* s_key_list = init_empty_session_key_list(); + SST_session_ctx_t* session_ctx = + server_secure_comm_setup(listener_priv->sst_ctx, connector_priv->socket_priv->socket_descriptor, s_key_list); + free_session_key_list_t(s_key_list); + connector_priv->session_ctx = session_ctx; + + // TODO: DONGHA + // Get the IP address of the other accepting client. This is used in two cases. + // 1) Decentralized coordination - handle_address_query() - Sends the port number and address of the federate. + // 2) Clock synchronization - send_physical_clock - Send through UDP. + struct sockaddr_in* pV4_addr = (struct sockaddr_in*)&client_fd; + connector_priv->socket_priv->server_ip_addr = pV4_addr->sin_addr; + return connector_nedrv; +} + +void create_connector(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + SST_ctx_t* ctx = init_SST((const char*)sst_config_path); + + sst_priv->sst_ctx = ctx; +} + +int connect_to_netdrv(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + session_key_list_t* s_key_list = get_session_key(sst_priv->sst_ctx, NULL); + // Does not increases RTI port number. + int sock = create_real_time_tcp_socket_errexit(); + // TODO: To support Decentralized, this should chagne. + int ret = connect_to_socket(sock, sst_priv->sst_ctx->config->entity_server_ip_addr, + atoi(sst_priv->sst_ctx->config->entity_server_port_num), + 0); // Not supporting user_specified_port yet. + if (ret != 0) { + return ret; + } + SST_session_ctx_t* session_ctx = secure_connect_to_server_with_socket(&s_key_list->s_key[0], sock); + sst_priv->session_ctx = session_ctx; + return 1; +} + +int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + if (buffer[0] == MSG_TYPE_FAILED) { + // Just return. + return 0; + } + return send_secure_message((char*)buffer, num_bytes, sst_priv->session_ctx); +} + +ssize_t read_from_netdrv(netdrv_t* drv, unsigned char* buffer, size_t buffer_length) { + if (drv == NULL) { + lf_print_warning("Netdriver is closed, returning -1."); + return -1; + } + if (buffer_length == 0) { + } // JUST TO PASS COMPILER. + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + unsigned char sst_buffer[1024]; + ssize_t bytes_read = 0; + unsigned int temp_length = 10; + + if (sst_priv->session_ctx->sock < 0) { + return -1; + } + + // Read 10 bytes first. + bytes_read = read(sst_priv->session_ctx->sock, sst_buffer, temp_length); + if (bytes_read == 0) { + // Connection closed. + return 0; + } else if (bytes_read < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { + // Add retry. + } + unsigned int payload_length; // Length of the payload of SST. + unsigned int var_length_buf_size; + // This fills payload_length and var_length_buf_size. + var_length_int_to_num(sst_buffer + sizeof(unsigned char), bytes_read, &payload_length, &var_length_buf_size); + unsigned int bytes_to_read = payload_length - (temp_length - (sizeof(unsigned char) + var_length_buf_size)); + + unsigned int second_read = 0; + ssize_t more = 0; + while (second_read != bytes_to_read) { + more = read(sst_priv->session_ctx->sock, sst_buffer + temp_length, bytes_to_read); + second_read += more; + if (more == 0) { + return 0; + } else if (more < 0 && errno != EAGAIN && errno != EWOULDBLOCK) { + continue; + } + bytes_read += second_read; + } + + unsigned int decrypted_buffer_length; + unsigned char* decrypted_buffer = + return_decrypted_buf(sst_buffer, bytes_read, &decrypted_buffer_length, sst_priv->session_ctx); + // Returned SEQ_NUM_BUFFER(8) + decrypted_buffer; + // Doing this because it should be freed. + memcpy(buffer, decrypted_buffer + 8, decrypted_buffer_length - 8); + free(decrypted_buffer); + return decrypted_buffer_length; +} + +// void netdrv_free(netdrv_t* drv) { +// sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; +// // free(priv); // Already freed on socket close() +// free_SST_ctx_t(sst_priv->sst_ctx); +// free(drv); +// } + +char* get_host_name(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + return sst_priv->socket_priv->server_hostname; +} +int32_t get_my_port(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + return sst_priv->socket_priv->port; +} +int32_t get_port(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + return (sst_priv->socket_priv == NULL) ? -1 : sst_priv->socket_priv->server_port; +} +// +struct in_addr* get_ip_addr(netdrv_t* drv) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + return &sst_priv->socket_priv->server_ip_addr; +} +void set_host_name(netdrv_t* drv, const char* hostname) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + memcpy(sst_priv->socket_priv->server_hostname, hostname, INET_ADDRSTRLEN); +} +void set_port(netdrv_t* drv, int port) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + sst_priv->socket_priv->server_port = port; +} + +void set_specified_port(netdrv_t* drv, int port) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + sst_priv->socket_priv->user_specified_port = port; +} + +// Unused. +void set_ip_addr(netdrv_t* drv, struct in_addr ip_addr) { + sst_priv_t* sst_priv = (sst_priv_t*)drv->priv; + sst_priv->socket_priv->server_ip_addr = ip_addr; +} + +ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (result == NULL) { + } // JUST TO PASS COMPILER. + return 0; +} + +void set_target_id(netdrv_t* drv, int federate_id) { + if (drv == NULL) { + } // JUST TO PASS COMPILER. + if (federate_id == 0) { + } // JUST TO PASS COMPILER. +} + +// ------------------Helper Functions------------------ // + +void lf_set_sst_config_path(const char* config_path) { sst_config_path = config_path; } +void lf_set_rti_sst_config_path(const char* config_path) { RTI_config_path = config_path; } + +static sst_priv_t* sst_priv_init() { + sst_priv_t* sst_priv = malloc(sizeof(sst_priv_t)); + if (!sst_priv) { + lf_print_error_and_exit("Falied to malloc sst_priv_t."); + } + memset(sst_priv, 0, sizeof(sst_priv_t)); + sst_priv->socket_priv = TCP_socket_priv_init(); + return sst_priv; +} + +static void var_length_int_to_num(unsigned char* buf, unsigned int buf_length, unsigned int* num, + unsigned int* var_len_int_buf_size) { + *num = 0; + *var_len_int_buf_size = 0; + for (unsigned int i = 0; i < buf_length; i++) { + *num |= (buf[i] & 127) << (7 * i); + if ((buf[i] & 128) == 0) { + *var_len_int_buf_size = i + 1; + break; + } + } +} diff --git a/core/federated/network/net_util.c b/core/federated/network/net_util.c index 61d4804bd..83744dafd 100644 --- a/core/federated/network/net_util.c +++ b/core/federated/network/net_util.c @@ -43,183 +43,14 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include // TCP_NODELAY #include "net_util.h" -#include "util.h" // Define socket functions only for federated execution. #ifdef FEDERATED -#include // Defines read(), write(), and close() #ifndef NUMBER_OF_FEDERATES #define NUMBER_OF_FEDERATES 1 #endif -/** Number of nanoseconds to sleep before retrying a socket read. */ -#define SOCKET_READ_RETRY_INTERVAL 1000000 - -// Mutex lock held while performing socket close operations. -// A deadlock can occur if two threads simulataneously attempt to close the same socket. -lf_mutex_t socket_mutex; - -int create_real_time_tcp_socket_errexit() { - int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (sock < 0) { - lf_print_error_system_failure("Could not open TCP socket."); - } - // Disable Nagle's algorithm which bundles together small TCP messages to - // reduce network traffic. - // TODO: Re-consider if we should do this, and whether disabling delayed ACKs - // is enough. - int flag = 1; - int result = setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(int)); - - if (result < 0) { - lf_print_error_system_failure("Failed to disable Nagle algorithm on socket server."); - } - -#if defined(PLATFORM_Linux) - // Disable delayed ACKs. Only possible on Linux - result = setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(int)); - - if (result < 0) { - lf_print_error_system_failure("Failed to disable Nagle algorithm on socket server."); - } -#endif // Linux - - return sock; -} - -int read_from_socket(int socket, size_t num_bytes, unsigned char* buffer) { - if (socket < 0) { - // Socket is not open. - errno = EBADF; - return -1; - } - ssize_t bytes_read = 0; - while (bytes_read < (ssize_t)num_bytes) { - ssize_t more = read(socket, buffer + bytes_read, num_bytes - (size_t)bytes_read); - if (more < 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { - // Those error codes set by the socket indicates - // that we should try again (@see man errno). - LF_PRINT_DEBUG("Reading from socket %d failed with error: `%s`. Will try again.", socket, strerror(errno)); - lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES); - continue; - } else if (more < 0) { - // A more serious error occurred. - lf_print_error("Reading from socket %d failed. With error: `%s`", socket, strerror(errno)); - return -1; - } else if (more == 0) { - // EOF received. - return 1; - } - bytes_read += more; - } - return 0; -} - -int read_from_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer) { - assert(socket); - int read_failed = read_from_socket(*socket, num_bytes, buffer); - if (read_failed) { - // Read failed. - // Socket has probably been closed from the other side. - // Shut down and close the socket from this side. - shutdown(*socket, SHUT_RDWR); - close(*socket); - // Mark the socket closed. - *socket = -1; - return -1; - } - return 0; -} - -void read_from_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, - char* format, ...) { - va_list args; - assert(socket); - int read_failed = read_from_socket_close_on_error(socket, num_bytes, buffer); - if (read_failed) { - // Read failed. - if (mutex != NULL) { - LF_MUTEX_UNLOCK(mutex); - } - if (format != NULL) { - va_start(args, format); - lf_print_error_system_failure(format, args); - va_end(args); - } else { - lf_print_error_system_failure("Failed to read from socket."); - } - } -} - -ssize_t peek_from_socket(int socket, unsigned char* result) { - ssize_t bytes_read = recv(socket, result, 1, MSG_DONTWAIT | MSG_PEEK); - if (bytes_read < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) - return 0; - else - return bytes_read; -} - -int write_to_socket(int socket, size_t num_bytes, unsigned char* buffer) { - if (socket < 0) { - // Socket is not open. - errno = EBADF; - return -1; - } - ssize_t bytes_written = 0; - while (bytes_written < (ssize_t)num_bytes) { - ssize_t more = write(socket, buffer + bytes_written, num_bytes - (size_t)bytes_written); - if (more <= 0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { - // The error codes EAGAIN or EWOULDBLOCK indicate - // that we should try again (@see man errno). - // The error code EINTR means the system call was interrupted before completing. - LF_PRINT_DEBUG("Writing to socket %d was blocked. Will try again.", socket); - lf_sleep(DELAY_BETWEEN_SOCKET_RETRIES); - continue; - } else if (more < 0) { - // A more serious error occurred. - lf_print_error("Writing to socket %d failed. With error: `%s`", socket, strerror(errno)); - return -1; - } - bytes_written += more; - } - return 0; -} - -int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer) { - assert(socket); - int result = write_to_socket(*socket, num_bytes, buffer); - if (result) { - // Write failed. - // Socket has probably been closed from the other side. - // Shut down and close the socket from this side. - shutdown(*socket, SHUT_RDWR); - close(*socket); - // Mark the socket closed. - *socket = -1; - } - return result; -} - -void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, - char* format, ...) { - va_list args; - assert(socket); - int result = write_to_socket_close_on_error(socket, num_bytes, buffer); - if (result) { - // Write failed. - if (mutex != NULL) { - LF_MUTEX_UNLOCK(mutex); - } - if (format != NULL) { - va_start(args, format); - lf_print_error_system_failure(format, args); - va_end(args); - } else { - lf_print_error("Failed to write to socket. Closing it."); - } - } -} #endif // FEDERATED diff --git a/core/federated/network/netdriver.c b/core/federated/network/netdriver.c new file mode 100644 index 000000000..5f82a6ecb --- /dev/null +++ b/core/federated/network/netdriver.c @@ -0,0 +1,135 @@ +#include +#include +#include + +#include "netdriver.h" +#include "util.h" + +// Mutex lock held while performing socket close operations. +// A deadlock can occur if two threads simulataneously attempt to close the same netdriver. +lf_mutex_t netdrv_mutex; + +/** + * Flag to prevent termination function from executing twice and to signal to background + * threads to terminate. + */ +bool _lf_termination_executed = false; + +netdrv_t* initialize_common_netdrv(int my_federate_id, const char* federation_id) { + netdrv_t* drv = malloc(sizeof(netdrv_t)); + if (!drv) { + lf_print_error_and_exit("Falied to malloc netdrv_t."); + } + memset(drv, 0, sizeof(netdrv_t)); + drv->read_remaining_bytes = 0; + drv->my_federate_id = my_federate_id; + drv->federation_id = federation_id; + return drv; +} + +/** + * Write the specified number of bytes to the specified netdriver using write_to_netdriver + * and close the netdriver if an error occurs. If an error occurs, this will change the + * netdriver pointed to by the first argument to -1 and will return -1. + * @param netdriver Pointer to the netdriver. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @return 0 for success, -1 for failure. + */ +int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer) { + int bytes_written = write_to_netdrv(drv, num_bytes, buffer); + if (bytes_written <= 0) { + // Write failed. + // Netdrv has probably been closed from the other side. + // Shut down and close the netdrv from this side. + close_netdrv(drv); + return -1; + } + return bytes_written; +} + +/** + * Write the specified number of bytes to the specified netdriver using + * write_to_netdriver_close_on_error and exit with an error code if an error occurs. + * If the mutex argument is non-NULL, release the mutex before exiting. If the + * format argument is non-null, then use it an any additional arguments to form + * the error message using printf conventions. Otherwise, print a generic error + * message. + * @param drv Pointer to the netdriver. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @param mutex If non-NULL, the mutex to unlock before exiting. + * @param format A format string for error messages, followed by any number of + * fields that will be used to fill the format string as in printf, or NULL + * to print a generic error message. + */ +void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, + char* format, ...) { + va_list args; + int bytes_written = write_to_netdrv_close_on_error(drv, num_bytes, buffer); + if (bytes_written <= 0) { + // Write failed. + if (mutex != NULL) { + lf_mutex_unlock(mutex); + } + if (format != NULL) { + lf_print_error_system_failure(format, args); + } else { + lf_print_error("Failed to write to netdriver. Closing it."); + } + } +} + +/** + * Read the specified number of bytes to the specified netdriver using read_from_netdriver + * and close the netdriver if an error occurs. If an error occurs, this will change the + * netdriver pointed to by the first argument to -1 and will return -1. + * @param netdriver Pointer to the netdriver. + * @param num_bytes The number of bytes to write. + * @param buffer The buffer from which to get the bytes. + * @return 0 for success, -1 for failure. + */ +ssize_t read_from_netdrv_close_on_error(netdrv_t* drv, unsigned char* buffer, size_t buffer_length) { + ssize_t bytes_read = read_from_netdrv(drv, buffer, buffer_length); + if (bytes_read < 0) { + return -1; + } + if (bytes_read == 0) { + // close_netdrv(drv); + return 0; + } + return bytes_read; +} + +/** + * Read the specified number of bytes from the specified netdriver into the + * specified buffer. If a disconnect or an EOF occurs during this + * reading, then if format is non-null, report an error and exit. + * If the mutex argument is non-NULL, release the mutex before exiting. + * If format is null, then report the error, but do not exit. + * This function takes a formatted string and additional optional arguments + * similar to printf(format, ...) that is appended to the error messages. + * @param netdriver The netdriver. + * @param num_bytes The number of bytes to read. + * @param buffer The buffer into which to put the bytes. + * @param format A printf-style format string, followed by arguments to + * fill the string, or NULL to not exit with an error message. + * @return The number of bytes read, or 0 if an EOF is received, or + * a negative number for an error. + */ +void read_from_netdrv_fail_on_error(netdrv_t* drv, unsigned char* buffer, size_t buffer_length, lf_mutex_t* mutex, + char* format, ...) { + va_list args; + ssize_t bytes_read = read_from_netdrv_close_on_error(drv, buffer, buffer_length); + if (bytes_read <= 0) { + // Read failed. + if (mutex != NULL) { + lf_mutex_unlock(mutex); + } + if (format != NULL) { + lf_print_error_system_failure(format, args); + } else { + lf_print_error_system_failure("Failed to read from netdrv."); + } + } +} diff --git a/core/federated/network/socket_common.c b/core/federated/network/socket_common.c new file mode 100644 index 000000000..fcd8a64a9 --- /dev/null +++ b/core/federated/network/socket_common.c @@ -0,0 +1,274 @@ +#include /* htons */ +#include +#include // IPPROTO_TCP, IPPROTO_UDP +#include // TCP_NODELAY +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "net_common.h" +#include "net_util.h" +#include "socket_common.h" + +// TODO: Not working... +// // Global variable defined in reactor_common.c: +// extern bool _lf_termination_executed; + +socket_priv_t* TCP_socket_priv_init() { + socket_priv_t* priv = malloc(sizeof(socket_priv_t)); + if (!priv) { + lf_print_error_and_exit("Falied to malloc socket_priv_t."); + } + memset(priv, 0, sizeof(socket_priv_t)); + + // federate initialization + strncpy(priv->server_hostname, "localhost", INET_ADDRSTRLEN); + priv->server_port = -1; + return priv; +} + +void TCP_socket_open(socket_priv_t* priv) { priv->socket_descriptor = create_real_time_tcp_socket_errexit(); } + +void TCP_socket_close(socket_priv_t* priv) { + if (priv->socket_descriptor > 0) { + shutdown(priv->socket_descriptor, SHUT_RDWR); + close(priv->socket_descriptor); + priv->socket_descriptor = -1; + } +} + +/** + * @brief Create an IPv4 TCP socket with Nagle's algorithm disabled + * (TCP_NODELAY) and Delayed ACKs disabled (TCP_QUICKACK). Exits application + * on any error. + * + * @return The socket ID (a file descriptor). + */ +int create_real_time_tcp_socket_errexit() { + // Timeout time for the communications of the server + struct timeval timeout_time = {.tv_sec = TCP_TIMEOUT_TIME / BILLION, .tv_usec = (TCP_TIMEOUT_TIME % BILLION) / 1000}; + int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (sock < 0) { + lf_print_error_system_failure("Could not open TCP socket."); + } + // Disable Nagle's algorithm which bundles together small TCP messages to + // reduce network traffic. + // TODO: Re-consider if we should do this, and whether disabling delayed ACKs + // is enough. + int flag = 1; + if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(int)) < 0) { + lf_print_error_system_failure("Failed to disable Nagle algorithm on socket server."); + } + + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(int32_t)) < 0) { + lf_print_error("RTI failed to set SO_REUSEADDR option on the socket: %s.", strerror(errno)); + } + // Set the timeout on the socket so that read and write operations don't block for too long + if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout_time, sizeof(timeout_time)) < 0) { + lf_print_error("RTI failed to set SO_RCVTIMEO option on the socket: %s.", strerror(errno)); + } + if (setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout_time, sizeof(timeout_time)) < 0) { + lf_print_error("RTI failed to set SO_SNDTIMEO option on the socket: %s.", strerror(errno)); + } + +#if defined(PLATFORM_Linux) + // Disable delayed ACKs. Only possible on Linux + if (setsockopt(sock, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(int)) < 0) { + lf_print_error_system_failure("Failed to disable Nagle algorithm on socket server."); + } +#endif // Linux + + return sock; +} + +// The port will be 0, for both RTI and federate server when nothing is specified. +// If the RTI port is 0 (not specified), it will use RTI_DEFAULT_PORT as default. +// If the federate server's port is 0 (not specified), the OS will assign the port. +int create_TCP_server(socket_priv_t* priv, int server_type, uint16_t port) { + uint16_t specified_port = port; + + // When server type is RTI, and port is not specified, set port as RTI_DEFAULT_PORT. + if (server_type == 0 && specified_port == 0) { // 0 for RTI + port = RTI_DEFAULT_PORT; + } + + // Create an IPv4 socket for TCP (not UDP) communication over IP (0). + priv->socket_descriptor = create_real_time_tcp_socket_errexit(); + + // Server file descriptor. + struct sockaddr_in server_fd; + // Zero out the server address structure. + bzero((char*)&server_fd, sizeof(server_fd)); + + server_fd.sin_family = AF_INET; // IPv4 + server_fd.sin_addr.s_addr = INADDR_ANY; // All interfaces, 0.0.0.0. + // Convert the port number from host byte order to network byte order. + server_fd.sin_port = htons(port); + + int result = bind(priv->socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); + + // If bind failed, try repeatedly to bind to a port. + int count = 1; + while (result != 0 && count++ < PORT_BIND_RETRY_LIMIT) { + // If port is not specified, we try the incremented port number. + if (specified_port == 0) { + lf_print_warning("Failed to get port %d.", port); + port++; + // If the RTI server's port is incremented until the upper bound, we retry the RTI_DEFAULT_PORT. + if (server_type == 0 && port >= RTI_DEFAULT_PORT + MAX_NUM_PORT_ADDRESSES) { + port = RTI_DEFAULT_PORT; + } + lf_print_warning("Try again with port %d.", port); + server_fd.sin_port = htons(port); + // Do not sleep. + } + // If the port is specified, we do not increment and try again after a PORT_BIND_RETRY_INTERVAL. + else { + lf_print("Failed to get port %d. Will try again.", port); + lf_sleep(PORT_BIND_RETRY_INTERVAL); + } + result = bind(priv->socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); + } + + if (result != 0) { + lf_print_error_and_exit("Failed to bind the socket. Port %d is not available. ", port); + } + // Enable listening for socket connections. + // The second argument is the maximum number of queued socket requests, + // which according to the Mac man page is limited to 128. + listen(priv->socket_descriptor, 128); + + // Set the port into priv->port. + // If the federate server has no specified port, we need to retrieve the port number assigned by the OS. + if (server_type == 1 && specified_port == 0) { // 1 for FED + struct sockaddr_in assigned; + socklen_t addr_len = sizeof(assigned); + if (getsockname(priv->socket_descriptor, (struct sockaddr*)&assigned, &addr_len) < 0) { + lf_print_error_and_exit("Failed to retrieve assigned port number."); + } + priv->port = ntohs(assigned.sin_port); + } else { + priv->port = port; + } + + LF_PRINT_LOG("Server for communicating with other federates started using port %d.", priv->port); + return 1; +} + +// Returns clock sync UDP socket. +int create_clock_sync_server(uint16_t* clock_sync_port) { + // Create UDP socket. + int socket_descriptor = -1; + socket_descriptor = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + // Set the appropriate timeout time for the communications of the server + struct timeval timeout_time = + (struct timeval){.tv_sec = UDP_TIMEOUT_TIME / BILLION, .tv_usec = (UDP_TIMEOUT_TIME % BILLION) / 1000}; + if (socket_descriptor < 0) { + lf_print_error_system_failure("Failed to create RTI socket."); + } + + // Set the option for this socket to reuse the same address + int true_variable = 1; // setsockopt() requires a reference to the value assigned to an option + if (setsockopt(socket_descriptor, SOL_SOCKET, SO_REUSEADDR, &true_variable, sizeof(int32_t)) < 0) { + lf_print_error("RTI failed to set SO_REUSEADDR option on the socket: %s.", strerror(errno)); + } + // Set the timeout on the socket so that read and write operations don't block for too long + if (setsockopt(socket_descriptor, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout_time, sizeof(timeout_time)) < 0) { + lf_print_error("RTI failed to set SO_RCVTIMEO option on the socket: %s.", strerror(errno)); + } + if (setsockopt(socket_descriptor, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout_time, sizeof(timeout_time)) < 0) { + lf_print_error("RTI failed to set SO_SNDTIMEO option on the socket: %s.", strerror(errno)); + } + + // Server file descriptor. + struct sockaddr_in server_fd; + // Zero out the server address structure. + bzero((char*)&server_fd, sizeof(server_fd)); + + uint16_t port = RTI_DEFAULT_UDP_PORT; // Default UDP port. + server_fd.sin_family = AF_INET; // IPv4 + server_fd.sin_addr.s_addr = INADDR_ANY; // All interfaces, 0.0.0.0. + // Convert the port number from host byte order to network byte order. + server_fd.sin_port = htons(port); + + int result = bind(socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); + + // Try repeatedly to bind to a port. If no specific port is specified, then + // increment the port number each time. + + int count = 1; + while (result != 0 && count++ < PORT_BIND_RETRY_LIMIT) { + lf_print_warning("RTI failed to get port %d.", port); + port++; + if (port >= RTI_DEFAULT_UDP_PORT + MAX_NUM_PORT_ADDRESSES) + port = RTI_DEFAULT_UDP_PORT; + lf_print_warning("RTI will try again with port %d.", port); + server_fd.sin_port = htons(port); + result = bind(socket_descriptor, (struct sockaddr*)&server_fd, sizeof(server_fd)); + } + if (result != 0) { + lf_print_error_and_exit("Failed to bind the RTI socket. Port %d is not available. ", port); + } + + // Update port number. + *clock_sync_port = port; + // No need to listen on the UDP socket + + return socket_descriptor; +} + +int connect_to_socket(int sock, char* hostname, int port, uint16_t user_specified_port) { + struct addrinfo hints; + struct addrinfo* result; + int ret = -1; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; /* Allow IPv4 */ + hints.ai_socktype = SOCK_STREAM; /* Stream socket */ + hints.ai_protocol = IPPROTO_TCP; /* TCP protocol */ + hints.ai_addr = NULL; + hints.ai_next = NULL; + hints.ai_flags = AI_NUMERICSERV; /* Allow only numeric port numbers */ + + int used_port = (user_specified_port == 0) ? port : user_specified_port; + + instant_t start_connect = lf_time_physical(); + // while (!_lf_termination_executed) { // Not working... + while (1) { + if (CHECK_TIMEOUT(start_connect, CONNECT_TIMEOUT)) { + lf_print_error("Failed to connect with timeout: " PRINTF_TIME ". Giving up.", CONNECT_TIMEOUT); + break; + } + // Convert port number to string. + char str[6]; + sprintf(str, "%u", used_port); + + // Get address structure matching hostname and hints criteria, and + // set port to the port number provided in str. There should only + // ever be one matching address structure, and we connect to that. + if (getaddrinfo(hostname, (const char*)&str, &hints, &result)) { + lf_print_error("No host matching given hostname: %s", hostname); + break; + } + ret = connect(sock, result->ai_addr, result->ai_addrlen); + if (ret < 0) { + lf_sleep(CONNECT_RETRY_INTERVAL); + if (user_specified_port == 0) { + used_port++; + } + lf_print_warning("Could not connect. Will try again every " PRINTF_TIME " nanoseconds.\n", + CONNECT_RETRY_INTERVAL); + continue; + } else { + break; + } + freeaddrinfo(result); + } + return ret; +} diff --git a/core/reactor_common.c b/core/reactor_common.c index 26a489bec..36d9599c1 100644 --- a/core/reactor_common.c +++ b/core/reactor_common.c @@ -35,6 +35,12 @@ #include "environment.h" #include "reactor_common.h" +#include "netdriver.h" +// #ifdef OPENSSL_REQUIRED +// #include +// #endif + + #if !defined(LF_SINGLE_THREADED) #include "watchdog.h" #endif @@ -873,6 +879,9 @@ void usage(int argc, const char* argv[]) { printf(" -l\n"); printf(" Send stdout to individual log files for each federate.\n\n"); #endif +#ifdef COMM_TYPE_SST + printf(" -sst, --sst \n"); +#endif printf("Command given:\n"); for (int i = 0; i < argc; i++) { @@ -1021,6 +1030,17 @@ int process_args(int argc, const char* argv[]) { return 0; } } +#endif +#ifdef COMM_TYPE_SST + else if (strcmp(arg, "-sst") == 0 || strcmp(arg, "--sst") == 0) { + if (argc < i + 1) { + lf_print_error("--sst needs a string argument."); + usage(argc, argv); + return 0; + } + const char* fid = argv[i++]; + lf_set_sst_config_path(fid); + } #endif else if (strcmp(arg, "--ros-args") == 0) { // FIXME: Ignore ROS arguments for now @@ -1098,7 +1118,8 @@ void initialize_global(void) { * Flag to prevent termination function from executing twice and to signal to background * threads to terminate. */ -bool _lf_termination_executed = false; +// bool _lf_termination_executed = false; +extern bool _lf_termination_executed; /** Flag used to disable cleanup operations on abnormal termination. */ bool _lf_normal_termination = false; diff --git a/core/utils/util.c b/core/utils/util.c index 62de9fd27..b5bb7a96e 100644 --- a/core/utils/util.c +++ b/core/utils/util.c @@ -51,9 +51,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define NUMBER_OF_FEDERATES 1 #endif -/** Number of nanoseconds to sleep before retrying a socket read. */ -#define SOCKET_READ_RETRY_INTERVAL 1000000 - /** * The ID of this federate. For a non-federated execution, this will be -1. * For a federated execution, it will be assigned in the generated code. diff --git a/include/core/federated/clock-sync.h b/include/core/federated/clock-sync.h index b003a3150..11b51a4be 100644 --- a/include/core/federated/clock-sync.h +++ b/include/core/federated/clock-sync.h @@ -34,6 +34,8 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define CLOCK_SYNC_H #include "low_level_platform.h" +#include "net_util.h" +#include "netdriver.h" // Clock synchronization defaults to performing clock synchronization only at initialization. #define LF_CLOCK_SYNC_OFF 1 @@ -148,7 +150,7 @@ void reset_socket_stat(struct socket_stat_t* socket_stat); * * @return port number to be sent to the RTI */ -uint16_t setup_clock_synchronization_with_rti(void); +uint16_t setup_clock_synchronization_with_rti(struct sockaddr_in* federate_UDP_addr); /** * Synchronize the initial physical clock with the RTI. @@ -165,7 +167,7 @@ uint16_t setup_clock_synchronization_with_rti(void); * * @param rti_socket_TCP Pointer to the RTI's socket */ -void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP); +void synchronize_initial_physical_clock_with_rti(netdrv_t* netdrv_to_rti); /** * Handle a clock synchroninzation message T1 coming from the RTI. @@ -178,7 +180,7 @@ void synchronize_initial_physical_clock_with_rti(int* rti_socket_TCP); * @param t2 The physical time at which the T1 message was received. * @return 0 if T3 reply is successfully sent, -1 otherwise. */ -int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2); +int handle_T1_clock_sync_message(unsigned char* buffer, void* netdrv_or_sock, netdrv_type_t netdrv_type, instant_t t2); /** * Handle a clock synchronization message T4 coming from the RTI. @@ -196,7 +198,7 @@ int handle_T1_clock_sync_message(unsigned char* buffer, int socket, instant_t t2 * @param socket The socket (either _lf_rti_socket_TCP or _lf_rti_socket_UDP). * @param r4 The physical time at which this T4 message was received. */ -void handle_T4_clock_sync_message(unsigned char* buffer, int socket, instant_t r4); +void handle_T4_clock_sync_message(unsigned char* buffer, void* netdrv_or_sock, netdrv_type_t netdrv_type, instant_t r4); /** * Thread that listens for UDP inputs from the RTI. diff --git a/include/core/federated/federate.h b/include/core/federated/federate.h index 230f3e277..4c547e1c7 100644 --- a/include/core/federated/federate.h +++ b/include/core/federated/federate.h @@ -19,6 +19,8 @@ #include "environment.h" #include "low_level_platform.h" +#include "netdriver.h" + #ifndef ADVANCE_MESSAGE_INTERVAL #define ADVANCE_MESSAGE_INTERVAL MSEC(10) #endif @@ -31,16 +33,16 @@ */ typedef struct federate_instance_t { /** - * The TCP socket descriptor for this federate to communicate with the RTI. + * The netdriver for this federate to communicate with the RTI. * This is set by lf_connect_to_rti(), which must be called before other * functions that communicate with the rti are called. */ - int socket_TCP_RTI; + netdrv_t* netdrv_to_rti; /** - * Thread listening for incoming TCP messages from the RTI. + * Thread listening for incoming messages from the RTI. */ - lf_thread_t RTI_socket_listener; + lf_thread_t RTI_netdrv_listener; /** * Number of inbound physical connections to the federate. @@ -54,7 +56,7 @@ typedef struct federate_instance_t { * This is NULL if there are none and otherwise has size given by * number_of_inbound_p2p_connections. */ - lf_thread_t* inbound_socket_listeners; + lf_thread_t* inbound_netdriv_listeners; /** * Number of outbound peer-to-peer connections from the federate. @@ -64,57 +66,50 @@ typedef struct federate_instance_t { size_t number_of_outbound_p2p_connections; /** - * An array that holds the socket descriptors for inbound + * An array that holds the netdriver descriptors for inbound * connections from each federate. The index will be the federate * ID of the remote sending federate. This is initialized at startup - * to -1 and is set to a socket ID by lf_handle_p2p_connections_from_federates() - * when the socket is opened. + * to -1 and is set to a socket ID by lf_handle_p2p_connections_from_federates() // TODO: check. + * when the netdriver is opened. * - * @note There will not be an inbound socket unless a physical connection + * @note There will not be an inbound netdriver unless a physical connection * or a p2p logical connection (by setting the coordination target property * to "distributed") is specified in the Lingua Franca program where this * federate is the destination. Multiple incoming p2p connections from the - * same remote federate will use the same socket. + * same remote federate will use the same netdriver. */ - int sockets_for_inbound_p2p_connections[NUMBER_OF_FEDERATES]; + netdrv_t* netdrv_for_inbound_p2p_connections[NUMBER_OF_FEDERATES]; /** - * An array that holds the socket descriptors for outbound direct + * An array that holds the netdrivers for outbound direct * connections to each remote federate. The index will be the federate * ID of the remote receiving federate. This is initialized at startup - * to -1 and is set to a socket ID by lf_connect_to_federate() - * when the socket is opened. + * to -1 and is set to a socket ID by lf_connect_to_federate() // TODO: check. + * when the netdriver is opened. * - * @note This federate will not open an outbound socket unless a physical + * @note This federate will not open an outbound netdriver unless a physical * connection or a p2p logical connection (by setting the coordination target * property to "distributed") is specified in the Lingua Franca * program where this federate acts as the source. Multiple outgoing p2p - * connections to the same remote federate will use the same socket. + * connections to the same remote federate will use the same netdriver. */ - int sockets_for_outbound_p2p_connections[NUMBER_OF_FEDERATES]; + netdrv_t* netdrv_for_outbound_p2p_connections[NUMBER_OF_FEDERATES]; /** - * Thread ID for a thread that accepts sockets and then supervises - * listening to those sockets for incoming P2P (physical) connections. + * Thread ID for a thread that accepts netdrivers and then supervises + * listening to those netdrivers for incoming P2P (physical) connections. */ lf_thread_t inbound_p2p_handling_thread_id; /** - * A socket descriptor for the socket server of the federate. + * A netdriver for the server of the federate. * This is assigned in lf_create_server(). - * This socket is used to listen to incoming physical connections from + * This netdriver is used to listen to incoming physical connections from * remote federates. Once an incoming connection is accepted, the - * opened socket will be stored in - * federate_sockets_for_inbound_p2p_connections. - */ - int server_socket; - - /** - * The port used for the server socket to listen for messages from other federates. - * The federate informs the RTI of this port once it has created its socket server by - * sending an ADDRESS_AD message (@see rti.h). + * opened netdriver will be stored in + * federate_netdrv_for_inbound_p2p_connections. */ - int server_port; + netdrv_t* my_netdrv; /** * Most recent tag advance grant (TAG) received from the RTI, or NEVER if none @@ -206,9 +201,9 @@ typedef enum parse_rti_code_t { SUCCESS, INVALID_PORT, INVALID_HOST, INVALID_USE // Global variables /** - * Mutex lock held while performing socket write and close operations. + * Mutex lock held while performing netdriver write and close operations. */ -extern lf_mutex_t lf_outbound_socket_mutex; +extern lf_mutex_t lf_outbound_netdrv_mutex; /** * Condition variable for blocking on unkonwn federate input ports. @@ -230,10 +225,10 @@ extern lf_cond_t lf_current_tag_changed; * to send messages directly to the specified federate. * This function first sends an MSG_TYPE_ADDRESS_QUERY message to the RTI to obtain * the IP address and port number of the specified federate. It then attempts - * to establish a socket connection to the specified federate. + * to establish a netdriver connection to the specified federate. * If this fails, the program exits. If it succeeds, it sets element [id] of - * the _fed.sockets_for_outbound_p2p_connections global array to - * refer to the socket for communicating directly with the federate. + * the _fed.netdrv_for_outbound_p2p_connections global array to + * refer to the netdriver for communicating directly with the federate. * @param remote_federate_id The ID of the remote federate. */ void lf_connect_to_federate(uint16_t); @@ -241,12 +236,12 @@ void lf_connect_to_federate(uint16_t); /** * @brief Connect to the RTI at the specified host and port. * - * This will return the socket descriptor for the connection. - * If port_number is 0, then start at DEFAULT_PORT and increment + * This will return the netdriver for the connection. + * If port_number is 0, then start at RTI_DEFAULT_PORT and increment * the port number on each attempt. If an attempt fails, wait CONNECT_RETRY_INTERVAL * and try again. If it fails after CONNECT_TIMEOUT, the program exits. - * If it succeeds, it sets the _fed.socket_TCP_RTI global variable to refer to - * the socket for communicating with the RTI. + * If it succeeds, it sets the _fed.netdrv_TCP_RTI global variable to refer to + * the netdriver for communicating with the RTI. * @param hostname A hostname, such as "localhost". * @param port_number A port number or 0 to start with the default. */ @@ -256,8 +251,8 @@ void lf_connect_to_rti(const char* hostname, int port_number); * @brief Create a server to listen to incoming P2P connections. * * Such connections are used for physical connections or any connection if using - * decentralized coordination. This function only handles the creation of the server socket. - * The bound port for the server socket is then sent to the RTI by sending an + * decentralized coordination. This function only handles the creation of the server netdriver. + * For TCP connections, the bound port for the server netdriver is then sent to the RTI by sending an * MSG_TYPE_ADDRESS_ADVERTISEMENT message (@see net_common.h). * This function expects no response from the RTI. * @@ -284,8 +279,8 @@ void lf_enqueue_port_absent_reactions(environment_t* env); * * This thread accepts connections from federates that send messages directly * to this one (not through the RTI). This thread starts a thread for - * each accepted socket connection to read messages and, once it has opened all expected - * sockets, exits. + * each accepted netdriver connection to read messages and, once it has opened all expected + * netdrivers, exits. * @param ignored No argument needed for this thread. */ void* lf_handle_p2p_connections_from_federates(void*); @@ -324,10 +319,10 @@ void lf_reset_status_fields_on_input_port_triggers(); * @brief Send a message to another federate. * * This function is used for physical connections - * between federates. If the socket connection to the remote federate or the RTI has been broken, + * between federates. If the netdriver connection to the remote federate or the RTI has been broken, * then this returns -1 without sending. Otherwise, it returns 0. * - * This method assumes that the caller does not hold the lf_outbound_socket_mutex lock, + * This method assumes that the caller does not hold the lf_outbound_netdrv_mutex lock, * which it acquires to perform the send. * * @param message_type The type of the message being sent (currently only MSG_TYPE_P2P_MESSAGE). @@ -350,7 +345,7 @@ int lf_send_message(int message_type, unsigned short port, unsigned short federa * information is needed for the RTI to perform the centralized coordination. * @see MSG_TYPE_NEIGHBOR_STRUCTURE in net_common.h */ -void lf_send_neighbor_structure_to_RTI(int); +void lf_send_neighbor_structure_to_RTI(netdrv_t* netdrv); /** * @brief Send a next event tag (NET) signal. @@ -429,7 +424,7 @@ void lf_send_port_absent_to_federate(environment_t* env, interval_t additional_d * * The payload is the specified tag plus one microstep. If this federate has previously * received a stop request from the RTI, then do not send the message and - * return 1. Return -1 if the socket is disconnected. Otherwise, return 0. + * return 1. Return -1 if the netdriver is disconnected. Otherwise, return 0. * @return 0 if the message is sent. */ int lf_send_stop_request_to_rti(tag_t stop_tag); @@ -441,7 +436,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag); * If the delayed tag falls after the timeout time, then the message is not sent and -1 is returned. * The caller can reuse or free the memory storing the message after this returns. * - * If the message fails to send (e.g. the socket connection is broken), then the + * If the message fails to send (e.g. the netdriver connection is broken), then the * response depends on the message_type. For MSG_TYPE_TAGGED_MESSAGE, the message is * supposed to go via the RTI, and failure to communicate with the RTI is a critical failure. * In this case, the program will exit with an error message. If the message type is @@ -450,7 +445,7 @@ int lf_send_stop_request_to_rti(tag_t stop_tag); * to believe that there were no messages forthcoming. In this case, on failure to send * the message, this function returns -11. * - * This method assumes that the caller does not hold the lf_outbound_socket_mutex lock, + * This method assumes that the caller does not hold the lf_outbound_netdrv_mutex lock, * which it acquires to perform the send. * * @param env The environment from which to get the current tag. @@ -506,7 +501,7 @@ void lf_stall_advance_level_federation_locked(size_t level); * @brief Synchronize the start with other federates via the RTI. * * This assumes that a connection to the RTI is already made - * and _lf_rti_socket_TCP is valid. It then sends the current logical + * and netdrv_to_rti is valid. It then sends the current logical * time to the RTI and waits for the RTI to respond with a specified * time. It starts a thread to listen for messages from the RTI. */ diff --git a/include/core/federated/network/CMakeLists.txt b/include/core/federated/network/CMakeLists.txt new file mode 100644 index 000000000..7b071d6f6 --- /dev/null +++ b/include/core/federated/network/CMakeLists.txt @@ -0,0 +1,10 @@ +add_library(lf-network-api INTERFACE) +add_library(lf::network-api ALIAS lf-network-api) +target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}) +target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/type) +target_include_directories(lf-network-api INTERFACE ${CMAKE_CURRENT_LIST_DIR}/../../utils) + +target_compile_definitions(lf-network-api INTERFACE COMM_TYPE_${COMM_TYPE}) +target_link_libraries(lf-network-api INTERFACE lf::tag-api) +target_link_libraries(lf-network-api INTERFACE lf::low-level-platform-api) +target_link_libraries(lf-network-api INTERFACE lf::low-level-platform-impl) diff --git a/include/core/federated/network/net_common.h b/include/core/federated/network/net_common.h index 202592aa8..a4b63bc22 100644 --- a/include/core/federated/network/net_common.h +++ b/include/core/federated/network/net_common.h @@ -39,7 +39,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * put into its code by the code generator (i.e., it attempts to * open a TCP connection). If an explicit port is given in the `at` clause * on the `federated reactor` statement, it will use that port. Otherwise, it will - * use DEFAULT_PORT. + * use RTI_DEFAULT_PORT. * * When it has successfully opened a TCP connection, the first message it sends * to the RTI is a MSG_TYPE_FED_IDS message, which contains the ID of this federate @@ -179,18 +179,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef NET_COMMON_H #define NET_COMMON_H -/** - * The timeout time in ns for TCP operations. - * Default value is 10 secs. - */ -#define TCP_TIMEOUT_TIME SEC(10) - -/** - * The timeout time in ns for UDP operations. - * Default value is 1 sec. - */ -#define UDP_TIMEOUT_TIME SEC(1) - /** * Size of the buffer used for messages sent between federates. * This is used by both the federates and the rti, so message lengths @@ -203,19 +191,21 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define CONNECT_RETRY_INTERVAL MSEC(500) + +//TODO: NEED TO ERASE!!! CHECK CONFLICTS. /** * Bound on the number of retries to connect to the RTI. - * A federate will retry every CONNECT_RETRY_INTERVAL seconds until - * CONNECTION_TIMEOUT expires. + * A federate will retry every CONNECT_RETRY_INTERVAL seconds + * this many times before giving up. */ -#define CONNECT_TIMEOUT MINUTES(1) +#define CONNECT_MAX_RETRIES 100 /** - * Maximum number of port addresses that a federate will try to connect to the RTI on. - * If you are using automatic ports begining at DEFAULT_PORT, this puts an upper bound - * on the number of RTIs that can be running on the same host. + * Bound on the number of retries to connect to the RTI. + * A federate will retry every CONNECT_RETRY_INTERVAL seconds until + * CONNECTION_TIMEOUT expires. */ -#define MAX_NUM_PORT_ADDRESSES 16 +#define CONNECT_TIMEOUT MINUTES(1) /** * Time that a federate waits before asking @@ -225,28 +215,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define ADDRESS_QUERY_RETRY_INTERVAL MSEC(250) -/** - * Time to wait before re-attempting to bind to a port. - * When a process closes, the network stack typically waits between 30 and 120 - * seconds before releasing the port. This is to allow for delayed packets so - * that a new process does not receive packets from a previous process. - * Here, we limit the retries to 60 seconds. - */ -#define PORT_BIND_RETRY_INTERVAL SEC(1) - -/** - * Number of attempts to bind to a port before giving up. - */ -#define PORT_BIND_RETRY_LIMIT 60 - -/** - * Default port number for the RTI. - * Unless a specific port has been specified by the LF program in the "at" - * for the RTI or on the command line, when the RTI starts up, it will attempt - * to open a socket server on this port. - */ -#define DEFAULT_PORT 15045u - /** * Delay the start of all federates by this amount. * This helps ensure that the federates do not start at the same time. @@ -347,6 +315,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define MSG_TYPE_FED_RESPONSE 102 +// TODO: Need to be moved. /** * The randomly created nonce size will be 8 bytes. */ @@ -366,6 +335,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #define MSG_TYPE_TIMESTAMP 2 #define MSG_TYPE_TIMESTAMP_LENGTH (1 + sizeof(int64_t)) +// TODO: Deprecated. /** Byte identifying a message to forward to another federate. * The next two bytes will be the ID of the destination port. * The next two bytes are the destination federate ID. @@ -670,6 +640,12 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #define MSG_TYPE_FAILED 25 +#define MSG_TYPE_MQTT_JOIN 26 + +#define MSG_TYPE_MQTT_ACCEPT 27 + +#define MSG_TYPE_MQTT_ACCEPT_ACK 28 + ///////////////////////////////////////////// //// Rejection codes diff --git a/include/core/federated/network/net_util.h b/include/core/federated/network/net_util.h index 24b4782f9..3bf2935a7 100644 --- a/include/core/federated/network/net_util.h +++ b/include/core/federated/network/net_util.h @@ -50,9 +50,7 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "low_level_platform.h" #include "tag.h" - -#define NUM_SOCKET_RETRIES 10 -#define DELAY_BETWEEN_SOCKET_RETRIES MSEC(100) +#include "util.h" #define HOST_LITTLE_ENDIAN 1 #define HOST_BIG_ENDIAN 2 @@ -63,125 +61,6 @@ THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ int host_is_big_endian(void); -#ifdef FEDERATED - -/** - * Mutex protecting socket close operations. - */ -extern lf_mutex_t socket_mutex; - -/** - * @brief Create an IPv4 TCP socket with Nagle's algorithm disabled - * (TCP_NODELAY) and Delayed ACKs disabled (TCP_QUICKACK). Exits application - * on any error. - * - * @return The socket ID (a file descriptor). - */ -int create_real_time_tcp_socket_errexit(); - -/** - * Read the specified number of bytes from the specified socket into the specified buffer. - * If an error occurs during this reading, return -1 and set errno to indicate - * the cause of the error. If the read succeeds in reading the specified number of bytes, - * return 0. If an EOF occurs before reading the specified number of bytes, return 1. - * This function repeats the read attempt until the specified number of bytes - * have been read, an EOF is read, or an error occurs. Specifically, errors EAGAIN, - * EWOULDBLOCK, and EINTR are not considered errors and instead trigger - * another attempt. A delay between attempts is given by DELAY_BETWEEN_SOCKET_RETRIES. - * @param socket The socket ID. - * @param num_bytes The number of bytes to read. - * @param buffer The buffer into which to put the bytes. - * @return 0 for success, 1 for EOF, and -1 for an error. - */ -int read_from_socket(int socket, size_t num_bytes, unsigned char* buffer); - -/** - * Read the specified number of bytes to the specified socket using read_from_socket - * and close the socket if an error occurs. If an error occurs, this will change the - * socket ID pointed to by the first argument to -1 and will return -1. - * @param socket Pointer to the socket ID. - * @param num_bytes The number of bytes to write. - * @param buffer The buffer from which to get the bytes. - * @return 0 for success, -1 for failure. - */ -int read_from_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer); - -/** - * Read the specified number of bytes from the specified socket into the - * specified buffer. If a disconnect or an EOF occurs during this - * reading, then if format is non-null, report an error and exit. - * If the mutex argument is non-NULL, release the mutex before exiting. - * If format is null, then report the error, but do not exit. - * This function takes a formatted string and additional optional arguments - * similar to printf(format, ...) that is appended to the error messages. - * @param socket The socket ID. - * @param num_bytes The number of bytes to read. - * @param buffer The buffer into which to put the bytes. - * @param format A printf-style format string, followed by arguments to - * fill the string, or NULL to not exit with an error message. - * @return The number of bytes read, or 0 if an EOF is received, or - * a negative number for an error. - */ -void read_from_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, - char* format, ...); - -/** - * Without blocking, peek at the specified socket and, if there is - * anything on the queue, put its first byte at the specified address and return 1. - * If there is nothing on the queue, return 0, and if an error occurs, - * return -1. - * @param socket The socket ID. - * @param result Pointer to where to put the first byte available on the socket. - */ -ssize_t peek_from_socket(int socket, unsigned char* result); - -/** - * Write the specified number of bytes to the specified socket from the - * specified buffer. If an error occurs, return -1 and set errno to indicate - * the cause of the error. If the write succeeds, return 0. - * This function repeats the attempt until the specified number of bytes - * have been written or an error occurs. Specifically, errors EAGAIN, - * EWOULDBLOCK, and EINTR are not considered errors and instead trigger - * another attempt. A delay between attempts is given by - * DELAY_BETWEEN_SOCKET_RETRIES. - * @param socket The socket ID. - * @param num_bytes The number of bytes to write. - * @param buffer The buffer from which to get the bytes. - * @return 0 for success, -1 for failure. - */ -int write_to_socket(int socket, size_t num_bytes, unsigned char* buffer); - -/** - * Write the specified number of bytes to the specified socket using write_to_socket - * and close the socket if an error occurs. If an error occurs, this will change the - * socket ID pointed to by the first argument to -1 and will return -1. - * @param socket Pointer to the socket ID. - * @param num_bytes The number of bytes to write. - * @param buffer The buffer from which to get the bytes. - * @return 0 for success, -1 for failure. - */ -int write_to_socket_close_on_error(int* socket, size_t num_bytes, unsigned char* buffer); - -/** - * Write the specified number of bytes to the specified socket using - * write_to_socket_close_on_error and exit with an error code if an error occurs. - * If the mutex argument is non-NULL, release the mutex before exiting. If the - * format argument is non-null, then use it an any additional arguments to form - * the error message using printf conventions. Otherwise, print a generic error - * message. - * @param socket Pointer to the socket ID. - * @param num_bytes The number of bytes to write. - * @param buffer The buffer from which to get the bytes. - * @param mutex If non-NULL, the mutex to unlock before exiting. - * @param format A format string for error messages, followed by any number of - * fields that will be used to fill the format string as in printf, or NULL - * to print a generic error message. - */ -void write_to_socket_fail_on_error(int* socket, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, - char* format, ...); - -#endif // FEDERATED - /** * Write the specified data as a sequence of bytes starting * at the specified address. This encodes the data in little-endian @@ -276,8 +155,19 @@ int64_t extract_int64(unsigned char* bytes); */ uint16_t extract_uint16(unsigned char* bytes); +/** + * Extract an uint32_t from the specified byte sequence. + * This will swap the order of the bytes if this machine is big endian. + * @param bytes The address of the start of the sequence of bytes. + */ +uint32_t extract_uint32(unsigned char* bytes); + #ifdef FEDERATED +/** + * Mutex protecting socket close operations. + */ + /** * Extract the core header information that all messages between * federates share. The core header information is two bytes with diff --git a/include/core/federated/network/netdriver.h b/include/core/federated/network/netdriver.h new file mode 100644 index 000000000..8e8e197af --- /dev/null +++ b/include/core/federated/network/netdriver.h @@ -0,0 +1,133 @@ +#ifndef NETDRIVER_H +#define NETDRIVER_H + +#include "low_level_platform.h" + +#include "socket_common.h" + +#if defined(COMM_TYPE_TCP) +#include "lf_socket_support.h" +#elif defined(COMM_TYPE_MQTT) +#include "lf_mqtt_support.h" +#elif defined(COMM_TYPE_SST) +#include "lf_sst_support.h" +#endif + +extern lf_mutex_t netdrv_mutex; + + +typedef enum netdrv_type_t { NETDRV, UDP } netdrv_type_t; + +// Just doing 0 for RTI, 1 for FED +typedef enum server_type_t { RTI, FED } server_type_t; + +typedef struct netdrv_t { + void* priv; + unsigned int read_remaining_bytes; + int my_federate_id; // The RTI is -1, and unitialized is -2. This must be int not uint16_t + const char* federation_id; +} netdrv_t; + +/** + * @brief Allocate memory for the netdriver, save my_federate_id, and federation_id used for the netdriver. + * If the netdriver belongs to the RTI, the federtae_id is -1. + * @param my_federate_id + * @param federation_id + * @return netdrv_t* + */ +netdrv_t* initialize_netdrv(int my_federate_id, const char* federation_id); + +netdrv_t* initialize_common_netdrv(int federate_id, const char* federation_id); + +/** + * @brief Close connections, and free allocated memory. + * + * @param drv + */ +void close_netdrv(netdrv_t* drv); + +// Port will be NULL on MQTT. +/** + * @brief Create a netdriver server. This is such as a server socket which accepts connections. However this is only the + * creation of the server netdriver. + * + * @param drv + * @param server_type + * @param port + * @return int + */ +int create_listener(netdrv_t* drv, server_type_t server_type, uint16_t port); + +/** + * @brief Creates a communication session. It uses the listener_netdrv to listen to connections, and return a + * connector_nedrv + * + * @param listener_netdrv + * @return netdrv_t* + */ +netdrv_t* establish_communication_session(netdrv_t* listener_netdrv); + +/** + * @brief Create a netdriver client. + * + * @param drv + */ +void create_connector(netdrv_t* drv); + +/** + * @brief Request connect to the target server. + * + * @param drv + * @return int + */ +int connect_to_netdrv(netdrv_t* drv); + +int write_to_netdrv(netdrv_t* drv, size_t num_bytes, unsigned char* buffer); + +int write_to_netdrv_close_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer); + +void write_to_netdrv_fail_on_error(netdrv_t* drv, size_t num_bytes, unsigned char* buffer, lf_mutex_t* mutex, + char* format, ...); + +// Return 0 when connection lost. -1 on error. > 0 bytes read. +ssize_t read_from_netdrv(netdrv_t* drv, unsigned char* buffer, size_t buffer_length); + +ssize_t read_from_netdrv_close_on_error(netdrv_t* drv, unsigned char* buffer, size_t buffer_length); + +void read_from_netdrv_fail_on_error(netdrv_t* drv, unsigned char* buffer, size_t buffer_length, lf_mutex_t* mutex, + char* format, ...); + +// Temporary ////////////////// +char* get_host_name(netdrv_t* drv); +int32_t get_my_port(netdrv_t* drv); +int32_t get_port(netdrv_t* drv); +struct in_addr* get_ip_addr(netdrv_t* drv); + +void set_host_name(netdrv_t* drv, const char* hostname); +void set_port(netdrv_t* drv, int port); +void set_specified_port(netdrv_t* drv, int port); +void set_ip_addr(netdrv_t* drv, struct in_addr ip_addr); +/** + * Without blocking, peek at the specified socket and, if there is + * anything on the queue, put its first byte at the specified address and return 1. + * If there is nothing on the queue, return 0, and if an error occurs, + * return -1. + * @param socket The socket ID. + * @param result Pointer to where to put the first byte available on the socket. + */ +ssize_t peek_from_netdrv(netdrv_t* drv, unsigned char* result); + +/** + * @brief Set the target id to connect. This is only used in MQTT. + * + * @param drv + * @param federate_id + */ +void set_target_id(netdrv_t* drv, int federate_id); + +// Returns socket number of clock_sync_server. +int create_clock_sync_server(uint16_t* clock_sync_port); + +//////////////////////////// + +#endif /* NETDRIVER_H */ diff --git a/include/core/federated/network/socket_common.h b/include/core/federated/network/socket_common.h new file mode 100644 index 000000000..a7b879e6f --- /dev/null +++ b/include/core/federated/network/socket_common.h @@ -0,0 +1,78 @@ +#ifndef SOCKET_COMMON_H +#define SOCKET_COMMON_H + +#include // IPPROTO_TCP, IPPROTO_UDP +#include // TCP_NODELAY + +/** + * The timeout time in ns for TCP operations. + * Default value is 10 secs. + */ +#define TCP_TIMEOUT_TIME SEC(10) + +/** + * The timeout time in ns for UDP operations. + * Default value is 1 sec. + */ +#define UDP_TIMEOUT_TIME SEC(1) + +/** + * Maximum number of port addresses that a federate will try to connect to the RTI on. + * If you are using automatic ports begining at DEFAULT_PORT, this puts an upper bound + * on the number of RTIs that can be running on the same host. + */ +#define MAX_NUM_PORT_ADDRESSES 16 + +/** + * Time to wait before re-attempting to bind to a port. + * When a process closes, the network stack typically waits between 30 and 120 + * seconds before releasing the port. This is to allow for delayed packets so + * that a new process does not receive packets from a previous process. + * Here, we limit the retries to 60 seconds. + */ +#define PORT_BIND_RETRY_INTERVAL SEC(1) + +/** + * Number of attempts to bind to a port before giving up. + */ +#define PORT_BIND_RETRY_LIMIT 60 + +/** + * Default port number for the RTI. + * Unless a specific port has been specified by the LF program in the "at" + * for the RTI or on the command line, when the RTI starts up, it will attempt + * to open a socket server on this port. + */ +#define RTI_DEFAULT_PORT 15045u + +#define RTI_DEFAULT_UDP_PORT 15061u + +typedef struct socket_priv_t { + int socket_descriptor; + uint16_t port; // my port number // TODO: Only used in federate.c to send federate's port. + uint16_t user_specified_port; + + // The connected other side's info. + char server_hostname[INET_ADDRSTRLEN]; // Human-readable IP address and + int32_t server_port; // port number of the socket server of the federate + // if it has any incoming direct connections from other federates. + // The port number will be -1 if there is no server or if the + // RTI has not been informed of the port number. + struct in_addr server_ip_addr; // Information about the IP address of the socket + // server of the federate. + + struct sockaddr_in UDP_addr; // The UDP address for the federate. +} socket_priv_t; + +socket_priv_t* TCP_socket_priv_init(); +void TCP_socket_close(socket_priv_t* priv); +void TCP_socket_open(socket_priv_t* priv); + + +int create_real_time_tcp_socket_errexit(); +// TODO: Check if it's fine to just use int. It's just an enum. Can't use server_type_t because not including netdriver.h +int create_TCP_server(socket_priv_t* priv, int server_type, uint16_t port); + +int connect_to_socket(int sock, char* hostname, int port, uint16_t user_specified_port); + +#endif /* SOCKET_COMMON_H */ \ No newline at end of file diff --git a/include/core/federated/network/type/lf_mqtt_support.h b/include/core/federated/network/type/lf_mqtt_support.h new file mode 100644 index 000000000..8c182edc0 --- /dev/null +++ b/include/core/federated/network/type/lf_mqtt_support.h @@ -0,0 +1,21 @@ +#ifndef LF_MQTT_SUPPORT_H +#define LF_MQTT_SUPPORT_H + +#include +#include + +#define MQTTkeepAliveInterval 20 +#define MQTTcleansession 1 +#define MQTTconnectTimeout 2 + +#define MQTT_RESIGNED 88 + +typedef struct MQTT_priv_t { + MQTTClient client; + MQTTClient_connectOptions conn_opts; // = MQTTClient_connectOptions_initializer; + char* topic_name_to_send; + char client_id[32]; + int target_id; // Must be int. Not uint_16_t. -1 stands for RTI, -2 means uninitialized. +} MQTT_priv_t; + +#endif // LF_MQTT_SUPPORT_H diff --git a/include/core/federated/network/type/lf_socket_support.h b/include/core/federated/network/type/lf_socket_support.h new file mode 100644 index 000000000..84f758754 --- /dev/null +++ b/include/core/federated/network/type/lf_socket_support.h @@ -0,0 +1,17 @@ +#ifndef LF_SOCKET_SUPPORT_H +#define LF_SOCKET_SUPPORT_H + +#define NUM_SOCKET_RETRIES 10 + +#define DELAY_BETWEEN_SOCKET_RETRIES MSEC(100) + +typedef enum { + HEADER_READ, + READ_MSG_TYPE_FED_IDS, + READ_MSG_TYPE_NEIGHBOR_STRUCTURE, + READ_MSG_TYPE_TAGGED_MESSAGE, + KEEP_READING, + FINISH_READ +} read_state_t; + +#endif // LF_SOCKET_SUPPORT_H diff --git a/include/core/federated/network/type/lf_sst_support.h b/include/core/federated/network/type/lf_sst_support.h new file mode 100644 index 000000000..a8f7b4ce5 --- /dev/null +++ b/include/core/federated/network/type/lf_sst_support.h @@ -0,0 +1,27 @@ +#ifndef LF_SST_SUPPORT_H +#define LF_SST_SUPPORT_H + +#include "socket_common.h" +#include +typedef struct sst_priv_t { + socket_priv_t* socket_priv; // Must be first variable. + SST_ctx_t* sst_ctx; + SST_session_ctx_t* session_ctx; + + // RTI: + // RTI_netdrv needs a sst_ctx, but not session_ctx; + // fed_netdrv needs only session_ctx not sst_ctx. + + // Centralized Federate: + // netdrv_to_rti needs both sst_ctx and session_ctx + // Decentralized Federate: + // my_netdrv needs a sst_ctx, but not session_ctx; + // netdrv_for_inbound_p2p_connections and outbound needs only session_ctx not sst_ctx. + +} sst_priv_t; + +void lf_set_sst_config_path(const char* config_path); +void lf_set_rti_sst_config_path(const char* config_path); + + +#endif // LF_SST_SUPPORT_H