diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..493605a --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,164 @@ +cmake_minimum_required(VERSION 3.11) + +# set the project name +project(norm VERSION 1.5.8) + +set(COMMON src/common) + +option(NORM_BUILD_EXAMPLES "Enables building of the examples in /examples." OFF) + +include(CheckCXXSymbolExists) +check_cxx_symbol_exists(dirfd "dirent.h" HAVE_DIRFD) +if(HAVE_DIRFD) + list(APPEND PLATFORM_DEFINITIONS HAVE_DIRFD) +endif() + +check_cxx_symbol_exists(lockf "unistd.h" HAVE_LOCKF) +if(HAVE_LOCKF) + list(APPEND PLATFORM_DEFINITIONS HAVE_LOCKF) +endif() + +check_cxx_symbol_exists(flock "sys/file.h" HAVE_FLOCK) +if(HAVE_FLOCK) + list(APPEND PLATFORM_DEFINITIONS HAVE_FLOCK) +endif() + +# Check for libraries +find_package(protokit QUIET) + +if(NOT protokit_FOUND) + include(FetchContent) + FetchContent_Declare( + protokit + GIT_REPOSITORY https://github.com/USNavalResearchLaboratory/protolib.git + GIT_TAG origin/master + ) + FetchContent_MakeAvailable(protokit) +endif() + +# List header files +list(APPEND PUBLIC_HEADER_FILES + include/galois.h + include/normApi.h + include/normEncoder.h + include/normEncoderMDP.h + include/normEncoderRS16.h + include/normEncoderRS8.h + include/normFile.h + include/normMessage.h + include/normNode.h + include/normObject.h + include/normPostProcess.h + include/normSegment.h + include/normSession.h + include/normSimAgent.h + include/normVersion.h +) + +# List platform-independent source files +list(APPEND COMMON_SOURCE_FILES + ${COMMON}/galois.cpp + ${COMMON}/normApi.cpp + ${COMMON}/normEncoder.cpp + ${COMMON}/normEncoderMDP.cpp + ${COMMON}/normEncoderRS16.cpp + ${COMMON}/normEncoderRS8.cpp + ${COMMON}/normFile.cpp + ${COMMON}/normMessage.cpp + ${COMMON}/normNode.cpp + ${COMMON}/normObject.cpp + ${COMMON}/normSegment.cpp + ${COMMON}/normSession.cpp ) + +# Setup platform independent include directory +list(APPEND INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/include ) + +# Setup platform dependent libraries, defines, source file and compiler flags +if(MSVC) + list(APPEND PLATFORM_LIBS Shell32) + list(APPEND PLATFORM_DEFINITIONS _CONSOLE) + list(APPEND PLATFORM_SOURCE_FILES src/win32/win32PostProcess.cpp) +elseif(UNIX) + list(APPEND PLATFORM_SOURCE_FILES src/unix/unixPostProcess.cpp) +endif() + +include(GNUInstallDirs) + +# Setup target +add_library(norm ${PLATFORM_SOURCE_FILES} ${COMMON_SOURCE_FILES} ${PUBLIC_HEADER_FILES}) +target_link_libraries(norm PRIVATE ${PLATFORM_LIBS} protokit::protokit) +target_compile_definitions(norm PUBLIC ${PLATFORM_DEFINITIONS}) +target_compile_options(norm PUBLIC ${PLATFORM_FLAGS}) +target_include_directories(norm PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(norm PUBLIC $ $) + +# Install target +install( TARGETS norm EXPORT normTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) + +set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/norm) + +install( EXPORT normTargets + FILE normTargets.cmake + NAMESPACE norm:: + DESTINATION ${INSTALL_CONFIGDIR} +) + +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/norm) + +# Create a ConfigVersion.cmake file +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/normConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion +) + +configure_package_config_file(${CMAKE_CURRENT_LIST_DIR}/cmake/normConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/normConfig.cmake + INSTALL_DESTINATION ${INSTALL_CONFIGDIR} +) + +# Install the config, configversion and custom find modules +install(FILES + ${CMAKE_CURRENT_BINARY_DIR}/normConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/normConfigVersion.cmake + DESTINATION ${INSTALL_CONFIGDIR} +) + +############################################## +# Exporting from the build tree +export(EXPORT normTargets + FILE ${CMAKE_CURRENT_BINARY_DIR}/normTargets.cmake + NAMESPACE norm::) + +# Register package in user's package registry +export(PACKAGE norm) + +if(NORM_BUILD_EXAMPLES) + # Setup examples + list(APPEND examples + #normDataExample + normDataRecv + normDataSend + normFileRecv + normFileSend + normStreamRecv + normStreamSend + normMsgr + normStreamer + normCast + normClient + normServer + #wintest + ) + + foreach(example ${examples}) + add_executable(${example} examples/${example}.cpp examples/normSocket.cpp) + target_link_libraries(${example} PRIVATE norm protokit::protokit) + endforeach() +endif() + diff --git a/cmake/normConfig.cmake.in b/cmake/normConfig.cmake.in new file mode 100644 index 0000000..a9f7552 --- /dev/null +++ b/cmake/normConfig.cmake.in @@ -0,0 +1,10 @@ + +include(GNUInstallDirs) + +include(CMakeFindDependencyMacro) + +# Same syntax as find_package +find_dependency(protokit REQUIRED) + +# Add the targets file +include("${CMAKE_CURRENT_LIST_DIR}/normTargets.cmake") \ No newline at end of file diff --git a/examples/normCast.cpp b/examples/normCast.cpp index 317c686..9a61d58 100644 --- a/examples/normCast.cpp +++ b/examples/normCast.cpp @@ -2,6 +2,7 @@ #include "normApi.h" #include // for printf(), etc #include // for atoi(), etc +#include #ifdef WIN32 #include diff --git a/examples/normFileRecv.cpp b/examples/normFileRecv.cpp index 97b04d6..1cc18b8 100644 --- a/examples/normFileRecv.cpp +++ b/examples/normFileRecv.cpp @@ -63,7 +63,7 @@ int main(int argc, char* argv[]) // NOTE: These are debugging routines available // (not necessary for normal app use) - // (Need to include "protolib/common/protoDebug.h" for this + // (Need to include "common/protoDebug.h" for this //SetDebugLevel(2); // Uncomment to turn on debug NORM message tracing //NormSetMessageTrace(session, true); diff --git a/examples/normFileSend.cpp b/examples/normFileSend.cpp index 84bcf1c..83eb566 100644 --- a/examples/normFileSend.cpp +++ b/examples/normFileSend.cpp @@ -71,7 +71,7 @@ int main(int argc, char* argv[]) // NOTE: These are some debugging routines available // (not necessary for normal app use) - // (Need to include "protolib/common/protoDebug.h" for this + // (Need to include "common/protoDebug.h" for this //SetDebugLevel(2); // Uncomment to turn on debug NORM message tracing NormSetMessageTrace(session, true); diff --git a/examples/normStreamRecv.cpp b/examples/normStreamRecv.cpp index 8fe95ca..e1c25ef 100644 --- a/examples/normStreamRecv.cpp +++ b/examples/normStreamRecv.cpp @@ -48,7 +48,7 @@ int main(int argc, char* argv[]) // NOTE: These are debugging routines available // (not necessary for normal app use) - // (Need to include "protolib/common/protoDebug.h" for this + // (Need to include "common/protoDebug.h" for this NormSetDebugLevel(3); // Uncomment to turn on debug NORM message tracing NormSetMessageTrace(session, true); diff --git a/examples/wintest.cpp b/examples/wintest.cpp index ef55f3c..9a76979 100644 --- a/examples/wintest.cpp +++ b/examples/wintest.cpp @@ -39,7 +39,7 @@ int main(int argc, char** argv) Win32InputHandler inputHandler; - inputHandler.Start(); + inputHandler.Open(); while (true) { @@ -51,7 +51,7 @@ int main(int argc, char** argv) DWORD dwWritten; BOOL fSuccess = WriteFile(hStdout, buffer, numBytes, &dwWritten, NULL); } - inputHandler.Stop(); + inputHandler.Close(); return 0; } // end main() diff --git a/src/common/normFile.cpp b/src/common/normFile.cpp index bb920b5..efa7124 100644 --- a/src/common/normFile.cpp +++ b/src/common/normFile.cpp @@ -17,6 +17,12 @@ static inline int dirfd(DIR *dir) {return (dir->dd_fd);} #endif // HAVE_DIRFD #endif // if/else WIN32 +#ifdef HAVE_FLOCK + #include +#elif defined(HAVE_LOCKF) + #include +#endif + #ifndef _WIN32_WCE #include #include diff --git a/src/common/normSession.cpp b/src/common/normSession.cpp index 37a0c8d..c02b777 100644 --- a/src/common/normSession.cpp +++ b/src/common/normSession.cpp @@ -4,127 +4,128 @@ #include "normEncoderRS8.h" // 8-bit Reed-Solomon encoder of RFC 5510 #include "normEncoderRS16.h" // 16-bit Reed-Solomon encoder of RFC 5510 -#include // for gmtime() in NormTrace() +#include // for gmtime() in NormTrace() #include "protoPktETH.h" #include "protoPktIP.h" -const UINT8 NormSession::DEFAULT_TTL = 255; -const double NormSession::DEFAULT_TRANSMIT_RATE = 64000.0; // bits/sec -const double NormSession::DEFAULT_GRTT_INTERVAL_MIN = 1.0; // sec -const double NormSession::DEFAULT_GRTT_INTERVAL_MAX = 30.0; // sec -const double NormSession::DEFAULT_GRTT_ESTIMATE = 0.25; // sec -const double NormSession::DEFAULT_GRTT_MAX = 10.0; // sec +const UINT8 NormSession::DEFAULT_TTL = 255; +const double NormSession::DEFAULT_TRANSMIT_RATE = 64000.0; // bits/sec +const double NormSession::DEFAULT_GRTT_INTERVAL_MIN = 1.0; // sec +const double NormSession::DEFAULT_GRTT_INTERVAL_MAX = 30.0; // sec +const double NormSession::DEFAULT_GRTT_ESTIMATE = 0.25; // sec +const double NormSession::DEFAULT_GRTT_MAX = 10.0; // sec const unsigned int NormSession::DEFAULT_GRTT_DECREASE_DELAY = 3; -const double NormSession::DEFAULT_BACKOFF_FACTOR = 4.0; -const double NormSession::DEFAULT_GSIZE_ESTIMATE = 1000.0; -const UINT16 NormSession::DEFAULT_NDATA = 64; -const UINT16 NormSession::DEFAULT_NPARITY = 8; +const double NormSession::DEFAULT_BACKOFF_FACTOR = 4.0; +const double NormSession::DEFAULT_GSIZE_ESTIMATE = 1000.0; +const UINT16 NormSession::DEFAULT_NDATA = 64; +const UINT16 NormSession::DEFAULT_NPARITY = 8; const UINT16 NormSession::DEFAULT_TX_CACHE_MIN = 8; const UINT16 NormSession::DEFAULT_TX_CACHE_MAX = 256; -const UINT32 NormSession::DEFAULT_TX_CACHE_SIZE = (UINT32)20*1024*1024; +const UINT32 NormSession::DEFAULT_TX_CACHE_SIZE = (UINT32)20 * 1024 * 1024; const double NormSession::DEFAULT_FLOW_CONTROL_FACTOR = 2.0; const UINT16 NormSession::DEFAULT_RX_CACHE_MAX = 256; - -const int NormSession::DEFAULT_ROBUST_FACTOR = 20; // default robust factor +const int NormSession::DEFAULT_ROBUST_FACTOR = 20; // default robust factor // This is extra stuff defined for NormSocket API extension purposes. As the NormSocket // extension is finalized, these may be refined/relocated -enum {NORM_SOCKET_VERSION = 1}; +enum +{ + NORM_SOCKET_VERSION = 1 +}; enum NormSocketCommand { - NORM_SOCKET_CMD_NULL = 0, // reserved, invalid/null command - NORM_SOCKET_CMD_REJECT, // sent by server-listener to reject invalid connection messages - NORM_SOCKET_CMD_ALIVE // TBD - for NormSocket "keep-alive" option? + NORM_SOCKET_CMD_NULL = 0, // reserved, invalid/null command + NORM_SOCKET_CMD_REJECT, // sent by server-listener to reject invalid connection messages + NORM_SOCKET_CMD_ALIVE // TBD - for NormSocket "keep-alive" option? }; -NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) - : session_mgr(sessionMgr), notify_pending(false), tx_port(0), tx_port_reuse(false), - tx_socket_actual(ProtoSocket::UDP), tx_socket(&tx_socket_actual), - rx_socket(ProtoSocket::UDP), rx_cap(NULL), rx_port_reuse(false), local_node_id(localNodeId), - ttl(DEFAULT_TTL), tos(0), loopback(false), mcast_loopback(false), fragmentation(false), ecn_enabled(false), - tx_rate(DEFAULT_TRANSMIT_RATE/8.0), tx_rate_min(-1.0), tx_rate_max(-1.0), tx_residual(0), - backoff_factor(DEFAULT_BACKOFF_FACTOR), is_sender(false), - tx_robust_factor(DEFAULT_ROBUST_FACTOR), instance_id(0), - ndata(DEFAULT_NDATA), nparity(DEFAULT_NPARITY), auto_parity(0), extra_parity(0), - sndr_emcon(false), tx_only(false), tx_connect(false), fti_mode(FTI_ALWAYS), encoder(NULL), - next_tx_object_id(0), - tx_cache_count_min(DEFAULT_TX_CACHE_MIN), - tx_cache_count_max(DEFAULT_TX_CACHE_MAX), - tx_cache_size_max(DEFAULT_TX_CACHE_SIZE), - posted_tx_queue_empty(false), posted_tx_rate_changed(false), posted_send_error(false), - acking_node_count(0), acking_auto_populate(TRACK_NONE), watermark_pending(false), watermark_flushes(false), - tx_repair_pending(false), advertise_repairs(false), - suppress_nonconfirmed(false), suppress_rate(-1.0), suppress_rtt(-1.0), - probe_proactive(true), probe_pending(false), probe_reset(true), probe_data_check(false), - grtt_interval(0.5), - grtt_interval_min(DEFAULT_GRTT_INTERVAL_MIN), - grtt_interval_max(DEFAULT_GRTT_INTERVAL_MAX), - grtt_max(DEFAULT_GRTT_MAX), - grtt_decrease_delay_count(DEFAULT_GRTT_DECREASE_DELAY), - grtt_response(false), grtt_current_peak(0.0), grtt_age(0.0), probe_count(1), - cc_enable(false), cc_adjust(true), cc_sequence(0), cc_slow_start(true), cc_active(false), - flow_control_factor(DEFAULT_FLOW_CONTROL_FACTOR), - cmd_count(0), cmd_buffer(NULL), cmd_length(0), syn_status(false), - ack_ex_buffer(NULL), ack_ex_length(0), - is_receiver(false), rx_robust_factor(DEFAULT_ROBUST_FACTOR), preset_sender(NULL), unicast_nacks(false), - receiver_silent(false), rcvr_ignore_info(false), rcvr_max_delay(-1), rcvr_realtime(false), - default_repair_boundary(NormSenderNode::BLOCK_BOUNDARY), - default_nacking_mode(NormObject::NACK_NORMAL), default_sync_policy(NormSenderNode::SYNC_CURRENT), - rx_cache_count_max(DEFAULT_RX_CACHE_MAX), is_server_listener(false), notify_on_grtt_update(true), - ecn_ignore_loss(false), - trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0), - user_data(NULL), next(NULL) +NormSession::NormSession(NormSessionMgr &sessionMgr, NormNodeId localNodeId) + : session_mgr(sessionMgr), notify_pending(false), tx_port(0), tx_port_reuse(false), + tx_socket_actual(ProtoSocket::UDP), tx_socket(&tx_socket_actual), + rx_socket(ProtoSocket::UDP), rx_cap(NULL), rx_port_reuse(false), local_node_id(localNodeId), + ttl(DEFAULT_TTL), tos(0), loopback(false), mcast_loopback(false), fragmentation(false), ecn_enabled(false), + tx_rate(DEFAULT_TRANSMIT_RATE / 8.0), tx_rate_min(-1.0), tx_rate_max(-1.0), tx_residual(0), + backoff_factor(DEFAULT_BACKOFF_FACTOR), is_sender(false), + tx_robust_factor(DEFAULT_ROBUST_FACTOR), instance_id(0), + ndata(DEFAULT_NDATA), nparity(DEFAULT_NPARITY), auto_parity(0), extra_parity(0), + sndr_emcon(false), tx_only(false), tx_connect(false), fti_mode(FTI_ALWAYS), encoder(NULL), + next_tx_object_id(0), + tx_cache_count_min(DEFAULT_TX_CACHE_MIN), + tx_cache_count_max(DEFAULT_TX_CACHE_MAX), + tx_cache_size_max(DEFAULT_TX_CACHE_SIZE), + posted_tx_queue_empty(false), posted_tx_rate_changed(false), posted_send_error(false), + acking_node_count(0), acking_auto_populate(TRACK_NONE), watermark_pending(false), watermark_flushes(false), + tx_repair_pending(false), advertise_repairs(false), + suppress_nonconfirmed(false), suppress_rate(-1.0), suppress_rtt(-1.0), + probe_proactive(true), probe_pending(false), probe_reset(true), probe_data_check(false), + grtt_interval(0.5), + grtt_interval_min(DEFAULT_GRTT_INTERVAL_MIN), + grtt_interval_max(DEFAULT_GRTT_INTERVAL_MAX), + grtt_max(DEFAULT_GRTT_MAX), + grtt_decrease_delay_count(DEFAULT_GRTT_DECREASE_DELAY), + grtt_response(false), grtt_current_peak(0.0), grtt_age(0.0), probe_count(1), + cc_enable(false), cc_adjust(true), cc_sequence(0), cc_slow_start(true), cc_active(false), + flow_control_factor(DEFAULT_FLOW_CONTROL_FACTOR), + cmd_count(0), cmd_buffer(NULL), cmd_length(0), syn_status(false), + ack_ex_buffer(NULL), ack_ex_length(0), + is_receiver(false), rx_robust_factor(DEFAULT_ROBUST_FACTOR), preset_sender(NULL), unicast_nacks(false), + receiver_silent(false), rcvr_ignore_info(false), rcvr_max_delay(-1), rcvr_realtime(false), + default_repair_boundary(NormSenderNode::BLOCK_BOUNDARY), + default_nacking_mode(NormObject::NACK_NORMAL), default_sync_policy(NormSenderNode::SYNC_CURRENT), + rx_cache_count_max(DEFAULT_RX_CACHE_MAX), is_server_listener(false), notify_on_grtt_update(true), + ecn_ignore_loss(false), + trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0), + user_data(NULL), next(NULL) { interface_name[0] = '\0'; tx_socket_actual.SetNotifier(&sessionMgr.GetSocketNotifier()); tx_socket_actual.SetListener(this, &NormSession::TxSocketRecvHandler); tx_address.Invalidate(); - + rx_socket.SetNotifier(&sessionMgr.GetSocketNotifier()); rx_socket.SetListener(this, &NormSession::RxSocketRecvHandler); - + tx_timer.SetListener(this, &NormSession::OnTxTimeout); tx_timer.SetInterval(0.0); tx_timer.SetRepeat(-1); - + repair_timer.SetListener(this, &NormSession::OnRepairTimeout); repair_timer.SetInterval(0.0); repair_timer.SetRepeat(1); - + flush_timer.SetListener(this, &NormSession::OnFlushTimeout); flush_timer.SetInterval(0.0); flush_timer.SetRepeat(0); - + flow_control_timer.SetListener(this, &NormSession::OnFlowControlTimeout); flow_control_timer.SetInterval(0.0); flow_control_timer.SetRepeat(0); - + cmd_timer.SetListener(this, &NormSession::OnCmdTimeout); cmd_timer.SetInterval(0.0); cmd_timer.SetRepeat(0); - + probe_timer.SetListener(this, &NormSession::OnProbeTimeout); probe_timer.SetInterval(0.0); probe_timer.SetRepeat(-1); probe_time_last.tv_sec = probe_time_last.tv_usec = 0; - + grtt_quantized = NormQuantizeRtt(DEFAULT_GRTT_ESTIMATE); grtt_measured = grtt_advertised = NormUnquantizeRtt(grtt_quantized); - + gsize_measured = DEFAULT_GSIZE_ESTIMATE; gsize_quantized = NormQuantizeGroupSize(DEFAULT_GSIZE_ESTIMATE); gsize_advertised = NormUnquantizeGroupSize(gsize_quantized); - - + // This timer is for printing out occasional status reports // (It may be used to trigger transmission of report messages // in the future for debugging, etc report_timer.SetListener(this, &NormSession::OnReportTimeout); report_timer.SetInterval(10.0); report_timer.SetRepeat(-1); - + user_timer.SetListener(this, &NormSession::OnUserTimeout); user_timer.SetInterval(0.0); user_timer.SetRepeat(0); @@ -132,7 +133,8 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) NormSession::~NormSession() { - if (user_timer.IsActive()) user_timer.Deactivate(); + if (user_timer.IsActive()) + user_timer.Deactivate(); if (NULL != preset_sender) { delete preset_sender; @@ -145,7 +147,7 @@ bool NormSession::Open() { ASSERT(address.IsValid()); if (!tx_socket->IsOpen()) - { + { // Make sure user wants a separate tx_socket if ((address.GetPort() != tx_port) || (tx_address.IsValid() && !address.HostIsEqual(tx_address))) @@ -153,7 +155,7 @@ bool NormSession::Open() if (!tx_socket->Open(tx_port, address.GetType(), false)) { PLOG(PL_FATAL, "NormSession::Open() tx_socket::Open() error\n"); - return false; + return false; } if (tx_port_reuse) { @@ -164,7 +166,7 @@ bool NormSession::Open() return false; } } - ProtoAddress* txBindAddress = tx_address.IsValid() ? &tx_address : NULL; + ProtoAddress *txBindAddress = tx_address.IsValid() ? &tx_address : NULL; if (!tx_socket->Bind(tx_port, txBindAddress)) { PLOG(PL_FATAL, "NormSession::Open() tx_socket::Bind() error\n"); @@ -185,7 +187,7 @@ bool NormSession::Open() } else { - tx_socket = &rx_socket; + tx_socket = &rx_socket; } } if (!rx_socket.IsOpen() && (!tx_only || (&rx_socket == tx_socket))) @@ -194,41 +196,41 @@ bool NormSession::Open() { PLOG(PL_FATAL, "NormSession::Open() rx_socket.Open() error\n"); Close(); - return false; + return false; } rx_socket.EnableRecvDstAddr(); if (rx_port_reuse) { - // Enable port/addr reuse and bind socket to destination address + // Enable port/addr reuse and bind socket to destination address if (!rx_socket.SetReuse(true)) { PLOG(PL_FATAL, "NormSession::Open() rx_socket::SetReuse() error\n"); Close(); - return false; + return false; } } - const ProtoAddress* bindAddr = NULL; + const ProtoAddress *bindAddr = NULL; if (rx_bind_addr.IsValid()) { #ifdef WIN32 - if (rx_bind_addr.IsMulticast()) // Win32 doesn't like to bind() mcast addr?? + if (rx_bind_addr.IsMulticast()) // Win32 doesn't like to bind() mcast addr?? PLOG(PL_WARN, "NormSession::Open() warning: WIN32 multicast bind() issue!\n"); else #endif bindAddr = &rx_bind_addr; } - if(!rx_socket.Bind(address.GetPort(), bindAddr)) + if (!rx_socket.Bind(address.GetPort(), bindAddr)) { PLOG(PL_FATAL, "NormSession::Open() error: rx_socket.Bind() error\n"); Close(); return false; } - if (rx_connect_addr.IsValid() && (0 != rx_connect_addr.GetPort())) + if (rx_connect_addr.IsValid() && (0 != rx_connect_addr.GetPort())) { - // For unicast, we use the "connect()" call to effectively + // For unicast, we use the "connect()" call to effectively // uniquely "bind" our rx_socket to the remote addr. - // (it _may_ be the case that "tx_port" == "address.GetPort()" - // for this to work?) + // (it _may_ be the case that "tx_port" == "address.GetPort()" + // for this to work?) if (!rx_socket.Connect(rx_connect_addr)) { PLOG(PL_FATAL, "NormSession::Open() rx_socket.Connect() error\n"); @@ -244,17 +246,16 @@ bool NormSession::Open() PLOG(PL_WARN, "NormSession::Open() warning: tx_socket.SetEcnEnable() error\n"); } } - + if (0 != tos) { if (!tx_socket->SetTOS(tos)) PLOG(PL_WARN, "NormSession::Open() warning: tx_socket.SetTOS() error\n"); } - + if (!tx_socket->SetFragmentation(fragmentation)) PLOG(PL_WARN, "NormSession::Open() warning: tx_socket.SetFragmentation() error\n"); - - + TRACE("NormSession::Open() address %s\n", address.GetHostString()); if (address.IsMulticast()) { @@ -271,7 +272,7 @@ bool NormSession::Open() Close(); return false; } - const char* interfaceName = NULL; + const char *interfaceName = NULL; if ('\0' != interface_name[0]) { bool result = tx_only ? true : rx_socket.SetMulticastInterface(interface_name); @@ -286,16 +287,16 @@ bool NormSession::Open() } if (!tx_only) { - if (!rx_socket.JoinGroup(address, interfaceName, ssm_source_addr.IsValid() ? &ssm_source_addr : NULL)) + if (!rx_socket.JoinGroup(address, interfaceName, ssm_source_addr.IsValid() ? &ssm_source_addr : NULL)) { PLOG(PL_FATAL, "NormSession::Open() rx_socket.JoinGroup error\n"); Close(); return false; - } + } } } - -#ifdef ECN_SUPPORT + +#ifdef ECN_SUPPORT // TBD - do this via UDP socket recvmsg() instead of raw packet capture // If raw packet capture is enabled, create/open ProtoCap device to do it if (ecn_enabled && !tx_only) @@ -315,8 +316,8 @@ bool NormSession::Open() PLOG(PL_FATAL, "NormSession::Open() error: unable to open ProtoCap device '%s'!\n", (('\0' != interface_name[0]) ? interface_name : "(null)")); Close(); return false; - } - rx_cap->StartInputNotification(); + } + rx_cap->StartInputNotification(); // Populate "dst_addr_list" with potential valid dst addrs for this host dst_addr_list.Destroy(); if (rx_bind_addr.IsValid()) @@ -352,14 +353,14 @@ bool NormSession::Open() return false; } } - rx_socket.StopInputNotification(); // Disable rx_socket (keep open so mcast JOIN holds) + rx_socket.StopInputNotification(); // Disable rx_socket (keep open so mcast JOIN holds) } -#endif // ECN_SUPPORT +#endif // ECN_SUPPORT if (message_pool.IsEmpty()) { for (unsigned int i = 0; i < DEFAULT_MESSAGE_POOL_DEPTH; i++) { - NormMsg* msg = new NormMsg(); + NormMsg *msg = new NormMsg(); if (msg) { message_pool.Append(msg); @@ -369,29 +370,34 @@ bool NormSession::Open() PLOG(PL_FATAL, "NormSession::Open() new message error: %s\n", GetErrorString()); Close(); return false; - } + } } } - if (!report_timer.IsActive()) ActivateTimer(report_timer); - + if (!report_timer.IsActive()) + ActivateTimer(report_timer); + return true; -} // end NormSession::Open() +} // end NormSession::Open() void NormSession::Close() { - if (report_timer.IsActive()) report_timer.Deactivate(); - if (is_sender) StopSender(); - if (is_receiver) StopReceiver(); - if (tx_timer.IsActive()) tx_timer.Deactivate(); + if (report_timer.IsActive()) + report_timer.Deactivate(); + if (is_sender) + StopSender(); + if (is_receiver) + StopReceiver(); + if (tx_timer.IsActive()) + tx_timer.Deactivate(); message_queue.Destroy(); message_pool.Destroy(); - if (tx_socket->IsOpen()) tx_socket->Close(); - if (rx_socket.IsOpen()) + if (tx_socket->IsOpen()) + tx_socket->Close(); + if (rx_socket.IsOpen()) { - if (address.IsMulticast()) + if (address.IsMulticast()) { - const char* interfaceName = ('\0' != interface_name[0]) ? - interface_name : NULL; + const char *interfaceName = ('\0' != interface_name[0]) ? interface_name : NULL; rx_socket.LeaveGroup(address, interfaceName, ssm_source_addr.IsValid() ? &ssm_source_addr : NULL); } rx_socket.Close(); @@ -404,10 +410,9 @@ void NormSession::Close() rx_cap = NULL; } #endif // ECN_SUPPORT -} // end NormSession::Close() +} // end NormSession::Close() - -bool NormSession::SetMulticastInterface(const char* interfaceName) +bool NormSession::SetMulticastInterface(const char *interfaceName) { if (NULL != interfaceName) { @@ -422,12 +427,12 @@ bool NormSession::SetMulticastInterface(const char* interfaceName) } else { - interface_name[0] = '\0'; - return true; + interface_name[0] = '\0'; + return true; } -} // end NormSession::SetMulticastInterface() +} // end NormSession::SetMulticastInterface() -bool NormSession::SetSSM(const char* sourceAddress) +bool NormSession::SetSSM(const char *sourceAddress) { if (NULL != sourceAddress) { @@ -439,23 +444,23 @@ bool NormSession::SetSSM(const char* sourceAddress) { PLOG(PL_ERROR, "NormSession::SetSSM() error: invalid source address\n"); return false; - } + } } else { ssm_source_addr.Invalidate(); return true; } -} // end NormSession::SetSSM() +} // end NormSession::SetSSM() // This must be called _before_ sender or receiver is started // (i.e., before socket(s) are opened) -bool NormSession::SetRxPortReuse(bool enableReuse, - const char* rxBindAddress, // bind() to / - const char* senderAddress, // connect() to / - UINT16 senderPort) +bool NormSession::SetRxPortReuse(bool enableReuse, + const char *rxBindAddress, // bind() to / + const char *senderAddress, // connect() to / + UINT16 senderPort) { - rx_port_reuse = enableReuse; // allow sessionPort reuse when true + rx_port_reuse = enableReuse; // allow sessionPort reuse when true bool result; if (NULL != rxBindAddress) { @@ -485,12 +490,12 @@ bool NormSession::SetRxPortReuse(bool enableReuse, } // TBD - if rx_socket.IsOpen(), should we do a Close()/Open() to rebind socket??? return result; -} // end NormSession::SetRxPortReuse() +} // end NormSession::SetRxPortReuse() -bool NormSession::SetTxPort(UINT16 txPort, bool enableReuse, const char* txAddress) +bool NormSession::SetTxPort(UINT16 txPort, bool enableReuse, const char *txAddress) { tx_port = txPort; - tx_port_reuse = enableReuse; + tx_port_reuse = enableReuse; bool result; if (NULL != txAddress) { @@ -506,9 +511,9 @@ bool NormSession::SetTxPort(UINT16 txPort, bool enableReuse, const char* txAddre { tx_address.Invalidate(); result = true; - } + } return result; -} // end NormSession::SetTxPort() +} // end NormSession::SetTxPort() UINT16 NormSession::GetTxPort() const { @@ -518,8 +523,7 @@ UINT16 NormSession::GetTxPort() const return tx_socket->GetPort(); else return 0; -} // end NormSession::GetTxPort() - +} // end NormSession::GetTxPort() UINT16 NormSession::GetRxPort() const { @@ -530,7 +534,7 @@ UINT16 NormSession::GetRxPort() const return rx_socket.GetPort(); else return address.GetPort(); -} // end NormSession::GetRxPort() +} // end NormSession::GetRxPort() void NormSession::SetTxOnly(bool txOnly, bool connectToSessionAddress) { @@ -540,8 +544,10 @@ void NormSession::SetTxOnly(bool txOnly, bool connectToSessionAddress) { if (txOnly) { - if (IsReceiver()) StopReceiver(); - if (rx_socket.IsOpen()) rx_socket.Close(); + if (IsReceiver()) + StopReceiver(); + if (rx_socket.IsOpen()) + rx_socket.Close(); #ifdef ECN_SUPPORT if (NULL != rx_cap) { @@ -560,7 +566,7 @@ void NormSession::SetTxOnly(bool txOnly, bool connectToSessionAddress) PLOG(PL_WARN, "NormSession::SetTxOnly() tx_socket connect() error: %s\n", ProtoSocket::GetErrorString()); } } -} // end NormSession::SetTxOnly() +} // end NormSession::SetTxOnly() double NormSession::GetTxRate() { @@ -568,15 +574,14 @@ double NormSession::GetTxRate() if (cc_enable && !cc_adjust) { // Return rate of CLR - const NormCCNode* clr = static_cast(cc_node_list.Head()); + const NormCCNode *clr = static_cast(cc_node_list.Head()); return ((NULL != clr) ? 8.0 * clr->GetRate() : 0.0); } else { - return (8.0*tx_rate); + return (8.0 * tx_rate); } -} // end NormSession::GetTxRate() - +} // end NormSession::GetTxRate() /* // This hack can be uncommented give us a tx rate interval that is POISSON instead of PERIODIC @@ -585,11 +590,11 @@ static double PoissonRand(double mean) return(-log(((double)rand())/((double)RAND_MAX))*mean); } */ - + static inline double GetTxInterval(unsigned int msgSize, double txRate) { double interval = (double)msgSize / txRate; - + //double jitterMax = 0.05*interval; //interval += UniformRand(jitterMax) - jitterMax/2.0; return interval; // PERIODIC interval based on rate @@ -598,12 +603,12 @@ static inline double GetTxInterval(unsigned int msgSize, double txRate) void NormSession::SetTxRateInternal(double txRate) { - if (!is_sender) + if (!is_sender) { tx_rate = txRate; return; } - if (txRate < 0.0) + if (txRate < 0.0) { PLOG(PL_FATAL, "NormSession::SetTxRateInternal() invalid transmit rate!\n"); return; @@ -612,8 +617,8 @@ void NormSession::SetTxRateInternal(double txRate) { if (txRate > 0.0) { - double adjustInterval = (tx_rate/txRate) * tx_timer.GetTimeRemaining(); - //adjustInterval = PoissonRand(adjustInterval); + double adjustInterval = (tx_rate / txRate) * tx_timer.GetTimeRemaining(); + //adjustInterval = PoissonRand(adjustInterval); if (adjustInterval > NORM_TICK_MIN) { tx_timer.SetInterval(adjustInterval); @@ -628,36 +633,37 @@ void NormSession::SetTxRateInternal(double txRate) else if ((0.0 == tx_rate) && IsOpen()) { tx_timer.SetInterval(0.0); - if (txRate > 0.0) ActivateTimer(tx_timer); + if (txRate > 0.0) + ActivateTimer(tx_timer); } - tx_rate = txRate; + tx_rate = txRate; if (tx_rate > 0.0) { unsigned char grttQuantizedOld = grtt_quantized; - double pktInterval = (double)(44+segment_size)/txRate; + double pktInterval = (double)(44 + segment_size) / txRate; if (grtt_measured < pktInterval) grtt_quantized = NormQuantizeRtt(pktInterval); else grtt_quantized = NormQuantizeRtt(grtt_measured); grtt_advertised = NormUnquantizeRtt(grtt_quantized); - + // What do we do when "pktInterval" > "grtt_max"? // We will take our lumps with some extra activity timeout NACKs when they happen? if (grtt_advertised > grtt_max) { - grtt_quantized = NormQuantizeRtt(grtt_max); - grtt_advertised = NormUnquantizeRtt(grtt_quantized); + grtt_quantized = NormQuantizeRtt(grtt_max); + grtt_advertised = NormUnquantizeRtt(grtt_quantized); } if (grttQuantizedOld != grtt_quantized) { PLOG(PL_DEBUG, "NormSession::SetTxRateInternal() node>%lu %s to new grtt to: %lf sec\n", - (unsigned long)LocalNodeId(), - (grttQuantizedOld < grtt_quantized) ? "increased" : "decreased", - grtt_advertised); + (unsigned long)LocalNodeId(), + (grttQuantizedOld < grtt_quantized) ? "increased" : "decreased", + grtt_advertised); if (notify_on_grtt_update) { notify_on_grtt_update = false; - Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::GRTT_UPDATED, (NormSenderNode *)NULL, (NormObject *)NULL); } } // wakeup grtt/cc probing if necessary @@ -669,7 +675,7 @@ void NormSession::SetTxRateInternal(double txRate) ActivateTimer(probe_timer); } } -} // end NormSession::SetTxRateInternal() +} // end NormSession::SetTxRateInternal() void NormSession::SetTxRateBounds(double rateMin, double rateMax) { @@ -681,19 +687,19 @@ void NormSession::SetTxRateBounds(double rateMin, double rateMax) { double temp = rateMin; rateMin = rateMax; - rateMax = temp; - } + rateMax = temp; + } } if (rateMin < 0.0) tx_rate_min = -1.0; else if (rateMin < 8.0) - tx_rate_min = 1.0; // one byte/second absolute minimum + tx_rate_min = 1.0; // one byte/second absolute minimum else - tx_rate_min = rateMin/8.0; // convert to bytes/second + tx_rate_min = rateMin / 8.0; // convert to bytes/second if (rateMax < 0.0) tx_rate_max = -1.0; else - tx_rate_max = rateMax/8.0; // convert to bytes/second + tx_rate_max = rateMax / 8.0; // convert to bytes/second if (cc_enable) { double txRate = tx_rate; @@ -701,28 +707,28 @@ void NormSession::SetTxRateBounds(double rateMin, double rateMax) txRate = tx_rate_min; if ((tx_rate_max >= 0.0) && (txRate > tx_rate_max)) txRate = tx_rate_max; - if (txRate != tx_rate) SetTxRateInternal(txRate); + if (txRate != tx_rate) + SetTxRateInternal(txRate); } -} // end NormSession::SetTxRateBounds() - +} // end NormSession::SetTxRateBounds() void NormSession::SetUserTimer(double seconds) { - if (user_timer.IsActive()) user_timer.Deactivate(); + if (user_timer.IsActive()) + user_timer.Deactivate(); if (seconds >= 0.0) { user_timer.SetInterval(seconds); ActivateTimer(user_timer); } -} // end NormSession::SetUserTimer() +} // end NormSession::SetUserTimer() - -bool NormSession::StartSender(UINT16 instanceId, - UINT32 bufferSpace, - UINT16 segmentSize, - UINT16 numData, - UINT16 numParity, - UINT8 fecId) +bool NormSession::StartSender(UINT16 instanceId, + UINT32 bufferSpace, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + UINT8 fecId) { UINT16 blockSize = numData + numParity; if (blockSize <= 255) @@ -740,69 +746,74 @@ bool NormSession::StartSender(UINT16 instanceId, return false; } } - if (!IsOpen()) + if (!IsOpen()) { - if (!Open()) return false; + if (!Open()) + return false; } - if (!tx_table.Init(tx_cache_count_max)) + if (!tx_table.Init(tx_cache_count_max)) { PLOG(PL_FATAL, "NormSession::StartSender() tx_table.Init() error!\n"); StopSender(); - return false; + return false; } if (!tx_pending_mask.Init(tx_cache_count_max, 0x0000ffff)) { PLOG(PL_FATAL, "NormSession::StartSender() tx_pending_mask.Init() error!\n"); StopSender(); - return false; + return false; } if (!tx_repair_mask.Init(tx_cache_count_max, 0x0000ffff)) { PLOG(PL_FATAL, "NormSession::StartSender() tx_repair_mask.Init() error!\n"); StopSender(); - return false; + return false; } - + // Calculate how much memory each buffered block will require unsigned long maskSize = blockSize >> 3; - if (0 != (blockSize & 0x07)) maskSize++; - unsigned long blockSpace = sizeof(NormBlock) + - blockSize * sizeof(char*) + - 2*maskSize + + if (0 != (blockSize & 0x07)) + maskSize++; + unsigned long blockSpace = sizeof(NormBlock) + + blockSize * sizeof(char *) + + 2 * maskSize + numParity * (segmentSize + NormDataMsg::GetStreamPayloadHeaderLength()); - + unsigned long numBlocks = bufferSpace / blockSpace; - if (bufferSpace > (numBlocks*blockSpace)) numBlocks++; - if (numBlocks < 2) numBlocks = 2; + if (bufferSpace > (numBlocks * blockSpace)) + numBlocks++; + if (numBlocks < 2) + numBlocks = 2; unsigned long numSegments = numBlocks * numParity; - + if (!block_pool.Init((UINT32)numBlocks, blockSize)) { PLOG(PL_FATAL, "NormSession::StartSender() block_pool init error\n"); StopSender(); return false; } - + if (!segment_pool.Init((unsigned int)numSegments, segmentSize + NormDataMsg::GetStreamPayloadHeaderLength())) { PLOG(PL_FATAL, "NormSession::StartSender() segment_pool init error\n"); StopSender(); return false; } - + if (numParity) { - if (NULL != encoder) delete encoder; - + if (NULL != encoder) + delete encoder; + if (blockSize <= 255) { -#ifdef ASSUME_MDP_FEC +#ifdef ASSUME_MDP_FEC if (NULL == (encoder = new NormEncoderMDP)) { PLOG(PL_FATAL, "NormSession::StartSender() new NormEncoderMDP error: %s\n", GetErrorString()); StopSender(); return false; - } + } fec_id = 129; fec_m = 8; #else @@ -811,13 +822,13 @@ bool NormSession::StartSender(UINT16 instanceId, PLOG(PL_FATAL, "NormSession::StartSender() new NormEncoderRS8 error: %s\n", GetErrorString()); StopSender(); return false; - } + } if (0 != fecId) fec_id = fecId; else fec_id = 5; fec_m = 8; -#endif +#endif } else //if (blockSize <= 65535) { @@ -826,7 +837,7 @@ bool NormSession::StartSender(UINT16 instanceId, PLOG(PL_FATAL, "NormSession::StartSender() new NormEncoderRS16 error: %s\n", GetErrorString()); StopSender(); return false; - } + } // TBD - Investigate if fec_id == 129 can also support 16-bit Reed Solomon fec_id = 2; fec_m = 16; @@ -837,13 +848,13 @@ bool NormSession::StartSender(UINT16 instanceId, StopSender(); return false; }*/ - + if (!encoder->Init(numData, numParity, segmentSize + NormDataMsg::GetStreamPayloadHeaderLength())) { PLOG(PL_FATAL, "NormSession::StartSender() encoder init error\n"); StopSender(); return false; - } + } } else { @@ -854,9 +865,9 @@ bool NormSession::StartSender(UINT16 instanceId, fec_id = 5; fec_m = 8; } - + fec_block_mask = NormPayloadId::GetFecBlockMask(fec_id, fec_m); - + // Initialize optional app-defined command state cmd_count = cmd_length = 0; if (NULL == (cmd_buffer = new char[segmentSize])) @@ -865,7 +876,7 @@ bool NormSession::StartSender(UINT16 instanceId, StopSender(); return false; } - + instance_id = instanceId; segment_size = segmentSize; sent_accumulator.Reset(); @@ -874,13 +885,13 @@ bool NormSession::StartSender(UINT16 instanceId, ndata = numData; nparity = numParity; is_sender = true; - + flush_count = (GetTxRobustFactor() < 0) ? 0 : (GetTxRobustFactor() + 1); - - if (cc_enable && cc_adjust) + + if (cc_enable && cc_adjust) { double txRate; - if(tx_rate_min > 0.0) + if (tx_rate_min > 0.0) { txRate = tx_rate_min; } @@ -893,7 +904,7 @@ bool NormSession::StartSender(UINT16 instanceId, } if ((tx_rate_max >= 0.0) && (tx_rate > tx_rate_max)) txRate = tx_rate_max; - SetTxRateInternal(txRate); // adjusts grtt_advertised as needed + SetTxRateInternal(txRate); // adjusts grtt_advertised as needed } else { @@ -901,24 +912,23 @@ bool NormSession::StartSender(UINT16 instanceId, } cc_slow_start = true; cc_active = false; - + grtt_age = 0.0; probe_pending = false; probe_data_check = false; if (probe_reset) - { + { probe_reset = false; OnProbeTimeout(probe_timer); if (!probe_timer.IsActive()) ActivateTimer(probe_timer); } return true; -} // end NormSession::StartSender() - +} // end NormSession::StartSender() void NormSession::StopSender() { - if (probe_timer.IsActive()) + if (probe_timer.IsActive()) { probe_timer.Deactivate(); probe_reset = true; @@ -934,21 +944,21 @@ void NormSession::StopSender() cmd_timer.Deactivate(); if (flow_control_timer.IsActive()) flow_control_timer.Deactivate(); - + if (NULL != ack_ex_buffer) { delete[] ack_ex_buffer; ack_ex_buffer = NULL; ack_ex_length = 0; } - + if (NULL != cmd_buffer) { delete[] cmd_buffer; cmd_buffer = NULL; cmd_length = 0; } - + if (NULL != encoder) { encoder->Destroy(); @@ -960,7 +970,7 @@ void NormSession::StopSender() // Iterate tx_table and release objects while (!tx_table.IsEmpty()) { - NormObject* obj = tx_table.Find(tx_table.RangeLo()); + NormObject *obj = tx_table.Find(tx_table.RangeLo()); ASSERT(NULL != obj); tx_table.Remove(obj); obj->Close(); @@ -973,8 +983,9 @@ void NormSession::StopSender() tx_repair_mask.Destroy(); tx_pending_mask.Destroy(); is_sender = false; - if (!IsReceiver()) Close(); -} // end NormSession::StopSender() + if (!IsReceiver()) + Close(); +} // end NormSession::StopSender() bool NormSession::StartReceiver(unsigned long bufferSize) { @@ -982,15 +993,16 @@ bool NormSession::StartReceiver(unsigned long bufferSize) tx_only = false; if (!rx_socket.IsOpen()) { - if (!Open()) return false; + if (!Open()) + return false; } is_receiver = true; remote_sender_buffer_size = bufferSize; return true; -} // end NormSession::StartReceiver() +} // end NormSession::StartReceiver() void NormSession::StopReceiver() -{ +{ // Iterate sender_tree and close/release sender nodes if (IsServerListener()) { @@ -998,21 +1010,22 @@ void NormSession::StopReceiver() } else { - NormSenderNode* senderNode = - static_cast(sender_tree.GetRoot()); + NormSenderNode *senderNode = + static_cast(sender_tree.GetRoot()); while (NULL != senderNode) { sender_tree.DetachNode(senderNode); senderNode->Close(); senderNode->Release(); - senderNode = static_cast(sender_tree.GetRoot()); + senderNode = static_cast(sender_tree.GetRoot()); } } is_receiver = false; - if (!is_sender) Close(); -} // end NormSession::StopReceiver() + if (!is_sender) + Close(); +} // end NormSession::StopReceiver() -void NormSession::DeleteRemoteSender(NormSenderNode& senderNode) +void NormSession::DeleteRemoteSender(NormSenderNode &senderNode) { // TBD - confirm that "senderNode" is valid??? if (IsServerListener()) @@ -1021,15 +1034,16 @@ void NormSession::DeleteRemoteSender(NormSenderNode& senderNode) sender_tree.DetachNode(&senderNode); senderNode.Close(); senderNode.Release(); -} // end NormSession::DeleteRemoteSender() +} // end NormSession::DeleteRemoteSender() -bool NormSession::PreallocateRemoteSender(unsigned int bufferSpace, - UINT16 segmentSize, - UINT16 numData, - UINT16 numParity, - unsigned int streamBufferSize) +bool NormSession::PreallocateRemoteSender(unsigned int bufferSpace, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + unsigned int streamBufferSize) { - if (NULL != preset_sender) delete preset_sender; + if (NULL != preset_sender) + delete preset_sender; preset_sender = new NormSenderNode(*this, NORM_NODE_ANY); if (!preset_sender->Open(0)) { @@ -1068,12 +1082,12 @@ bool NormSession::PreallocateRemoteSender(unsigned int bufferSpace, } } return true; -} // end NormSession::PreallocateRemoteSender() +} // end NormSession::PreallocateRemoteSender() bool NormSession::SetPresetFtiData(unsigned int objectSize, - UINT16 segmentSize, - UINT16 numData, - UINT16 numParity) + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity) { UINT16 blockSize = numData + numParity; UINT8 fecM; @@ -1099,39 +1113,40 @@ bool NormSession::SetPresetFtiData(unsigned int objectSize, preset_fti.SetFecFieldSize(fecM); preset_fti.SetFecInstanceId(0); return true; -} // end NormSession::SetPresetFtiData() +} // end NormSession::SetPresetFtiData() void NormSession::Serve() { - // Only send new data when no other messages are queued for transmission - if (!message_queue.IsEmpty()) + // Only send new data when no other messages are queued for transmission + if (!message_queue.IsEmpty()) { ASSERT(tx_timer.IsActive()); return; } - + // Queue next sender message NormObjectId objectId; - NormObject* obj = NULL; + NormObject *obj = NULL; if (SenderGetFirstPending(objectId)) { obj = tx_table.Find(objectId); ASSERT(NULL != obj); } - + // If any app-defined command is pending, enqueue it for transmission - if ((0 != cmd_count) && !(cmd_timer.IsActive())) + if ((0 != cmd_count) && !(cmd_timer.IsActive())) { // If command is enqueued - if (SenderQueueAppCmd()) return; + if (SenderQueueAppCmd()) + return; } - + bool watermarkJustCompleted = false; if (watermark_pending && !flush_timer.IsActive()) { PLOG(PL_DEBUG, "NormSession::Serve() watermark status check ...\n"); // Determine next message (objectId::blockId::segmentId) to be sent - NormObject* nextObj; + NormObject *nextObj; NormObjectId nextObjectId = next_tx_object_id; NormBlockId nextBlockId = 0; NormSegmentId nextSegmentId = 0; @@ -1142,15 +1157,16 @@ void NormSession::Serve() nextObjectId = objectId; if (nextObj->IsPending()) { - if(nextObj->GetFirstPending(nextBlockId)) + if (nextObj->GetFirstPending(nextBlockId)) { - NormBlock* block = nextObj->FindBlock(nextBlockId); - if (block) + NormBlock *block = nextObj->FindBlock(nextBlockId); + if (block) { block->GetFirstPending(nextSegmentId); // Adjust so watermark segmentId < block length UINT16 nextBlockSize = nextObj->GetBlockSize(nextBlockId); - if (nextSegmentId >= nextBlockSize) nextSegmentId = nextBlockSize - 1; + if (nextSegmentId >= nextBlockSize) + nextSegmentId = nextBlockSize - 1; } } else @@ -1160,24 +1176,24 @@ void NormSession::Serve() } else { - // Must be an active, but non-pending stream object + // Must be an active, but non-pending stream object ASSERT(nextObj->IsStream()); - nextBlockId = static_cast(nextObj)->GetNextBlockId(); - nextSegmentId = static_cast(nextObj)->GetNextSegmentId(); - } + nextBlockId = static_cast(nextObj)->GetNextBlockId(); + nextSegmentId = static_cast(nextObj)->GetNextSegmentId(); + } } - PLOG(PL_DEBUG, " nextPending index>%hu:%lu:%hu\n", - (UINT16)nextObjectId, - (unsigned long)nextBlockId.GetValue(), - (UINT16)nextSegmentId); - + PLOG(PL_DEBUG, " nextPending index>%hu:%lu:%hu\n", + (UINT16)nextObjectId, + (unsigned long)nextBlockId.GetValue(), + (UINT16)nextSegmentId); + if (tx_repair_pending) { - - PLOG(PL_DEBUG, " tx_repair index>%hu:%lu:%hu\n", - (UINT16)tx_repair_object_min, - (unsigned long)tx_repair_block_min.GetValue(), - (UINT16)tx_repair_segment_min); + + PLOG(PL_DEBUG, " tx_repair index>%hu:%lu:%hu\n", + (UINT16)tx_repair_object_min, + (unsigned long)tx_repair_block_min.GetValue(), + (UINT16)tx_repair_segment_min); if ((tx_repair_object_min < nextObjectId) || ((tx_repair_object_min == nextObjectId) && //((tx_repair_block_min < nextBlockId) || @@ -1188,45 +1204,44 @@ void NormSession::Serve() nextObjectId = tx_repair_object_min; nextBlockId = tx_repair_block_min; nextSegmentId = tx_repair_segment_min; - PLOG(PL_DEBUG, " updated nextPending index>%hu:%lu:%hu\n", - (UINT16)nextObjectId, - (unsigned long)nextBlockId.GetValue(), - (UINT16)nextSegmentId); - + PLOG(PL_DEBUG, " updated nextPending index>%hu:%lu:%hu\n", + (UINT16)nextObjectId, + (unsigned long)nextBlockId.GetValue(), + (UINT16)nextSegmentId); } - } // end if (tx_repair_pending) - + } // end if (tx_repair_pending) + ASSERT(nextBlockId.GetValue() <= (UINT32)0x00ffffff); - + PLOG(PL_DEBUG, " watermark>%hu:%lu:%hu check against next pending index>%hu:%lu:%hu\n", - (UINT16)watermark_object_id, (unsigned long)watermark_block_id.GetValue(), (UINT16)watermark_segment_id, - (UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), (UINT16)nextSegmentId); + (UINT16)watermark_object_id, (unsigned long)watermark_block_id.GetValue(), (UINT16)watermark_segment_id, + (UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), (UINT16)nextSegmentId); if ((nextObjectId > watermark_object_id) || ((nextObjectId == watermark_object_id) && //((nextBlockId > watermark_block_id) || ((Compare(nextBlockId, watermark_block_id) > 0) || ((nextBlockId == watermark_block_id) && - (nextSegmentId > watermark_segment_id))))) + (nextSegmentId > watermark_segment_id))))) { PLOG(PL_DEBUG, " calling SenderQueueWatermarkFlush() ...\n"); // The sender tx position is > watermark - if (SenderQueueWatermarkFlush()) + if (SenderQueueWatermarkFlush()) { watermark_active = true; return; } else { - // (TBD) optionally return here to have ack collection temporarily + // (TBD) optionally return here to have ack collection temporarily // suspend forward progress of data transmission //return; - + // If the app has set the property to "truncated_flushing" because // there is explicit positive acknowledge from everyone in the group - // (or unicast destination), we can safely provide an _early_ + // (or unicast destination), we can safely provide an _early_ // termination of flushing at this point iff: // - // "(false == watermark_pending) && + // "(false == watermark_pending) && // (NULL == obj) && // (watermark_object_id == last_tx_object_id) // @@ -1245,22 +1260,22 @@ void NormSession::Serve() else { // The sender tx position is < watermark - // Reset non-acked acking nodes since sender has rewound + // Reset non-acked acking nodes since sender has rewound // TBD - notify application that watermark ack has been reset ??? if (watermark_active) { watermark_active = false; NormNodeTreeIterator iterator(acking_node_tree); - NormAckingNode* next; - while ((next = static_cast(iterator.GetNextNode()))) + NormAckingNode *next; + while ((next = static_cast(iterator.GetNextNode()))) next->ResetReqCount(GetTxRobustFactor()); - } + } } - } // end if (watermark_pending && !flush_timer.IsActive()) - + } // end if (watermark_pending && !flush_timer.IsActive()) + if (NULL != obj) { - NormObjectMsg* msg = (NormObjectMsg*)GetMessageFromPool(); + NormObjectMsg *msg = (NormObjectMsg *)GetMessageFromPool(); if (msg) { if (obj->NextSenderMsg(msg)) @@ -1272,11 +1287,11 @@ void NormSession::Serve() { double elapsed = probe_timer.GetInterval() - probe_timer.GetTimeRemaining(); double probeInterval = GetProbeInterval(); - if (elapsed > probeInterval) + if (elapsed > probeInterval) probe_timer.SetInterval(0.0); else probe_timer.SetInterval(probeInterval - elapsed); - probe_timer.Reschedule(); + probe_timer.Reschedule(); } } msg->SetDestination(address); @@ -1299,7 +1314,7 @@ void NormSession::Serve() { // Tell the app we would like to send more data ... posted_tx_queue_empty = true; - Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode *)NULL, (NormObject *)NULL); // (TBD) Was session deleted? } } @@ -1309,7 +1324,7 @@ void NormSession::Serve() ReturnMessageToPool(msg); if (obj->IsStream()) { - NormStreamObject* stream = static_cast(obj); + NormStreamObject *stream = static_cast(obj); if (stream->IsFlushPending() || stream->IsClosing()) { // Queue flush message @@ -1319,25 +1334,25 @@ void NormSession::Serve() { SenderQueueFlush(); } - else if (GetTxRobustFactor() == flush_count) + else if (GetTxRobustFactor() == flush_count) { - + PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender stream flush complete ...\n", - (unsigned long)LocalNodeId()); - Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode*)NULL, stream); + (unsigned long)LocalNodeId()); + Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode *)NULL, stream); flush_count++; - data_active = false; + data_active = false; if (stream->IsClosing()) { // If the stream just failed end-of-stream watermark flush, we don't // close the stream yet, but instead give app chance to reset watermark - bool watermarkFailed = watermarkJustCompleted && (obj->GetId() == watermark_object_id) && + bool watermarkFailed = watermarkJustCompleted && (obj->GetId() == watermark_object_id) && (ACK_FAILURE == SenderGetAckingStatus(NORM_NODE_ANY)); if (!watermarkFailed) { // end of stream was successfully acknowledged stream->Close(); - DeleteTxObject(stream, true); + DeleteTxObject(stream, true); obj = NULL; } } @@ -1346,18 +1361,18 @@ void NormSession::Serve() } //ASSERT(stream->IsPending() || stream->IsRepairPending() || stream->IsClosing()); if (!posted_tx_queue_empty && !stream->IsClosing() && stream->IsPending()) - // post if pending || !repair_timer.IsActive() || (repair_timer.GetRepeatCount() == 0) ??? + // post if pending || !repair_timer.IsActive() || (repair_timer.GetRepeatCount() == 0) ??? { //data_active = false; posted_tx_queue_empty = true; - Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, obj); + Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode *)NULL, obj); // (TBD) Was session deleted? return; } } else { - PLOG(PL_ERROR, "NormSession::Serve() pending non-stream obj, no message?.\n"); + PLOG(PL_ERROR, "NormSession::Serve() pending non-stream obj, no message?.\n"); //ASSERT(repair_timer.IsActive()); } } @@ -1365,7 +1380,7 @@ void NormSession::Serve() else { PLOG(PL_ERROR, "NormSession::Serve() node>%lu Warning! message_pool empty.\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } } else @@ -1374,34 +1389,34 @@ void NormSession::Serve() if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) { // Queue flush message - if (!tx_repair_pending) // don't queue flush if repair pending + if (!tx_repair_pending) // don't queue flush if repair pending SenderQueueFlush(); else PLOG(PL_DETAIL, "NormSession::Serve() node>%lu NORM_CMD(FLUSH) deferred by pending repairs ...\n", - (unsigned long)LocalNodeId()); - } + (unsigned long)LocalNodeId()); + } else if (GetTxRobustFactor() == flush_count) { - PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", - (unsigned long)LocalNodeId()); + PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", + (unsigned long)LocalNodeId()); Notify(NormController::TX_FLUSH_COMPLETED, - (NormSenderNode*)NULL, - (NormObject*)NULL); - flush_count++; + (NormSenderNode *)NULL, + (NormObject *)NULL); + flush_count++; data_active = false; - } + } } -} // end NormSession::Serve() +} // end NormSession::Serve() -bool NormSession::SenderSetWatermark(NormObjectId objectId, - NormBlockId blockId, +bool NormSession::SenderSetWatermark(NormObjectId objectId, + NormBlockId blockId, NormSegmentId segmentId, - bool overrideFlush, - const char* appAckReq, - unsigned int appAckReqLen) + bool overrideFlush, + const char *appAckReq, + unsigned int appAckReqLen) { PLOG(PL_DEBUG, "NormSession::SenderSetWatermark() watermark>%hu:%lu:%hu\n", - (UINT16)objectId, (unsigned long)blockId.GetValue(), (UINT16)segmentId); + (UINT16)objectId, (unsigned long)blockId.GetValue(), (UINT16)segmentId); watermark_flushes = overrideFlush; watermark_pending = true; watermark_active = false; @@ -1411,16 +1426,16 @@ bool NormSession::SenderSetWatermark(NormObjectId objectId, acking_success_count = 0; // Reset acking_node_list NormNodeTreeIterator iterator(acking_node_tree); - NormNode* next; + NormNode *next; int robustFactor = GetTxRobustFactor(); while ((next = iterator.GetNextNode())) - static_cast(next)->Reset(robustFactor); - + static_cast(next)->Reset(robustFactor); + if (NULL != appAckReq) { if (appAckReqLen != ack_ex_length) { - if (NULL != ack_ex_buffer) + if (NULL != ack_ex_buffer) { delete[] ack_ex_buffer; ack_ex_buffer = NULL; @@ -1451,16 +1466,16 @@ bool NormSession::SenderSetWatermark(NormObjectId objectId, } PromptSender(); return true; -} // end NormSession::SenderSetWatermark() +} // end NormSession::SenderSetWatermark() void NormSession::SenderResetWatermark() { NormNodeTreeIterator iterator(acking_node_tree); - NormNode* next; + NormNode *next; int robustFactor = GetTxRobustFactor(); while ((next = iterator.GetNextNode())) { - NormAckingNode* node = static_cast(next); + NormAckingNode *node = static_cast(next); if ((NORM_NODE_NONE == node->GetId()) || (!node->AckReceived())) { node->Reset(robustFactor); @@ -1469,16 +1484,16 @@ void NormSession::SenderResetWatermark() } } PromptSender(); -} // end NormSession::SenderResetWatermark() +} // end NormSession::SenderResetWatermark() void NormSession::SenderCancelWatermark() { watermark_pending = false; -} // end NormSession::SenderCancelWatermark() +} // end NormSession::SenderCancelWatermark() -NormAckingNode* NormSession::SenderAddAckingNode(NormNodeId nodeId, const ProtoAddress* srcAddress) +NormAckingNode *NormSession::SenderAddAckingNode(NormNodeId nodeId, const ProtoAddress *srcAddress) { - NormAckingNode* theNode = static_cast(acking_node_tree.FindNodeById(nodeId)); + NormAckingNode *theNode = static_cast(acking_node_tree.FindNodeById(nodeId)); if (NULL == theNode) { theNode = new NormAckingNode(*this, nodeId); @@ -1492,21 +1507,22 @@ NormAckingNode* NormSession::SenderAddAckingNode(NormNodeId nodeId, const ProtoA { PLOG(PL_ERROR, "NormSession::SenderAddAckingNode() new NormAckingNode error: %s\n", GetErrorString()); return NULL; - } + } } else { PLOG(PL_WARN, "NormSession::SenderAddAckingNode() warning: node already in list!?\n"); } - if (NULL != srcAddress) theNode->SetAddress(*srcAddress); + if (NULL != srcAddress) + theNode->SetAddress(*srcAddress); return theNode; -} // end NormSession::AddAckingNode(NormNodeId nodeId) +} // end NormSession::AddAckingNode(NormNodeId nodeId) void NormSession::SenderRemoveAckingNode(NormNodeId nodeId) { - NormAckingNode* theNode = - static_cast(acking_node_tree.FindNodeById(nodeId)); - if (NULL != theNode) + NormAckingNode *theNode = + static_cast(acking_node_tree.FindNodeById(nodeId)); + if (NULL != theNode) { acking_node_tree.DetachNode(theNode); theNode->Release(); @@ -1514,7 +1530,7 @@ void NormSession::SenderRemoveAckingNode(NormNodeId nodeId) // non-pending acker, can we immediately issue WATERMARK_COMPLETED? acking_node_count--; } -} // end NormSession::RemoveAckingNode() +} // end NormSession::RemoveAckingNode() NormSession::AckingStatus NormSession::SenderGetAckingStatus(NormNodeId nodeId) { @@ -1535,8 +1551,8 @@ NormSession::AckingStatus NormSession::SenderGetAckingStatus(NormNodeId nodeId) } else { - NormAckingNode* theNode = - static_cast(acking_node_tree.FindNodeById(nodeId)); + NormAckingNode *theNode = + static_cast(acking_node_tree.FindNodeById(nodeId)); if (NULL != theNode) { if (theNode->IsPending()) @@ -1551,23 +1567,22 @@ NormSession::AckingStatus NormSession::SenderGetAckingStatus(NormNodeId nodeId) else { return ACK_INVALID; - } - } -} // end NormSession::SenderGetAckingStatus() + } + } +} // end NormSession::SenderGetAckingStatus() - -bool NormSession::SenderGetNextAckingNode(NormNodeId& prevNodeId, AckingStatus* ackingStatus) +bool NormSession::SenderGetNextAckingNode(NormNodeId &prevNodeId, AckingStatus *ackingStatus) { - NormNode* prevNode = NULL; + NormNode *prevNode = NULL; if (NORM_NODE_NONE != prevNodeId) prevNode = acking_node_tree.FindNodeById(prevNodeId); NormNodeTreeIterator iterator(acking_node_tree, prevNode); - NormAckingNode* nextNode = static_cast(iterator.GetNextNode()); + NormAckingNode *nextNode = static_cast(iterator.GetNextNode()); // Note we skip NORM_NODE_NONE even though it may be in the tree // (This method only returns the id / status of _actual_ nodes) // TBD - we could return NORM_NODE_ANY as a proxy id for a NORM_NODE_NONE entry if ((NULL != nextNode) && (NORM_NODE_NONE == nextNode->GetId())) - nextNode = static_cast(iterator.GetNextNode()); + nextNode = static_cast(iterator.GetNextNode()); if (NULL != nextNode) { prevNodeId = nextNode->GetId(); @@ -1591,31 +1606,33 @@ bool NormSession::SenderGetNextAckingNode(NormNodeId& prevNodeId, AckingStatus* *ackingStatus = ACK_INVALID; return false; } -} // end NormSession::SenderGetNextAckingNode() +} // end NormSession::SenderGetNextAckingNode() -bool NormSession::SenderGetAckEx(NormNodeId nodeId, char* buffer, unsigned int* buflen) +bool NormSession::SenderGetAckEx(NormNodeId nodeId, char *buffer, unsigned int *buflen) { - NormAckingNode* theNode = - static_cast(acking_node_tree.FindNodeById(nodeId)); + NormAckingNode *theNode = + static_cast(acking_node_tree.FindNodeById(nodeId)); if (NULL != theNode) { return theNode->GetAckEx(buffer, buflen); } else { - if (NULL != buflen) *buflen = 0; + if (NULL != buflen) + *buflen = 0; return false; } -} // end NormSession::SenderGetAckEx() +} // end NormSession::SenderGetAckEx() bool NormSession::SenderQueueWatermarkFlush() { - if (flush_timer.IsActive()) return false; - NormCmdFlushMsg* flush = static_cast(GetMessageFromPool()); + if (flush_timer.IsActive()) + return false; + NormCmdFlushMsg *flush = static_cast(GetMessageFromPool()); if (flush) { flush->Init(); - + flush->SetDestination(address); flush->SetGrtt(grtt_quantized); flush->SetBackoffFactor((unsigned char)backoff_factor); @@ -1623,7 +1640,7 @@ bool NormSession::SenderQueueWatermarkFlush() flush->SetObjectId(watermark_object_id); // _Attempt_ to set the fec_payload_id source block length field appropriately UINT16 blockLen; - NormObject* obj = tx_table.Find(watermark_object_id); + NormObject *obj = tx_table.Find(watermark_object_id); if (NULL != obj) blockLen = obj->GetBlockSize(watermark_block_id); else if (watermark_segment_id < ndata) @@ -1631,7 +1648,7 @@ bool NormSession::SenderQueueWatermarkFlush() else blockLen = watermark_segment_id; flush->SetFecPayloadId(fec_id, watermark_block_id.GetValue(), watermark_segment_id, blockLen, fec_m); - + if (0 != ack_ex_length) { NormAppAckExtension ext; @@ -1639,18 +1656,18 @@ bool NormSession::SenderQueueWatermarkFlush() ext.SetContent(ack_ex_buffer, ack_ex_length); flush->PackExtension(ext); } - + NormNodeTreeIterator iterator(acking_node_tree); - NormAckingNode* next; + NormAckingNode *next; watermark_pending = false; - NormAckingNode* nodeNone = NULL; + NormAckingNode *nodeNone = NULL; acking_success_count = 0; - while (NULL != (next = static_cast(iterator.GetNextNode()))) + while (NULL != (next = static_cast(iterator.GetNextNode()))) { // Save NORM_NODE_NONE for last - if (NORM_NODE_NONE == next->GetId()) + if (NORM_NODE_NONE == next->GetId()) { - if (next->IsPending()) + if (next->IsPending()) nodeNone = next; else acking_success_count++; // implicit success for NORM_NODE_NONE @@ -1658,11 +1675,11 @@ bool NormSession::SenderQueueWatermarkFlush() } if (next->AckReceived()) { - acking_success_count++; // ACK was received for this node + acking_success_count++; // ACK was received for this node } else if (next->IsPending()) { - // Add node to list + // Add node to list if (flush->AppendAckingNode(next->GetId(), segment_size)) { next->DecrementReqCount(); @@ -1672,8 +1689,8 @@ bool NormSession::SenderQueueWatermarkFlush() { PLOG(PL_FATAL, "NormSession::ServeQueueWatermarkFlush() full cmd ...\n"); nodeNone = NULL; - break; - } + break; + } } } if (NULL != nodeNone) @@ -1690,22 +1707,22 @@ bool NormSession::SenderQueueWatermarkFlush() } if (watermark_pending) { - + // (TBD) we should increment the "flush_count" here only iff the watermark // corresponds to our "last_tx_object_id", etc //if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) // flush_count++; QueueMessage(flush); - PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu cmd queued ...\n", - (unsigned long)LocalNodeId()); + PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu cmd queued ...\n", + (unsigned long)LocalNodeId()); } else if (NULL != acking_node_tree.GetRoot()) { ReturnMessageToPool(flush); PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu watermark ack finished.\n", - (unsigned long)LocalNodeId()); - Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); - return false; + (unsigned long)LocalNodeId()); + Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode *)NULL, (NormObject *)NULL); + return false; } else { @@ -1717,19 +1734,20 @@ bool NormSession::SenderQueueWatermarkFlush() else { PLOG(PL_ERROR, "NormSession::SenderQueueWatermarkRequest() node>%lu message_pool exhausted! (couldn't req)\n", - (unsigned long)LocalNodeId()); - } - PLOG(PL_DEBUG, "NormSession::SenderQueueWatermarkFlush() starting flush timeout: %lf sec ....\n", 2*grtt_advertised); - flush_timer.SetInterval(2*grtt_advertised); + (unsigned long)LocalNodeId()); + } + PLOG(PL_DEBUG, "NormSession::SenderQueueWatermarkFlush() starting flush timeout: %lf sec ....\n", 2 * grtt_advertised); + flush_timer.SetInterval(2 * grtt_advertised); ActivateTimer(flush_timer); return true; -} // end NormSession::SenderQueueWatermarkFlush() - +} // end NormSession::SenderQueueWatermarkFlush() + void NormSession::SenderQueueFlush() { // (TBD) Don't enqueue a new flush if there is already one in our tx_queue! - if (flush_timer.IsActive()) return; - NormObject* obj = tx_table.Find(tx_table.RangeHi()); + if (flush_timer.IsActive()) + return; + NormObject *obj = tx_table.Find(tx_table.RangeHi()); NormObjectId objectId; NormBlockId blockId; NormSegmentId segmentId; @@ -1737,7 +1755,7 @@ void NormSession::SenderQueueFlush() { if (obj->IsStream()) { - NormStreamObject* stream = (NormStreamObject*)obj; + NormStreamObject *stream = (NormStreamObject *)obj; objectId = stream->GetId(); blockId = stream->FlushBlockId(); segmentId = stream->FlushSegmentId(); @@ -1748,7 +1766,7 @@ void NormSession::SenderQueueFlush() blockId = obj->GetFinalBlockId(); segmentId = obj->GetBlockSize(blockId) - 1; } - NormCmdFlushMsg* flush = (NormCmdFlushMsg*)GetMessageFromPool(); + NormCmdFlushMsg *flush = (NormCmdFlushMsg *)GetMessageFromPool(); if (flush) { flush->Init(); @@ -1757,19 +1775,19 @@ void NormSession::SenderQueueFlush() flush->SetBackoffFactor((unsigned char)backoff_factor); flush->SetGroupSize(gsize_quantized); flush->SetObjectId(objectId); - + flush->SetFecPayloadId(fec_id, blockId.GetValue(), segmentId, obj->GetBlockSize(blockId), fec_m); - + QueueMessage(flush); - if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) + if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) flush_count++; PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() node>%lu, flush queued (flush_count:%u)...\n", - (unsigned long)LocalNodeId(), flush_count); + (unsigned long)LocalNodeId(), flush_count); } else { PLOG(PL_ERROR, "NormSession::SenderQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } } else @@ -1783,34 +1801,34 @@ void NormSession::SenderQueueFlush() // if all tx object state is gone ... if (SenderQueueSquelch(next_tx_object_id)) { - if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) + if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) flush_count++; PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() node>%lu squelch queued (flush_count:%u)...\n", - (unsigned long)LocalNodeId(), flush_count); + (unsigned long)LocalNodeId(), flush_count); } else { PLOG(PL_ERROR, "NormSession::SenderQueueFlush() warning: node>%lu unable to queue squelch\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } } - PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() starting flush timeout: %lf sec ....\n", 2*grtt_advertised); - flush_timer.SetInterval(2*grtt_advertised); + PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() starting flush timeout: %lf sec ....\n", 2 * grtt_advertised); + flush_timer.SetInterval(2 * grtt_advertised); ActivateTimer(flush_timer); -} // end NormSession::SenderQueueFlush() +} // end NormSession::SenderQueueFlush() -bool NormSession::OnFlushTimeout(ProtoTimer& /*theTimer*/) +bool NormSession::OnFlushTimeout(ProtoTimer & /*theTimer*/) { PLOG(PL_DEBUG, "NormSession::OnFlushTimeout() deactivating flush_timer ....\n"); flush_timer.Deactivate(); PromptSender(); - return false; -} // NormSession::OnFlushTimeout() - -void NormSession::QueueMessage(NormMsg* msg) + return false; +} // NormSession::OnFlushTimeout() + +void NormSession::QueueMessage(NormMsg *msg) { -/* A little test jig + /* A little test jig static struct timeval lastTime = {0,0}; struct timeval currentTime; ProtoSystemTime(currentTime); @@ -1823,40 +1841,39 @@ void NormSession::QueueMessage(NormMsg* msg) } lastTime = currentTime; */ - // (TBD) if (0.0 == tx_rate), should we just dump the - // message rather than queueing it? - if (!tx_timer.IsActive() && (tx_rate > 0.0)) + // (TBD) if (0.0 == tx_rate), should we just dump the + // message rather than queueing it? + if (!tx_timer.IsActive() && (tx_rate > 0.0)) { tx_timer.SetInterval(0.0); - ActivateTimer(tx_timer); + ActivateTimer(tx_timer); } - if (NULL != msg) message_queue.Append(msg); -} // end NormSesssion::QueueMessage(NormMsg& msg) + if (NULL != msg) + message_queue.Append(msg); +} // end NormSesssion::QueueMessage(NormMsg& msg) - - -NormFileObject* NormSession::QueueTxFile(const char* path, - const char* infoPtr, - UINT16 infoLen) +NormFileObject *NormSession::QueueTxFile(const char *path, + const char *infoPtr, + UINT16 infoLen) { if (!IsSender()) { PLOG(PL_FATAL, "NormSession::QueueTxFile() Error: sender is closed\n"); return NULL; - } - NormFileObject* file = new NormFileObject(*this, (NormSenderNode*)NULL, next_tx_object_id); + } + NormFileObject *file = new NormFileObject(*this, (NormSenderNode *)NULL, next_tx_object_id); if (NULL == file) { PLOG(PL_FATAL, "NormSession::QueueTxFile() new file object error: %s\n", - GetErrorString()); - return NULL; + GetErrorString()); + return NULL; } if (!file->Open(path, infoPtr, infoLen)) { - PLOG(PL_FATAL, "NormSession::QueueTxFile() file open error\n"); - file->Release(); - return NULL; - } + PLOG(PL_FATAL, "NormSession::QueueTxFile() file open error\n"); + file->Release(); + return NULL; + } if (QueueTxObject(file)) { return file; @@ -1867,31 +1884,31 @@ NormFileObject* NormSession::QueueTxFile(const char* path, file->Release(); return NULL; } -} // end NormSession::QueueTxFile() +} // end NormSession::QueueTxFile() -NormDataObject* NormSession::QueueTxData(const char* dataPtr, - UINT32 dataLen, - const char* infoPtr, - UINT16 infoLen) +NormDataObject *NormSession::QueueTxData(const char *dataPtr, + UINT32 dataLen, + const char *infoPtr, + UINT16 infoLen) { if (!IsSender()) { PLOG(PL_FATAL, "NormSession::QueueTxData() Error: sender is closed\n"); return NULL; - } - NormDataObject* obj = new NormDataObject(*this, (NormSenderNode*)NULL, next_tx_object_id, session_mgr.GetDataFreeFunction()); + } + NormDataObject *obj = new NormDataObject(*this, (NormSenderNode *)NULL, next_tx_object_id, session_mgr.GetDataFreeFunction()); if (!obj) { PLOG(PL_FATAL, "NormSession::QueueTxData() new data object error: %s\n", - GetErrorString()); - return NULL; + GetErrorString()); + return NULL; } - if (!obj->Open((char*)dataPtr, dataLen, false, infoPtr, infoLen)) + if (!obj->Open((char *)dataPtr, dataLen, false, infoPtr, infoLen)) { - PLOG(PL_FATAL, "NormSession::QueueTxData() object open error\n"); - obj->Release(); - return NULL; - } + PLOG(PL_FATAL, "NormSession::QueueTxData() object open error\n"); + obj->Release(); + return NULL; + } if (QueueTxObject(obj)) { return obj; @@ -1902,25 +1919,24 @@ NormDataObject* NormSession::QueueTxData(const char* dataPtr, obj->Release(); return NULL; } -} // end NormSession::QueueTxData() +} // end NormSession::QueueTxData() - -NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, - bool doubleBuffer, - const char* infoPtr, - UINT16 infoLen) +NormStreamObject *NormSession::QueueTxStream(UINT32 bufferSize, + bool doubleBuffer, + const char *infoPtr, + UINT16 infoLen) { if (!IsSender()) { PLOG(PL_FATAL, "NormSession::QueueTxStream() Error: sender is closed\n"); return NULL; - } - NormStreamObject* stream = new NormStreamObject(*this, (NormSenderNode*)NULL, next_tx_object_id); + } + NormStreamObject *stream = new NormStreamObject(*this, (NormSenderNode *)NULL, next_tx_object_id); if (!stream) { PLOG(PL_FATAL, "NormSession::QueueTxStream() new stream object error: %s\n", - GetErrorString()); - return NULL; + GetErrorString()); + return NULL; } if (!stream->Open(bufferSize, doubleBuffer, infoPtr, infoLen)) { @@ -1940,25 +1956,24 @@ NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, stream->Release(); return NULL; } -} // end NormSession::QueueTxStream() - +} // end NormSession::QueueTxStream() #ifdef SIMULATE -NormSimObject* NormSession::QueueTxSim(unsigned long objectSize) +NormSimObject *NormSession::QueueTxSim(unsigned long objectSize) { if (!IsSender()) { PLOG(PL_FATAL, "NormSession::QueueTxSim() Error: sender is closed\n"); return NULL; } - NormSimObject* simObject = new NormSimObject(*this, NULL, next_tx_object_id); + NormSimObject *simObject = new NormSimObject(*this, NULL, next_tx_object_id); if (!simObject) { PLOG(PL_FATAL, "NormSession::QueueTxSim() new sim object error: %s\n", - GetErrorString()); - return NULL; - } - + GetErrorString()); + return NULL; + } + if (!simObject->Open(objectSize)) { PLOG(PL_FATAL, "NormSession::QueueTxSim() open error\n"); @@ -1974,23 +1989,23 @@ NormSimObject* NormSession::QueueTxSim(unsigned long objectSize) simObject->Release(); return NULL; } -} // end NormSession::QueueTxSim() +} // end NormSession::QueueTxSim() #endif // SIMULATE -bool NormSession::QueueTxObject(NormObject* obj) +bool NormSession::QueueTxObject(NormObject *obj) { if (!IsSender()) { PLOG(PL_FATAL, "NormSession::QueueTxObject() non-sender session error!?\n"); return false; } - + if (preset_fti.IsValid() && (obj->GetSize() != preset_fti.GetObjectSize())) { PLOG(PL_FATAL, "NormSession::QueueTxObject() preset object info mismatch!\n"); return false; } - + // Manage tx_table min/max count and max size bounds // Depending on tx cache bounds _and_ what has been // enqueued/dequeued, we may need to prune the @@ -2007,21 +2022,22 @@ bool NormSession::QueueTxObject(NormObject* obj) ((newCount > tx_cache_count_max) || ((tx_table.GetSize() + obj->GetSize()) > tx_cache_size_max)))) { - // Remove oldest non-pending - NormObject* oldest = tx_table.Find(tx_table.RangeLo()); + // Remove oldest non-pending + NormObject *oldest = tx_table.Find(tx_table.RangeLo()); if (oldest->IsRepairPending() || oldest->IsPending()) { PLOG(PL_ALWAYS, "NormSession::QueueTxObject() all held objects repair pending:%d (repair active:%d) pending:%d\n", - oldest->IsRepairPending(), repair_timer.IsActive(), oldest->IsPending()); + oldest->IsRepairPending(), repair_timer.IsActive(), oldest->IsPending()); posted_tx_queue_empty = false; return false; } - else + else { double delay = GetFlowControlDelay() - oldest->GetNackAge(); if (delay < 1.0e-06) { - if (FlowControlIsActive()) DeactivateFlowControl(); + if (FlowControlIsActive()) + DeactivateFlowControl(); DeleteTxObject(oldest, true); } else @@ -2032,9 +2048,9 @@ bool NormSession::QueueTxObject(NormObject* obj) posted_tx_queue_empty = false; return false; } - } - newCount = tx_table.GetCount() + 1; - } + } + newCount = tx_table.GetCount() + 1; + } // Attempt to queue the object (note it gets "retained" by the tx_table) if (!tx_table.Insert(obj)) { @@ -2047,16 +2063,16 @@ bool NormSession::QueueTxObject(NormObject* obj) next_tx_object_id++; TouchSender(); return true; -} // end NormSession::QueueTxObject() +} // end NormSession::QueueTxObject() -bool NormSession::RequeueTxObject(NormObject* obj) +bool NormSession::RequeueTxObject(NormObject *obj) { ASSERT(NULL != obj); if (obj->IsStream()) { // (TBD) allow buffered stream to be reset? PLOG(PL_FATAL, "NormSession::RequeueTxObject() error: can't requeue NORM_OBJECT_STREAM\n"); - return false; + return false; } NormObjectId objectId = obj->GetId(); if (tx_table.Find(objectId) == obj) @@ -2066,7 +2082,7 @@ bool NormSession::RequeueTxObject(NormObject* obj) obj->TxReset(0, true); TouchSender(); return true; - } + } else { PLOG(PL_FATAL, "NormSession::RequeueTxObject() error: couldn't set object as pending\n"); @@ -2078,36 +2094,38 @@ bool NormSession::RequeueTxObject(NormObject* obj) PLOG(PL_FATAL, "NormSession::RequeueTxObject() error: couldn't find object\n"); return false; } -} // end NormSession::RequeueTxObject() +} // end NormSession::RequeueTxObject() -void NormSession::DeleteTxObject(NormObject* obj, bool notify) +void NormSession::DeleteTxObject(NormObject *obj, bool notify) { ASSERT(NULL != obj); if (tx_table.Remove(obj)) { - Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, obj); + Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode *)NULL, obj); NormObjectId objectId = obj->GetId(); tx_pending_mask.Unset(objectId); tx_repair_mask.Unset(objectId); obj->Close(); obj->Release(); } -} // end NormSession::DeleteTxObject() +} // end NormSession::DeleteTxObject() -bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, - unsigned long countMin, - unsigned long countMax) +bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, + unsigned long countMin, + unsigned long countMax) { bool result = true; tx_cache_size_max = sizeMax; tx_cache_count_min = (unsigned int)((countMin < countMax) ? countMin : countMax); - if (tx_cache_count_min < 1) tx_cache_count_min = 1; + if (tx_cache_count_min < 1) + tx_cache_count_min = 1; tx_cache_count_max = (unsigned int)((countMax > countMin) ? countMax : countMin); - if (tx_cache_count_max < 1) tx_cache_count_max = 1; - - tx_cache_count_min &= 0x00007fff; // limited to one-half of 16-bit NormObjectId space + if (tx_cache_count_max < 1) + tx_cache_count_max = 1; + + tx_cache_count_min &= 0x00007fff; // limited to one-half of 16-bit NormObjectId space tx_cache_count_max &= 0x00007fff; - + if (IsSender()) { // Trim/resize the tx_table and tx masks as needed @@ -2117,7 +2135,7 @@ bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, (tx_table.GetSize() > tx_cache_size_max))) { // Remove oldest (hopefully non-pending ) object - NormObject* oldest = tx_table.Find(tx_table.RangeLo()); + NormObject *oldest = tx_table.Find(tx_table.RangeLo()); ASSERT(NULL != oldest); DeleteTxObject(oldest, true); count = tx_table.GetCount(); @@ -2135,7 +2153,7 @@ bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, { countMax = tx_pending_mask.GetSize(); if (tx_repair_mask.GetSize() < countMax) - countMax = tx_repair_mask.GetSize(); + countMax = tx_repair_mask.GetSize(); if (tx_cache_count_max > countMax) tx_cache_count_max = (unsigned int)countMax; if (tx_cache_count_min > tx_cache_count_max) @@ -2144,25 +2162,25 @@ bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, } } return result; -} // end NormSession::SetTxCacheBounds() +} // end NormSession::SetTxCacheBounds() -NormBlock* NormSession::SenderGetFreeBlock(NormObjectId objectId, - NormBlockId blockId) +NormBlock *NormSession::SenderGetFreeBlock(NormObjectId objectId, + NormBlockId blockId) { // First, try to get one from our block pool - NormBlock* b = block_pool.Get(); + NormBlock *b = block_pool.Get(); // Second, try to steal oldest non-pending block if (!b) { NormObjectTable::Iterator iterator(tx_table); - NormObject* obj; + NormObject *obj; while ((obj = iterator.GetNextObject())) - { + { if (obj->GetId() == objectId) b = obj->StealNonPendingBlock(true, blockId); else b = obj->StealNonPendingBlock(false); - if (b) + if (b) { b->EmptyToPool(segment_pool); break; @@ -2174,45 +2192,45 @@ NormBlock* NormSession::SenderGetFreeBlock(NormObjectId objectId, { // reverse iteration to find newest object with resources NormObjectTable::Iterator iterator(tx_table); - NormObject* obj; + NormObject *obj; while ((obj = iterator.GetPrevObject())) { - if (obj->GetId() < objectId) + if (obj->GetId() < objectId) { break; } else { if (obj->GetId() > objectId) - b = obj->StealNewestBlock(false); - else + b = obj->StealNewestBlock(false); + else b = obj->StealNewestBlock(true, blockId); - if (b) + if (b) { b->EmptyToPool(segment_pool); break; } } - } + } } return b; -} // end NormSession::SenderGetFreeBlock() +} // end NormSession::SenderGetFreeBlock() -char* NormSession::SenderGetFreeSegment(NormObjectId objectId, - NormBlockId blockId) +char *NormSession::SenderGetFreeSegment(NormObjectId objectId, + NormBlockId blockId) { while (segment_pool.IsEmpty()) - { - NormBlock* b = SenderGetFreeBlock(objectId, blockId); + { + NormBlock *b = SenderGetFreeBlock(objectId, blockId); if (b) block_pool.Put(b); else return NULL; } return segment_pool.Get(); -} // end NormSession::SenderGetFreeSegment() +} // end NormSession::SenderGetFreeSegment() -void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket, +void NormSession::TxSocketRecvHandler(ProtoSocket &theSocket, ProtoSocket::Event theEvent) { if (ProtoSocket::RECV == theEvent) @@ -2221,12 +2239,13 @@ void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket, unsigned int msgLength = NormMsg::MAX_SIZE; while (true) { - - if (theSocket.RecvFrom(msg.AccessBuffer(), - msgLength, + + if (theSocket.RecvFrom(msg.AccessBuffer(), + msgLength, msg.AccessAddress())) { - if (0 == msgLength) break; // no more data to read + if (0 == msgLength) + break; // no more data to read if (msg.InitFromBuffer(msgLength)) { // Since it arrived on the tx_socket, we know it was unicast @@ -2235,7 +2254,7 @@ void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket, } else { - PLOG(PL_ERROR, "NormSession::TxSocketRecvHandler() warning: received bad message\n"); + PLOG(PL_ERROR, "NormSession::TxSocketRecvHandler() warning: received bad message\n"); } } else @@ -2255,13 +2274,15 @@ void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket, { // This is a little cheesy, but ... theSocket.StopOutputNotification(); - if (tx_timer.IsActive()) tx_timer.Deactivate(); + if (tx_timer.IsActive()) + tx_timer.Deactivate(); if (OnTxTimeout(tx_timer)) { - if (!tx_timer.IsActive()) ActivateTimer(tx_timer); + if (!tx_timer.IsActive()) + ActivateTimer(tx_timer); } } -} // end NormSession::TxSocketRecvHandler() +} // end NormSession::TxSocketRecvHandler() //#define RX_MEASURE_ONLY #ifdef RX_MEASURE_ONLY @@ -2274,7 +2295,7 @@ UINT16 rxMeasureSeqPrev = 0; int rxMeasureGapMax = 0; #endif // RX_MEASURE_ONLY -void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, +void NormSession::RxSocketRecvHandler(ProtoSocket &theSocket, ProtoSocket::Event theEvent) { if (ProtoSocket::RECV == theEvent) @@ -2284,13 +2305,14 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, unsigned int msgLength = NormMsg::MAX_SIZE; while (true) { - ProtoAddress destAddr; // we get the pkt destAddr to determine unicast/multicast + ProtoAddress destAddr; // we get the pkt destAddr to determine unicast/multicast if (theSocket.RecvFrom(msg.AccessBuffer(), - msgLength, + msgLength, msg.AccessAddress(), destAddr)) { - if (0 == msgLength) break; + if (0 == msgLength) + break; if (msg.InitFromBuffer(msgLength)) { #ifdef RX_MEASURE_ONLY @@ -2305,13 +2327,13 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, rxMeasurePktCount = rxMeasurePktTotal = 1; rxMeasureByteTotal = msgLength; rxMeasureInit = false; - return; + return; } int seqDelta = (int)seq - (int)rxMeasureSeqPrev; ASSERT(seqDelta > 0); - rxMeasurePktTotal += seqDelta; // total should have received. - rxMeasurePktCount++; // total actually received + rxMeasurePktTotal += seqDelta; // total should have received. + rxMeasurePktCount++; // total actually received rxMeasureByteTotal += msgLength; if (seqDelta > rxMeasureGapMax) @@ -2324,7 +2346,7 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, deltaSec += 1.0e-06 * (double)(currentTime.tv_usec - rxMeasureRefTime.tv_usec); else deltaSec -= 1.0e-06 * (double)(rxMeasureRefTime.tv_usec - currentTime.tv_usec); - double rxRate = (8.0/1000.0) * (double)rxMeasureByteTotal / (double)deltaSec; + double rxRate = (8.0 / 1000.0) * (double)rxMeasureByteTotal / (double)deltaSec; double rxLoss = 100.0 * (1.0 - (double)rxMeasurePktCount / (double)rxMeasurePktTotal); rxMeasureRefTime = currentTime; @@ -2332,12 +2354,12 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, } rxMeasureSeqPrev = seq; return; -#endif // RX_MEASURE_ONLY +#endif // RX_MEASURE_ONLY bool ecnStatus = false; #ifdef SIMULATE ecnStatus = theSocket.GetEcnStatus(); //if (ecnStatus) TRACE("NORM RECEIVED PACKET W/ ECN BIT SET!!!!!\n"); -#endif // SIMULATE +#endif // SIMULATE bool wasUnicast; if (destAddr.IsValid()) wasUnicast = destAddr.IsUnicast(); @@ -2348,7 +2370,7 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, } else { - PLOG(PL_ERROR, "NormSession::RxSocketRecvHandler() warning: received bad message\n"); + PLOG(PL_ERROR, "NormSession::RxSocketRecvHandler() warning: received bad message\n"); } // If our system gets very busy reading sockets, we should occasionally // execute any timeouts to keep protocol operation smooth (i.e., sending feedback) @@ -2361,13 +2383,13 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, } } else - { + { TRACE("NormSession::RxSocketRecvHandler() RecvFrom error address:%s (unicast:%d)\n", Address().GetHostString(), Address().IsUnicast()); // Probably an ICMP "port unreachable" error // Note we purposefull do _not_ set the "posted_send_error" // status here because we do not want this notification // cleared due to SEND_OK status since it's receiver driven - if (Address().IsUnicast()) + if (Address().IsUnicast()) { TRACE("posting send error ...\n"); Notify(NormController::SEND_ERROR, NULL, NULL); @@ -2380,22 +2402,24 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, { // This is a little cheesy, but ... theSocket.StopOutputNotification(); - if (tx_timer.IsActive()) tx_timer.Deactivate(); + if (tx_timer.IsActive()) + tx_timer.Deactivate(); if (OnTxTimeout(tx_timer)) { - if (!tx_timer.IsActive()) ActivateTimer(tx_timer); + if (!tx_timer.IsActive()) + ActivateTimer(tx_timer); } - } // end if/else (theEvent == RECV/SEND) -} // end NormSession::RxSocketRecvHandler() - + } // end if/else (theEvent == RECV/SEND) +} // end NormSession::RxSocketRecvHandler() #ifndef SIMULATE -void NormSession::OnPktCapture(ProtoChannel& theChannel, - ProtoChannel::Notification notifyType) +void NormSession::OnPktCapture(ProtoChannel &theChannel, + ProtoChannel::Notification notifyType) { // We only care about NOTIFY_INPUT events (all we should get anyway) - if (ProtoChannel::NOTIFY_INPUT != notifyType) return; - while(1) + if (ProtoChannel::NOTIFY_INPUT != notifyType) + return; + while (1) { ProtoCap::Direction direction; // Note: We offset the buffer by 2 bytes since Ethernet header is 14 bytes @@ -2403,22 +2427,23 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, // This gives us a properly aligned buffer for 32-bit aligned IP packets // (The 256*sizeof(UINT32) bytes are for potential "smfPkt" message header use) const int BUFFER_MAX = 4096; - UINT32 alignedBuffer[BUFFER_MAX/sizeof(UINT32)]; - UINT16* ethBuffer = ((UINT16*)alignedBuffer) + 1; // offset by 2-bytes so IP content is 32-bit aligned - UINT32* ipBuffer = alignedBuffer + 4; // offset by ETHER header size + 2 bytes - unsigned int numBytes = (sizeof(UINT32) * (BUFFER_MAX/sizeof(UINT32))) - 2; - - ProtoCap& cap = static_cast(theChannel); - - if (!cap.Recv((char*)ethBuffer, numBytes, &direction)) + UINT32 alignedBuffer[BUFFER_MAX / sizeof(UINT32)]; + UINT16 *ethBuffer = ((UINT16 *)alignedBuffer) + 1; // offset by 2-bytes so IP content is 32-bit aligned + UINT32 *ipBuffer = alignedBuffer + 4; // offset by ETHER header size + 2 bytes + unsigned int numBytes = (sizeof(UINT32) * (BUFFER_MAX / sizeof(UINT32))) - 2; + + ProtoCap &cap = static_cast(theChannel); + + if (!cap.Recv((char *)ethBuffer, numBytes, &direction)) { - PLOG(PL_ERROR, "NormSession::OnPktCapture() ProtoCap::Recv() error\n"); - break; + PLOG(PL_ERROR, "NormSession::OnPktCapture() ProtoCap::Recv() error\n"); + break; } - if (numBytes == 0) break; // no more packets to receive - + if (numBytes == 0) + break; // no more packets to receive + // Map ProtoPktETH instance into buffer and init for processing - ProtoPktETH ethPkt((UINT32*)((void*)ethBuffer), BUFFER_MAX - 2); + ProtoPktETH ethPkt((UINT32 *)((void *)ethBuffer), BUFFER_MAX - 2); if (!ethPkt.InitFromBuffer(numBytes)) { PLOG(PL_ERROR, "NormSession::OnPktCapture() error: bad Ether frame\n"); @@ -2426,8 +2451,8 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, } // Only process IP packets (skip others) UINT16 ethType = ethPkt.GetType(); - if ((ethType != 0x0800) && (ethType != 0x86dd)) - continue; // go read next packet + if ((ethType != 0x0800) && (ethType != 0x86dd)) + continue; // go read next packet // Map ProtoPktIP instance into buffer and init for processing. ProtoPktIP ipPkt(ipBuffer, BUFFER_MAX - 16); if (!ipPkt.InitFromBuffer(ethPkt.GetPayloadLength())) @@ -2435,35 +2460,35 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, PLOG(PL_ERROR, "NormSession::OnPktCapture() error: bad IP packet\n"); continue; } - + // Does this packet match any of our valid destination addrs? ProtoAddress dstIp; ProtoAddress srcIp; ProtoSocket::EcnStatus ecnStatus = ProtoSocket::ECN_NONE; switch (ipPkt.GetVersion()) { - case 4: - { - ProtoPktIPv4 ip4Pkt(ipPkt); - ip4Pkt.GetDstAddr(dstIp); - ip4Pkt.GetSrcAddr(srcIp); - ecnStatus = (ProtoSocket::EcnStatus)(ip4Pkt.GetTOS() & ProtoSocket::ECN_CE); - break; - } - case 6: - { - ProtoPktIPv6 ip6Pkt(ipPkt); - ip6Pkt.GetDstAddr(dstIp); - ip6Pkt.GetSrcAddr(srcIp); - ecnStatus = (ProtoSocket::EcnStatus)(ip6Pkt.GetTrafficClass() & ProtoSocket::ECN_CE); - break; - } - default: - PLOG(PL_ERROR, "NormSession::OnPktCapture() error: recvd IP packet w/ bad version number\n"); - continue; // go read next packet + case 4: + { + ProtoPktIPv4 ip4Pkt(ipPkt); + ip4Pkt.GetDstAddr(dstIp); + ip4Pkt.GetSrcAddr(srcIp); + ecnStatus = (ProtoSocket::EcnStatus)(ip4Pkt.GetTOS() & ProtoSocket::ECN_CE); + break; + } + case 6: + { + ProtoPktIPv6 ip6Pkt(ipPkt); + ip6Pkt.GetDstAddr(dstIp); + ip6Pkt.GetSrcAddr(srcIp); + ecnStatus = (ProtoSocket::EcnStatus)(ip6Pkt.GetTrafficClass() & ProtoSocket::ECN_CE); + break; + } + default: + PLOG(PL_ERROR, "NormSession::OnPktCapture() error: recvd IP packet w/ bad version number\n"); + continue; // go read next packet } if (!dst_addr_list.Contains(dstIp)) - continue; // not a matching dst addr, go read next packet + continue; // not a matching dst addr, go read next packet // Is this a UDP packet for our session dst port? int dstPort = -1; ProtoPktUDP udpPkt; @@ -2471,7 +2496,7 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, dstPort = udpPkt.GetDstPort(); //if (dstPort != address.GetPort()) if (dstPort != rx_socket.GetPort()) - continue; // not a UDP packet for our session, go read next packet + continue; // not a UDP packet for our session, go read next packet // If our rx_socket is "connected", make sure source addr/port matches srcIp.SetPort(udpPkt.GetSrcPort()); // if socket is connected, validate that the packet's from the specified source addr @@ -2479,34 +2504,35 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, { if (0 != rx_connect_addr.GetPort()) { - // check host addr component only for match - if (!rx_connect_addr.HostIsEqual(srcIp)) continue; - - + // check host addr component only for match + if (!rx_connect_addr.HostIsEqual(srcIp)) + continue; } else { // check for addr _and_ port match - if (!rx_connect_addr.IsEqual(srcIp)) continue; + if (!rx_connect_addr.IsEqual(srcIp)) + continue; } } // if we are using SSM multicast make sure it's the right source addr - if (ssm_source_addr.IsValid() && !ssm_source_addr.HostIsEqual(srcIp)) continue; - + if (ssm_source_addr.IsValid() && !ssm_source_addr.HostIsEqual(srcIp)) + continue; + // IMPORTANT NOTE: We ignore the checksum for OUTBOUND packets since these // are often computed by the Ethernet hardware these days if ((ProtoCap::INBOUND == direction) && !udpPkt.ChecksumIsValid(ipPkt)) { - PLOG(PL_WARN, "NormSession::OnPktCapture() error: recvd UDP packet w/ bad checksum: %04x (computed: %04x)\n", - (UINT16)udpPkt.GetChecksum(), udpPkt.ComputeChecksum(ipPkt)); - continue; // go read next packet + PLOG(PL_WARN, "NormSession::OnPktCapture() error: recvd UDP packet w/ bad checksum: %04x (computed: %04x)\n", + (UINT16)udpPkt.GetChecksum(), udpPkt.ComputeChecksum(ipPkt)); + continue; // go read next packet } - + // TBD - we can avoid this copy NormMsg msg; - if (msg.CopyFromBuffer((const char*)udpPkt.GetPayload(), udpPkt.GetPayloadLength())) + if (msg.CopyFromBuffer((const char *)udpPkt.GetPayload(), udpPkt.GetPayloadLength())) { - + msg.AccessAddress() = srcIp; HandleReceiveMessage(msg, dstIp.IsUnicast(), (ProtoSocket::ECN_CE == ecnStatus)); } @@ -2514,52 +2540,49 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, { PLOG(PL_WARN, "NormSession::OnPktCapture() error: recvd bad NORM packet?!\n"); } - } // end while(1) -} // end NormSession::OnPktCapture() + } // end while(1) +} // end NormSession::OnPktCapture() #endif // !SIMULATE // TBD - move this to its own cpp file??? -void NormTrace(const struct timeval& currentTime, - NormNodeId localId, - const NormMsg& msg, - bool sent, - UINT8 fecM, - UINT16 instId) +void NormTrace(const struct timeval ¤tTime, + NormNodeId localId, + const NormMsg &msg, + bool sent, + UINT8 fecM, + UINT16 instId) { - static const char* MSG_NAME[] = - { - "INVALID", - "INFO", - "DATA", - "CMD", - "NACK", - "ACK", - "REPORT" - }; - static const char* CMD_NAME[] = - { - "CMD(INVALID)", - "CMD(FLUSH)", - "CMD(EOT)", - "CMD(SQUELCH)", - "CMD(CC)", - "CMD(REPAIR_ADV)", - "CMD(ACK_REQ)", - "CMD(APP)" - }; - static const char* REQ_NAME[] = - { - "INVALID", - "WATERMARK", - "RTT", - "APP" - }; - + static const char *MSG_NAME[] = + { + "INVALID", + "INFO", + "DATA", + "CMD", + "NACK", + "ACK", + "REPORT"}; + static const char *CMD_NAME[] = + { + "CMD(INVALID)", + "CMD(FLUSH)", + "CMD(EOT)", + "CMD(SQUELCH)", + "CMD(CC)", + "CMD(REPAIR_ADV)", + "CMD(ACK_REQ)", + "CMD(APP)"}; + static const char *REQ_NAME[] = + { + "INVALID", + "WATERMARK", + "RTT", + "APP"}; + NormMsg::Type msgType = msg.GetType(); UINT16 length = msg.GetLength(); - const char* status = sent ? "dst" : "src"; - const ProtoAddress& addr = sent ? msg.GetDestination() : msg.GetSource(); - + const char *status = sent ? "dst" : "src"; + const ProtoAddress &addr = sent ? msg.GetDestination() : msg.GetSource(); + UINT16 seq = msg.GetSequence(); #ifdef _WIN32_WCE @@ -2567,44 +2590,44 @@ void NormTrace(const struct timeval& currentTime, timeStruct.tm_hour = currentTime.tv_sec / 3600; unsigned long hourSecs = 3600 * timeStruct.tm_hour; timeStruct.tm_min = (currentTime.tv_sec - (hourSecs)) / 60; - timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60*timeStruct.tm_min); + timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60 * timeStruct.tm_min); timeStruct.tm_hour = timeStruct.tm_hour % 24; - struct tm* ct = &timeStruct; -#else + struct tm *ct = &timeStruct; +#else time_t secs = (time_t)currentTime.tv_sec; - struct tm* ct = gmtime(&secs); + struct tm *ct = gmtime(&secs); #endif // if/else _WIN32_WCE PLOG(PL_ALWAYS, "trace>%02d:%02d:%02d.%06lu ", - (int)ct->tm_hour, (int)ct->tm_min, (int)ct->tm_sec, (unsigned int)currentTime.tv_usec); + (int)ct->tm_hour, (int)ct->tm_min, (int)ct->tm_sec, (unsigned int)currentTime.tv_usec); PLOG(PL_ALWAYS, "node>%lu %s>%s/%hu ", (unsigned long)localId, status, addr.GetHostString(), addr.GetPort()); - + bool clrFlag = false; switch (msgType) { - case NormMsg::INFO: + case NormMsg::INFO: + { + const NormInfoMsg &info = (const NormInfoMsg &)msg; + PLOG(PL_ALWAYS, "inst>%hu seq>%hu INFO obj>%hu ", + info.GetInstanceId(), seq, (UINT16)info.GetObjectId()); + break; + } + case NormMsg::DATA: + { + const NormDataMsg &data = (const NormDataMsg &)msg; + PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%lu seg>%hu ", + data.GetInstanceId(), + seq, + //data.IsData() ? "DATA" : "PRTY", + (UINT16)data.GetObjectId(), + (unsigned long)data.GetFecBlockId(fecM).GetValue(), + (UINT16)data.GetFecSymbolId(fecM)); + + if (data.IsStream()) { - const NormInfoMsg& info = (const NormInfoMsg&)msg; - PLOG(PL_ALWAYS, "inst>%hu seq>%hu INFO obj>%hu ", - info.GetInstanceId(), seq, (UINT16)info.GetObjectId()); - break; - } - case NormMsg::DATA: - { - const NormDataMsg& data = (const NormDataMsg&)msg; - PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%lu seg>%hu ", - data.GetInstanceId(), - seq, - //data.IsData() ? "DATA" : "PRTY", - (UINT16)data.GetObjectId(), - (unsigned long)data.GetFecBlockId(fecM).GetValue(), - (UINT16)data.GetFecSymbolId(fecM)); - - if (data.IsStream()) - { - UINT32 offset = NormDataMsg::ReadStreamPayloadOffset(data.GetPayload()); - PLOG(PL_ALWAYS, "offset>%lu ", (unsigned long)offset); - - /*if (data.GetFecSymbolId(fecM) < 32) + UINT32 offset = NormDataMsg::ReadStreamPayloadOffset(data.GetPayload()); + PLOG(PL_ALWAYS, "offset>%lu ", (unsigned long)offset); + + /*if (data.GetFecSymbolId(fecM) < 32) { //if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_MSG_START)) UINT16 msgStartOffset = NormDataMsg::ReadStreamPayloadMsgStart(data.GetPayload()); @@ -2617,156 +2640,155 @@ void NormTrace(const struct timeval& currentTime, PLOG(PL_ALWAYS, "(stream end) "); } */ - } + } + break; + } + case NormMsg::CMD: + { + const NormCmdMsg &cmd = static_cast(msg); + NormCmdMsg::Flavor flavor = cmd.GetFlavor(); + PLOG(PL_ALWAYS, "inst>%hu seq>%hu %s ", cmd.GetInstanceId(), seq, CMD_NAME[flavor]); + switch (flavor) + { + case NormCmdMsg::ACK_REQ: + { + int index = ((const NormCmdAckReqMsg &)msg).GetAckType(); + index = MIN(index, 3); + PLOG(PL_ALWAYS, "(%s) ", REQ_NAME[index]); break; } - case NormMsg::CMD: + case NormCmdMsg::SQUELCH: { - const NormCmdMsg& cmd = static_cast(msg); - NormCmdMsg::Flavor flavor = cmd.GetFlavor(); - PLOG(PL_ALWAYS, "inst>%hu seq>%hu %s ", cmd.GetInstanceId(), seq, CMD_NAME[flavor]); - switch (flavor) - { - case NormCmdMsg::ACK_REQ: - { - int index = ((const NormCmdAckReqMsg&)msg).GetAckType(); - index = MIN(index, 3); - PLOG(PL_ALWAYS, "(%s) ", REQ_NAME[index]); - break; - } - case NormCmdMsg::SQUELCH: - { - const NormCmdSquelchMsg& squelch = - static_cast(msg); - PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", - (UINT16)squelch.GetObjectId(), - (unsigned long)squelch.GetFecBlockId(fecM).GetValue(), - (UINT16)squelch.GetFecSymbolId(fecM)); - break; - } - case NormCmdMsg::FLUSH: - { - const NormCmdFlushMsg& flush = - static_cast(msg); - PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", - (UINT16)flush.GetObjectId(), - (unsigned long)flush.GetFecBlockId(fecM).GetValue(), - (UINT16)flush.GetFecSymbolId(fecM)); - - if (0 != flush.GetAckingNodeCount()) - PLOG(PL_ALWAYS, "(WATERMARK) "); // ACK requested - break; - } - case NormCmdMsg::CC: - { - const NormCmdCCMsg& cc = static_cast(msg); - PLOG(PL_ALWAYS, " seq>%u ", cc.GetCCSequence()); - NormHeaderExtension ext; - while (cc.GetNextExtension(ext)) - { - if (NormHeaderExtension::CC_RATE == ext.GetType()) - { - UINT16 sendRate = ((NormCCRateExtension&)ext).GetSendRate(); - PLOG(PL_ALWAYS, " rate>%f ", 8.0e-03 * NormUnquantizeRate(sendRate)); - break; - } - } - break; - } - default: - break; - } + const NormCmdSquelchMsg &squelch = + static_cast(msg); + PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", + (UINT16)squelch.GetObjectId(), + (unsigned long)squelch.GetFecBlockId(fecM).GetValue(), + (UINT16)squelch.GetFecSymbolId(fecM)); break; } - - case NormMsg::ACK: - case NormMsg::NACK: + case NormCmdMsg::FLUSH: { - PLOG(PL_ALWAYS, "inst>%hu ", instId); - // look for NormCCFeedback extension + const NormCmdFlushMsg &flush = + static_cast(msg); + PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", + (UINT16)flush.GetObjectId(), + (unsigned long)flush.GetFecBlockId(fecM).GetValue(), + (UINT16)flush.GetFecSymbolId(fecM)); + + if (0 != flush.GetAckingNodeCount()) + PLOG(PL_ALWAYS, "(WATERMARK) "); // ACK requested + break; + } + case NormCmdMsg::CC: + { + const NormCmdCCMsg &cc = static_cast(msg); + PLOG(PL_ALWAYS, " seq>%u ", cc.GetCCSequence()); NormHeaderExtension ext; - while (msg.GetNextExtension(ext)) + while (cc.GetNextExtension(ext)) { - if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) + if (NormHeaderExtension::CC_RATE == ext.GetType()) { - clrFlag = ((NormCCFeedbackExtension&)ext).CCFlagIsSet(NormCC::CLR); + UINT16 sendRate = ((NormCCRateExtension &)ext).GetSendRate(); + PLOG(PL_ALWAYS, " rate>%f ", 8.0e-03 * NormUnquantizeRate(sendRate)); break; } } - if (NormMsg::ACK == msgType) + break; + } + default: + break; + } + break; + } + + case NormMsg::ACK: + case NormMsg::NACK: + { + PLOG(PL_ALWAYS, "inst>%hu ", instId); + // look for NormCCFeedback extension + NormHeaderExtension ext; + while (msg.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) { - const NormAckMsg& ack = static_cast(msg); - if (NormAck::FLUSH == ack.GetAckType()) - { - const NormAckFlushMsg& flushAck = static_cast(ack); - PLOG(PL_ALWAYS, "ACK(FLUSH) obj>%hu blk>%lu seg>%hu ", - (UINT16)flushAck.GetObjectId(), - (unsigned long)flushAck.GetFecBlockId(fecM).GetValue(), - (UINT16)flushAck.GetFecSymbolId(fecM)); - } - else if (NormAck::CC == ack.GetAckType()) - { - PLOG(PL_ALWAYS, "ACK(CC) "); - } - else - { - PLOG(PL_ALWAYS, "ACK(ZZZ) "); - } + clrFlag = ((NormCCFeedbackExtension &)ext).CCFlagIsSet(NormCC::CLR); + break; + } + } + if (NormMsg::ACK == msgType) + { + const NormAckMsg &ack = static_cast(msg); + if (NormAck::FLUSH == ack.GetAckType()) + { + const NormAckFlushMsg &flushAck = static_cast(ack); + PLOG(PL_ALWAYS, "ACK(FLUSH) obj>%hu blk>%lu seg>%hu ", + (UINT16)flushAck.GetObjectId(), + (unsigned long)flushAck.GetFecBlockId(fecM).GetValue(), + (UINT16)flushAck.GetFecSymbolId(fecM)); + } + else if (NormAck::CC == ack.GetAckType()) + { + PLOG(PL_ALWAYS, "ACK(CC) "); } else { - PLOG(PL_ALWAYS, "NACK "); - // TBD - provide deeper NACK inspection? + PLOG(PL_ALWAYS, "ACK(ZZZ) "); } - break; } - - default: - PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); - break; - } // end switch (msgType) + else + { + PLOG(PL_ALWAYS, "NACK "); + // TBD - provide deeper NACK inspection? + } + break; + } + + default: + PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); + break; + } // end switch (msgType) PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); -} // end NormTrace(); +} // end NormTrace(); - -void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnStatus) -{ +void NormSession::HandleReceiveMessage(NormMsg &msg, bool wasUnicast, bool ecnStatus) +{ // Ignore messages from ourself unless "loopback" is enabled if ((msg.GetSourceId() == LocalNodeId()) && !loopback) return; // Drop some rx messages for testing - if ((rx_loss_rate > 0) && (UniformRand(100.0) < rx_loss_rate)) + if ((rx_loss_rate > 0) && (UniformRand(100.0) < rx_loss_rate)) return; - + struct timeval currentTime; ::ProtoSystemTime(currentTime); - - if (trace) + + if (trace) { - // Initially assume it's a message we generated (or similarly configured sender) + // Initially assume it's a message we generated (or similarly configured sender) UINT8 fecM = fec_m; - UINT16 instId = instance_id; - NormNodeId senderId; - switch (msg.GetType()) - { - case NormMsg::ACK: - senderId = static_cast(msg).GetSenderId(); - break; - case NormMsg::NACK: - senderId = static_cast(msg).GetSenderId(); - break; - default: - senderId = msg.GetSourceId(); - break; - } - if (IsReceiver() && (senderId != LocalNodeId())) - { + UINT16 instId = instance_id; + NormNodeId senderId; + switch (msg.GetType()) + { + case NormMsg::ACK: + senderId = static_cast(msg).GetSenderId(); + break; + case NormMsg::NACK: + senderId = static_cast(msg).GetSenderId(); + break; + default: + senderId = msg.GetSourceId(); + break; + } + if (IsReceiver() && (senderId != LocalNodeId())) + { // Use our receiver state to look up sender if possible - NormSenderNode* sender; + NormSenderNode *sender; if (IsServerListener()) sender = client_tree.FindNodeByAddress(msg.GetSource()); else - sender = static_cast(sender_tree.FindNodeById(senderId)); + sender = static_cast(sender_tree.FindNodeById(senderId)); if (NULL != sender) { fecM = sender->GetFecFieldSize(); @@ -2777,12 +2799,12 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt fecM = 16; // reasonable assumption instId = 0; } - } - NormTrace(currentTime, LocalNodeId(), msg, false, fecM, instId); // TBD don't assume m == 16 (i.e. for fec_id == 2) - } // end if (trace) - + } + NormTrace(currentTime, LocalNodeId(), msg, false, fecM, instId); // TBD don't assume m == 16 (i.e. for fec_id == 2) + } // end if (trace) + NormMsg::Type msgType = msg.GetType(); - + if (IsServerListener()) { // Only pay attention to packets with FLAG_SYN set @@ -2792,28 +2814,29 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt bool senderMsg = true; switch (msgType) { - case NormMsg::CMD: + case NormMsg::CMD: + { + NormCmdMsg &cmd = static_cast(msg); + if ((NormCmdMsg::CC == cmd.GetFlavor()) && + static_cast(cmd).SynIsSet()) { - NormCmdMsg& cmd = static_cast(msg); - if ((NormCmdMsg::CC == cmd.GetFlavor()) && - static_cast(cmd).SynIsSet()) - { - syn = true; - } - break; + syn = true; } - case NormMsg::INFO: - case NormMsg::DATA: - if (static_cast(msg).FlagIsSet(NormObjectMsg::FLAG_SYN)) - { - syn = true; - } - break; - default: - senderMsg = false; - // Receiver messages are ignored by unicast server-listener, - // but multicast server needs to process ACKS/NACKS from client receivers - if (!Address().IsMulticast()) return; + break; + } + case NormMsg::INFO: + case NormMsg::DATA: + if (static_cast(msg).FlagIsSet(NormObjectMsg::FLAG_SYN)) + { + syn = true; + } + break; + default: + senderMsg = false; + // Receiver messages are ignored by unicast server-listener, + // but multicast server needs to process ACKS/NACKS from client receivers + if (!Address().IsMulticast()) + return; } if (senderMsg) { @@ -2828,24 +2851,24 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt } } } - + // Add newly detected nodes to acking list _before_ processing message if (IsSender() && (TRACK_NONE != acking_auto_populate)) { bool addNode = false; switch (acking_auto_populate) { - case TRACK_ALL: - addNode = true; - break; - case TRACK_RECEIVERS: - addNode = (NormMsg::NACK == msgType) || (NormMsg::ACK == msgType); - break; - case TRACK_SENDERS: - addNode = (NormMsg::NACK != msgType) && (NormMsg::ACK != msgType); - break; - default: - break; + case TRACK_ALL: + addNode = true; + break; + case TRACK_RECEIVERS: + addNode = (NormMsg::NACK == msgType) || (NormMsg::ACK == msgType); + break; + case TRACK_SENDERS: + addNode = (NormMsg::NACK != msgType) && (NormMsg::ACK != msgType); + break; + default: + break; } if (addNode) { @@ -2854,84 +2877,88 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt { if (!SenderAddAckingNode(msg.GetSourceId(), &msg.GetSource())) PLOG(PL_ERROR, "NormSession::HandleReceiveMessage() error: unable to add acking node!\n"); - NormAckingNode* acker = (NormAckingNode*)acking_node_tree.FindNodeById(sourceId); + NormAckingNode *acker = (NormAckingNode *)acking_node_tree.FindNodeById(sourceId); Notify(NormController::ACKING_NODE_NEW, acker, NULL); } } } - + switch (msg.GetType()) { - case NormMsg::INFO: - //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::INFO)\n"); - if (IsReceiver()) ReceiverHandleObjectMessage(currentTime, (NormObjectMsg&)msg, ecnStatus); - break; - case NormMsg::DATA: - //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::DATA) ...\n"); - if (IsReceiver()) ReceiverHandleObjectMessage(currentTime, (NormObjectMsg&)msg, ecnStatus); - break; - case NormMsg::CMD: - //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::CMD) ...\n"); - if (IsReceiver()) ReceiverHandleCommand(currentTime, (NormCmdMsg&)msg, ecnStatus); - break; - case NormMsg::NACK: - if (IsSender() && (((NormNackMsg&)msg).GetSenderId() == LocalNodeId())) - { - SenderHandleNackMessage(currentTime, (NormNackMsg&)msg); - if (wasUnicast && (backoff_factor > 0.5) && Address().IsMulticast()) - { - // for suppression of unicast nack feedback - advertise_repairs = true; - QueueMessage(NULL); // to prompt transmit timeout - } + case NormMsg::INFO: + //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::INFO)\n"); + if (IsReceiver()) + ReceiverHandleObjectMessage(currentTime, (NormObjectMsg &)msg, ecnStatus); + break; + case NormMsg::DATA: + //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::DATA) ...\n"); + if (IsReceiver()) + ReceiverHandleObjectMessage(currentTime, (NormObjectMsg &)msg, ecnStatus); + break; + case NormMsg::CMD: + //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::CMD) ...\n"); + if (IsReceiver()) + ReceiverHandleCommand(currentTime, (NormCmdMsg &)msg, ecnStatus); + break; + case NormMsg::NACK: + if (IsSender() && (((NormNackMsg &)msg).GetSenderId() == LocalNodeId())) + { + SenderHandleNackMessage(currentTime, (NormNackMsg &)msg); + if (wasUnicast && (backoff_factor > 0.5) && Address().IsMulticast()) + { + // for suppression of unicast nack feedback + advertise_repairs = true; + QueueMessage(NULL); // to prompt transmit timeout } - if (IsReceiver()) ReceiverHandleNackMessage((NormNackMsg&)msg); - break; - case NormMsg::ACK: - if (IsSender() && (((NormAckMsg&)msg).GetSenderId() == LocalNodeId())) - SenderHandleAckMessage(currentTime, (NormAckMsg&)msg, wasUnicast); - if (IsReceiver()) ReceiverHandleAckMessage((NormAckMsg&)msg); - break; - - case NormMsg::REPORT: - case NormMsg::INVALID: - PLOG(PL_ERROR, "NormSession::HandleReceiveMessage(NormMsg::INVALID)\n"); - break; + } + if (IsReceiver()) + ReceiverHandleNackMessage((NormNackMsg &)msg); + break; + case NormMsg::ACK: + if (IsSender() && (((NormAckMsg &)msg).GetSenderId() == LocalNodeId())) + SenderHandleAckMessage(currentTime, (NormAckMsg &)msg, wasUnicast); + if (IsReceiver()) + ReceiverHandleAckMessage((NormAckMsg &)msg); + break; + + case NormMsg::REPORT: + case NormMsg::INVALID: + PLOG(PL_ERROR, "NormSession::HandleReceiveMessage(NormMsg::INVALID)\n"); + break; } -} // end NormSession::HandleReceiveMessage() +} // end NormSession::HandleReceiveMessage() - -void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTime, - const NormObjectMsg& msg, - bool ecnStatus) +void NormSession::ReceiverHandleObjectMessage(const struct timeval ¤tTime, + const NormObjectMsg &msg, + bool ecnStatus) { // Do common updates for senders we already know. NormNodeId sourceId = msg.GetSourceId(); - NormSenderNode* theSender; + NormSenderNode *theSender; if (IsServerListener()) theSender = client_tree.FindNodeByAddress(msg.GetSource()); else - theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId); + theSender = (NormSenderNode *)sender_tree.FindNodeById(sourceId); if (theSender) { if (msg.GetInstanceId() != theSender->GetInstanceId()) { PLOG(PL_INFO, "NormSession::ReceiverHandleObjectMessage() node>%lu sender>%lu instanceId change - resyncing.\n", - (unsigned long)LocalNodeId(), (unsigned long)theSender->GetId()); + (unsigned long)LocalNodeId(), (unsigned long)theSender->GetId()); theSender->Close(); Notify(NormController::REMOTE_SENDER_RESET, theSender, NULL); if (!theSender->Open(msg.GetInstanceId())) { PLOG(PL_ERROR, "NormSession::ReceiverHandleObjectMessage() node>%lu error re-opening NormSenderNode\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); // (TBD) notify application of error - return; - } + return; + } } } else { - if (NULL != preset_sender) + if (NULL != preset_sender) { theSender = preset_sender; preset_sender = NULL; @@ -2943,7 +2970,7 @@ void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTim else sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n", - (unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId()); Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else if (NULL != (theSender = new NormSenderNode(*this, msg.GetSourceId()))) @@ -2956,24 +2983,24 @@ void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTim else sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n", - (unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId()); } else { PLOG(PL_FATAL, "NormSession::ReceiverHandleObjectMessage() node>%lu error opening NormSenderNode\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); // (TBD) notify application of error - return; - } + return; + } Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else { PLOG(PL_ERROR, "NormSession::ReceiverHandleObjectMessage() new NormSenderNode error: %s\n", - GetErrorString()); + GetErrorString()); // (TBD) notify application of error - return; - } + return; + } } theSender->Activate(true); if (!theSender->GetAddress().IsEqual(msg.GetSource())) @@ -2984,47 +3011,47 @@ void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTim } theSender->UpdateRecvRate(currentTime, msg.GetLength()); //if (ecnStatus) TRACE("UPDATING LOSS EST w/ ECN pkt\n"); - theSender->UpdateLossEstimate(currentTime, msg.GetSequence(), ecnStatus); + theSender->UpdateLossEstimate(currentTime, msg.GetSequence(), ecnStatus); theSender->IncrementRecvTotal(msg.GetLength()); // for statistics only (TBD) #ifdef NORM_DEBUG theSender->HandleObjectMessage(msg); - theSender->CheckCCFeedback(); // this cues immediate CLR cc feedback if loss was detected - // and cc feedback was not provided in response otherwise - -} // end NormSession::ReceiverHandleObjectMessage() + theSender->CheckCCFeedback(); // this cues immediate CLR cc feedback if loss was detected + // and cc feedback was not provided in response otherwise -void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, - const NormCmdMsg& cmd, - bool ecnStatus) +} // end NormSession::ReceiverHandleObjectMessage() + +void NormSession::ReceiverHandleCommand(const struct timeval ¤tTime, + const NormCmdMsg &cmd, + bool ecnStatus) { // Do common updates for senders we already know. NormNodeId sourceId = cmd.GetSourceId(); - NormSenderNode* theSender; + NormSenderNode *theSender; if (IsServerListener()) theSender = client_tree.FindNodeByAddress(cmd.GetSource()); else - theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId); + theSender = (NormSenderNode *)sender_tree.FindNodeById(sourceId); if (NULL != theSender) { if (cmd.GetInstanceId() != theSender->GetInstanceId()) { PLOG(PL_INFO, "NormSession::ReceiverHandleCommand() node>%lu sender>%lu instanceId change - resyncing.\n", - (unsigned long)LocalNodeId(), theSender->GetId()); - theSender->Close(); + (unsigned long)LocalNodeId(), theSender->GetId()); + theSender->Close(); Notify(NormController::REMOTE_SENDER_RESET, theSender, NULL); if (!theSender->Open(cmd.GetInstanceId())) { PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() node>%lu error re-opening NormSenderNode\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); // (TBD) notify application of error - return; - } + return; + } } } else { //DMSG(0, "NormSession::ReceiverHandleCommand() node>%lu recvd command from unknown sender ...\n", - // (unsigned long)LocalNodeId()); - if (NULL != preset_sender) + // (unsigned long)LocalNodeId()); + if (NULL != preset_sender) { theSender = preset_sender; preset_sender = NULL; @@ -3036,7 +3063,7 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, else sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleCommand() node>%lu new remote sender:%lu ...\n", - (unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId()); Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else if ((theSender = new NormSenderNode(*this, cmd.GetSourceId()))) @@ -3049,7 +3076,7 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, else sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleCommand() node>%lu new remote sender:%lu ...\n", - (unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId()); } else { @@ -3062,10 +3089,10 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, else { PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() new NormSenderNode node>%lu error: %s\n", - (unsigned long)LocalNodeId(), GetErrorString()); + (unsigned long)LocalNodeId(), GetErrorString()); // (TBD) notify application of error - return; - } + return; + } } // We should "re-activate" senders on NORM_CMD(FLUSH) if (NormCmdMsg::FLUSH == cmd.GetFlavor()) @@ -3079,19 +3106,20 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, Notify(NormController::REMOTE_SENDER_ADDRESS, theSender, NULL); } theSender->UpdateRecvRate(currentTime, cmd.GetLength()); - theSender->UpdateLossEstimate(currentTime, cmd.GetSequence(), ecnStatus); + theSender->UpdateLossEstimate(currentTime, cmd.GetSequence(), ecnStatus); theSender->IncrementRecvTotal(cmd.GetLength()); // for statistics only (TBD) #ifdef NORM_DEBUG - theSender->HandleCommand(currentTime, cmd); - theSender->CheckCCFeedback(); // this cues immediate CLR cc feedback if loss was detected - // and cc feedback was not provided in response otherwise -} // end NormSession::ReceiverHandleCommand() + theSender->HandleCommand(currentTime, cmd); + theSender->CheckCCFeedback(); // this cues immediate CLR cc feedback if loss was detected + // and cc feedback was not provided in response otherwise +} // end NormSession::ReceiverHandleCommand() -bool NormSession::InsertRemoteSender(NormSenderNode& sender) +bool NormSession::InsertRemoteSender(NormSenderNode &sender) { - // Build a NORM_CMD(CC) message with information from + // Build a NORM_CMD(CC) message with information from // a "sender" being inserted from another NormSession - // (supports NormSocket server operations) - if (!IsReceiver()) return false; + // (supports NormSocket server operations) + if (!IsReceiver()) + return false; NormCmdCCMsg cmd; cmd.Init(); cmd.SetSequence(sender.GetCurrentSequence()); @@ -3100,7 +3128,7 @@ bool NormSession::InsertRemoteSender(NormSenderNode& sender) cmd.SetInstanceId(sender.GetInstanceId()); cmd.SetGrtt(sender.GetGrttQuantized()); cmd.SetBackoffFactor(sender.GetBackoffFactor()); - cmd.SetGroupSize(sender.GetGroupSizeQuantized()); + cmd.SetGroupSize(sender.GetGroupSizeQuantized()); cmd.SetCCSequence(sender.GetCCSequence()); // Adjust send time for any current hold time // since it will be "rehandled" @@ -3109,22 +3137,21 @@ bool NormSession::InsertRemoteSender(NormSenderNode& sender) ::ProtoSystemTime(currentTime); sender.CalculateGrttResponse(currentTime, adjustedSendTime); cmd.SetSendTime(adjustedSendTime); - + // Insert NORM-CC header extension, if applicable // (Note we set the extension "rate" _after_ AdjustRate() done below) NormCCRateExtension ext; cmd.AttachExtension(ext); ext.SetSendRate(NormQuantizeRate(sender.GetSendRate())); - + HandleReceiveMessage(cmd, false); - - return true; // TBD - confirm the node was added - -} // end NormSession::InsertRemoteSender() + return true; // TBD - confirm the node was added -double NormSession::CalculateRtt(const struct timeval& currentTime, - const struct timeval& grttResponse) +} // end NormSession::InsertRemoteSender() + +double NormSession::CalculateRtt(const struct timeval ¤tTime, + const struct timeval &grttResponse) { if (grttResponse.tv_sec || grttResponse.tv_usec) { @@ -3132,18 +3159,18 @@ double NormSession::CalculateRtt(const struct timeval& currentTime, // Calculate rtt estimate for this receiver and process the response if (currentTime.tv_usec < grttResponse.tv_usec) { - rcvrRtt = + rcvrRtt = (double)(currentTime.tv_sec - grttResponse.tv_sec - 1); - rcvrRtt += + rcvrRtt += ((double)(1000000 - (grttResponse.tv_usec - currentTime.tv_usec))) / 1.0e06; } else { - rcvrRtt = + rcvrRtt = (double)(currentTime.tv_sec - grttResponse.tv_sec); - rcvrRtt += + rcvrRtt += ((double)(currentTime.tv_usec - grttResponse.tv_usec)) / 1.0e06; - } + } // Lower limit on RTT (because of coarse timer resolution on some systems, // this can sometimes actually end up a negative value!) // (TBD) this should be system clock granularity? @@ -3153,7 +3180,7 @@ double NormSession::CalculateRtt(const struct timeval& currentTime, { return -1.0; } -} // end NormSession::CalculateRtt() +} // end NormSession::CalculateRtt() void NormSession::SenderUpdateGrttEstimate(double receiverRtt) { @@ -3162,11 +3189,12 @@ void NormSession::SenderUpdateGrttEstimate(double receiverRtt) { // Immediately incorporate bigger RTT's grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; - grtt_measured = 0.25 * grtt_measured + 0.75 * receiverRtt; - //grtt_measured = 0.9 * grtt_measured + 0.1 * receiverRtt; - if (grtt_measured > grtt_max) grtt_measured = grtt_max; + grtt_measured = 0.25 * grtt_measured + 0.75 * receiverRtt; + //grtt_measured = 0.9 * grtt_measured + 0.1 * receiverRtt; + if (grtt_measured > grtt_max) + grtt_measured = grtt_max; UINT8 grttQuantizedOld = grtt_quantized; - double pktInterval = ((double)(44+segment_size))/tx_rate; + double pktInterval = ((double)(44 + segment_size)) / tx_rate; if (grtt_measured < pktInterval) grtt_quantized = NormQuantizeRtt(pktInterval); else @@ -3184,20 +3212,18 @@ void NormSession::SenderUpdateGrttEstimate(double receiverRtt) if (notify_on_grtt_update) { notify_on_grtt_update = false; - Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::GRTT_UPDATED, (NormSenderNode *)NULL, (NormObject *)NULL); } - Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::GRTT_UPDATED, (NormSenderNode *)NULL, (NormObject *)NULL); PLOG(PL_DEBUG, "NormSession::SenderUpdateGrttEstimate() node>%lu increased to new grtt>%lf sec\n", - (unsigned long)LocalNodeId(), grtt_advertised); + (unsigned long)LocalNodeId(), grtt_advertised); } - } - else if (receiverRtt > grtt_current_peak) + } + else if (receiverRtt > grtt_current_peak) { grtt_current_peak = receiverRtt; } -} // end NormSession::SenderUpdateGrttEstimate() - - +} // end NormSession::SenderUpdateGrttEstimate() double NormSession::CalculateRate(double size, double rtt, double loss) { @@ -3206,25 +3232,25 @@ double NormSession::CalculateRate(double size, double rtt, double loss) // rtt * (sqrt(2*loss/3) + 12*loss*(1 + 32*loss*loss)*sqrt(3*loss/8)) // // notes: "b" = 1 and "t_RTO" = 4*rtt where "b" is number of TCP pkts/ACK - - double denom = rtt * (sqrt((2.0/3.0)*loss) + - (12.0 * sqrt((3.0/8.0)*loss) * loss * - (1.0 + 32.0*loss*loss))); - return (size / denom); -} // end NormSession::CalculateRate() -void NormSession::SenderHandleCCFeedback(struct timeval currentTime, - NormNodeId nodeId, - UINT8 ccFlags, - double ccRtt, - double ccLoss, - double ccRate, - UINT16 ccSequence) + double denom = rtt * (sqrt((2.0 / 3.0) * loss) + + (12.0 * sqrt((3.0 / 8.0) * loss) * loss * + (1.0 + 32.0 * loss * loss))); + return (size / denom); +} // end NormSession::CalculateRate() + +void NormSession::SenderHandleCCFeedback(struct timeval currentTime, + NormNodeId nodeId, + UINT8 ccFlags, + double ccRtt, + double ccLoss, + double ccRate, + UINT16 ccSequence) { - - PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() cc feedback recvd at time %lu.%lf ccRate:%9.3lf ccRtt:%lf ccLoss:%lf ccFlags:%02x\n", - (unsigned long)currentTime.tv_sec, ((double)currentTime.tv_usec)*1.0e-06, - ccRate*8.0/1000.0, ccRtt, ccLoss, ccFlags); + + PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() cc feedback recvd at time %lu.%lf ccRate:%9.3lf ccRtt:%lf ccLoss:%lf ccFlags:%02x\n", + (unsigned long)currentTime.tv_sec, ((double)currentTime.tv_usec) * 1.0e-06, + ccRate * 8.0 / 1000.0, ccRtt, ccLoss, ccFlags); // Keep track of current suppressing feedback // (non-CLR, lowest rate, unconfirmed RTT) if (0 == (ccFlags & NormCC::CLR)) @@ -3237,27 +3263,32 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, } else { - if (ccRate < suppress_rate) suppress_rate = ccRate; - if (ccRtt > suppress_rtt) suppress_rtt = ccRtt; - if (0 == (ccFlags & NormCC::RTT)) suppress_nonconfirmed = true; - } + if (ccRate < suppress_rate) + suppress_rate = ccRate; + if (ccRtt > suppress_rtt) + suppress_rtt = ccRtt; + if (0 == (ccFlags & NormCC::RTT)) + suppress_nonconfirmed = true; + } } - if (!cc_enable) return; - + if (!cc_enable) + return; + // Adjust ccRtt if we already have state on this nodeId - NormCCNode* node = (NormCCNode*)cc_node_list.FindNodeById(nodeId); - if (node) ccRtt = node->UpdateRtt(ccRtt); - + NormCCNode *node = (NormCCNode *)cc_node_list.FindNodeById(nodeId); + if (node) + ccRtt = node->UpdateRtt(ccRtt); + bool ccSlowStart = (0 != (ccFlags & NormCC::START)); - + if (!ccSlowStart) { - double calcRate = CalculateRate(nominal_packet_size, ccRtt, ccLoss); -#ifdef LIMIT_CC_RATE + double calcRate = CalculateRate(nominal_packet_size, ccRtt, ccLoss); +#ifdef LIMIT_CC_RATE // Experimental modification to NORM-CC where congestion control rate is limited // to MIN(2.0*measured recv rate, calculated rate). This might prevent large rate - // overshoot in conditions where the loss measurement (perhaps initial loss) is - // very low due to big network packet buffers, etc + // overshoot in conditions where the loss measurement (perhaps initial loss) is + // very low due to big network packet buffers, etc // Note that when the NORM_CC_FLAG_LIMIT is set, this indicates the receiver // has set the rate field to 2.0 * measured recv rate instead of calculated rate. if (0 != (ccFlags & NormCC::LIMIT)) @@ -3268,33 +3299,34 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, ccRate = calcRate; } else -#endif // LIMIT_CC_RATE +#endif // LIMIT_CC_RATE { ccRate = calcRate; - } + } } - + PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() node>%lu rate>%lf (rtt>%lf loss>%lf slow_start>%d limit>%d)\n", - (unsigned long)nodeId, ccRate * 8.0 / 1000.0, ccRtt, ccLoss, (0 != (ccFlags & NormCC::START)), - (0 != (ccFlags & NormCC::LIMIT))); - + (unsigned long)nodeId, ccRate * 8.0 / 1000.0, ccRtt, ccLoss, (0 != (ccFlags & NormCC::START)), + (0 != (ccFlags & NormCC::LIMIT))); + // Keep the active CLR (if there is one) at the head of the list NormNodeListIterator iterator(cc_node_list); - NormCCNode* next = (NormCCNode*)iterator.GetNextNode(); + NormCCNode *next = (NormCCNode *)iterator.GetNextNode(); // 1) Does this response replace the active CLR? if (next && next->IsActive()) { - // First, make sure this is _new_ non-duplicative + // First, make sure this is _new_ non-duplicative // feedback for the given "nodeId" if (next->GetId() == nodeId) { INT16 ccDelta = ccSequence - next->GetCCSequence(); - if (ccDelta <= 0) return; + if (ccDelta <= 0) + return; } if ((nodeId == next->GetId()) || (ccRate < next->GetRate()) || - ((ccRate < (next->GetRate() * 1.1)) && - (ccRtt > next->GetRtt()))) // use Rtt as tie-breaker if close + ((ccRate < (next->GetRate() * 1.1)) && + (ccRtt > next->GetRtt()))) // use Rtt as tie-breaker if close { NormNodeId savedId = next->GetId(); bool savedRttStatus = next->HasRtt(); @@ -3303,20 +3335,20 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, double savedRate = next->GetRate(); UINT16 savedSequence = next->GetCCSequence(); struct timeval savedTime = next->GetFeedbackTime(); - + next->SetId(nodeId); next->SetClrStatus(true); next->SetRttStatus(0 != (ccFlags & NormCC::RTT)); - + next->SetLoss(ccLoss); next->SetRate(ccRate); next->SetCCSequence(ccSequence); next->SetActive(true); next->SetFeedbackTime(currentTime); - cc_slow_start = ccSlowStart; // use CLR status for our slow_start state + cc_slow_start = ccSlowStart; // use CLR status for our slow_start state if (savedId == nodeId) { - // This was feedback from the current CLR + // This was feedback from the current CLR AdjustRate(true); return; } @@ -3336,22 +3368,22 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, currentTime = savedTime; } } - else + else { // There was no active CLR if (!next) { - if ((next = new NormCCNode(*this, nodeId))) + if ((next = new NormCCNode(*this, nodeId))) { cc_node_list.Append(next); } else { - PLOG(PL_FATAL, "NormSession::SenderHandleCCFeedback() memory allocation error: %s\n", - GetErrorString()); - return; - } - } + PLOG(PL_FATAL, "NormSession::SenderHandleCCFeedback() memory allocation error: %s\n", + GetErrorString()); + return; + } + } next->SetId(nodeId); next->SetClrStatus(true); //next->SetPlrStatus(false); @@ -3365,24 +3397,24 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, AdjustRate(true); return; } - + // 2) Go through cc_node_list and find lowest priority candidate - NormCCNode* candidate = NULL; + NormCCNode *candidate = NULL; if (cc_node_list.GetCount() < 5) { if ((candidate = new NormCCNode(*this, nodeId))) { cc_node_list.Append(candidate); - } + } else { PLOG(PL_FATAL, "NormSession::SenderHandleCCFeedback() memory allocation error: %s\n", - GetErrorString()); + GetErrorString()); } } else { - while ((next = (NormCCNode*)iterator.GetNextNode())) + while ((next = (NormCCNode *)iterator.GetNextNode())) { if (next->GetId() == nodeId) { @@ -3394,9 +3426,9 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, if (candidate->IsActive() && !next->IsActive()) { candidate = next; - continue; + continue; } - if (!next->HasRtt() && candidate->HasRtt()) + if (!next->HasRtt() && candidate->HasRtt()) continue; else if (!candidate->HasRtt() && next->HasRtt()) candidate = next; @@ -3410,7 +3442,7 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, } } } - + // 3) Replace candidate if this response is higher precedence if (candidate) { @@ -3432,7 +3464,7 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, { candidate->SetId(nodeId); candidate->SetClrStatus(false); - //candidate->SetPlrStatus(true); // do this only + //candidate->SetPlrStatus(true); // do this only candidate->SetRttStatus(0 != (ccFlags & NormCC::RTT)); candidate->SetRtt(ccRtt); candidate->SetLoss(ccLoss); @@ -3441,20 +3473,20 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, candidate->SetActive(true); } } -} // end NormSession::SenderHandleCCFeedback() - +} // end NormSession::SenderHandleCCFeedback() -void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, const NormAckMsg& ack, bool wasUnicast) +void NormSession::SenderHandleAckMessage(const struct timeval ¤tTime, const NormAckMsg &ack, bool wasUnicast) { // Update GRTT estimate struct timeval grttResponse; ack.GetGrttResponse(grttResponse); double receiverRtt = CalculateRtt(currentTime, grttResponse); PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu sender received ACK from node>%lu rtt>%lf\n", - (unsigned long)LocalNodeId(), (unsigned long)ack.GetSourceId(), receiverRtt); - - if (receiverRtt >= 0.0) SenderUpdateGrttEstimate(receiverRtt); - + (unsigned long)LocalNodeId(), (unsigned long)ack.GetSourceId(), receiverRtt); + + if (receiverRtt >= 0.0) + SenderUpdateGrttEstimate(receiverRtt); + // Look for NORM-CC Feedback header extension NormCCFeedbackExtension ext; while (ack.GetNextExtension(ext)) @@ -3464,16 +3496,15 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons SenderHandleCCFeedback(currentTime, ack.GetSourceId(), ext.GetCCFlags(), - receiverRtt >= 0.0 ? - receiverRtt : NormUnquantizeRtt(ext.GetCCRtt()), + receiverRtt >= 0.0 ? receiverRtt : NormUnquantizeRtt(ext.GetCCRtt()), NormUnquantizeLoss32(ext.GetCCLoss32()), NormUnquantizeRate(ext.GetCCRate()), ext.GetCCSequence()); - if (wasUnicast && probe_proactive && Address().IsMulticast()) + if (wasUnicast && probe_proactive && Address().IsMulticast()) { // if it's the CLR, it doesn't suppress anyone, don't advertise if (!ext.CCFlagIsSet(NormCC::CLR)) - { + { // for suppression of unicast cc feedback advertise_repairs = true; QueueMessage(NULL); @@ -3482,46 +3513,46 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons break; } } - + switch (ack.GetAckType()) { - case NormAck::CC: - // Everything is in the ACK header or extension for this one - break; - - case NormAck::FLUSH: - if (watermark_pending) + case NormAck::CC: + // Everything is in the ACK header or extension for this one + break; + + case NormAck::FLUSH: + if (watermark_pending) + { + NormAckingNode *acker = + static_cast(acking_node_tree.FindNodeById(ack.GetSourceId())); + if (NULL != acker) { - NormAckingNode* acker = - static_cast(acking_node_tree.FindNodeById(ack.GetSourceId())); - if (NULL != acker) + if (!acker->AckReceived()) { - if (!acker->AckReceived()) + const NormAckFlushMsg &flushAck = static_cast(ack); + if (flushAck.GetFecId() != fec_id) { - const NormAckFlushMsg& flushAck = static_cast(ack); - if (flushAck.GetFecId() != fec_id) + PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received watermark ACK with wrong fec_id?!\n"); + } + else if ((watermark_object_id == flushAck.GetObjectId()) && + (watermark_block_id == flushAck.GetFecBlockId(fec_m)) && + (watermark_segment_id == flushAck.GetFecSymbolId(fec_m))) + { + // Cache any application-defined extended ACK content for this acker + NormAppAckExtension ext; + while (ack.GetNextExtension(ext)) { - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received watermark ACK with wrong fec_id?!\n"); - } - else if ((watermark_object_id == flushAck.GetObjectId()) && - (watermark_block_id == flushAck.GetFecBlockId(fec_m)) && - (watermark_segment_id == flushAck.GetFecSymbolId(fec_m))) - { - // Cache any application-defined extended ACK content for this acker - NormAppAckExtension ext; - while (ack.GetNextExtension(ext)) + if (NormHeaderExtension::APP_ACK == ext.GetType()) { - if (NormHeaderExtension::APP_ACK == ext.GetType()) + if (!acker->SetAckEx(ext.GetContent(), ext.GetContentLength())) { - if (!acker->SetAckEx(ext.GetContent(), ext.GetContentLength())) - { - // TBD - notify app of error - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() error: unable to cache application-defined ACK content!\n"); - } + // TBD - notify app of error + PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() error: unable to cache application-defined ACK content!\n"); } } - acker->MarkAckReceived(); - /* This code was an attempt to expedite delivery of the TX_WATERMARK_COMPLETED + } + acker->MarkAckReceived(); + /* This code was an attempt to expedite delivery of the TX_WATERMARK_COMPLETED notification to the application, but breaks some other desired behavior. watermark_pending = false; acking_success_count = 0; @@ -3541,55 +3572,56 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); } */ - } - else - { - // This can happen when new watermarks are set when an old watermark is still - // pending (i.e. receivers may still be in the process of replying) - PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received old/wrong watermark ACK?!\n"); - } } else { - PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received redundant watermark ACK?!\n"); + // This can happen when new watermarks are set when an old watermark is still + // pending (i.e. receivers may still be in the process of replying) + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received old/wrong watermark ACK?!\n"); } } else { - PLOG(PL_WARN, "NormSession::SenderHandleAckMessage() received watermark ACK from unknown acker?!\n"); + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received redundant watermark ACK?!\n"); } } else { - PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received unsolicited watermark ACK?!\n"); + PLOG(PL_WARN, "NormSession::SenderHandleAckMessage() received watermark ACK from unknown acker?!\n"); } - break; - - // (TBD) Handle other acknowledgement types - default: - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() node>%lu received unsupported ack type:%d\n", - (unsigned long)LocalNodeId(), ack.GetAckType()); - break; - } -} // end SenderHandleAckMessage() + } + else + { + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received unsolicited watermark ACK?!\n"); + } + break; -void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, NormNackMsg& nack) + // (TBD) Handle other acknowledgement types + default: + PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() node>%lu received unsupported ack type:%d\n", + (unsigned long)LocalNodeId(), ack.GetAckType()); + break; + } +} // end SenderHandleAckMessage() + +void NormSession::SenderHandleNackMessage(const struct timeval ¤tTime, NormNackMsg &nack) { struct timeval grttResponse; nack.GetGrttResponse(grttResponse); double receiverRtt = CalculateRtt(currentTime, grttResponse); - if (GetDebugLevel() >= PL_DEBUG) + if (GetDebugLevel() >= PL_DEBUG) { - PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu sender received NACK message from node>%lu rtt>%lf (tactive>%d) with content:\n", - (unsigned long)LocalNodeId(), (unsigned long)nack.GetSourceId(), receiverRtt, repair_timer.IsActive()); + PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu sender received NACK message from node>%lu rtt>%lf (tactive>%d) with content:\n", + (unsigned long)LocalNodeId(), (unsigned long)nack.GetSourceId(), receiverRtt, repair_timer.IsActive()); LogRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength(), fec_id, fec_m); PLOG(PL_ALWAYS, "\n"); } // (TBD) maintain average of "numErasures" for SEGMENT repair requests // to use as input to a future automatic "auto parity" adjustor??? // Update GRTT estimate - if (receiverRtt >= 0.0) SenderUpdateGrttEstimate(receiverRtt); - + if (receiverRtt >= 0.0) + SenderUpdateGrttEstimate(receiverRtt); + // Look for NORM-CC Feedback header extension NormCCFeedbackExtension ext; while (nack.GetNextExtension(ext)) @@ -3599,37 +3631,36 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor SenderHandleCCFeedback(currentTime, nack.GetSourceId(), ext.GetCCFlags(), - receiverRtt >= 0.0 ? - receiverRtt : NormUnquantizeRtt(ext.GetCCRtt()), - NormUnquantizeLoss32(ext.GetCCLoss32()), // note using extended precision loss value here + receiverRtt >= 0.0 ? receiverRtt : NormUnquantizeRtt(ext.GetCCRtt()), + NormUnquantizeLoss32(ext.GetCCLoss32()), // note using extended precision loss value here NormUnquantizeRate(ext.GetCCRate()), ext.GetCCSequence()); } break; } - - // Parse and process NACK + + // Parse and process NACK UINT16 requestOffset = 0; UINT16 requestLength = 0; NormRepairRequest req; - NormObject* object = NULL; + NormObject *object = NULL; bool freshObject = true; NormObjectId prevObjectId = 0; - NormBlock* block = NULL; + NormBlock *block = NULL; bool freshBlock = true; NormBlockId prevBlockId = 0; - + bool startTimer = false; UINT16 numErasures = extra_parity; - + bool squelchQueued = false; - + // Get the index of our next pending NORM_DATA transmission NormObjectId txObjectIndex; NormBlockId txBlockIndex; if (SenderGetFirstPending(txObjectIndex)) { - NormObject* obj = tx_table.Find(txObjectIndex); + NormObject *obj = tx_table.Find(txObjectIndex); ASSERT(NULL != obj); if (obj->IsPendingInfo()) { @@ -3650,9 +3681,15 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor txObjectIndex = next_tx_object_id; txBlockIndex = 0; } - - bool holdoff = (repair_timer.IsActive() && !repair_timer.GetRepeatCount()); - enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; + + bool holdoff = (repair_timer.IsActive() && !repair_timer.GetRepeatCount()); + enum NormRequestLevel + { + SEGMENT, + BLOCK, + INFO, + OBJECT + }; while (0 != (requestLength = nack.UnpackRepairRequest(req, requestOffset))) { NormRepairRequest::Form requestForm = req.GetForm(); @@ -3677,27 +3714,27 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor else { PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu recvd repair request w/ invalid repair level\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); continue; } - + NormRepairRequest::Iterator iterator(req, fec_id, fec_m); NormObjectId nextObjectId, lastObjectId; NormBlockId nextBlockId, lastBlockId; UINT16 nextBlockLen, lastBlockLen; NormSegmentId nextSegmentId, lastSegmentId; - while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, + while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, &nextBlockLen, &nextSegmentId)) { if (NormRepairRequest::RANGES == requestForm) { - if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, + if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, &lastBlockLen, &lastSegmentId)) { PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu recvd incomplete RANGE request!\n", - (unsigned long)LocalNodeId()); - continue; // (TBD) break/return instead??? - } + (unsigned long)LocalNodeId()); + continue; // (TBD) break/return instead??? + } // (TBD) test for valid range form/level } else @@ -3707,19 +3744,21 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor lastBlockLen = nextBlockLen; lastSegmentId = nextSegmentId; } - + bool inRange = true; while (inRange) { - if (nextObjectId != prevObjectId) freshObject = true; + if (nextObjectId != prevObjectId) + freshObject = true; if (freshObject) { freshBlock = true; if (!(object = tx_table.Find(nextObjectId))) { PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu recvd repair request " - "for unknown object ...\n", (unsigned long)LocalNodeId()); - if (!squelchQueued) + "for unknown object ...\n", + (unsigned long)LocalNodeId()); + if (!squelchQueued) { SenderQueueSquelch(nextObjectId); squelchQueued = true; @@ -3727,7 +3766,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor if ((OBJECT == requestLevel) || (INFO == requestLevel)) { nextObjectId++; - if (nextObjectId > lastObjectId) + if (nextObjectId > lastObjectId) inRange = false; } else @@ -3735,11 +3774,11 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor inRange = false; } continue; - } + } prevObjectId = nextObjectId; freshObject = false; // Deal with INFO request if applicable - if (req.FlagIsSet(NormRepairRequest::INFO)) + if (req.FlagIsSet(NormRepairRequest::INFO)) { if (holdoff) { @@ -3763,435 +3802,434 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor tx_repair_pending = true; tx_repair_object_min = nextObjectId; tx_repair_block_min = 0; - tx_repair_segment_min = 0; + tx_repair_segment_min = 0; } object->HandleInfoRequest(false); startTimer = true; } } - } // end if (freshObject) + } // end if (freshObject) ASSERT(NULL != object); object->SetLastNackTime(ProtoTime(currentTime)); - + switch (requestLevel) { - case OBJECT: - PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(OBJECT) objs>%hu:%hu\n", - (UINT16)nextObjectId, (UINT16)lastObjectId); - if (holdoff) + case OBJECT: + PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(OBJECT) objs>%hu:%hu\n", + (UINT16)nextObjectId, (UINT16)lastObjectId); + if (holdoff) + { + if (nextObjectId > txObjectIndex) { - if (nextObjectId > txObjectIndex) + if (object->IsStream()) + object->TxReset(((NormStreamObject *)object)->StreamBufferLo()); + else + object->TxReset(); + if (!tx_pending_mask.Set(nextObjectId)) + PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (1)\n", + (UINT16)nextObjectId); + } + } + else + { + // Update our minimum tx repair index as needed + if (tx_repair_pending) + { + if (nextObjectId <= tx_repair_object_min) { - if (object->IsStream()) - object->TxReset(((NormStreamObject*)object)->StreamBufferLo()); - else - object->TxReset(); - if (!tx_pending_mask.Set(nextObjectId)) - PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (1)\n", - (UINT16)nextObjectId); + tx_repair_object_min = nextObjectId; + tx_repair_block_min = 0; + tx_repair_segment_min = 0; } } else { - // Update our minimum tx repair index as needed - if (tx_repair_pending) + tx_repair_pending = true; + tx_repair_object_min = nextObjectId; + tx_repair_block_min = 0; + tx_repair_segment_min = 0; + } + tx_repair_mask.Set(nextObjectId); + startTimer = true; + } + nextObjectId++; + if (nextObjectId > lastObjectId) + inRange = false; + break; + case BLOCK: + PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(BLOCK) obj>%hu blks>%lu:%lu\n", + (UINT16)nextObjectId, + (unsigned long)nextBlockId.GetValue(), + (unsigned long)lastBlockId.GetValue()); + inRange = false; // BLOCK requests are processed in one pass + // (TBD) if entire object is TxReset(), continue + if (object->IsStream()) + { + // mark nack time for potential flow control + static_cast(object)->SetLastNackTime(nextBlockId, ProtoTime(currentTime)); + bool attemptLock = true; + NormBlockId firstLockId = nextBlockId; + if (holdoff) + { + // Only lock blocks for which we're going to accept the repair request + if (nextObjectId == txObjectIndex) { - if (nextObjectId <= tx_repair_object_min) + //if (lastBlockId < txBlockIndex) + if (Compare(lastBlockId, txBlockIndex) < 0) + attemptLock = false; + //else if (nextBlockId < txBlockIndex) + else if (Compare(nextBlockId, txBlockIndex) < 0) + firstLockId = txBlockIndex; + } + else if (nextObjectId < txObjectIndex) + { + attemptLock = false; // NACK arrived too late to be useful + } + } + + // Make sure the stream' pending_mask can be set as needed + // (TBD) + + // Lock stream_buffer pending for block data retransmissions + if (attemptLock) + { + if (!((NormStreamObject *)object)->LockBlocks(firstLockId, lastBlockId, currentTime)) + { + PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu LockBlocks() failure\n", + (unsigned long)LocalNodeId()); + if (!squelchQueued) { - tx_repair_object_min = nextObjectId; - tx_repair_block_min = 0; + SenderQueueSquelch(nextObjectId); + squelchQueued = true; + } + break; + } + } + else + { + break; // ignore late arriving NACK + } + } // end if (object->IsStream() + if (holdoff) + { + if (nextObjectId == txObjectIndex) + { + //if (nextBlockId >= txBlockIndex) + if (Compare(nextBlockId, txBlockIndex) >= 0) + object->TxResetBlocks(nextBlockId, lastBlockId); + //else if (lastBlockId >= txBlockIndex) + else if (Compare(lastBlockId, txBlockIndex) >= 0) + object->TxResetBlocks(txBlockIndex, lastBlockId); + } + else if (nextObjectId > txObjectIndex) + { + if (object->TxResetBlocks(nextBlockId, lastBlockId)) + { + if (!tx_pending_mask.Set(nextObjectId)) + PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (2)\n", + (UINT16)nextObjectId); + } + } + } + else + { + // Update our minimum tx repair index as needed + if (tx_repair_pending) + { + if (nextObjectId < tx_repair_object_min) + { + tx_repair_object_min = nextObjectId; + tx_repair_block_min = nextBlockId; + tx_repair_segment_min = 0; + } + else if (nextObjectId == tx_repair_object_min) + { + //if (nextBlockId <= tx_repair_block_min) + if (Compare(nextBlockId, tx_repair_block_min) <= 0) + { + tx_repair_block_min = nextBlockId; tx_repair_segment_min = 0; } } + } + else + { + tx_repair_pending = true; + tx_repair_object_min = nextObjectId; + tx_repair_block_min = nextBlockId; + tx_repair_segment_min = 0; + } + if (!object->HandleBlockRequest(nextBlockId, lastBlockId)) + { + if (!squelchQueued) + { + SenderQueueSquelch(nextObjectId); + squelchQueued = true; + } + } + startTimer = true; + } + break; + case SEGMENT: + PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(SEGMENT) obj>%hu blk>%lu segs>%hu:%hu\n", + (UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), + (UINT16)nextSegmentId, (UINT16)lastSegmentId); + inRange = false; // SEGMENT repairs are also handled in one pass + if (nextBlockId != prevBlockId) + freshBlock = true; + if (freshBlock) + { + // Is this entire block already repair pending? + if (object->IsRepairSet(nextBlockId)) + continue; + if (NULL == (block = object->FindBlock(nextBlockId))) + { + // Is this entire block already tx pending? + if (object->IsPendingSet(nextBlockId)) + { + // Entire block already tx pending, don't worry about individual segments + PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu " + "recvd SEGMENT repair request for pending block.\n", + (unsigned long)LocalNodeId()); + continue; + } else { - tx_repair_pending = true; - tx_repair_object_min = nextObjectId; - tx_repair_block_min = 0; - tx_repair_segment_min = 0; + // Try to recover block including parity calculation + if (NULL == (block = object->SenderRecoverBlock(nextBlockId))) + { + if (NormObject::STREAM == object->GetType()) + { + PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu " + "recvd repair request for old stream block(%lu) ...\n", + (unsigned long)LocalNodeId(), + (unsigned long)nextBlockId.GetValue()); + if (!squelchQueued) + { + SenderQueueSquelch(nextObjectId); + squelchQueued = true; + } + } + else + { + // Resource constrained, move on to next repair request + PLOG(PL_INFO, "NormSession::SenderHandleNackMessage() node>%lu " + "Warning - sender is resource constrained ...\n", + (unsigned long)LocalNodeId()); + } + continue; + } } - tx_repair_mask.Set(nextObjectId); - startTimer = true; } - nextObjectId++; - if (nextObjectId > lastObjectId) inRange = false; - break; - case BLOCK: - PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(BLOCK) obj>%hu blks>%lu:%lu\n", - (UINT16)nextObjectId, - (unsigned long)nextBlockId.GetValue(), - (unsigned long)lastBlockId.GetValue()); - inRange = false; // BLOCK requests are processed in one pass - // (TBD) if entire object is TxReset(), continue - if (object->IsStream()) + freshBlock = false; + numErasures = extra_parity; + prevBlockId = nextBlockId; + } // end if (freshBlock) + ASSERT(NULL != block); + + // If stream && explicit data repair, lock the data for retransmission + // (TBD) this use of "ndata" needs to be replaced for dynamically shortened blocks + if (object->IsStream()) + { + // mark nack time for potential flow control + static_cast(object)->SetLastNackTime(nextBlockId, ProtoTime(currentTime)); + if (nextSegmentId < ndata) { - // mark nack time for potential flow control - static_cast(object)->SetLastNackTime(nextBlockId, ProtoTime(currentTime)); bool attemptLock = true; - NormBlockId firstLockId = nextBlockId; + NormSegmentId firstLockId = nextSegmentId; + NormSegmentId lastLockId = ndata - 1; + lastLockId = MIN(lastLockId, lastSegmentId); if (holdoff) { - // Only lock blocks for which we're going to accept the repair request if (nextObjectId == txObjectIndex) { - //if (lastBlockId < txBlockIndex) - if (Compare(lastBlockId, txBlockIndex) < 0) - attemptLock = false; - //else if (nextBlockId < txBlockIndex) - else if (Compare(nextBlockId, txBlockIndex) < 0) - firstLockId = txBlockIndex; + //if (nextBlockId < txBlockIndex) + if (Compare(nextBlockId, txBlockIndex) < 0) + { + //if (1 == (txBlockIndex - nextBlockId)) + if (1 == (UINT32)Difference(txBlockIndex, nextBlockId)) + { + // We're currently sending this block + if (block->IsPending()) + { + NormSegmentId firstPending = 0; + block->GetFirstPending(firstPending); + if (lastLockId <= firstPending) + attemptLock = false; + else if (nextSegmentId < firstPending) + firstLockId = firstPending; + } + else + { + // block was just recovered + } + } + else + { + attemptLock = false; // NACK arrived way too late + } + } } else if (nextObjectId < txObjectIndex) { - attemptLock = false; // NACK arrived too late to be useful + attemptLock = false; // NACK arrived too late } - } - - // Make sure the stream' pending_mask can be set as needed - // (TBD) - - // Lock stream_buffer pending for block data retransmissions + } // end if (holdoff) if (attemptLock) { - if (!((NormStreamObject*)object)->LockBlocks(firstLockId, lastBlockId, currentTime)) + if (!((NormStreamObject *)object)->LockSegments(nextBlockId, firstLockId, lastLockId)) { - PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu LockBlocks() failure\n", - (unsigned long)LocalNodeId()); - if (!squelchQueued) + PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu " + "LockSegments() failure\n", + (unsigned long)LocalNodeId()); + if (!squelchQueued) { SenderQueueSquelch(nextObjectId); squelchQueued = true; } break; - } + } } else { - break; // ignore late arriving NACK - } - } // end if (object->IsStream() - if (holdoff) - { - if (nextObjectId == txObjectIndex) - { - //if (nextBlockId >= txBlockIndex) - if (Compare(nextBlockId, txBlockIndex) >= 0) - object->TxResetBlocks(nextBlockId, lastBlockId); - //else if (lastBlockId >= txBlockIndex) - else if (Compare(lastBlockId, txBlockIndex) >= 0) - object->TxResetBlocks(txBlockIndex, lastBlockId); + break; // ignore late arriving NACK } - else if (nextObjectId > txObjectIndex) + } // end if (nextSegmentId < ndata) + } // end if (object->IsStream()) + + // With a series of SEGMENT repair requests for a block, "numErasures" will + // eventually total the number of missing segments in the block. + numErasures += (lastSegmentId - nextSegmentId + 1); + if (holdoff) + { + if (nextObjectId > txObjectIndex) + { + if (object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures)) { - if (object->TxResetBlocks(nextBlockId, lastBlockId)) + if (!tx_pending_mask.Set(nextObjectId)) + PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (3)\n", + (UINT16)nextObjectId); + } + } + else if (nextObjectId == txObjectIndex) + { + //if (nextBlockId >= txBlockIndex) + if (Compare(nextBlockId, txBlockIndex) >= 0) + { + object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); + } + //else if (1 == (txBlockIndex - nextBlockId)) + else if (1 == (UINT32)Difference(txBlockIndex, nextBlockId)) + { + NormSegmentId firstPending = 0; + if (block->GetFirstPending(firstPending)) { - if (!tx_pending_mask.Set(nextObjectId)) - PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (2)\n", - (UINT16)nextObjectId); + if (nextSegmentId > firstPending) + object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); + else if (lastSegmentId > firstPending) + object->TxUpdateBlock(block, firstPending, lastSegmentId, numErasures); + else if (numErasures > block->ParityCount()) + object->TxUpdateBlock(block, firstPending, firstPending, numErasures); + } + else + { + // This block was just recovered, so do full update + object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); + } + } + } + } + else // !holdoff + { + // Update our minimum tx repair index as needed + ASSERT(nextBlockId == block->GetId()); + UINT16 nextBlockSize = object->GetBlockSize(nextBlockId); + if (tx_repair_pending) + { + if (nextObjectId < tx_repair_object_min) + { + tx_repair_block_min = nextBlockId; + tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? nextSegmentId : (nextBlockSize - 1); + } + else if (nextObjectId == tx_repair_object_min) + { + //if (nextBlockId < tx_repair_block_min) + if (Compare(nextBlockId, tx_repair_block_min) < 0) + { + tx_repair_block_min = nextBlockId; + tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? nextSegmentId : (nextBlockSize - 1); + } + else if (nextBlockId == tx_repair_block_min) + { + if (nextSegmentId < tx_repair_segment_min) + tx_repair_segment_min = nextSegmentId; } } } else { - // Update our minimum tx repair index as needed - if (tx_repair_pending) - { - if (nextObjectId < tx_repair_object_min) - { - tx_repair_object_min = nextObjectId; - tx_repair_block_min = nextBlockId; - tx_repair_segment_min = 0; - } - else if (nextObjectId == tx_repair_object_min) - { - //if (nextBlockId <= tx_repair_block_min) - if (Compare(nextBlockId, tx_repair_block_min) <= 0) - { - tx_repair_block_min = nextBlockId; - tx_repair_segment_min = 0; - } - } - } - else - { - tx_repair_pending = true; - tx_repair_object_min = nextObjectId; - tx_repair_block_min = nextBlockId; - tx_repair_segment_min = 0; - } - if (!object->HandleBlockRequest(nextBlockId, lastBlockId)) - { - if (!squelchQueued) - { - SenderQueueSquelch(nextObjectId); - squelchQueued = true; - } - } - startTimer = true; + tx_repair_pending = true; + tx_repair_object_min = nextObjectId; + tx_repair_block_min = nextBlockId; + tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? nextSegmentId : (nextBlockSize - 1); } - break; - case SEGMENT: - PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(SEGMENT) obj>%hu blk>%lu segs>%hu:%hu\n", - (UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), - (UINT16)nextSegmentId, (UINT16)lastSegmentId); - inRange = false; // SEGMENT repairs are also handled in one pass - if (nextBlockId != prevBlockId) freshBlock = true; - if (freshBlock) - { - // Is this entire block already repair pending? - if (object->IsRepairSet(nextBlockId)) - continue; - if (NULL == (block = object->FindBlock(nextBlockId))) - { - // Is this entire block already tx pending? - if (object->IsPendingSet(nextBlockId)) - { - // Entire block already tx pending, don't worry about individual segments - PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu " - "recvd SEGMENT repair request for pending block.\n", - (unsigned long)LocalNodeId()); - continue; - } - else - { - // Try to recover block including parity calculation - if (NULL == (block = object->SenderRecoverBlock(nextBlockId))) - { - if (NormObject::STREAM == object->GetType()) - { - PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu " - "recvd repair request for old stream block(%lu) ...\n", - (unsigned long)LocalNodeId(), - (unsigned long)nextBlockId.GetValue()); - if (!squelchQueued) - { - SenderQueueSquelch(nextObjectId); - squelchQueued = true; - } - } - else - { - // Resource constrained, move on to next repair request - PLOG(PL_INFO, "NormSession::SenderHandleNackMessage() node>%lu " - "Warning - sender is resource constrained ...\n", - (unsigned long)LocalNodeId()); - } - continue; - } - } - } - freshBlock = false; - numErasures = extra_parity; - prevBlockId = nextBlockId; - } // end if (freshBlock) - ASSERT(NULL != block); - - // If stream && explicit data repair, lock the data for retransmission - // (TBD) this use of "ndata" needs to be replaced for dynamically shortened blocks - if (object->IsStream()) - { - // mark nack time for potential flow control - static_cast(object)->SetLastNackTime(nextBlockId, ProtoTime(currentTime)); - if (nextSegmentId < ndata) - { - bool attemptLock = true; - NormSegmentId firstLockId = nextSegmentId; - NormSegmentId lastLockId = ndata - 1; - lastLockId = MIN(lastLockId, lastSegmentId); - if (holdoff) - { - if (nextObjectId == txObjectIndex) - { - //if (nextBlockId < txBlockIndex) - if (Compare(nextBlockId, txBlockIndex) < 0) - { - //if (1 == (txBlockIndex - nextBlockId)) - if (1 == (UINT32)Difference(txBlockIndex, nextBlockId)) - { - // We're currently sending this block - if (block->IsPending()) - { - NormSegmentId firstPending = 0; - block->GetFirstPending(firstPending); - if (lastLockId <= firstPending) - attemptLock = false; - else if (nextSegmentId < firstPending) - firstLockId = firstPending; - } - else - { - // block was just recovered - } - } - else - { - attemptLock = false; // NACK arrived way too late - } - } - } - else if (nextObjectId < txObjectIndex) - { - attemptLock = false; // NACK arrived too late - } - } // end if (holdoff) - if (attemptLock) - { - if (!((NormStreamObject*)object)->LockSegments(nextBlockId, firstLockId, lastLockId)) - { - PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu " - "LockSegments() failure\n", (unsigned long)LocalNodeId()); - if (!squelchQueued) - { - SenderQueueSquelch(nextObjectId); - squelchQueued = true; - } - break; - } - } - else - { - break; // ignore late arriving NACK - } - } // end if (nextSegmentId < ndata) - } // end if (object->IsStream()) - - // With a series of SEGMENT repair requests for a block, "numErasures" will - // eventually total the number of missing segments in the block. - numErasures += (lastSegmentId - nextSegmentId + 1); - if (holdoff) - { - if (nextObjectId > txObjectIndex) - { - if (object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures)) - { - if (!tx_pending_mask.Set(nextObjectId)) - PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (3)\n", - (UINT16)nextObjectId); - } - } - else if (nextObjectId == txObjectIndex) - { - //if (nextBlockId >= txBlockIndex) - if (Compare(nextBlockId, txBlockIndex) >= 0) - { - object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); - } - //else if (1 == (txBlockIndex - nextBlockId)) - else if (1 == (UINT32)Difference(txBlockIndex, nextBlockId)) - { - NormSegmentId firstPending = 0; - if (block->GetFirstPending(firstPending)) - { - if (nextSegmentId > firstPending) - object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); - else if (lastSegmentId > firstPending) - object->TxUpdateBlock(block, firstPending, lastSegmentId, numErasures); - else if (numErasures > block->ParityCount()) - object->TxUpdateBlock(block, firstPending, firstPending, numErasures); - } - else - { - // This block was just recovered, so do full update - object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); - } - } - } - } - else // !holdoff - { - // Update our minimum tx repair index as needed - ASSERT(nextBlockId == block->GetId()); - UINT16 nextBlockSize = object->GetBlockSize(nextBlockId); - if (tx_repair_pending) - { - if (nextObjectId < tx_repair_object_min) - { - tx_repair_block_min = nextBlockId; - tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? - nextSegmentId : (nextBlockSize - 1); - } - else if (nextObjectId == tx_repair_object_min) - { - //if (nextBlockId < tx_repair_block_min) - if (Compare(nextBlockId, tx_repair_block_min) < 0) - { - tx_repair_block_min = nextBlockId; - tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? - nextSegmentId : (nextBlockSize - 1); - } - else if (nextBlockId == tx_repair_block_min) - { - if (nextSegmentId < tx_repair_segment_min) - tx_repair_segment_min = nextSegmentId; - } - } - } - else - { - tx_repair_pending = true; - tx_repair_object_min = nextObjectId; - tx_repair_block_min = nextBlockId; - tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? - nextSegmentId : (nextBlockSize - 1); - } - block->HandleSegmentRequest(nextSegmentId, lastSegmentId, - nextBlockSize, nparity, - numErasures); - startTimer = true; - } // end if/else (holdoff) - break; - case INFO: - // We already dealt with INFO request above with respect to initiating repair - nextObjectId++; - if (nextObjectId > lastObjectId) inRange = false; - break; - } // end switch(requestLevel) - } // end while(inRange) - } // end while(NextRepairItem()) - } // end while(UnpackRepairRequest()) + block->HandleSegmentRequest(nextSegmentId, lastSegmentId, + nextBlockSize, nparity, + numErasures); + startTimer = true; + } // end if/else (holdoff) + break; + case INFO: + // We already dealt with INFO request above with respect to initiating repair + nextObjectId++; + if (nextObjectId > lastObjectId) + inRange = false; + break; + } // end switch(requestLevel) + } // end while(inRange) + } // end while(NextRepairItem()) + } // end while(UnpackRepairRequest()) if (startTimer && !repair_timer.IsActive()) { // BACKOFF related code - double aggregateInterval = address.IsMulticast() ? - grtt_advertised * (backoff_factor + 1.0) : 0.0; + double aggregateInterval = address.IsMulticast() ? grtt_advertised * (backoff_factor + 1.0) : 0.0; // Uncommenting the line below treats ((0 == ndata) && 0.0 == backoff_factor) // as a special case (sets zero sender aggregateInterval) aggregateInterval = ((0 != nparity) || (backoff_factor > 0.0)) ? aggregateInterval : 0.0; - + // TBD - why did we do this thing here to limit the min aggregateInterval??? // (I think to allow "11th hour NACKs to be incorporated .. so this should be // for mcast only) if (tx_timer.IsActive() && address.IsMulticast()) { double txTimeout = tx_timer.GetTimeRemaining() - 1.0e-06; - aggregateInterval = MAX(txTimeout, aggregateInterval); - } - repair_timer.SetInterval(aggregateInterval); + aggregateInterval = MAX(txTimeout, aggregateInterval); + } + repair_timer.SetInterval(aggregateInterval); PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu starting sender " - "NACK aggregation timer (%lf sec)...\n", - (unsigned long)LocalNodeId(), aggregateInterval); - ActivateTimer(repair_timer); + "NACK aggregation timer (%lf sec)...\n", + (unsigned long)LocalNodeId(), aggregateInterval); + ActivateTimer(repair_timer); } -} // end NormSession::SenderHandleNackMessage() +} // end NormSession::SenderHandleNackMessage() - -void NormSession::ReceiverHandleAckMessage(const NormAckMsg& ack) +void NormSession::ReceiverHandleAckMessage(const NormAckMsg &ack) { - NormSenderNode* theSender = (NormSenderNode*)sender_tree.FindNodeById(ack.GetSenderId()); + NormSenderNode *theSender = (NormSenderNode *)sender_tree.FindNodeById(ack.GetSenderId()); if (theSender) { - theSender->HandleAckMessage(ack); + theSender->HandleAckMessage(ack); } else if (ack.GetSenderId() != LocalNodeId()) { PLOG(PL_DEBUG, "NormSession::ReceiverHandleAckMessage() node>%lu heard ACK for unknown sender>%lu\n", - (unsigned long)LocalNodeId(), (unsigned long)ack.GetSenderId(), IsServerListener()); + (unsigned long)LocalNodeId(), (unsigned long)ack.GetSenderId(), IsServerListener()); } -} // end NormSession::ReceiverHandleAckMessage() +} // end NormSession::ReceiverHandleAckMessage() -void NormSession::ReceiverHandleNackMessage(const NormNackMsg& nack) +void NormSession::ReceiverHandleNackMessage(const NormNackMsg &nack) { - NormSenderNode* theSender = (NormSenderNode*)sender_tree.FindNodeById(nack.GetSenderId()); + NormSenderNode *theSender = (NormSenderNode *)sender_tree.FindNodeById(nack.GetSenderId()); if (theSender) { theSender->HandleNackMessage(nack); @@ -4199,29 +4237,28 @@ void NormSession::ReceiverHandleNackMessage(const NormNackMsg& nack) else if (nack.GetSenderId() != LocalNodeId()) { PLOG(PL_DEBUG, "NormSession::ReceiverHandleNackMessage() node>%lu heard NACK for unknown sender\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } -} // end NormSession::ReceiverHandleNackMessage() - +} // end NormSession::ReceiverHandleNackMessage() bool NormSession::SenderQueueSquelch(NormObjectId objectId) { // If a squelch is already queued, update it if (objectId < squelch->objectId) bool doEnqueue = true; - NormCmdSquelchMsg* squelch = NULL; - NormMsg* msg = message_queue.GetHead(); + NormCmdSquelchMsg *squelch = NULL; + NormMsg *msg = message_queue.GetHead(); while (NULL != msg) { // (TBD) we need to depreceate the whole "message_pool" idea and // have messages be built on demand in NormSession::Serve() according // to some state variables (i.e. that dictate when to send a command - // instead of data, etc). This will simplify alot of stuff and + // instead of data, etc). This will simplify alot of stuff and // probably improve performance some too. if (NormMsg::CMD == msg->GetType()) { - if (NormCmdMsg::SQUELCH == static_cast(msg)->GetFlavor()) + if (NormCmdMsg::SQUELCH == static_cast(msg)->GetFlavor()) { - squelch = static_cast(msg); + squelch = static_cast(msg); break; } } @@ -4235,7 +4272,7 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) } else { - squelch = (NormCmdSquelchMsg*)GetMessageFromPool(); + squelch = (NormCmdSquelchMsg *)GetMessageFromPool(); } if (squelch) { @@ -4244,7 +4281,7 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) squelch->SetGrtt(grtt_quantized); squelch->SetBackoffFactor((unsigned char)backoff_factor); squelch->SetGroupSize(gsize_quantized); - NormObject* obj = tx_table.Find(objectId); + NormObject *obj = tx_table.Find(objectId); NormObjectTable::Iterator iterator(tx_table); NormObjectId nextId; if (NULL != obj) @@ -4252,10 +4289,11 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) ASSERT(NormObject::STREAM == obj->GetType()); squelch->SetObjectId(objectId); //NormBlockId blockId = static_cast(obj)->StreamBufferLo(); - NormBlockId blockId = static_cast(obj)->RepairWindowLo(); + NormBlockId blockId = static_cast(obj)->RepairWindowLo(); squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m); while ((obj = iterator.GetNextObject())) - if (objectId == obj->GetId()) break; + if (objectId == obj->GetId()) + break; nextId = objectId + 1; } else @@ -4263,15 +4301,15 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) obj = iterator.GetNextObject(); if (NULL != obj) { - squelch->SetObjectId(obj->GetId()); - NormBlockId blockId; - if (obj->IsStream()) - //blockId =static_cast(obj)->StreamBufferLo(); - blockId = static_cast(obj)->RepairWindowLo(); - else - blockId = NormBlockId(0); - squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m); - nextId = obj->GetId() + 1; + squelch->SetObjectId(obj->GetId()); + NormBlockId blockId; + if (obj->IsStream()) + //blockId =static_cast(obj)->StreamBufferLo(); + blockId = static_cast(obj)->RepairWindowLo(); + else + blockId = NormBlockId(0); + squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m); + nextId = obj->GetId() + 1; } else { @@ -4300,24 +4338,24 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) { QueueMessage(squelch); PLOG(PL_DEBUG, "NormSession::SenderQueueSquelch() node>%lu sender queued squelch ...\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } else { PLOG(PL_DEBUG, "NormSession::SenderQueueSquelch() node>%lu sender updated squelch ...\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } return true; } else { PLOG(PL_FATAL, "NormSession::SenderQueueSquelch() node>%lu message_pool exhausted! (couldn't squelch)\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); return false; } -} // end NormSession::SenderQueueSquelch() +} // end NormSession::SenderQueueSquelch() -bool NormSession::SenderSendCmd(const char* cmdBuffer, unsigned int cmdLength, bool robust) +bool NormSession::SenderSendCmd(const char *cmdBuffer, unsigned int cmdLength, bool robust) { if (!is_sender) { @@ -4337,21 +4375,23 @@ bool NormSession::SenderSendCmd(const char* cmdBuffer, unsigned int cmdLength, b memcpy(cmd_buffer, cmdBuffer, cmdLength); cmd_length = cmdLength; cmd_count = robust ? tx_robust_factor : 1; - if (!tx_timer.IsActive()) PromptSender(); + if (!tx_timer.IsActive()) + PromptSender(); return true; -} // end NormSession::SenderSendCmd() +} // end NormSession::SenderSendCmd() void NormSession::SenderCancelCmd() { if (0 != cmd_count) { - if (cmd_timer.IsActive()) cmd_timer.Deactivate(); + if (cmd_timer.IsActive()) + cmd_timer.Deactivate(); cmd_count = 0; cmd_length = 0; } -} // end NormSession::SenderCancelCmd() +} // end NormSession::SenderCancelCmd() -bool NormSession::SenderSendAppCmd(const char* buffer, unsigned int length, const ProtoAddress& dst) +bool NormSession::SenderSendAppCmd(const char *buffer, unsigned int length, const ProtoAddress &dst) { // Build/immediately send a NORM_CMD(APPLICATION) message NormCmdAppMsg appMsg; @@ -4365,19 +4405,20 @@ bool NormSession::SenderSendAppCmd(const char* buffer, unsigned int length, cons appMsg.SetDestination(dst); if (MSG_SEND_OK != SendMessage(appMsg)) PLOG(PL_ERROR, "NormSession::SenderSendAppCmd() node>%lu sender unable to send app-defined cmd ...\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); else PLOG(PL_DEBUG, "NormSession::SenderSendAppCmd() node>%lu sender sending app-defined cmd len:%u...\n", - (unsigned long)LocalNodeId(), appMsg.GetLength()); + (unsigned long)LocalNodeId(), appMsg.GetLength()); return true; -} // end NormSession::SenderSendAppCmd() +} // end NormSession::SenderSendAppCmd() -bool NormSession::SenderQueueAppCmd() +bool NormSession::SenderQueueAppCmd() { - if (0 == cmd_count) return false; - ASSERT(!cmd_timer.IsActive()); + if (0 == cmd_count) + return false; + ASSERT(!cmd_timer.IsActive()); // 1) Build a NORM_CMD(APPLICATION) message - NormCmdAppMsg* appMsg = static_cast(GetMessageFromPool()); + NormCmdAppMsg *appMsg = static_cast(GetMessageFromPool()); if (NULL != appMsg) { appMsg->Init(); @@ -4388,7 +4429,7 @@ bool NormSession::SenderQueueAppCmd() appMsg->SetContent(cmd_buffer, cmd_length, segment_size); QueueMessage(appMsg); PLOG(PL_DEBUG, "NormSession::SenderQueueAppCmd() node>%lu sender queued app-defined cmd ...\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); cmd_count--; if (0 != cmd_count) { @@ -4398,7 +4439,7 @@ bool NormSession::SenderQueueAppCmd() else { PLOG(PL_DEBUG, "NormSession::SenderQueueAppCmd() node>%lu cmd transmission completed ...\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); Notify(NormController::TX_CMD_SENT, NULL, NULL); } return true; @@ -4406,16 +4447,17 @@ bool NormSession::SenderQueueAppCmd() else { PLOG(PL_FATAL, "NormSession::SenderQueueAppCmd() node>%lu message_pool exhausted!\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); return false; } -} // end NormSession::SenderQueueAppCmd() +} // end NormSession::SenderQueueAppCmd() -bool NormSession::OnCmdTimeout(ProtoTimer& theTimer) +bool NormSession::OnCmdTimeout(ProtoTimer &theTimer) { - if (!tx_timer.IsActive()) PromptSender(); + if (!tx_timer.IsActive()) + PromptSender(); return true; -} // end NormSession::OnCmdTimeout() +} // end NormSession::OnCmdTimeout() void NormSession::ActivateFlowControl(double delay, NormObjectId objectId, NormController::Event event) { @@ -4423,14 +4465,14 @@ void NormSession::ActivateFlowControl(double delay, NormObjectId objectId, NormC flow_control_event = event; flow_control_timer.SetInterval(delay); if (flow_control_timer.IsActive()) - flow_control_timer.Reschedule(); + flow_control_timer.Reschedule(); else ActivateTimer(flow_control_timer); -} // end NormSession::ActivateFlowControl() +} // end NormSession::ActivateFlowControl() -bool NormSession::OnFlowControlTimeout(ProtoTimer& theTimer) +bool NormSession::OnFlowControlTimeout(ProtoTimer &theTimer) { - NormObject* object = tx_table.Find(flow_control_object); + NormObject *object = tx_table.Find(flow_control_object); if (NULL == object) { PLOG(PL_WARN, "NormSession::OnFlowControlTimeout() flow_control_object removed?!\n"); @@ -4442,13 +4484,13 @@ bool NormSession::OnFlowControlTimeout(ProtoTimer& theTimer) if (object->IsStream()) { // A stream was flow-controlled, so check its stream nack age, etc - NormBlock* block = static_cast(object)->StreamBlockLo(); + NormBlock *block = static_cast(object)->StreamBlockLo(); if (NULL == block) { // No blocks in stream buffer, thus it is actually empty // Notify(NormController::TX_QUEUE_VACANCY, (NormSenderNode*)NULL, object); posted_tx_queue_empty = true; - Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, object);//(NormObject*)NULL); + Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode *)NULL, object); //(NormObject*)NULL); return true; } deltaTime = GetFlowControlDelay() - block->GetNackAge(); @@ -4459,7 +4501,7 @@ bool NormSession::OnFlowControlTimeout(ProtoTimer& theTimer) if (!block->IsPending()) { posted_tx_queue_empty = (NormController::TX_QUEUE_EMPTY == flow_control_event); - Notify(flow_control_event, (NormSenderNode*)NULL, object); + Notify(flow_control_event, (NormSenderNode *)NULL, object); } return true; } @@ -4474,7 +4516,7 @@ bool NormSession::OnFlowControlTimeout(ProtoTimer& theTimer) if (!object->IsRepairPending() && !object->IsPending()) { posted_tx_queue_empty = (NormController::TX_QUEUE_EMPTY == flow_control_event); - Notify(flow_control_event, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(flow_control_event, (NormSenderNode *)NULL, (NormObject *)NULL); } return true; } @@ -4482,14 +4524,13 @@ bool NormSession::OnFlowControlTimeout(ProtoTimer& theTimer) // Extend flow control timeout due to recent activity // NOTE that above we limited the minimum "deltaTime" to 1.0-06 ... otherwise // ProtoTime 1usec precision limitation put us into an infinite loop in - // _simulation_ environments where perfect scheduling occurs w/ zero processing time + // _simulation_ environments where perfect scheduling occurs w/ zero processing time theTimer.SetInterval(deltaTime); theTimer.Reschedule(); return false; -} // end NormSession::OnFlowControlTimeout() +} // end NormSession::OnFlowControlTimeout() - -bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) +bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg &cmd) { // Build a NORM_CMD(REPAIR_ADV) message with current pending repair state. NormRepairRequest req; @@ -4498,36 +4539,37 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) NormObjectId firstId; UINT16 objectCount = 0; NormObjectTable::Iterator iterator(tx_table); - NormObject* nextObject = iterator.GetNextObject(); + NormObject *nextObject = iterator.GetNextObject(); while (NULL != nextObject) { - NormObject* currentObject = nextObject; + NormObject *currentObject = nextObject; nextObject = iterator.GetNextObject(); NormObjectId currentId = currentObject->GetId(); bool repairEntireObject = tx_repair_mask.Test(currentId); if (repairEntireObject) { - if (!objectCount) firstId = currentId; // set first OBJECT level repair id - objectCount++; // increment consecutive OBJECT level repair count. + if (!objectCount) + firstId = currentId; // set first OBJECT level repair id + objectCount++; // increment consecutive OBJECT level repair count. } - + // Check for non-OBJECT level request or end if (objectCount && (!repairEntireObject || (NULL != nextObject))) { NormRepairRequest::Form form; switch (objectCount) { - case 0: - form = NormRepairRequest::INVALID; - break; - case 1: - case 2: - form = NormRepairRequest::ITEMS; - break; - default: - form = NormRepairRequest::RANGES; - break; - } + case 0: + form = NormRepairRequest::INVALID; + break; + case 1: + case 2: + form = NormRepairRequest::ITEMS; + break; + default: + form = NormRepairRequest::RANGES; + break; + } if (form != prevForm) { if (NormRepairRequest::INVALID != prevForm) @@ -4538,7 +4580,7 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case break; - } + } } req.SetForm(form); cmd.AttachRepairRequest(req, segment_size); @@ -4546,22 +4588,22 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) } switch (form) { - case 0: - ASSERT(0); // can't happen - break; - case 1: - case 2: - req.SetForm(NormRepairRequest::ITEMS); - req.AppendRepairItem(fec_id, fec_m, firstId, 0, ndata, 0); // (TBD) error check - if (2 == objectCount) - req.AppendRepairItem(fec_id, fec_m, currentId, 0, ndata, 0); // (TBD) error check - break; - default: - req.SetForm(NormRepairRequest::RANGES); - req.AppendRepairRange(fec_id, fec_m, firstId, 0, ndata, 0, // (TBD) error check - currentId, 0, ndata, 0); - break; - } + case 0: + ASSERT(0); // can't happen + break; + case 1: + case 2: + req.SetForm(NormRepairRequest::ITEMS); + req.AppendRepairItem(fec_id, fec_m, firstId, 0, ndata, 0); // (TBD) error check + if (2 == objectCount) + req.AppendRepairItem(fec_id, fec_m, currentId, 0, ndata, 0); // (TBD) error check + break; + default: + req.SetForm(NormRepairRequest::RANGES); + req.AppendRepairRange(fec_id, fec_m, firstId, 0, ndata, 0, // (TBD) error check + currentId, 0, ndata, 0); + break; + } prevForm = NormRepairRequest::INVALID; if (0 == cmd.PackRepairRequest(req)) { @@ -4572,10 +4614,10 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) objectCount = 0; } if (!repairEntireObject) - { + { if (currentObject->IsRepairPending()) { - if (NormRepairRequest::INVALID != prevForm)// && currentObject->IsRepairPending()) + if (NormRepairRequest::INVALID != prevForm) // && currentObject->IsRepairPending()) { prevForm = NormRepairRequest::INVALID; if (0 == cmd.PackRepairRequest(req)) @@ -4585,11 +4627,12 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) break; } } - if (!currentObject->AppendRepairAdv(cmd)) break; + if (!currentObject->AppendRepairAdv(cmd)) + break; } - objectCount = 0; // this is probably redundant + objectCount = 0; // this is probably redundant } - } // end while (nextObject) + } // end while (nextObject) if (NormRepairRequest::INVALID != prevForm) { if (0 == cmd.PackRepairRequest(req)) @@ -4597,94 +4640,93 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case } return true; -} // end NormSession::SenderBuildRepairAdv() +} // end NormSession::SenderBuildRepairAdv() -bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/) +bool NormSession::OnRepairTimeout(ProtoTimer & /*theTimer*/) { tx_repair_pending = false; if (0 != repair_timer.GetRepeatCount()) { // NACK aggregation period has ended. (incorporate accumulated repair requests) PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu sender NACK aggregation time ended.\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); NormObjectTable::Iterator iterator(tx_table); - NormObject* obj; + NormObject *obj; while ((obj = iterator.GetNextObject())) { NormObjectId objectId = obj->GetId(); if (tx_repair_mask.Test(objectId)) { PLOG(PL_TRACE, "NormSession::OnRepairTimeout() node>%lu tx reset obj>%hu ...\n", - (unsigned long)LocalNodeId(), (UINT16)objectId); + (unsigned long)LocalNodeId(), (UINT16)objectId); if (obj->IsStream()) - obj->TxReset(((NormStreamObject*)obj)->RepairWindowLo()); + obj->TxReset(((NormStreamObject *)obj)->RepairWindowLo()); else obj->TxReset(); tx_repair_mask.Unset(objectId); if (!tx_pending_mask.Set(objectId)) { PLOG(PL_ERROR, "NormSession::OnRepairTimeout() tx_pending_mask.Set(%hu) error (1)\n", - (UINT16)objectId); + (UINT16)objectId); } - } + } else { PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu activating obj>%hu repairs ...\n", - (unsigned long)LocalNodeId(), (UINT16)objectId); - if (obj->ActivateRepairs()) + (unsigned long)LocalNodeId(), (UINT16)objectId); + if (obj->ActivateRepairs()) { PLOG(PL_TRACE, "NormSession::OnRepairTimeout() node>%lu activated obj>%hu repairs ...\n", - (unsigned long)LocalNodeId(), (UINT16)objectId); + (unsigned long)LocalNodeId(), (UINT16)objectId); if (!tx_pending_mask.Set(objectId)) PLOG(PL_ERROR, "NormSession::OnRepairTimeout() node>%lu tx_pending_mask.Set(%hu) error (2)\n", - (unsigned long)LocalNodeId(), (UINT16)objectId); - } - } - } // end while (iterator.GetNextObject()) + (unsigned long)LocalNodeId(), (UINT16)objectId); + } + } + } // end while (iterator.GetNextObject()) PromptSender(); // BACKOFF related code - // Holdoff initiation of new repair cycle for one GRTT + // Holdoff initiation of new repair cycle for one GRTT // (TBD) for unicast sessions, use CLR RTT ??? - //double holdoffInterval = backoff_factor > 0.0 ? grtt_advertised : 0.0; - double holdoffInterval = grtt_advertised; + //double holdoffInterval = backoff_factor > 0.0 ? grtt_advertised : 0.0; + double holdoffInterval = grtt_advertised; repair_timer.SetInterval(holdoffInterval); // repair holdoff interval = 1*GRTT PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu starting sender " - "NACK holdoff timer (%lf sec)...\n", - (unsigned long)LocalNodeId(), holdoffInterval); + "NACK holdoff timer (%lf sec)...\n", + (unsigned long)LocalNodeId(), holdoffInterval); } else { // REPAIR holdoff interval has now ended. PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu sender holdoff time ended.\n", - (unsigned long)LocalNodeId()); + (unsigned long)LocalNodeId()); } return true; -} // end NormSession::OnRepairTimeout() - +} // end NormSession::OnRepairTimeout() // (TBD) Should pass current system time to ProtoTimer timeout handlers // for more efficiency ... -bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) +bool NormSession::OnTxTimeout(ProtoTimer & /*theTimer*/) { - NormMsg* msg; - - // Note: sometimes need RepairAdv even when cc_enable is false ... - NormCmdRepairAdvMsg adv; - if (advertise_repairs && (probe_proactive || (repair_timer.IsActive() && + NormMsg *msg; + + // Note: sometimes need RepairAdv even when cc_enable is false ... + NormCmdRepairAdvMsg adv; + if (advertise_repairs && (probe_proactive || (repair_timer.IsActive() && repair_timer.GetRepeatCount()))) { - // Build a NORM_CMD(NACK_ADV) in response to - // receipt of unicast NACK or CC update + // Build a NORM_CMD(NACK_ADV) in response to + // receipt of unicast NACK or CC update adv.Init(); adv.SetGrtt(grtt_quantized); adv.SetBackoffFactor((unsigned char)backoff_factor); adv.SetGroupSize(gsize_quantized); adv.SetDestination(address); - + // Fill in congestion control header extension NormCCFeedbackExtension ext; adv.AttachExtension(ext); - + if (suppress_rate < 0.0) { ext.SetCCFlag(NormCC::RTT); @@ -4693,26 +4735,27 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) } else { - if (!suppress_nonconfirmed) ext.SetCCFlag(NormCC::RTT); + if (!suppress_nonconfirmed) + ext.SetCCFlag(NormCC::RTT); ext.SetCCRtt(NormQuantizeRtt(suppress_rtt)); ext.SetCCRate(NormQuantizeRate(suppress_rate)); } - + SenderBuildRepairAdv(adv); - - msg = (NormMsg*)&adv; + + msg = (NormMsg *)&adv; } else { msg = message_queue.RemoveHead(); advertise_repairs = false; } - + if (NULL != msg) { // Do "packet pairing of NORM_CMD(CC) and subsequent message (usually NORM_DATA), if any //unsigned int msgLength = msg->GetLength(); - + unsigned int msgLength = tx_residual; /* Uncomment this section of code to instate CMD(CC) / NORM_DATA "packet pairing" tx_residual = 0; @@ -4726,56 +4769,58 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) { msgLength += msg->GetLength(); } - + switch (SendMessage(*msg)) { - case MSG_SEND_OK: - if (tx_rate > 0.0) - tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate)); - if (advertise_repairs) - { - advertise_repairs = false; - suppress_rate = -1.0; // reset cc feedback suppression rate - } - else - { - ReturnMessageToPool(msg); - } - // Pre-serve to allow pre-prompt for empty tx queue - // (TBD) do this in a better way ??? There is a slight chance - // that with this approach some new data may get pre-queued - // when an interim repair request should be serviced first - // instead ??? - //if (message_queue.IsEmpty() && IsSender()) Serve(); - return true; // reinstall tx_timer - - case MSG_SEND_BLOCKED: - // Message was not sent due to to EWOULDBLOCK, so we invoke async i/o output notification - if (!advertise_repairs) - message_queue.Prepend(msg); - if (tx_timer.IsActive()) - tx_timer.Deactivate(); - //TRACE("starting output notification ...\n"); - tx_socket->StartOutputNotification(); - return false; // since timer was deactivated - - case MSG_SEND_FAILED: - // Message was not sent due to socket error (no route, etc), so so just timeout and try again - // (TBD - is there something smarter we should do) - //TRACE("MSG_SEND_FAILED\n"); - if (!advertise_repairs) message_queue.Prepend(msg); - if (tx_rate > 0.0) - tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate)); - else if (0.0 == tx_timer.GetInterval()) - tx_timer.SetInterval(0.001); - return true; // timer will be reactivated + case MSG_SEND_OK: + if (tx_rate > 0.0) + tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate)); + if (advertise_repairs) + { + advertise_repairs = false; + suppress_rate = -1.0; // reset cc feedback suppression rate + } + else + { + ReturnMessageToPool(msg); + } + // Pre-serve to allow pre-prompt for empty tx queue + // (TBD) do this in a better way ??? There is a slight chance + // that with this approach some new data may get pre-queued + // when an interim repair request should be serviced first + // instead ??? + //if (message_queue.IsEmpty() && IsSender()) Serve(); + return true; // reinstall tx_timer + + case MSG_SEND_BLOCKED: + // Message was not sent due to to EWOULDBLOCK, so we invoke async i/o output notification + if (!advertise_repairs) + message_queue.Prepend(msg); + if (tx_timer.IsActive()) + tx_timer.Deactivate(); + //TRACE("starting output notification ...\n"); + tx_socket->StartOutputNotification(); + return false; // since timer was deactivated + + case MSG_SEND_FAILED: + // Message was not sent due to socket error (no route, etc), so so just timeout and try again + // (TBD - is there something smarter we should do) + //TRACE("MSG_SEND_FAILED\n"); + if (!advertise_repairs) + message_queue.Prepend(msg); + if (tx_rate > 0.0) + tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate)); + else if (0.0 == tx_timer.GetInterval()) + tx_timer.SetInterval(0.001); + return true; // timer will be reactivated } } else - { + { // 1) Prompt for next sender message - if (IsSender()) Serve(); - + if (IsSender()) + Serve(); + if (message_queue.IsEmpty()) { if (tx_timer.IsActive()) @@ -4792,103 +4837,104 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) return OnTxTimeout(tx_timer); } } - return true; // actually will never get here but compiler thinks it's needed -} // end NormSession::OnTxTimeout() + return true; // actually will never get here but compiler thinks it's needed +} // end NormSession::OnTxTimeout() -NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) -{ +NormSession::MessageStatus NormSession::SendMessage(NormMsg &msg) +{ //TRACE("sending message length %hu\n", msg.GetLength()); bool isReceiverMsg = false; bool isProbe = false; - + // Fill in any last minute timestamps // (TBD) fill in InstanceId fields on all messages as needed // We need "fec_m" for the message for NormTrace() purposes - UINT8 fecM = fec_m; // assume it's a sender message (will be overridden otherwise) - UINT16 instId = instance_id; // assume it's a sender message (will be overridden otherwise) + UINT8 fecM = fec_m; // assume it's a sender message (will be overridden otherwise) + UINT16 instId = instance_id; // assume it's a sender message (will be overridden otherwise) switch (msg.GetType()) { - case NormMsg::INFO: - case NormMsg::DATA: + case NormMsg::INFO: + case NormMsg::DATA: + { + NormObjectMsg &objMsg = static_cast(msg); + objMsg.SetInstanceId(instId); + msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs + if (syn_status) + objMsg.SetFlag(NormObjectMsg::FLAG_SYN); + break; + } + case NormMsg::CMD: + { + NormCmdMsg &cmd = static_cast(msg); + ((NormCmdMsg &)msg).SetInstanceId(instId); + switch (cmd.GetFlavor()) { - NormObjectMsg& objMsg = static_cast(msg); - objMsg.SetInstanceId(instId); - msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs - if (syn_status) objMsg.SetFlag(NormObjectMsg::FLAG_SYN); - break; - } - case NormMsg::CMD: + case NormCmdMsg::CC: { - NormCmdMsg& cmd = static_cast(msg); - ((NormCmdMsg&)msg).SetInstanceId(instId); - switch (cmd.GetFlavor()) - { - case NormCmdMsg::CC: - { - NormCmdCCMsg& ccMsg = static_cast(cmd); - struct timeval currentTime; - ProtoSystemTime(currentTime); - ccMsg.SetSendTime(currentTime); - isProbe = true; - if (syn_status) ccMsg.SetSyn(); - break; - } - case NormCmdMsg::SQUELCH: - break; - default: - break; - } - msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs - break; - } - case NormMsg::NACK: - { - msg.SetSequence(0); // TBD - set per destination - isReceiverMsg = true; - NormNackMsg& nack = (NormNackMsg&)msg; - NormSenderNode* theSender = - (NormSenderNode*)sender_tree.FindNodeById(nack.GetSenderId()); - ASSERT(NULL != theSender); - fecM = theSender->GetFecFieldSize(); - instId = theSender->GetInstanceId(); + NormCmdCCMsg &ccMsg = static_cast(cmd); struct timeval currentTime; - ProtoSystemTime(currentTime); - struct timeval grttResponse; - theSender->CalculateGrttResponse(currentTime, grttResponse); - nack.SetGrttResponse(grttResponse); + ProtoSystemTime(currentTime); + ccMsg.SetSendTime(currentTime); + isProbe = true; + if (syn_status) + ccMsg.SetSyn(); break; } - case NormMsg::ACK: - { - msg.SetSequence(0); // TBD - set per destination - isReceiverMsg = true; - NormAckMsg& ack = (NormAckMsg&)msg; - NormSenderNode* theSender; - if (IsServerListener()) - theSender = client_tree.FindNodeByAddress(ack.GetDestination()); - else - theSender = (NormSenderNode*)sender_tree.FindNodeById(ack.GetSenderId()); - ASSERT(NULL != theSender); - fecM = theSender->GetFecFieldSize(); - instId = theSender->GetInstanceId(); - struct timeval grttResponse; - struct timeval currentTime; - ProtoSystemTime(currentTime); - theSender->CalculateGrttResponse(currentTime, grttResponse); - ack.SetGrttResponse(grttResponse); + case NormCmdMsg::SQUELCH: break; - } default: break; + } + msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs + break; + } + case NormMsg::NACK: + { + msg.SetSequence(0); // TBD - set per destination + isReceiverMsg = true; + NormNackMsg &nack = (NormNackMsg &)msg; + NormSenderNode *theSender = + (NormSenderNode *)sender_tree.FindNodeById(nack.GetSenderId()); + ASSERT(NULL != theSender); + fecM = theSender->GetFecFieldSize(); + instId = theSender->GetInstanceId(); + struct timeval currentTime; + ProtoSystemTime(currentTime); + struct timeval grttResponse; + theSender->CalculateGrttResponse(currentTime, grttResponse); + nack.SetGrttResponse(grttResponse); + break; + } + case NormMsg::ACK: + { + msg.SetSequence(0); // TBD - set per destination + isReceiverMsg = true; + NormAckMsg &ack = (NormAckMsg &)msg; + NormSenderNode *theSender; + if (IsServerListener()) + theSender = client_tree.FindNodeByAddress(ack.GetDestination()); + else + theSender = (NormSenderNode *)sender_tree.FindNodeById(ack.GetSenderId()); + ASSERT(NULL != theSender); + fecM = theSender->GetFecFieldSize(); + instId = theSender->GetInstanceId(); + struct timeval grttResponse; + struct timeval currentTime; + ProtoSystemTime(currentTime); + theSender->CalculateGrttResponse(currentTime, grttResponse); + ack.SetGrttResponse(grttResponse); + break; + } + default: + break; } // Fill in common message fields msg.SetSourceId(local_node_id); UINT16 msgSize = msg.GetLength(); // Possibly drop some tx messages for testing purposes - - + bool drop = (tx_loss_rate > 0.0) ? (UniformRand(100.0) < tx_loss_rate) : false; - + if (isReceiverMsg && receiver_silent) { // don't send receiver messages if "silent receiver" @@ -4900,23 +4946,23 @@ NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) } else if (drop) { - //DMSG(0, "TX MESSAGE DROPPED! (tx_loss_rate:%lf\n", tx_loss_rate); + //DMSG(0, "TX MESSAGE DROPPED! (tx_loss_rate:%lf\n", tx_loss_rate); // "Pretend" like dropped message was sent for trace and timing purposes - if (trace) + if (trace) { struct timeval currentTime; - ProtoSystemTime(currentTime); + ProtoSystemTime(currentTime); NormTrace(currentTime, LocalNodeId(), msg, true, fecM, instId); } // Update sent rate tracker even if dropped (for testing/debugging) sent_accumulator.Increment(msgSize); - nominal_packet_size += 0.01 * (((double)msgSize) - nominal_packet_size); - } + nominal_packet_size += 0.01 * (((double)msgSize) - nominal_packet_size); + } else { unsigned int numBytes = msgSize; - bool result = tx_socket->SendTo(msg.GetBuffer(), numBytes, msg.GetDestination()); - + bool result = tx_socket->SendTo(msg.GetBuffer(), numBytes, msg.GetDestination()); + if (result) { if (numBytes == msgSize) @@ -4926,18 +4972,18 @@ NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) // Clear SEND_ERROR indication posted_send_error = false; Notify(NormController::SEND_OK, NULL, NULL); - } + } // Separate send/recv tracing - if (trace) + if (trace) { struct timeval currentTime; - ProtoSystemTime(currentTime); + ProtoSystemTime(currentTime); NormTrace(currentTime, LocalNodeId(), msg, true, fecM, instId); } - // To keep track of _actual_ sent rate + // To keep track of _actual_ sent rate sent_accumulator.Increment(msgSize); // Update nominal packet size - nominal_packet_size += 0.01 * (((double)msgSize) - nominal_packet_size); + nominal_packet_size += 0.01 * (((double)msgSize) - nominal_packet_size); } else { @@ -4945,7 +4991,7 @@ NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) tx_sequence--; // TBD - is PL_WARN too verbose here PLOG(PL_WARN, "NormSession::SendMessage() sendto(%s/%hu) 'blocked' warning: %s\n", - msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); + msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); return MSG_SEND_BLOCKED; } } @@ -4954,7 +5000,7 @@ NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) // packet not sent tx_sequence--; PLOG(PL_WARN, "NormSession::SendMessage() sendto(%s/%hu) 'failed' warning: %s\n", - msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); + msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); if (!posted_send_error) { // Post a Notify(NormController::SEND_ERROR, NULL, NULL); @@ -4968,11 +5014,11 @@ NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) { probe_pending = false; probe_data_check = true; - if (probe_reset) + if (probe_reset) { probe_reset = false; if (!probe_timer.IsActive()) - ActivateTimer(probe_timer); + ActivateTimer(probe_timer); } } else if (!isReceiverMsg && IsSender()) @@ -4987,19 +5033,22 @@ NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) } } return MSG_SEND_OK; -} // end NormSession::SendMessage() +} // end NormSession::SendMessage() void NormSession::SetGrttProbingInterval(double intervalMin, double intervalMax) { - if ((intervalMin < 0.0) || (intervalMax < 0.0)) return; + if ((intervalMin < 0.0) || (intervalMax < 0.0)) + return; double temp = intervalMin; if (temp > intervalMax) { intervalMin = intervalMax; intervalMax = temp; } - if (intervalMin < NORM_TICK_MIN) intervalMin = NORM_TICK_MIN; - if (intervalMax < NORM_TICK_MIN) intervalMax = NORM_TICK_MIN; + if (intervalMin < NORM_TICK_MIN) + intervalMin = NORM_TICK_MIN; + if (intervalMax < NORM_TICK_MIN) + intervalMax = NORM_TICK_MIN; grtt_interval_min = intervalMin; grtt_interval_max = intervalMax; if (grtt_interval < grtt_interval_min) @@ -5010,62 +5059,63 @@ void NormSession::SetGrttProbingInterval(double intervalMin, double intervalMax) if (probe_timer.IsActive() && !cc_enable) { double elapsed = probe_timer.GetInterval() - probe_timer.GetTimeRemaining(); - if (elapsed < 0.0) elapsed = 0.0; + if (elapsed < 0.0) + elapsed = 0.0; if (elapsed > grtt_interval) probe_timer.SetInterval(0.0); - else + else probe_timer.SetInterval(grtt_interval - elapsed); - probe_timer.Reschedule(); - } - } -} // end NormSession::SetGrttProbingInterval() + probe_timer.Reschedule(); + } + } +} // end NormSession::SetGrttProbingInterval() void NormSession::SetGrttProbingMode(ProbingMode probingMode) { - if (cc_enable) return; // can't change probing mode when cc is enabled! - // (cc _requires_ probing mode == PROBE_ACTIVE) + if (cc_enable) + return; // can't change probing mode when cc is enabled! + // (cc _requires_ probing mode == PROBE_ACTIVE) switch (probingMode) { - case PROBE_NONE: - probe_reset = false; - if (probe_timer.IsActive()) - probe_timer.Deactivate(); - break; - case PROBE_PASSIVE: - probe_proactive = false; - if (IsSender()) + case PROBE_NONE: + probe_reset = false; + if (probe_timer.IsActive()) + probe_timer.Deactivate(); + break; + case PROBE_PASSIVE: + probe_proactive = false; + if (IsSender()) + { + if (!probe_timer.IsActive()) { - if (!probe_timer.IsActive()) - { - probe_timer.SetInterval(0.0); - ActivateTimer(probe_timer); - } + probe_timer.SetInterval(0.0); + ActivateTimer(probe_timer); } - else + } + else + { + probe_reset = true; + } + break; + case PROBE_ACTIVE: + probe_proactive = true; + if (IsSender()) + { + if (!probe_timer.IsActive()) { - probe_reset = true; + probe_timer.SetInterval(0.0); + ActivateTimer(probe_timer); } - break; - case PROBE_ACTIVE: - probe_proactive = true; - if (IsSender()) - { - if (!probe_timer.IsActive()) - { - probe_timer.SetInterval(0.0); - ActivateTimer(probe_timer); - } - } - else - { - probe_reset = true; - } - break; + } + else + { + probe_reset = true; + } + break; } -} // end NormSession::SetGrttProbingMode() +} // end NormSession::SetGrttProbingMode() - -bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) +bool NormSession::OnProbeTimeout(ProtoTimer & /*theTimer*/) { // 1) Temporarily kill probe_timer if CMD(CC) not yet tx'd // (or if data has not been sent since last probe) @@ -5075,11 +5125,11 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) if (probe_timer.IsActive()) probe_timer.Deactivate(); return false; - } - + } + // 2) Update grtt_estimate _if_ sufficient time elapsed. // This new code allows more liberal downward adjustment of - // of grtt when congestion control is enabled. + // of grtt when congestion control is enabled. // We have to keep track of the _actual_ deltaTime instead // of relying on the probe_timer interval because in real- @@ -5095,9 +5145,9 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) { double deltaTime = currentTime.tv_sec - probe_time_last.tv_sec; if (currentTime.tv_usec > probe_time_last.tv_usec) - deltaTime += 1.0e-06*((double)(currentTime.tv_usec - probe_time_last.tv_usec)); + deltaTime += 1.0e-06 * ((double)(currentTime.tv_usec - probe_time_last.tv_usec)); else - deltaTime -= 1.0e-06*((double)(probe_time_last.tv_usec - currentTime.tv_usec)); + deltaTime -= 1.0e-06 * ((double)(probe_time_last.tv_usec - currentTime.tv_usec)); grtt_age += deltaTime; } probe_time_last = currentTime; @@ -5108,14 +5158,14 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) // how the grtt estimate is descreasing ... this is most notable at // startup and thus the hack here to allow the grtt estimate to more // rapidly decrease during "slow start" - double ageMax = grtt_advertised; - if (!cc_enable && !cc_slow_start) + double ageMax = grtt_advertised; + if (!cc_enable && !cc_slow_start) ageMax = ageMax > grtt_interval_min ? ageMax : grtt_interval_min; if (grtt_age >= ageMax) { if (grtt_response) { - // Update grtt estimate + // Update grtt estimate if (grtt_current_peak < grtt_measured) { grtt_measured *= 0.9; @@ -5134,18 +5184,18 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) { // Increase already incorporated grtt_current_peak = 0.0; - grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; } if (grtt_measured < NORM_GRTT_MIN) grtt_measured = NORM_GRTT_MIN; else if (grtt_measured > grtt_max) grtt_measured = grtt_max; UINT8 grttQuantizedOld = grtt_quantized; - double pktInterval = (double)(44+segment_size)/tx_rate; + double pktInterval = (double)(44 + segment_size) / tx_rate; if (grtt_measured < pktInterval) grtt_quantized = NormQuantizeRtt(pktInterval); else - grtt_quantized = NormQuantizeRtt(grtt_measured); + grtt_quantized = NormQuantizeRtt(grtt_measured); // Recalculate grtt_advertise since quantization rounds upward grtt_advertised = NormUnquantizeRtt(grtt_quantized); if (grtt_advertised > grtt_max) @@ -5155,58 +5205,58 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) } if (grttQuantizedOld != grtt_quantized) { - Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::GRTT_UPDATED, (NormSenderNode *)NULL, (NormObject *)NULL); PLOG(PL_DEBUG, "NormSession::OnProbeTimeout() node>%lu decreased to new grtt to: %lf sec\n", - (unsigned long)LocalNodeId(), grtt_advertised); + (unsigned long)LocalNodeId(), grtt_advertised); } - grtt_response = false; // reset + grtt_response = false; // reset } grtt_age = 0.0; } - + if (grtt_interval < grtt_interval_min) grtt_interval = grtt_interval_min; else grtt_interval *= 1.5; if (grtt_interval > grtt_interval_max) - grtt_interval = grtt_interval_max; - + grtt_interval = grtt_interval_max; + // 3) Build a NORM_CMD(CC) message - NormCmdCCMsg* cmd = (NormCmdCCMsg*)GetMessageFromPool(); + NormCmdCCMsg *cmd = (NormCmdCCMsg *)GetMessageFromPool(); if (!cmd) { PLOG(PL_FATAL, "NormSession::OnProbeTimeout() node>%lu message_pool empty! can't probe\n", - (unsigned long)LocalNodeId()); - ASSERT(0); + (unsigned long)LocalNodeId()); + ASSERT(0); return true; - } + } cmd->Init(); cmd->SetDestination(address); cmd->SetGrtt(grtt_quantized); cmd->SetBackoffFactor((unsigned char)backoff_factor); - cmd->SetGroupSize(gsize_quantized); + cmd->SetGroupSize(gsize_quantized); // defer SetSendTime() to when message is being sent (in OnTxTimeout()) cmd->SetCCSequence(cc_sequence++); - + // Insert NORM-CC header extension, if applicable // (Note we set the extension "rate" _after_ AdjustRate() done below) NormCCRateExtension ext; if (probe_proactive) cmd->AttachExtension(ext); - + if (cc_enable) { // Iterate over cc_node_list and append cc_nodes ... // (we also check cc_node "activity status here) NormNodeListIterator iterator(cc_node_list); - NormCCNode* next; - while ((next = (NormCCNode*)iterator.GetNextNode())) + NormCCNode *next; + while ((next = (NormCCNode *)iterator.GetNextNode())) { if (next->IsActive()) { UINT8 ccFlags = 0; if (next->IsClr()) - { + { ccFlags |= (UINT8)NormCC::CLR; } else if (next->IsPlr()) @@ -5215,12 +5265,13 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) } ccFlags |= (UINT8)NormCC::RTT; UINT8 rttQuantized = NormQuantizeRtt(next->GetRtt()); - if (cc_slow_start) ccFlags |= (UINT8)NormCC::START; + if (cc_slow_start) + ccFlags |= (UINT8)NormCC::START; UINT16 rateQuantized = NormQuantizeRate(next->GetRate()); // (TBD) check result - cmd->AppendCCNode(segment_size, - next->GetId(), - ccFlags, + cmd->AppendCCNode(segment_size, + next->GetId(), + ccFlags, rttQuantized, rateQuantized); //if (!next->IsClr()) next->SetActive(false); @@ -5228,7 +5279,7 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) struct timeval feedbackTime = next->GetFeedbackTime(); double feedbackAge = currentTime.tv_sec - feedbackTime.tv_sec; feedbackAge += 1.0e-06 * ((double)((currentTime.tv_usec - feedbackTime.tv_usec))); - + /*if (currentTime.tv_usec > feedbackTime.tv_usec) feedbackAge += 1.0e-06*((double)(currentTime.tv_usec - feedbackTime.tv_usec)); else @@ -5236,46 +5287,45 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) double maxFeedbackAge = 20 * MAX(grtt_advertised, next->GetRtt()); // Safety bound to compensate for computer clock coarseness // and possible sluggish feedback from slower machines - // at higher norm data rates (keeps rate from being + // at higher norm data rates (keeps rate from being // prematurely reduced) - if (maxFeedbackAge <(10*NORM_TICK_MIN)) maxFeedbackAge = (10*NORM_TICK_MIN); - INT16 ccSeqDelta = cc_sequence - next->GetCCSequence(); + if (maxFeedbackAge < (10 * NORM_TICK_MIN)) + maxFeedbackAge = (10 * NORM_TICK_MIN); + INT16 ccSeqDelta = cc_sequence - next->GetCCSequence(); if ((feedbackAge > maxFeedbackAge) && (ccSeqDelta > (INT16)(20 * probe_count))) { PLOG(PL_DEBUG, "Deactivating cc node feedbackAge:%lf sec maxAge:%lf sec ccSeqDelta:%u\n", - feedbackAge, maxFeedbackAge, ccSeqDelta); + feedbackAge, maxFeedbackAge, ccSeqDelta); next->SetActive(false); } - } + } } AdjustRate(false); - } // end if (cc_enable) - + } // end if (cc_enable) + if (probe_proactive) ext.SetSendRate(NormQuantizeRate(tx_rate)); - - + double probeInterval = GetProbeInterval(); /*// perhaps this instead of the commented out probe_reset case??? double nominalInterval = ((double)segment_size)/((double)tx_rate); if (nominalInterval > grtt_max) nominalInterval = grtt_max; if (nominalInterval > probeInterval) probeInterval = nominalInterval; */ - + // Set probe_timer interval for next probe probe_timer.SetInterval(probeInterval); - - QueueMessage(cmd); - probe_pending = true; - - return true; -} // end NormSession::OnProbeTimeout() + QueueMessage(cmd); + probe_pending = true; + + return true; +} // end NormSession::OnProbeTimeout() double NormSession::GetProbeInterval() { if (cc_enable && data_active) { - const NormCCNode* clr = static_cast(cc_node_list.Head()); + const NormCCNode *clr = static_cast(cc_node_list.Head()); if (NULL != clr) { double probeInterval = (clr->IsActive() ? MIN(grtt_advertised, clr->GetRtt()) : grtt_advertised); @@ -5286,21 +5336,25 @@ double NormSession::GetProbeInterval() // (although we floor the probeCount at 1 (i.e., the usual 1 probe per RTT) // Note that no more than a few (e.g. 3) probes per RTT provides performance benefit unsigned int probeCount = (unsigned int)(0.25 * tx_rate * probeInterval / (double)segment_size); - if (probeCount < 1) probeCount = 1; + if (probeCount < 1) + probeCount = 1; if (clr->GetRtt() > 0.200) { - if (probeCount > 3) probeCount = 3; + if (probeCount > 3) + probeCount = 3; } else if (clr->GetRtt() > 0.100) { - if (probeCount > 2) probeCount = 2; + if (probeCount > 2) + probeCount = 2; } else { probeCount = 1; } - if (1 != probe_count) probeCount = probe_count; - + if (1 != probe_count) + probeCount = probe_count; + // Don't send more than one CLR probe per RTT during slow_start return (cc_slow_start ? probeInterval : (probeInterval / (double)probeCount)); } @@ -5313,11 +5367,11 @@ double NormSession::GetProbeInterval() { return grtt_interval; } -} // end NormSession::GetProbeInterval() +} // end NormSession::GetProbeInterval() void NormSession::AdjustRate(bool onResponse) { - const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head(); + const NormCCNode *clr = (const NormCCNode *)cc_node_list.Head(); double ccRtt = clr ? clr->GetRtt() : grtt_measured; double ccLoss = clr ? clr->GetLoss() : 0.0; double txRate = tx_rate; @@ -5328,7 +5382,7 @@ void NormSession::AdjustRate(bool onResponse) cc_active = true; Notify(NormController::CC_ACTIVE, NULL, NULL); } - if (data_active) // adjust only if actively transmitting + if (data_active) // adjust only if actively transmitting { // Adjust rate based on CLR feedback and // adjust probe schedule @@ -5339,39 +5393,39 @@ void NormSession::AdjustRate(bool onResponse) txRate = clr->GetRate(); if (GetDebugLevel() >= 6) { - double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / (report_timer.GetInterval() - report_timer.GetTimeRemaining())); + double sentRate = 8.0e-03 * sent_accumulator.GetScaledValue(1.0 / (report_timer.GetInterval() - report_timer.GetTimeRemaining())); PLOG(PL_DETAIL, "NormSession::AdjustRate(slow start) clr>%lu newRate>%lf (oldRate>%lf sentRate>%lf clrRate>%lf\n", - (unsigned long)clr->GetId(), 8.0e-03*txRate, 8.0e-03*tx_rate, sentRate, 8.0e-03*clr->GetRate()); - } + (unsigned long)clr->GetId(), 8.0e-03 * txRate, 8.0e-03 * tx_rate, sentRate, 8.0e-03 * clr->GetRate()); + } } else { double clrRate = clr->GetRate(); if (clrRate > txRate) { - double maxRate = txRate*2; + double maxRate = txRate * 2; txRate = MIN(clrRate, maxRate); } else { txRate = clrRate; } - + // Here, we use the most recent CLR rtt sample to "damp" oscillation double damper = clr->GetRttSqMean() / sqrt(clr->GetRttSample()); if (damper < 0.5) damper = 0.5; - else if (damper > 2.0) + else if (damper > 2.0) damper = 2.0; txRate *= damper; PLOG(PL_DETAIL, "NormSession::AdjustRate(stdy state) clr>%lu newRate>%lf (rtt>%lf loss>%lf)\n", - (unsigned long)clr->GetId(), 8.0e-03*txRate, clr->GetRtt(), clr->GetLoss()); + (unsigned long)clr->GetId(), 8.0e-03 * txRate, clr->GetRtt(), clr->GetLoss()); } } if (!address.IsMulticast()) { - // For unicast, adjust the probe timeout right away - double probeInterval = GetProbeInterval(); // based on CLR RTT, etc + // For unicast, adjust the probe timeout right away + double probeInterval = GetProbeInterval(); // based on CLR RTT, etc if (probe_timer.GetInterval() > probeInterval) { // reduce to speed up rate increase @@ -5381,7 +5435,8 @@ void NormSession::AdjustRate(bool onResponse) else probeInterval = 0.0; probe_timer.SetInterval(probeInterval); - if (probe_timer.IsActive()) probe_timer.Reschedule(); + if (probe_timer.IsActive()) + probe_timer.Reschedule(); } } } @@ -5416,11 +5471,10 @@ void NormSession::AdjustRate(bool onResponse) // reduce rate by half if no active clr txRate *= 0.5; } - - + // Keep "tx_rate" within default or user set rate bounds (if any) double minRate; - if(tx_rate_min > 0.0) + if (tx_rate_min > 0.0) { minRate = tx_rate_min; } @@ -5432,7 +5486,7 @@ void NormSession::AdjustRate(bool onResponse) else minRate = (double)(segment_size); } - if (txRate <= minRate) + if (txRate <= minRate) { txRate = minRate; if ((NULL == clr) || (!clr->IsActive())) @@ -5447,9 +5501,10 @@ void NormSession::AdjustRate(bool onResponse) } if ((tx_rate_max >= 0.0) && (txRate > tx_rate_max)) txRate = tx_rate_max; - if (txRate != tx_rate) + if (txRate != tx_rate) { - if (cc_adjust) SetTxRateInternal(txRate); + if (cc_adjust) + SetTxRateInternal(txRate); if (!posted_tx_rate_changed) { // TBD - make API notification filtering more consistent @@ -5457,23 +5512,23 @@ void NormSession::AdjustRate(bool onResponse) // putting API code in charge of resetting these API // state variables). posted_tx_rate_changed = true; - Notify(NormController::TX_RATE_CHANGED, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::TX_RATE_CHANGED, (NormSenderNode *)NULL, (NormObject *)NULL); } } - + struct timeval currentTime; ::ProtoSystemTime(currentTime); double theTime = (double)currentTime.tv_sec + 1.0e-06 * ((double)currentTime.tv_usec); - PLOG(PL_DEBUG, "SenderRateTracking time>%lf rate>%lf rtt>%lf loss>%lf\n", theTime, 8.0e-03*txRate, ccRtt, ccLoss); + PLOG(PL_DEBUG, "SenderRateTracking time>%lf rate>%lf rtt>%lf loss>%lf\n", theTime, 8.0e-03 * txRate, ccRtt, ccLoss); //TRACE("SenderRateTracking time>%lf rate>%lf rtt>%lf loss>%lf\n", theTime, 8.0e-03*txRate, ccRtt, ccLoss); //double calcRate = NormSession::CalculateRate(nominal_packet_size, ccRtt, ccLoss); - //TRACE("SenderRateTracking time>%lf rate>%lf clrRate>%lf rtt>%lf loss>%lf calcRate>%lf size>%lf%s\n\n", - // theTime, 8.0e-03*txRate, clr ? 8.0e-03*clr->GetRate() : 0.0, ccRtt, ccLoss, calcRate*8.0e-03, + //TRACE("SenderRateTracking time>%lf rate>%lf clrRate>%lf rtt>%lf loss>%lf calcRate>%lf size>%lf%s\n\n", + // theTime, 8.0e-03*txRate, clr ? 8.0e-03*clr->GetRate() : 0.0, ccRtt, ccLoss, calcRate*8.0e-03, // nominal_packet_size, cc_slow_start ? " (slow start)" : ""); //ASSERT((NULL == clr) || (txRate <= clr->GetRate())); -} // end NormSession::AdjustRate() +} // end NormSession::AdjustRate() -bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/) +bool NormSession::OnReportTimeout(ProtoTimer & /*theTimer*/) { // Receiver reporting (just print out for now) struct timeval currentTime; @@ -5483,97 +5538,94 @@ bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/) timeStruct.tm_hour = currentTime.tv_sec / 3600; unsigned long hourSecs = 3600 * timeStruct.tm_hour; timeStruct.tm_min = (currentTime.tv_sec - (hourSecs)) / 60; - timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60*timeStruct.tm_min); + timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60 * timeStruct.tm_min); timeStruct.tm_hour = timeStruct.tm_hour % 24; - struct tm* ct = &timeStruct; -#else + struct tm *ct = &timeStruct; +#else time_t secs = (time_t)currentTime.tv_sec; struct tm timeStruct; #ifdef WIN32 - gmtime_s(&timeStruct, &secs); - struct tm* ct = &timeStruct; + gmtime_s(&timeStruct, &secs); + struct tm *ct = &timeStruct; #else - struct tm* ct = gmtime_r(&secs, &timeStruct); + struct tm *ct = gmtime_r(&secs, &timeStruct); #endif - + #endif // if/else _WIN32_WCE ASSERT(NULL != ct); ProtoDebugLevel reportDebugLevel = PL_INFO; - PLOG(reportDebugLevel, "REPORT time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", - ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, (unsigned long)LocalNodeId()); + PLOG(reportDebugLevel, "REPORT time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", + ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, (unsigned long)LocalNodeId()); if (IsSender()) { PLOG(reportDebugLevel, "Local status:\n"); - double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / report_timer.GetInterval()); // kbps + double sentRate = 8.0e-03 * sent_accumulator.GetScaledValue(1.0 / report_timer.GetInterval()); // kbps sent_accumulator.Reset(); - PLOG(reportDebugLevel, " txRate>%9.3lf kbps sentRate>%9.3lf grtt>%lf\n", - 8.0e-03*tx_rate, sentRate, grtt_advertised); + PLOG(reportDebugLevel, " txRate>%9.3lf kbps sentRate>%9.3lf grtt>%lf\n", + 8.0e-03 * tx_rate, sentRate, grtt_advertised); if (cc_enable) { - const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head(); - if (clr) + const NormCCNode *clr = (const NormCCNode *)cc_node_list.Head(); + if (clr) { - PLOG(reportDebugLevel, " clr>%lu rate>%9.3lf rtt>%lf loss>%lf %s\n", - (unsigned long)clr->GetId(), 8.0e-03*clr->GetRate(), - clr->GetRtt(), clr->GetLoss(), cc_slow_start ? "(slow_start)" : ""); + PLOG(reportDebugLevel, " clr>%lu rate>%9.3lf rtt>%lf loss>%lf %s\n", + (unsigned long)clr->GetId(), 8.0e-03 * clr->GetRate(), + clr->GetRtt(), clr->GetLoss(), cc_slow_start ? "(slow_start)" : ""); } - } + } } if (IsReceiver()) { NormNodeTreeIterator iterator(sender_tree); - NormSenderNode* next; - while ((next = (NormSenderNode*)iterator.GetNextNode())) + NormSenderNode *next; + while ((next = (NormSenderNode *)iterator.GetNextNode())) { - PLOG(reportDebugLevel, "Remote sender>%lu grtt>%lf sec loss>%lf\n", (unsigned long)next->GetId(), - next->GetGrttEstimate(), next->LossEstimate()); + PLOG(reportDebugLevel, "Remote sender>%lu grtt>%lf sec loss>%lf\n", (unsigned long)next->GetId(), + next->GetGrttEstimate(), next->LossEstimate()); // TBD - Output sender congestion control status if cc is enabled - double rxRate = 8.0e-03*next->GetRecvRate(report_timer.GetInterval()); // kbps - double rxGoodput = 8.0e-03*next->GetRecvGoodput(report_timer.GetInterval()); // kbps + double rxRate = 8.0e-03 * next->GetRecvRate(report_timer.GetInterval()); // kbps + double rxGoodput = 8.0e-03 * next->GetRecvGoodput(report_timer.GetInterval()); // kbps next->ResetRecvStats(); PLOG(reportDebugLevel, " rxRate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput); - PLOG(reportDebugLevel, " rxObjects> completed>%lu pending>%lu failed>%lu\n", - next->CompletionCount(), next->PendingCount(), next->FailureCount()); - PLOG(reportDebugLevel, " fecBufferUsage> current>%lu peak>%lu overuns>%lu\n", - next->CurrentBufferUsage(), next->PeakBufferUsage(), - next->BufferOverunCount()); - PLOG(reportDebugLevel, " strBufferUsage> current>%lu peak>%lu overuns>%lu\n", - next->CurrentStreamBufferUsage(), next->PeakStreamBufferUsage(), - next->StreamBufferOverunCount()); - PLOG(reportDebugLevel, " resyncs>%lu nacks>%lu suppressed>%lu\n", - next->ResyncCount() ? next->ResyncCount() - 1 : 0, // "ResyncCount()" is really "SyncCount()" - next->NackCount(), next->SuppressCount()); + PLOG(reportDebugLevel, " rxObjects> completed>%lu pending>%lu failed>%lu\n", + next->CompletionCount(), next->PendingCount(), next->FailureCount()); + PLOG(reportDebugLevel, " fecBufferUsage> current>%lu peak>%lu overuns>%lu\n", + next->CurrentBufferUsage(), next->PeakBufferUsage(), + next->BufferOverunCount()); + PLOG(reportDebugLevel, " strBufferUsage> current>%lu peak>%lu overuns>%lu\n", + next->CurrentStreamBufferUsage(), next->PeakStreamBufferUsage(), + next->StreamBufferOverunCount()); + PLOG(reportDebugLevel, " resyncs>%lu nacks>%lu suppressed>%lu\n", + next->ResyncCount() ? next->ResyncCount() - 1 : 0, // "ResyncCount()" is really "SyncCount()" + next->NackCount(), next->SuppressCount()); // Some stream status for current receive stream (if applicable) - NormObject* obj = next->GetNextPendingObject(); + NormObject *obj = next->GetNextPendingObject(); if ((NULL != obj) && obj->IsStream()) { - NormStreamObject* stream = (NormStreamObject*)obj; + NormStreamObject *stream = (NormStreamObject *)obj; PLOG(reportDebugLevel, " stream_sync_id>%lu stream_next_id>%lu read_index:%lu.%hu\n", - (unsigned long)stream->GetSyncId().GetValue(), - (unsigned long)stream->GetNextId().GetValue(), - (unsigned long)stream->GetNextBlockId().GetValue(), - (UINT16)stream->GetNextSegmentId()); + (unsigned long)stream->GetSyncId().GetValue(), + (unsigned long)stream->GetNextId().GetValue(), + (unsigned long)stream->GetNextBlockId().GetValue(), + (UINT16)stream->GetNextSegmentId()); } - } - } // end if (IsReceiver()) + } // end if (IsReceiver()) PLOG(reportDebugLevel, "***************************************************************************\n"); return true; -} // end NormSession::OnReportTimeout() +} // end NormSession::OnReportTimeout() -bool NormSession::OnUserTimeout(ProtoTimer& /*theTimer*/) +bool NormSession::OnUserTimeout(ProtoTimer & /*theTimer*/) { - Notify(NormController::USER_TIMEOUT, (NormSenderNode*)NULL, (NormObject*)NULL); + Notify(NormController::USER_TIMEOUT, (NormSenderNode *)NULL, (NormObject *)NULL); return true; -} // end NormSession::OnUserTimeout( +} // end NormSession::OnUserTimeout( - - -NormSessionMgr::NormSessionMgr(ProtoTimerMgr& timerMgr, - ProtoSocket::Notifier& socketNotifier, - ProtoChannel::Notifier* channelNotifier) - : timer_mgr(timerMgr), socket_notifier(socketNotifier), channel_notifier(channelNotifier), - controller(NULL), data_free_func(NULL), top_session(NULL) +NormSessionMgr::NormSessionMgr(ProtoTimerMgr &timerMgr, + ProtoSocket::Notifier &socketNotifier, + ProtoChannel::Notifier *channelNotifier) + : timer_mgr(timerMgr), socket_notifier(socketNotifier), channel_notifier(channelNotifier), + controller(NULL), data_free_func(NULL), top_session(NULL) { } @@ -5584,17 +5636,17 @@ NormSessionMgr::~NormSessionMgr() void NormSessionMgr::Destroy() { - NormSession* next; + NormSession *next; while ((next = top_session)) { top_session = next->next; delete next; } -} // end NormSessionMgr::Destroy() +} // end NormSessionMgr::Destroy() -NormSession* NormSessionMgr::NewSession(const char* sessionAddress, - UINT16 sessionPort, - NormNodeId localNodeId) +NormSession *NormSessionMgr::NewSession(const char *sessionAddress, + UINT16 sessionPort, + NormNodeId localNodeId) { if ((NORM_NODE_ANY == localNodeId) || (NORM_NODE_NONE == localNodeId)) { @@ -5604,50 +5656,49 @@ NormSession* NormSessionMgr::NewSession(const char* sessionAddress, if (!localAddr.ResolveLocalAddress()) { PLOG(PL_ERROR, "NormSessionMgr::NewSession() local address lookup error\n"); - return ((NormSession*)NULL); - } + return ((NormSession *)NULL); + } // (TBD) test IPv6 "EndIdentifier" ??? localNodeId = localAddr.EndIdentifier(); #else - localNodeId = NORM_NODE_ANY - 1; + localNodeId = NORM_NODE_ANY - 1; #endif } ProtoAddress theAddress; if (!theAddress.ResolveFromString(sessionAddress)) { PLOG(PL_ERROR, "NormSessionMgr::NewSession() session address \"%s\" lookup error!\n", sessionAddress); - return ((NormSession*)NULL); + return ((NormSession *)NULL); } - theAddress.SetPort(sessionPort); - NormSession* theSession = new NormSession(*this, localNodeId); + theAddress.SetPort(sessionPort); + NormSession *theSession = new NormSession(*this, localNodeId); if (!theSession) { - PLOG(PL_ERROR, "NormSessionMgr::NewSession() new session error: %s\n", GetErrorString()); - return ((NormSession*)NULL); - } + PLOG(PL_ERROR, "NormSessionMgr::NewSession() new session error: %s\n", GetErrorString()); + return ((NormSession *)NULL); + } theSession->SetAddress(theAddress); // Add new session to our session list theSession->next = top_session; top_session = theSession; return theSession; -} // end NormSessionMgr::NewSession() +} // end NormSessionMgr::NewSession() -void NormSessionMgr::DeleteSession(class NormSession* theSession) +void NormSessionMgr::DeleteSession(class NormSession *theSession) { - NormSession* prev = NULL; - NormSession* next = top_session; + NormSession *prev = NULL; + NormSession *next = top_session; while (next && (next != theSession)) { prev = next; - next = next->next; + next = next->next; } if (next) { if (prev) prev->next = theSession->next; else - top_session = theSession->next; + top_session = theSession->next; delete theSession; } -} // end NormSessionMgr::DeleteSession() - +} // end NormSessionMgr::DeleteSession() diff --git a/src/common/pcap2norm.cpp b/src/common/pcap2norm.cpp index 0a8888f..9612adf 100644 --- a/src/common/pcap2norm.cpp +++ b/src/common/pcap2norm.cpp @@ -2,7 +2,6 @@ // Assumes UDP packets in tcpdump trace file (pcap file) are // MGEN packets and parses to build an MGEN log file - #include #include #include // for PF_ types (protocol family) @@ -12,92 +11,95 @@ #include "normSession.h" -void NormTrace2(const struct timeval& currentTime, - const NormMsg& msg, - const ProtoAddress& srcAddr, - const ProtoAddress& dstAddr); +void NormTrace2(const struct timeval ¤tTime, + const NormMsg &msg, + const ProtoAddress &srcAddr, + const ProtoAddress &dstAddr); void Usage() { fprintf(stderr, "pcap2norm [pcapInputFile [outputFile]]\n"); } -int main(int argc, char* argv[]) +int main(int argc, char *argv[]) { // Use stdin/stdout by default - FILE* infile = stdin; - FILE* outfile = stdout; - switch(argc) + FILE *infile = stdin; + FILE *outfile = stdout; + switch (argc) { - case 1: - // using default stdin/stdout - break; - case 2: - // using named input pcap file and stdout - if (NULL == (infile = fopen(argv[1], "r"))) - { - perror("pcap2norm: error opening input file"); - return -1; - } - break; - case 3: - // use name input and output files - if (NULL == (infile = fopen(argv[1], "r"))) - { - perror("pcap2norm: error opening input file"); - return -1; - } - if (NULL == (outfile = fopen(argv[2], "w+"))) - { - perror("pcap2norm: error opening output file"); - return -1; - } - break; - default: - fprintf(stderr, "pcap2norm: error: too many arguments!\n"); - Usage(); + case 1: + // using default stdin/stdout + break; + case 2: + // using named input pcap file and stdout + if (NULL == (infile = fopen(argv[1], "r"))) + { + perror("pcap2norm: error opening input file"); return -1; - } // end switch(argc) - - char pcapErrBuf[PCAP_ERRBUF_SIZE+1]; + } + break; + case 3: + // use name input and output files + if (NULL == (infile = fopen(argv[1], "r"))) + { + perror("pcap2norm: error opening input file"); + return -1; + } + if (NULL == (outfile = fopen(argv[2], "w+"))) + { + perror("pcap2norm: error opening output file"); + return -1; + } + break; + default: + fprintf(stderr, "pcap2norm: error: too many arguments!\n"); + Usage(); + return -1; + } // end switch(argc) + + char pcapErrBuf[PCAP_ERRBUF_SIZE + 1]; pcapErrBuf[PCAP_ERRBUF_SIZE] = '\0'; - pcap_t* pcapDevice = pcap_fopen_offline(infile, pcapErrBuf); + pcap_t *pcapDevice = pcap_fopen_offline(infile, pcapErrBuf); if (NULL == pcapDevice) { fprintf(stderr, "pcap2norm: pcap_fopen_offline() error: %s\n", pcapErrBuf); - if (stdin != infile) fclose(infile); - if (stdout != outfile) fclose(outfile); + if (stdin != infile) + fclose(infile); + if (stdout != outfile) + fclose(outfile); return -1; } - + int deviceType = pcap_datalink(pcapDevice); - - UINT32 alignedBuffer[4096/4]; // 4096 byte buffer for packet parsing - UINT16* ethBuffer = ((UINT16*)alignedBuffer) + 1; - unsigned int maxBytes = 4096 - 2; // due to offset, can only use 4094 bytes of buffer - + + UINT32 alignedBuffer[4096 / 4]; // 4096 byte buffer for packet parsing + UINT16 *ethBuffer = ((UINT16 *)alignedBuffer) + 1; + unsigned int maxBytes = 4096 - 2; // due to offset, can only use 4094 bytes of buffer + pcap_pkthdr hdr; - const u_char* pktData; - while(NULL != (pktData = pcap_next(pcapDevice, &hdr))) + const u_char *pktData; + while (NULL != (pktData = pcap_next(pcapDevice, &hdr))) { unsigned int numBytes = maxBytes; - if (hdr.caplen < numBytes) numBytes = hdr.caplen; + if (hdr.caplen < numBytes) + numBytes = hdr.caplen; ProtoPktETH::Type ethType; unsigned int payloadLength; - UINT32* payloadPtr; + UINT32 *payloadPtr; if (DLT_NULL == deviceType) { // pcap was captured from "loopback" device memcpy(alignedBuffer, pktData, numBytes); switch (alignedBuffer[0]) { - case PF_INET: - ethType = ProtoPktETH::IP; - break; - case PF_INET6: - ethType = ProtoPktETH::IPv6; - break; - default: - continue; // not an IP packet + case PF_INET: + ethType = ProtoPktETH::IP; + break; + case PF_INET6: + ethType = ProtoPktETH::IPv6; + break; + default: + continue; // not an IP packet } payloadLength = numBytes - 4; payloadPtr = alignedBuffer + 1; @@ -110,14 +112,14 @@ int main(int argc, char* argv[]) { fprintf(stderr, "pcap2norm error: invalid Ether frame in pcap file\n"); continue; - } + } ethType = ethPkt.GetType(); payloadLength = ethPkt.GetPayloadLength(); // This is done know we offset the ethBuffer above - payloadPtr = alignedBuffer + (2 + ethPkt.GetLength() - ethPkt.GetPayloadLength())/4; + payloadPtr = alignedBuffer + (2 + ethPkt.GetLength() - ethPkt.GetPayloadLength()) / 4; //payloadPtr = (UINT32*)ethPkt.AccessPayload(); } - + ProtoPktIP ipPkt; ProtoAddress srcAddr, dstAddr; if ((ProtoPktETH::IP == ethType) || @@ -130,25 +132,25 @@ int main(int argc, char* argv[]) } switch (ipPkt.GetVersion()) { - case 4: - { - ProtoPktIPv4 ip4Pkt(ipPkt); - ip4Pkt.GetDstAddr(dstAddr); - ip4Pkt.GetSrcAddr(srcAddr); - break; - } - case 6: - { - ProtoPktIPv6 ip6Pkt(ipPkt); - ip6Pkt.GetDstAddr(dstAddr); - ip6Pkt.GetSrcAddr(srcAddr); - break; - } - default: - { - PLOG(PL_ERROR,"pcap2norm Error: Invalid IP pkt version.\n"); - break; - } + case 4: + { + ProtoPktIPv4 ip4Pkt(ipPkt); + ip4Pkt.GetDstAddr(dstAddr); + ip4Pkt.GetSrcAddr(srcAddr); + break; + } + case 6: + { + ProtoPktIPv6 ip6Pkt(ipPkt); + ip6Pkt.GetDstAddr(dstAddr); + ip6Pkt.GetSrcAddr(srcAddr); + break; + } + default: + { + PLOG(PL_ERROR, "pcap2norm Error: Invalid IP pkt version.\n"); + break; + } } //PLOG(PL_ALWAYS, "pcap2norm IP packet dst>%s ", dstAddr.GetHostString()); //PLOG(PL_ALWAYS," src>%s length>%d\n", srcAddr.GetHostString(), ipPkt.GetLength()); @@ -157,13 +159,15 @@ int main(int argc, char* argv[]) { fprintf(stderr, "eth type = %d\n", ethType); } - if (!srcAddr.IsValid()) continue; // wasn't an IP packet - + if (!srcAddr.IsValid()) + continue; // wasn't an IP packet + ProtoPktUDP udpPkt; - if (!udpPkt.InitFromPacket(ipPkt)) continue; // not a UDP packet - + if (!udpPkt.InitFromPacket(ipPkt)) + continue; // not a UDP packet + NormMsg msg; - if (msg.CopyFromBuffer((const char*)udpPkt.GetPayload(), udpPkt.GetPayloadLength())) + if (msg.CopyFromBuffer((const char *)udpPkt.GetPayload(), udpPkt.GetPayloadLength())) { srcAddr.SetPort(udpPkt.GetSrcPort()); msg.AccessAddress() = srcAddr; @@ -173,51 +177,47 @@ int main(int argc, char* argv[]) else { fprintf(stderr, "pcap2norm warning: UDP packet not an MGEN packet?\n"); - } - } // end while (pcap_next()) - -} // end main() + } + } // end while (pcap_next()) +} // end main() static UINT8 lastFecId = 0; -void NormTrace2(const struct timeval& currentTime, - const NormMsg& msg, - const ProtoAddress& srcAddr, - const ProtoAddress& dstAddr) +void NormTrace2(const struct timeval ¤tTime, + const NormMsg &msg, + const ProtoAddress &srcAddr, + const ProtoAddress &dstAddr) { - - UINT8 fecM = 8; // NOTE - this assumes 16-bit RS code for fec_id == 2 - - static const char* MSG_NAME[] = - { - "INVALID", - "INFO", - "DATA", - "CMD", - "NACK", - "ACK", - "REPORT" - }; - static const char* CMD_NAME[] = - { - "CMD(INVALID)", - "CMD(FLUSH)", - "CMD(EOT)", - "CMD(SQUELCH)", - "CMD(CC)", - "CMD(REPAIR_ADV)", - "CMD(ACK_REQ)", - "CMD(APP)" - }; - static const char* REQ_NAME[] = - { - "INVALID", - "WATERMARK", - "RTT", - "APP" - }; - + + UINT8 fecM = 8; // NOTE - this assumes 16-bit RS code for fec_id == 2 + + static const char *MSG_NAME[] = + { + "INVALID", + "INFO", + "DATA", + "CMD", + "NACK", + "ACK", + "REPORT"}; + static const char *CMD_NAME[] = + { + "CMD(INVALID)", + "CMD(FLUSH)", + "CMD(EOT)", + "CMD(SQUELCH)", + "CMD(CC)", + "CMD(REPAIR_ADV)", + "CMD(ACK_REQ)", + "CMD(APP)"}; + static const char *REQ_NAME[] = + { + "INVALID", + "WATERMARK", + "RTT", + "APP"}; + NormMsg::Type msgType = msg.GetType(); UINT16 length = msg.GetLength(); UINT16 seq = msg.GetSequence(); @@ -225,56 +225,53 @@ void NormTrace2(const struct timeval& currentTime, src[63] = dst[63] = '\0'; srcAddr.GetHostString(src, 63); dstAddr.GetHostString(dst, 63); - - #ifdef _WIN32_WCE struct tm timeStruct; timeStruct.tm_hour = currentTime.tv_sec / 3600; unsigned long hourSecs = 3600 * timeStruct.tm_hour; timeStruct.tm_min = (currentTime.tv_sec - (hourSecs)) / 60; - timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60*timeStruct.tm_min); + timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60 * timeStruct.tm_min); timeStruct.tm_hour = timeStruct.tm_hour % 24; - struct tm* ct = &timeStruct; -#else + struct tm *ct = &timeStruct; +#else time_t secs = (time_t)currentTime.tv_sec; - struct tm* ct = gmtime(&secs); + struct tm *ct = gmtime(&secs); #endif // if/else _WIN32_WCE - - + PLOG(PL_ALWAYS, "trace>%02d:%02d:%02d.%06lu ", - (int)ct->tm_hour, (int)ct->tm_min, (int)ct->tm_sec, (unsigned int)currentTime.tv_usec); + (int)ct->tm_hour, (int)ct->tm_min, (int)ct->tm_sec, (unsigned int)currentTime.tv_usec); PLOG(PL_ALWAYS, "src>%s/%hu dst>%s/%hu id>0x%08x ", src, srcAddr.GetPort(), dst, dstAddr.GetPort(), (UINT32)msg.GetSourceId()); - + bool clrFlag = false; switch (msgType) { - case NormMsg::INFO: + case NormMsg::INFO: + { + const NormInfoMsg &info = (const NormInfoMsg &)msg; + lastFecId = info.GetFecId(); + PLOG(PL_ALWAYS, "inst>%hu seq>%hu INFO obj>%hu ", + info.GetInstanceId(), seq, (UINT16)info.GetObjectId()); + break; + } + case NormMsg::DATA: + { + const NormDataMsg &data = (const NormDataMsg &)msg; + lastFecId = data.GetFecId(); + PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%u seg>%04hu ", + data.GetInstanceId(), + seq, + //data.IsData() ? "DATA" : "PRTY", + (UINT16)data.GetObjectId(), + (UINT32)data.GetFecBlockId(fecM).GetValue(), + (UINT16)data.GetFecSymbolId(fecM)); + + if (data.IsStream()) { - const NormInfoMsg& info = (const NormInfoMsg&)msg; - lastFecId = info.GetFecId(); - PLOG(PL_ALWAYS, "inst>%hu seq>%hu INFO obj>%hu ", - info.GetInstanceId(), seq, (UINT16)info.GetObjectId()); - break; + UINT32 offset = NormDataMsg::ReadStreamPayloadOffset(data.GetPayload()); + PLOG(PL_ALWAYS, "offset>%lu ", offset); } - case NormMsg::DATA: - { - const NormDataMsg& data = (const NormDataMsg&)msg; - lastFecId = data.GetFecId(); - PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%u seg>%04hu ", - data.GetInstanceId(), - seq, - //data.IsData() ? "DATA" : "PRTY", - (UINT16)data.GetObjectId(), - (UINT32)data.GetFecBlockId(fecM).GetValue(), - (UINT16)data.GetFecSymbolId(fecM)); - - if (data.IsStream()) - { - UINT32 offset = NormDataMsg::ReadStreamPayloadOffset(data.GetPayload()); - PLOG(PL_ALWAYS, "offset>%lu ", offset); - } - /* + /* if (data.IsData() && data.IsStream()) { //if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_MSG_START)) @@ -288,124 +285,124 @@ void NormTrace2(const struct timeval& currentTime, PLOG(PL_ALWAYS, "(stream end) "); } */ - break; - } - case NormMsg::CMD: + break; + } + case NormMsg::CMD: + { + const NormCmdMsg &cmd = static_cast(msg); + NormCmdMsg::Flavor flavor = cmd.GetFlavor(); + PLOG(PL_ALWAYS, "inst>%hu seq>%hu %s ", cmd.GetInstanceId(), seq, CMD_NAME[flavor]); + switch (flavor) { - const NormCmdMsg& cmd = static_cast(msg); - NormCmdMsg::Flavor flavor = cmd.GetFlavor(); - PLOG(PL_ALWAYS, "inst>%hu seq>%hu %s ", cmd.GetInstanceId(), seq, CMD_NAME[flavor]); - switch (flavor) - { - case NormCmdMsg::ACK_REQ: - { - int index = ((const NormCmdAckReqMsg&)msg).GetAckType(); - index = MIN(index, 3); - PLOG(PL_ALWAYS, "(%s) ", REQ_NAME[index]); - break; - } - case NormCmdMsg::SQUELCH: - { - const NormCmdSquelchMsg& squelch = - static_cast(msg); - PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", - (UINT16)squelch.GetObjectId(), - (UINT32)squelch.GetFecBlockId(fecM).GetValue(), - (UINT16)squelch.GetFecSymbolId(fecM)); - break; - } - case NormCmdMsg::FLUSH: - { - const NormCmdFlushMsg& flush = - static_cast(msg); - PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", - (UINT16)flush.GetObjectId(), - (UINT32)flush.GetFecBlockId(fecM).GetValue(), - (UINT16)flush.GetFecSymbolId(fecM)); - - // Print acking node list (if any) - UINT16 nodeCount = flush.GetAckingNodeCount(); - if (nodeCount > 0) - { - PLOG(PL_ALWAYS, "ackers>"); - for (UINT16 i = 0; i < nodeCount; i++) - { - if (i > 0) PLOG(PL_ALWAYS, ","); - PLOG(PL_ALWAYS,"0x%08x", (UINT32)flush.GetAckingNodeId(i)); - } - PLOG(PL_ALWAYS, " "); - } - break; - } - case NormCmdMsg::CC: - { - const NormCmdCCMsg& cc = static_cast(msg); - PLOG(PL_ALWAYS, " seq>%u ", cc.GetCCSequence()); - NormHeaderExtension ext; - while (cc.GetNextExtension(ext)) - { - if (NormHeaderExtension::CC_RATE == ext.GetType()) - { - UINT16 sendRate = ((NormCCRateExtension&)ext).GetSendRate(); - PLOG(PL_ALWAYS, " rate>%f ", 8.0e-03 * NormUnquantizeRate(sendRate)); - break; - } - } - struct timeval sendTime; - cc.GetSendTime(sendTime); - double delay = ProtoTime::Delta(ProtoTime(currentTime), ProtoTime(sendTime)); - PLOG(PL_ALWAYS, "delay>%lf ", delay); - break; - } - default: - break; - } - break; - } - - case NormMsg::ACK: - case NormMsg::NACK: + case NormCmdMsg::ACK_REQ: { - PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); - // look for NormCCFeedback extension - NormHeaderExtension ext; - while (msg.GetNextExtension(ext)) - { - if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) - { - clrFlag = ((NormCCFeedbackExtension&)ext).CCFlagIsSet(NormCC::CLR); - // Print ccRtt (only valid if pcap file is from sender node) - double ccRtt = NormUnquantizeRtt(((NormCCFeedbackExtension&)ext).GetCCRtt()); - double ccLoss = NormUnquantizeLoss32(((NormCCFeedbackExtension&)ext).GetCCLoss32()); - PLOG(PL_ALWAYS, "ccRtt:%lf ccLoss:%lf ", ccRtt, ccLoss); - break; - } - } - // Print locally measured rtt (only valid if pcap file is from sender node) - struct timeval grttResponse; - if (NormMsg::NACK == msgType) - static_cast(msg).GetGrttResponse(grttResponse); - else - static_cast(msg).GetGrttResponse(grttResponse); - - double rtt = ProtoTime::Delta(ProtoTime(currentTime), ProtoTime(grttResponse)); - PLOG(PL_ALWAYS, "rtt:%lf ", rtt); - - PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); - if (NormMsg::NACK == msgType) - { - const NormNackMsg& nack = static_cast(msg); - PLOG(PL_ALWAYS, "repair content for sender id 0x%08x)\n", nack.GetSenderId()); - LogRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength(), lastFecId, fecM); - } - return; + int index = ((const NormCmdAckReqMsg &)msg).GetAckType(); + index = MIN(index, 3); + PLOG(PL_ALWAYS, "(%s) ", REQ_NAME[index]); break; } - - default: - PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); + case NormCmdMsg::SQUELCH: + { + const NormCmdSquelchMsg &squelch = + static_cast(msg); + PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", + (UINT16)squelch.GetObjectId(), + (UINT32)squelch.GetFecBlockId(fecM).GetValue(), + (UINT16)squelch.GetFecSymbolId(fecM)); break; - } - PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); -} // end NormTrace2(); + } + case NormCmdMsg::FLUSH: + { + const NormCmdFlushMsg &flush = + static_cast(msg); + PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", + (UINT16)flush.GetObjectId(), + (UINT32)flush.GetFecBlockId(fecM).GetValue(), + (UINT16)flush.GetFecSymbolId(fecM)); + // Print acking node list (if any) + UINT16 nodeCount = flush.GetAckingNodeCount(); + if (nodeCount > 0) + { + PLOG(PL_ALWAYS, "ackers>"); + for (UINT16 i = 0; i < nodeCount; i++) + { + if (i > 0) + PLOG(PL_ALWAYS, ","); + PLOG(PL_ALWAYS, "0x%08x", (UINT32)flush.GetAckingNodeId(i)); + } + PLOG(PL_ALWAYS, " "); + } + break; + } + case NormCmdMsg::CC: + { + const NormCmdCCMsg &cc = static_cast(msg); + PLOG(PL_ALWAYS, " seq>%u ", cc.GetCCSequence()); + NormHeaderExtension ext; + while (cc.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_RATE == ext.GetType()) + { + UINT16 sendRate = ((NormCCRateExtension &)ext).GetSendRate(); + PLOG(PL_ALWAYS, " rate>%f ", 8.0e-03 * NormUnquantizeRate(sendRate)); + break; + } + } + struct timeval sendTime; + cc.GetSendTime(sendTime); + double delay = ProtoTime::Delta(ProtoTime(currentTime), ProtoTime(sendTime)); + PLOG(PL_ALWAYS, "delay>%lf ", delay); + break; + } + default: + break; + } + break; + } + + case NormMsg::ACK: + case NormMsg::NACK: + { + PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); + // look for NormCCFeedback extension + NormHeaderExtension ext; + while (msg.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) + { + clrFlag = ((NormCCFeedbackExtension &)ext).CCFlagIsSet(NormCC::CLR); + // Print ccRtt (only valid if pcap file is from sender node) + double ccRtt = NormUnquantizeRtt(((NormCCFeedbackExtension &)ext).GetCCRtt()); + double ccLoss = NormUnquantizeLoss32(((NormCCFeedbackExtension &)ext).GetCCLoss32()); + PLOG(PL_ALWAYS, "ccRtt:%lf ccLoss:%lf ", ccRtt, ccLoss); + break; + } + } + // Print locally measured rtt (only valid if pcap file is from sender node) + struct timeval grttResponse; + if (NormMsg::NACK == msgType) + static_cast(msg).GetGrttResponse(grttResponse); + else + static_cast(msg).GetGrttResponse(grttResponse); + + double rtt = ProtoTime::Delta(ProtoTime(currentTime), ProtoTime(grttResponse)); + PLOG(PL_ALWAYS, "rtt:%lf ", rtt); + + PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); + if (NormMsg::NACK == msgType) + { + const NormNackMsg &nack = static_cast(msg); + PLOG(PL_ALWAYS, "repair content for sender id 0x%08x)\n", nack.GetSenderId()); + LogRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength(), lastFecId, fecM); + } + return; + break; + } + + default: + PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); + break; + } + PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); +} // end NormTrace2(); diff --git a/src/unix/unixPostProcess.cpp b/src/unix/unixPostProcess.cpp index dfa79d3..a3c36f9 100644 --- a/src/unix/unixPostProcess.cpp +++ b/src/unix/unixPostProcess.cpp @@ -38,7 +38,7 @@ */ // Some portions of this code -// Copyright © 1996 Netscape Communications Corporation, +// Copyright � 1996 Netscape Communications Corporation, // all rights reserved. #include "normPostProcess.h"