BodgeMaster
/
NORM-mirror
mirror of https://github.com/USNavalResearchLaboratory/norm.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
NORM-mirror/src/common/normSession.cpp

5631 lines
218 KiB
C++
Raw Blame History

#include "normSession.h"
#include "normEncoderMDP.h" // "legacy" MDP Reed-Solomon encoder
#include "normEncoderRS8.h" // 8-bit Reed-Solomon encoder of RFC 5510
#include "normEncoderRS16.h" // 16-bit Reed-Solomon encoder of RFC 5510
#include <time.h> // for gmtime() in NormTrace()
#include "protoPktETH.h"
#include "protoPktIP.h"
const UINT8 NormSession::DEFAULT_TTL = 255;
const double NormSession::DEFAULT_TRANSMIT_RATE = 64000.0; // bits/sec
const double NormSession::DEFAULT_GRTT_INTERVAL_MIN = 1.0; // sec
const double NormSession::DEFAULT_GRTT_INTERVAL_MAX = 30.0; // sec
const double NormSession::DEFAULT_GRTT_ESTIMATE = 0.25; // sec
const double NormSession::DEFAULT_GRTT_MAX = 10.0; // sec
const unsigned int NormSession::DEFAULT_GRTT_DECREASE_DELAY = 3;
const double NormSession::DEFAULT_BACKOFF_FACTOR = 4.0;
const double NormSession::DEFAULT_GSIZE_ESTIMATE = 1000.0;
const UINT16 NormSession::DEFAULT_NDATA = 64;
const UINT16 NormSession::DEFAULT_NPARITY = 8;
const UINT16 NormSession::DEFAULT_TX_CACHE_MIN = 8;
const UINT16 NormSession::DEFAULT_TX_CACHE_MAX = 256;
const UINT32 NormSession::DEFAULT_TX_CACHE_SIZE = (UINT32)20*1024*1024;
const double NormSession::DEFAULT_FLOW_CONTROL_FACTOR = 2.0;
const UINT16 NormSession::DEFAULT_RX_CACHE_MAX = 256;
const int NormSession::DEFAULT_ROBUST_FACTOR = 20; // default robust factor
// This is extra stuff defined for NormSocket API extension purposes. As the NormSocket
// extension is finalized, these may be refined/relocated
enum {NORM_SOCKET_VERSION = 1};
enum NormSocketCommand
{
NORM_SOCKET_CMD_NULL = 0, // reserved, invalid/null command
NORM_SOCKET_CMD_REJECT, // sent by server-listener to reject invalid connection messages
NORM_SOCKET_CMD_ALIVE // TBD - for NormSocket "keep-alive" option?
};
NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId)
: session_mgr(sessionMgr), notify_pending(false), tx_port(0), tx_port_reuse(false),
tx_socket_actual(ProtoSocket::UDP), tx_socket(&tx_socket_actual),
rx_socket(ProtoSocket::UDP), rx_cap(NULL), rx_port_reuse(false), local_node_id(localNodeId),
ttl(DEFAULT_TTL), tos(0), loopback(false), mcast_loopback(false), fragmentation(false), ecn_enabled(false),
tx_rate(DEFAULT_TRANSMIT_RATE/8.0), tx_rate_min(-1.0), tx_rate_max(-1.0), tx_residual(0),
backoff_factor(DEFAULT_BACKOFF_FACTOR), is_sender(false),
tx_robust_factor(DEFAULT_ROBUST_FACTOR), instance_id(0),
ndata(DEFAULT_NDATA), nparity(DEFAULT_NPARITY), auto_parity(0), extra_parity(0),
sndr_emcon(false), tx_only(false), tx_connect(false), fti_mode(FTI_ALWAYS), encoder(NULL),
next_tx_object_id(0),
tx_cache_count_min(DEFAULT_TX_CACHE_MIN),
tx_cache_count_max(DEFAULT_TX_CACHE_MAX),
tx_cache_size_max(DEFAULT_TX_CACHE_SIZE),
posted_tx_queue_empty(false), posted_tx_rate_changed(false),
acking_node_count(0), acking_auto_populate(TRACK_NONE), watermark_pending(false), watermark_flushes(false),
tx_repair_pending(false), advertise_repairs(false),
suppress_nonconfirmed(false), suppress_rate(-1.0), suppress_rtt(-1.0),
probe_proactive(true), probe_pending(false), probe_reset(true), probe_data_check(false),
grtt_interval(0.5),
grtt_interval_min(DEFAULT_GRTT_INTERVAL_MIN),
grtt_interval_max(DEFAULT_GRTT_INTERVAL_MAX),
grtt_max(DEFAULT_GRTT_MAX),
grtt_decrease_delay_count(DEFAULT_GRTT_DECREASE_DELAY),
grtt_response(false), grtt_current_peak(0.0), grtt_age(0.0), probe_count(1),
cc_enable(false), cc_adjust(true), cc_sequence(0), cc_slow_start(true), cc_active(false),
flow_control_factor(DEFAULT_FLOW_CONTROL_FACTOR),
cmd_count(0), cmd_buffer(NULL), cmd_length(0), syn_status(false),
ack_ex_buffer(NULL), ack_ex_length(0),
is_receiver(false), rx_robust_factor(DEFAULT_ROBUST_FACTOR), preset_sender(NULL), unicast_nacks(false),
receiver_silent(false), rcvr_ignore_info(false), rcvr_max_delay(-1), rcvr_realtime(false),
default_repair_boundary(NormSenderNode::BLOCK_BOUNDARY),
default_nacking_mode(NormObject::NACK_NORMAL), default_sync_policy(NormSenderNode::SYNC_CURRENT),
rx_cache_count_max(DEFAULT_RX_CACHE_MAX), is_server_listener(false), notify_on_grtt_update(true),
ecn_ignore_loss(false),
trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0),
user_data(NULL), next(NULL)
{
interface_name[0] = '\0';
tx_socket_actual.SetNotifier(&sessionMgr.GetSocketNotifier());
tx_socket_actual.SetListener(this, &NormSession::TxSocketRecvHandler);
tx_address.Invalidate();
rx_socket.SetNotifier(&sessionMgr.GetSocketNotifier());
rx_socket.SetListener(this, &NormSession::RxSocketRecvHandler);
tx_timer.SetListener(this, &NormSession::OnTxTimeout);
tx_timer.SetInterval(0.0);
tx_timer.SetRepeat(-1);
repair_timer.SetListener(this, &NormSession::OnRepairTimeout);
repair_timer.SetInterval(0.0);
repair_timer.SetRepeat(1);
flush_timer.SetListener(this, &NormSession::OnFlushTimeout);
flush_timer.SetInterval(0.0);
flush_timer.SetRepeat(0);
flow_control_timer.SetListener(this, &NormSession::OnFlowControlTimeout);
flow_control_timer.SetInterval(0.0);
flow_control_timer.SetRepeat(0);
cmd_timer.SetListener(this, &NormSession::OnCmdTimeout);
cmd_timer.SetInterval(0.0);
cmd_timer.SetRepeat(0);
probe_timer.SetListener(this, &NormSession::OnProbeTimeout);
probe_timer.SetInterval(0.0);
probe_timer.SetRepeat(-1);
probe_time_last.tv_sec = probe_time_last.tv_usec = 0;
grtt_quantized = NormQuantizeRtt(DEFAULT_GRTT_ESTIMATE);
grtt_measured = grtt_advertised = NormUnquantizeRtt(grtt_quantized);
gsize_measured = DEFAULT_GSIZE_ESTIMATE;
gsize_quantized = NormQuantizeGroupSize(DEFAULT_GSIZE_ESTIMATE);
gsize_advertised = NormUnquantizeGroupSize(gsize_quantized);
// This timer is for printing out occasional status reports
// (It may be used to trigger transmission of report messages
// in the future for debugging, etc
report_timer.SetListener(this, &NormSession::OnReportTimeout);
report_timer.SetInterval(10.0);
report_timer.SetRepeat(-1);
user_timer.SetListener(this, &NormSession::OnUserTimeout);
user_timer.SetInterval(0.0);
user_timer.SetRepeat(0);
}
NormSession::~NormSession()
{
if (user_timer.IsActive()) user_timer.Deactivate();
if (NULL != preset_sender)
{
delete preset_sender;
preset_sender = NULL;
}
Close();
}
bool NormSession::Open()
{
ASSERT(address.IsValid());
if (!tx_socket->IsOpen())
{
// Make sure user wants a separate tx_socket
if ((address.GetPort() != tx_port) ||
(tx_address.IsValid() && !address.HostIsEqual(tx_address)))
{
if (!tx_socket->Open(tx_port, address.GetType(), false))
{
PLOG(PL_FATAL, "NormSession::Open() tx_socket::Open() error\n");
return false;
}
if (tx_port_reuse)
{
if (!tx_socket->SetReuse(true))
{
PLOG(PL_FATAL, "NormSession::Open() tx_socket::SetReuse() error\n");
Close();
return false;
}
}
ProtoAddress* txBindAddress = tx_address.IsValid() ? &tx_address : NULL;
if (!tx_socket->Bind(tx_port, txBindAddress))
{
PLOG(PL_FATAL, "NormSession::Open() tx_socket::Bind() error\n");
Close();
return false;
}
// Connecting unicast sockets can help us get unicast NACKs
// when tx_port_reuse and tx_only is requested
if (tx_connect && !address.IsMulticast())
{
if (!tx_socket->Connect(address))
{
PLOG(PL_FATAL, "NormSession::Open() tx_socket::Connect() error\n");
Close();
return false;
}
}
}
else
{
tx_socket = &rx_socket;
}
}
if (!rx_socket.IsOpen() && (!tx_only || (&rx_socket == tx_socket)))
{
if (!rx_socket.Open(0, address.GetType(), false))
{
PLOG(PL_FATAL, "NormSession::Open() rx_socket.Open() error\n");
Close();
return false;
}
rx_socket.EnableRecvDstAddr();
if (rx_port_reuse)
{
// Enable port/addr reuse and bind socket to destination address
if (!rx_socket.SetReuse(true))
{
PLOG(PL_FATAL, "NormSession::Open() rx_socket::SetReuse() error\n");
Close();
return false;
}
}
const ProtoAddress* bindAddr = NULL;
if (rx_bind_addr.IsValid())
{
#ifdef WIN32
if (rx_bind_addr.IsMulticast()) // Win32 doesn't like to bind() mcast addr??
PLOG(PL_WARN, "NormSession::Open() warning: WIN32 multicast bind() issue!\n");
else
#endif
bindAddr = &rx_bind_addr;
}
if(!rx_socket.Bind(address.GetPort(), bindAddr))
{
PLOG(PL_FATAL, "NormSession::Open() error: rx_socket.Bind() error\n");
Close();
return false;
}
if (rx_connect_addr.IsValid() && (0 != rx_connect_addr.GetPort()))
{
// For unicast, we use the "connect()" call to effectively
// uniquely "bind" our rx_socket to the remote addr.
// (it _may_ be the case that "tx_port" == "address.GetPort()"
// for this to work?)
if (!rx_socket.Connect(rx_connect_addr))
{
PLOG(PL_FATAL, "NormSession::Open() rx_socket.Connect() error\n");
Close();
return false;
}
}
}
if (ecn_enabled)
{
if (!tx_socket->SetEcnCapable(true))
{
PLOG(PL_WARN, "NormSession::Open() warning: tx_socket.SetEcnEnable() error\n");
}
}
if (0 != tos)
{
if (!tx_socket->SetTOS(tos))
PLOG(PL_WARN, "NormSession::Open() warning: tx_socket.SetTOS() error\n");
}
if (!tx_socket->SetFragmentation(fragmentation))
PLOG(PL_WARN, "NormSession::Open() warning: tx_socket.SetFragmentation() error\n");
if (address.IsMulticast())
{
if (!tx_socket->SetTTL(ttl))
{
PLOG(PL_FATAL, "NormSession::Open() tx_socket.SetTTL() error\n");
Close();
return false;
}
if (!tx_socket->SetLoopback(mcast_loopback))
{
// TBD - Should this be set on the rx_socket instead???
PLOG(PL_FATAL, "NormSession::Open() tx_socket.SetLoopback() error\n");
Close();
return false;
}
const char* interfaceName = NULL;
if ('\0' != interface_name[0])
{
bool result = tx_only ? true : rx_socket.SetMulticastInterface(interface_name);
result &= tx_socket->SetMulticastInterface(interface_name);
if (!result)
{
PLOG(PL_FATAL, "NormSession::Open() tx_socket::SetMulticastInterface() error\n");
Close();
return false;
}
interfaceName = interface_name;
}
if (!tx_only)
{
if (!rx_socket.JoinGroup(address, interfaceName, ssm_source_addr.IsValid() ? &ssm_source_addr : NULL))
{
PLOG(PL_FATAL, "NormSession::Open() rx_socket.JoinGroup error\n");
Close();
return false;
}
}
}
#ifdef ECN_SUPPORT
// TBD - do this via UDP socket recvmsg() instead of raw packet capture
// If raw packet capture is enabled, create/open ProtoCap device to do it
if (ecn_enabled && !tx_only)
{
if (NULL == rx_cap)
{
if (NULL == (rx_cap = ProtoCap::Create()))
{
PLOG(PL_FATAL, "NormSession::Open() error: unable to create ProtoCap device!\n");
Close();
return false;
}
rx_cap->SetListener(this, &NormSession::OnPktCapture);
rx_cap->SetNotifier(session_mgr.GetChannelNotifier());
if (!rx_cap->Open(('\0' != interface_name[0]) ? interface_name : NULL))
{
PLOG(PL_FATAL, "NormSession::Open() error: unable to open ProtoCap device '%s'!\n", (('\0' != interface_name[0]) ? interface_name : "(null)"));
Close();
return false;
}
rx_cap->StartInputNotification();
// Populate "dst_addr_list" with potential valid dst addrs for this host
dst_addr_list.Destroy();
if (rx_bind_addr.IsValid())
{
if (!dst_addr_list.Insert(rx_bind_addr))
{
PLOG(PL_FATAL, "NormSession::Open() error: unable to add rx_bind_addr to dst_addr_list!!\n");
Close();
return false;
}
}
else
{
// Put all local unicast addrs in list
if (!ProtoSocket::GetHostAddressList(ProtoAddress::IPv4, dst_addr_list))
PLOG(PL_WARN, "NormSession::Open() warning: incomplete IPv4 host address list\n");
if (!ProtoSocket::GetHostAddressList(ProtoAddress::IPv6, dst_addr_list))
PLOG(PL_WARN, "NormSession::Open() warning: incomplete IPv6 host address list\n");
}
if (address.IsMulticast() && !address.HostIsEqual(rx_bind_addr))
{
if (!dst_addr_list.Insert(address))
{
PLOG(PL_FATAL, "NormSession::Open() error: unable to add session addr to dst_addr_list!!\n");
Close();
return false;
}
}
if (dst_addr_list.IsEmpty())
{
PLOG(PL_FATAL, "NormSession::Open() error: unable to add any addresses to dst_addr_list!!\n");
Close();
return false;
}
}
rx_socket.StopInputNotification(); // Disable rx_socket (keep open so mcast JOIN holds)
}
#endif // ECN_SUPPORT
if (message_pool.IsEmpty())
{
for (unsigned int i = 0; i < DEFAULT_MESSAGE_POOL_DEPTH; i++)
{
NormMsg* msg = new NormMsg();
if (msg)
{
message_pool.Append(msg);
}
else
{
PLOG(PL_FATAL, "NormSession::Open() new message error: %s\n", GetErrorString());
Close();
return false;
}
}
}
if (!report_timer.IsActive()) ActivateTimer(report_timer);
return true;
} // end NormSession::Open()
void NormSession::Close()
{
if (report_timer.IsActive()) report_timer.Deactivate();
if (is_sender) StopSender();
if (is_receiver) StopReceiver();
if (tx_timer.IsActive()) tx_timer.Deactivate();
message_queue.Destroy();
message_pool.Destroy();
if (tx_socket->IsOpen()) tx_socket->Close();
if (rx_socket.IsOpen())
{
if (address.IsMulticast())
{
const char* interfaceName = ('\0' != interface_name[0]) ?
interface_name : NULL;
rx_socket.LeaveGroup(address, interfaceName, ssm_source_addr.IsValid() ? &ssm_source_addr : NULL);
}
rx_socket.Close();
}
#ifdef ECN_SUPPORT
if (NULL != rx_cap)
{
rx_cap->Close();
delete rx_cap;
rx_cap = NULL;
}
#endif // ECN_SUPPORT
} // end NormSession::Close()
bool NormSession::SetMulticastInterface(const char* interfaceName)
{
if (NULL != interfaceName)
{
bool result = true;
if (rx_socket.IsOpen())
result &= rx_socket.SetMulticastInterface(interfaceName);
if (tx_socket->IsOpen())
result &= tx_socket->SetMulticastInterface(interfaceName);
strncpy(interface_name, interfaceName, IFACE_NAME_MAX);
interface_name[IFACE_NAME_MAX] = '\0';
return result;
}
else
{
interface_name[0] = '\0';
return true;
}
} // end NormSession::SetMulticastInterface()
bool NormSession::SetSSM(const char* sourceAddress)
{
if (NULL != sourceAddress)
{
if (ssm_source_addr.ResolveFromString(sourceAddress))
{
return true;
}
else
{
PLOG(PL_ERROR, "NormSession::SetSSM() error: invalid source address\n");
return false;
}
}
else
{
ssm_source_addr.Invalidate();
return true;
}
} // end NormSession::SetSSM()
// This must be called _before_ sender or receiver is started
// (i.e., before socket(s) are opened)
bool NormSession::SetRxPortReuse(bool enableReuse,
const char* rxBindAddress, // bind() to <rxBindAddress>/<sessionPort>
const char* senderAddress, // connect() to <senderAddress>/<senderPort>
UINT16 senderPort)
{
rx_port_reuse = enableReuse; // allow sessionPort reuse when true
bool result;
if (NULL != rxBindAddress)
{
result = rx_bind_addr.ResolveFromString(rxBindAddress);
}
else
{
rx_bind_addr.Invalidate();
result = true;
}
if (NULL != senderAddress)
{
// TBD - if open, connect() to sender?
if (rx_connect_addr.ResolveFromString(senderAddress))
{
rx_connect_addr.SetPort(senderPort);
result &= true;
}
else
{
result = false;
}
}
else
{
rx_connect_addr.Invalidate();
}
// TBD - if rx_socket.IsOpen(), should we do a Close()/Open() to rebind socket???
return result;
} // end NormSession::SetRxPortReuse()
bool NormSession::SetTxPort(UINT16 txPort, bool enableReuse, const char* txAddress)
{
tx_port = txPort;
tx_port_reuse = enableReuse;
bool result;
if (NULL != txAddress)
{
result = tx_address.ResolveFromString(txAddress);
// Automatically set port reuse if same tx/rx port nums but diff addr bindings
if (result)
{
if ((tx_port == GetRxPort()) && !tx_address.HostIsEqual(address))
tx_port_reuse = rx_port_reuse = true;
}
}
else
{
tx_address.Invalidate();
result = true;
}
return result;
} // end NormSession::SetTxPort()
UINT16 NormSession::GetTxPort() const
{
if (0 != tx_port)
return tx_port;
else if (tx_socket->IsOpen())
return tx_socket->GetPort();
else
return 0;
} // end NormSession::GetTxPort()
UINT16 NormSession::GetRxPort() const
{
// Note that rx port _can_ be different than session port
// if the session destination address was changed via
// NormSession::SetAddress()
if (rx_socket.IsOpen())
return rx_socket.GetPort();
else
return address.GetPort();
} // end NormSession::GetRxPort()
void NormSession::SetTxOnly(bool txOnly, bool connectToSessionAddress)
{
tx_only = txOnly;
tx_connect = connectToSessionAddress;
if (IsOpen())
{
if (txOnly)
{
if (IsReceiver()) StopReceiver();
if (rx_socket.IsOpen()) rx_socket.Close();
#ifdef ECN_SUPPORT
if (NULL != rx_cap)
{
rx_cap->Close();
delete rx_cap;
rx_cap = NULL;
}
#endif // ECN_SUPPORT
}
// We connect tx_only session sockets when tx port
// reuse is set _and_ it is a unicast session
// (This makes sure unicast NACKs get back to the right tx_socket!)
if (connectToSessionAddress && !address.IsMulticast())
{
if (!tx_socket->Connect(address))
PLOG(PL_WARN, "NormSession::SetTxOnly() tx_socket connect() error: %s\n", ProtoSocket::GetErrorString());
}
}
} // end NormSession::SetTxOnly()
double NormSession::GetTxRate()
{
posted_tx_rate_changed = false;
if (cc_enable && !cc_adjust)
{
// Return rate of CLR
const NormCCNode* clr = static_cast<const NormCCNode*>(cc_node_list.Head());
return ((NULL != clr) ? 8.0 * clr->GetRate() : 0.0);
}
else
{
return (8.0*tx_rate);
}
} // end NormSession::GetTxRate()
/*
// This hack can be uncommented give us a tx rate interval that is POISSON instead of PERIODIC
static double PoissonRand(double mean)
{
return(-log(((double)rand())/((double)RAND_MAX))*mean);
}
*/
static inline double GetTxInterval(unsigned int msgSize, double txRate)
{
double interval = (double)msgSize / txRate;
//double jitterMax = 0.05*interval;
//interval += UniformRand(jitterMax) - jitterMax/2.0;
return interval; // PERIODIC interval based on rate
//return PoissonRand(interval);
}
void NormSession::SetTxRateInternal(double txRate)
{
if (!is_sender)
{
tx_rate = txRate;
return;
}
if (txRate < 0.0)
{
PLOG(PL_FATAL, "NormSession::SetTxRateInternal() invalid transmit rate!\n");
return;
}
if (tx_timer.IsActive())
{
if (txRate > 0.0)
{
double adjustInterval = (tx_rate/txRate) * tx_timer.GetTimeRemaining();
//adjustInterval = PoissonRand(adjustInterval);
if (adjustInterval > NORM_TICK_MIN)
{
tx_timer.SetInterval(adjustInterval);
tx_timer.Reschedule();
}
}
else
{
tx_timer.Deactivate();
}
}
else if ((0.0 == tx_rate) && IsOpen())
{
tx_timer.SetInterval(0.0);
if (txRate > 0.0) ActivateTimer(tx_timer);
}
tx_rate = txRate;
if (tx_rate > 0.0)
{
unsigned char grttQuantizedOld = grtt_quantized;
double pktInterval = (double)(44+segment_size)/txRate;
if (grtt_measured < pktInterval)
grtt_quantized = NormQuantizeRtt(pktInterval);
else
grtt_quantized = NormQuantizeRtt(grtt_measured);
grtt_advertised = NormUnquantizeRtt(grtt_quantized);
// What do we do when "pktInterval" > "grtt_max"?
// We will take our lumps with some extra activity timeout NACKs when they happen?
if (grtt_advertised > grtt_max)
{
grtt_quantized = NormQuantizeRtt(grtt_max);
grtt_advertised = NormUnquantizeRtt(grtt_quantized);
}
if (grttQuantizedOld != grtt_quantized)
{
PLOG(PL_DEBUG, "NormSession::SetTxRateInternal() node>%lu %s to new grtt to: %lf sec\n",
(unsigned long)LocalNodeId(),
(grttQuantizedOld < grtt_quantized) ? "increased" : "decreased",
grtt_advertised);
if (notify_on_grtt_update)
{
notify_on_grtt_update = false;
Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL);
}
}
// wakeup grtt/cc probing if necessary
if (probe_reset)
{
probe_reset = false;
OnProbeTimeout(probe_timer);
if (!probe_timer.IsActive())
ActivateTimer(probe_timer);
}
}
} // end NormSession::SetTxRateInternal()
void NormSession::SetTxRateBounds(double rateMin, double rateMax)
{
posted_tx_rate_changed = false;
// Make sure min <= max
if ((rateMin >= 0.0) && (rateMax >= 0.0))
{
if (rateMin > rateMax)
{
double temp = rateMin;
rateMin = rateMax;
rateMax = temp;
}
}
if (rateMin < 0.0)
tx_rate_min = -1.0;
else if (rateMin < 8.0)
tx_rate_min = 1.0; // one byte/second absolute minimum
else
tx_rate_min = rateMin/8.0; // convert to bytes/second
if (rateMax < 0.0)
tx_rate_max = -1.0;
else
tx_rate_max = rateMax/8.0; // convert to bytes/second
if (cc_enable)
{
double txRate = tx_rate;
if ((tx_rate_min > 0.0) && (txRate < tx_rate_min))
txRate = tx_rate_min;
if ((tx_rate_max >= 0.0) && (txRate > tx_rate_max))
txRate = tx_rate_max;
if (txRate != tx_rate) SetTxRateInternal(txRate);
}
} // end NormSession::SetTxRateBounds()
void NormSession::SetUserTimer(double seconds)
{
if (user_timer.IsActive()) user_timer.Deactivate();
if (seconds >= 0.0)
{
user_timer.SetInterval(seconds);
ActivateTimer(user_timer);
}
} // end NormSession::SetUserTimer()
bool NormSession::StartSender(UINT16 instanceId,
UINT32 bufferSpace,
UINT16 segmentSize,
UINT16 numData,
UINT16 numParity,
UINT8 fecId)
{
UINT16 blockSize = numData + numParity;
if (blockSize <= 255)
fec_m = 8;
else
fec_m = 16;
if (preset_fti.IsValid())
{
if ((preset_fti.GetSegmentSize() != segmentSize) ||
(preset_fti.GetFecMaxBlockLen() != numData) ||
(preset_fti.GetFecNumParity() != numParity) ||
(preset_fti.GetFecFieldSize() != fec_m))
{
PLOG(PL_FATAL, "NormSession::StartSender() preset FTI mismatch error!\n");
return false;
}
}
if (!IsOpen())
{
if (!Open()) return false;
}
if (!tx_table.Init(tx_cache_count_max))
{
PLOG(PL_FATAL, "NormSession::StartSender() tx_table.Init() error!\n");
StopSender();
return false;
}
if (!tx_pending_mask.Init(tx_cache_count_max, 0x0000ffff))
{
PLOG(PL_FATAL, "NormSession::StartSender() tx_pending_mask.Init() error!\n");
StopSender();
return false;
}
if (!tx_repair_mask.Init(tx_cache_count_max, 0x0000ffff))
{
PLOG(PL_FATAL, "NormSession::StartSender() tx_repair_mask.Init() error!\n");
StopSender();
return false;
}
// Calculate how much memory each buffered block will require
unsigned long maskSize = blockSize >> 3;
if (0 != (blockSize & 0x07)) maskSize++;
unsigned long blockSpace = sizeof(NormBlock) +
blockSize * sizeof(char*) +
2*maskSize +
numParity * (segmentSize + NormDataMsg::GetStreamPayloadHeaderLength());
unsigned long numBlocks = bufferSpace / blockSpace;
if (bufferSpace > (numBlocks*blockSpace)) numBlocks++;
if (numBlocks < 2) numBlocks = 2;
unsigned long numSegments = numBlocks * numParity;
if (!block_pool.Init(numBlocks, blockSize))
{
PLOG(PL_FATAL, "NormSession::StartSender() block_pool init error\n");
StopSender();
return false;
}
if (!segment_pool.Init(numSegments, segmentSize + NormDataMsg::GetStreamPayloadHeaderLength()))
{
PLOG(PL_FATAL, "NormSession::StartSender() segment_pool init error\n");
StopSender();
return false;
}
if (numParity)
{
if (NULL != encoder) delete encoder;
if (blockSize <= 255)
{
#ifdef ASSUME_MDP_FEC
if (NULL == (encoder = new NormEncoderMDP))
{
PLOG(PL_FATAL, "NormSession::StartSender() new NormEncoderMDP error: %s\n", GetErrorString());
StopSender();
return false;
}
fec_id = 129;
fec_m = 8;
#else
if (NULL == (encoder = new NormEncoderRS8))
{
PLOG(PL_FATAL, "NormSession::StartSender() new NormEncoderRS8 error: %s\n", GetErrorString());
StopSender();
return false;
}
if (0 != fecId)
fec_id = fecId;
else
fec_id = 5;
fec_m = 8;
#endif
}
else //if (blockSize <= 65535)
{
if (NULL == (encoder = new NormEncoderRS16))
{
PLOG(PL_FATAL, "NormSession::StartSender() new NormEncoderRS16 error: %s\n", GetErrorString());
StopSender();
return false;
}
// TBD - Investigate if fec_id == 129 can also support 16-bit Reed Solomon
fec_id = 2;
fec_m = 16;
}
/*else
{
PLOG(PL_FATAL, "NormSession::StartSender() error: invalid FEC block size\n");
StopSender();
return false;
}*/
if (!encoder->Init(numData, numParity, segmentSize + NormDataMsg::GetStreamPayloadHeaderLength()))
{
PLOG(PL_FATAL, "NormSession::StartSender() encoder init error\n");
StopSender();
return false;
}
}
else
{
// for now use RS8 fec_id with no parity (TBD - support "compact" null FEC type)
if (0 != fecId)
fec_id = fecId;
else
fec_id = 5;
fec_m = 8;
}
fec_block_mask = NormPayloadId::GetFecBlockMask(fec_id, fec_m);
// Initialize optional app-defined command state
cmd_count = cmd_length = 0;
if (NULL == (cmd_buffer = new char[segmentSize]))
{
PLOG(PL_FATAL, "NormSession::StartSender() error: unable to allocate cmd_buffer: %s\n", GetErrorString());
StopSender();
return false;
}
instance_id = instanceId;
segment_size = segmentSize;
sent_accumulator.Reset();
nominal_packet_size = (double)segmentSize;
data_active = false;
ndata = numData;
nparity = numParity;
is_sender = true;
flush_count = (GetTxRobustFactor() < 0) ? 0 : (GetTxRobustFactor() + 1);
if (cc_enable && cc_adjust)
{
double txRate;
if(tx_rate_min > 0.0)
{
txRate = tx_rate_min;
}
else
{
// Don't let txRate below MIN(one segment per grtt, one segment per second)
txRate = ((double)segment_size) / grtt_measured;
if (txRate > ((double)(segment_size)))
txRate = (double)(segment_size);
}
if ((tx_rate_max >= 0.0) && (tx_rate > tx_rate_max))
txRate = tx_rate_max;
SetTxRateInternal(txRate); // adjusts grtt_advertised as needed
}
else
{
SetTxRateInternal(tx_rate); // takes segment size into account, etc on sender start
}
cc_slow_start = true;
cc_active = false;
grtt_age = 0.0;
probe_pending = false;
probe_data_check = false;
if (probe_reset)
{
probe_reset = false;
OnProbeTimeout(probe_timer);
if (!probe_timer.IsActive())
ActivateTimer(probe_timer);
}
return true;
} // end NormSession::StartSender()
void NormSession::StopSender()
{
if (probe_timer.IsActive())
{
probe_timer.Deactivate();
probe_reset = true;
}
if (repair_timer.IsActive())
{
repair_timer.Deactivate();
tx_repair_pending = false;
}
if (flush_timer.IsActive())
flush_timer.Deactivate();
if (cmd_timer.IsActive())
cmd_timer.Deactivate();
if (flow_control_timer.IsActive())
flow_control_timer.Deactivate();
if (NULL != ack_ex_buffer)
{
delete[] ack_ex_buffer;
ack_ex_buffer = NULL;
ack_ex_length = 0;
}
if (NULL != cmd_buffer)
{
delete[] cmd_buffer;
cmd_buffer = NULL;
cmd_length = 0;
}
if (NULL != encoder)
{
encoder->Destroy();
delete encoder;
encoder = NULL;
}
acking_node_tree.Destroy();
cc_node_list.Destroy();
// Iterate tx_table and release objects
while (!tx_table.IsEmpty())
{
NormObject* obj = tx_table.Find(tx_table.RangeLo());
ASSERT(NULL != obj);
tx_table.Remove(obj);
obj->Close();
obj->Release();
}
// Then destroy table
tx_table.Destroy();
block_pool.Destroy();
segment_pool.Destroy();
tx_repair_mask.Destroy();
tx_pending_mask.Destroy();
is_sender = false;
if (!IsReceiver()) Close();
} // end NormSession::StopSender()
bool NormSession::StartReceiver(unsigned long bufferSize)
{
//if (tx_only) return false;
tx_only = false;
if (!rx_socket.IsOpen())
{
if (!Open()) return false;
}
is_receiver = true;
remote_sender_buffer_size = bufferSize;
return true;
} // end NormSession::StartReceiver()
void NormSession::StopReceiver()
{
// Iterate sender_tree and close/release sender nodes
if (IsServerListener())
{
client_tree.Destroy();
}
else
{
NormSenderNode* senderNode =
static_cast<NormSenderNode*>(sender_tree.GetRoot());
while (NULL != senderNode)
{
sender_tree.DetachNode(senderNode);
senderNode->Close();
senderNode->Release();
senderNode = static_cast<NormSenderNode*>(sender_tree.GetRoot());
}
}
is_receiver = false;
if (!is_sender) Close();
} // end NormSession::StopReceiver()
void NormSession::DeleteRemoteSender(NormSenderNode& senderNode)
{
// TBD - confirm that "senderNode" is valid???
if (IsServerListener())
client_tree.RemoveNode(senderNode);
else
sender_tree.DetachNode(&senderNode);
senderNode.Close();
senderNode.Release();
} // end NormSession::DeleteRemoteSender()
bool NormSession::PreallocateRemoteSender(unsigned int bufferSpace,
UINT16 segmentSize,
UINT16 numData,
UINT16 numParity,
unsigned int streamBufferSize)
{
if (NULL != preset_sender) delete preset_sender;
preset_sender = new NormSenderNode(*this, NORM_NODE_ANY);
if (!preset_sender->Open(0))
{
PLOG(PL_ERROR, "NormSession::PreallocateRemoteSender() error: NormSenderNode::Open() failure!\n");
delete preset_sender;
preset_sender = NULL;
return false;
}
UINT16 blockSize = numData + numParity;
UINT8 fecId;
UINT8 fecM = 8;
if (blockSize > 255)
{
fecId = 2;
fecM = 16;
}
else
{
fecId = 5;
}
if (!preset_sender->AllocateBuffers(bufferSpace, fecId, 0, fecM, segmentSize, numData, numParity))
{
PLOG(PL_ERROR, "NormSession::PreallocateRemoteSender() error: buffer allocation failure!\n");
delete preset_sender;
preset_sender = NULL;
return false;
}
if (0 != streamBufferSize)
{
if (!preset_sender->PreallocateRxStream(streamBufferSize, segmentSize, numData, numParity))
{
PLOG(PL_ERROR, "NormSession::PreallocateRemoteSender() error: preset_stream allocation failure!\n");
delete preset_sender;
preset_sender = NULL;
return false;
}
}
return true;
} // end NormSession::PreallocateRemoteSender()
bool NormSession::SetPresetFtiData(unsigned int objectSize,
UINT16 segmentSize,
UINT16 numData,
UINT16 numParity)
{
UINT16 blockSize = numData + numParity;
UINT8 fecM;
if (blockSize > 255)
fecM = 16;
else
fecM = 8;
if (IsSender())
{
if ((segmentSize != segment_size) ||
(numData != ndata) ||
(numParity != nparity) ||
(fecM != fec_m))
{
PLOG(PL_FATAL, "NormSession::SetPresetFtiData() sender FTI mismatch error!\n");
return false;
}
}
preset_fti.SetObjectSize(NormObjectSize(objectSize));
preset_fti.SetSegmentSize(segmentSize);
preset_fti.SetFecMaxBlockLen(numData);
preset_fti.SetFecNumParity(numParity);
preset_fti.SetFecFieldSize(fecM);
preset_fti.SetFecInstanceId(0);
return true;
} // end NormSession::SetPresetFtiData()
void NormSession::Serve()
{
// Only send new data when no other messages are queued for transmission
if (!message_queue.IsEmpty())
{
ASSERT(tx_timer.IsActive());
return;
}
// Queue next sender message
NormObjectId objectId;
NormObject* obj = NULL;
if (SenderGetFirstPending(objectId))
{
obj = tx_table.Find(objectId);
ASSERT(NULL != obj);
}
// If any app-defined command is pending, enqueue it for transmission
if ((0 != cmd_count) && !(cmd_timer.IsActive()))
{
// If command is enqueued
if (SenderQueueAppCmd()) return;
}
bool watermarkJustCompleted = false;
if (watermark_pending && !flush_timer.IsActive())
{
PLOG(PL_DEBUG, "NormSession::Serve() watermark status check ...\n");
// Determine next message (objectId::blockId::segmentId) to be sent
NormObject* nextObj;
NormObjectId nextObjectId = next_tx_object_id;
NormBlockId nextBlockId = 0;
NormSegmentId nextSegmentId = 0;
if (obj)
{
// Get index (objectId::blockId::segmentId) of next transmit pending segment
nextObj = obj;
nextObjectId = objectId;
if (nextObj->IsPending())
{
if(nextObj->GetFirstPending(nextBlockId))
{
NormBlock* block = nextObj->FindBlock(nextBlockId);
if (block)
{
#ifdef PROTO_DEBUG
ASSERT(block->GetFirstPending(nextSegmentId));
#else
block->GetFirstPending(nextSegmentId);
#endif // if/else PROTO_DEBUG
// Adjust so watermark segmentId < block length
UINT16 nextBlockSize = nextObj->GetBlockSize(nextBlockId);
if (nextSegmentId >= nextBlockSize) nextSegmentId = nextBlockSize - 1;
}
}
else
{
// info only pending; so blockId = segmentId = 0 (as inited)
}
}
else
{
// Must be an active, but non-pending stream object
ASSERT(nextObj->IsStream());
nextBlockId = static_cast<NormStreamObject*>(nextObj)->GetNextBlockId();
nextSegmentId = static_cast<NormStreamObject*>(nextObj)->GetNextSegmentId();
}
}
PLOG(PL_DEBUG, " nextPending index>%hu:%lu:%hu\n",
(UINT16)nextObjectId,
(unsigned long)nextBlockId.GetValue(),
(UINT16)nextSegmentId);
if (tx_repair_pending)
{
PLOG(PL_DEBUG, " tx_repair index>%hu:%lu:%hu\n",
(UINT16)tx_repair_object_min,
(unsigned long)tx_repair_block_min.GetValue(),
(UINT16)tx_repair_segment_min);
if ((tx_repair_object_min < nextObjectId) ||
((tx_repair_object_min == nextObjectId) &&
//((tx_repair_block_min < nextBlockId) ||
((Compare(tx_repair_block_min, nextBlockId) < 0) ||
((tx_repair_block_min == nextBlockId) &&
(tx_repair_segment_min < nextSegmentId)))))
{
nextObjectId = tx_repair_object_min;
nextBlockId = tx_repair_block_min;
nextSegmentId = tx_repair_segment_min;
PLOG(PL_DEBUG, " updated nextPending index>%hu:%lu:%hu\n",
(UINT16)nextObjectId,
(unsigned long)nextBlockId.GetValue(),
(UINT16)nextSegmentId);
}
} // end if (tx_repair_pending)
ASSERT(nextBlockId.GetValue() <= (UINT32)0x00ffffff);
PLOG(PL_DEBUG, " watermark>%hu:%lu:%hu check against next pending index>%hu:%lu:%hu\n",
(UINT16)watermark_object_id, (unsigned long)watermark_block_id.GetValue(), (UINT16)watermark_segment_id,
(UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), (UINT16)nextSegmentId);
if ((nextObjectId > watermark_object_id) ||
((nextObjectId == watermark_object_id) &&
//((nextBlockId > watermark_block_id) ||
((Compare(nextBlockId, watermark_block_id) > 0) ||
((nextBlockId == watermark_block_id) &&
(nextSegmentId > watermark_segment_id)))))
{
PLOG(PL_DEBUG, " calling SenderQueueWatermarkFlush() ...\n");
// The sender tx position is > watermark
if (SenderQueueWatermarkFlush())
{
watermark_active = true;
return;
}
else
{
// (TBD) optionally return here to have ack collection temporarily
// suspend forward progress of data transmission
//return;
// If the app has set the property to "truncated_flushing" because
// there is explicit positive acknowledge from everyone in the group
// (or unicast destination), we can safely provide an _early_
// termination of flushing at this point iff:
//
// "(false == watermark_pending) &&
// (NULL == obj) &&
// (watermark_object_id == last_tx_object_id)
//
// These conditions indicate that the watermark ack has completed,
// there is no more data to send, _and_ the watermark was the
// ordinally highest object that has been enqueued ....
if (!watermark_pending)
{
// watermark flush just completed
if (watermark_flushes)
flush_count = GetTxRobustFactor();
watermarkJustCompleted = true;
}
}
}
else
{
// The sender tx position is < watermark
// Reset non-acked acking nodes since sender has rewound
// TBD - notify application that watermark ack has been reset ???
if (watermark_active)
{
watermark_active = false;
NormNodeTreeIterator iterator(acking_node_tree);
NormAckingNode* next;
while ((next = static_cast<NormAckingNode*>(iterator.GetNextNode())))
next->ResetReqCount(GetTxRobustFactor());
}
}
} // end if (watermark_pending && !flush_timer.IsActive())
if (NULL != obj)
{
NormObjectMsg* msg = (NormObjectMsg*)GetMessageFromPool();
if (msg)
{
if (obj->NextSenderMsg(msg))
{
if (cc_enable && !data_active)
{
data_active = true;
if (probe_timer.IsActive())
{
double elapsed = probe_timer.GetInterval() - probe_timer.GetTimeRemaining();
double probeInterval = GetProbeInterval();
if (elapsed > probeInterval)
probe_timer.SetInterval(0.0);
else
probe_timer.SetInterval(probeInterval - elapsed);
probe_timer.Reschedule();
}
}
msg->SetDestination(address);
msg->SetGrtt(grtt_quantized);
msg->SetBackoffFactor((unsigned char)backoff_factor);
msg->SetGroupSize(gsize_quantized);
QueueMessage(msg);
flush_count = 0;
// (TBD) ??? should streams every allowed to be non-pending?
// we _could_ re-architect streams a little bit and allow
// for this by having NormStreamObject::Write() control
// stream advancement ... I think it would be cleaner.
// (mod NormStreamObject::StreamAdvance() to depend upon
// what has been written and conversely set some pending
// state as calls to NormStreamObject::Write() are made.
if (!obj->IsPending() && !obj->IsStream())
{
tx_pending_mask.Unset(obj->GetId());
if (!tx_pending_mask.IsSet() && !posted_tx_queue_empty)
{
// Tell the app we would like to send more data ...
posted_tx_queue_empty = true;
Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, (NormObject*)NULL);
// (TBD) Was session deleted?
}
}
}
else
{
ReturnMessageToPool(msg);
if (obj->IsStream())
{
NormStreamObject* stream = static_cast<NormStreamObject*>(obj);
if (stream->IsFlushPending() || stream->IsClosing())
{
// Queue flush message
if (!flush_timer.IsActive())
{
if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor()))
{
SenderQueueFlush();
}
else if (GetTxRobustFactor() == flush_count)
{
PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender stream flush complete ...\n",
(unsigned long)LocalNodeId());
Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode*)NULL, stream);
flush_count++;
data_active = false;
if (stream->IsClosing())
{
// If the stream just failed end-of-stream watermark flush, we don't
// close the stream yet, but instead give app chance to reset watermark
bool watermarkFailed = watermarkJustCompleted && (obj->GetId() == watermark_object_id) &&
(ACK_FAILURE == SenderGetAckingStatus(NORM_NODE_ANY));
if (!watermarkFailed)
{
// end of stream was successfully acknowledged
stream->Close();
DeleteTxObject(stream, true);
obj = NULL;
}
}
}
}
}
//ASSERT(stream->IsPending() || stream->IsRepairPending() || stream->IsClosing());
if (!posted_tx_queue_empty && !stream->IsClosing() && stream->IsPending())
// post if pending || !repair_timer.IsActive() || (repair_timer.GetRepeatCount() == 0) ???
{
//data_active = false;
posted_tx_queue_empty = true;
Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, obj);
// (TBD) Was session deleted?
return;
}
}
else
{
PLOG(PL_ERROR, "NormSession::Serve() pending non-stream obj, no message?.\n");
//ASSERT(repair_timer.IsActive());
}
}
}
else
{
PLOG(PL_ERROR, "NormSession::Serve() node>%lu Warning! message_pool empty.\n",
(unsigned long)LocalNodeId());
}
}
else
{
// No pending objects or positive acknowledgement request
if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor()))
{
// Queue flush message
if (!tx_repair_pending) // don't queue flush if repair pending
SenderQueueFlush();
else
PLOG(PL_DETAIL, "NormSession::Serve() node>%lu NORM_CMD(FLUSH) deferred by pending repairs ...\n",
(unsigned long)LocalNodeId());
}
else if (GetTxRobustFactor() == flush_count)
{
PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n",
(unsigned long)LocalNodeId());
Notify(NormController::TX_FLUSH_COMPLETED,
(NormSenderNode*)NULL,
(NormObject*)NULL);
flush_count++;
data_active = false;
}
}
} // end NormSession::Serve()
bool NormSession::SenderSetWatermark(NormObjectId objectId,
NormBlockId blockId,
NormSegmentId segmentId,
bool overrideFlush,
const char* appAckReq,
unsigned int appAckReqLen)
{
PLOG(PL_DEBUG, "NormSession::SenderSetWatermark() watermark>%hu:%lu:%hu\n",
(UINT16)objectId, (unsigned long)blockId.GetValue(), (UINT16)segmentId);
watermark_flushes = overrideFlush;
watermark_pending = true;
watermark_active = false;
watermark_object_id = objectId;
watermark_block_id = blockId;
watermark_segment_id = segmentId;
acking_success_count = 0;
// Reset acking_node_list
NormNodeTreeIterator iterator(acking_node_tree);
NormNode* next;
int robustFactor = GetTxRobustFactor();
while ((next = iterator.GetNextNode()))
static_cast<NormAckingNode*>(next)->Reset(robustFactor);
if (NULL != appAckReq)
{
if (appAckReqLen != ack_ex_length)
{
if (NULL != ack_ex_buffer)
{
delete[] ack_ex_buffer;
ack_ex_buffer = NULL;
ack_ex_length = 0;
}
// Make sure there is room left for at least one acker NormNodeId in
if ((NormHeaderExtension::GetContentOffset() + appAckReqLen) > (segment_size - sizeof(NormNodeId)))
{
PLOG(PL_ERROR, "NormSession::SenderSetWatermark() error: application-defined ACK_REQ content too large!\n");
watermark_pending = false;
return false;
}
else if (NULL == (ack_ex_buffer = new char[appAckReqLen]))
{
PLOG(PL_ERROR, "NormSession::SenderSetWatermark() new app_req_buffer error: %s\n", GetErrorString());
watermark_pending = false;
return false;
}
}
memcpy(ack_ex_buffer, appAckReq, appAckReqLen);
ack_ex_length = appAckReqLen;
}
else if (NULL != ack_ex_buffer)
{
delete[] ack_ex_buffer;
ack_ex_buffer = NULL;
ack_ex_length = 0;
}
PromptSender();
return true;
} // end NormSession::SenderSetWatermark()
void NormSession::SenderResetWatermark()
{
NormNodeTreeIterator iterator(acking_node_tree);
NormNode* next;
int robustFactor = GetTxRobustFactor();
while ((next = iterator.GetNextNode()))
{
NormAckingNode* node = static_cast<NormAckingNode*>(next);
if ((NORM_NODE_NONE == node->GetId()) || (!node->AckReceived()))
{
node->Reset(robustFactor);
watermark_pending = true;
watermark_active = false;
}
}
PromptSender();
} // end NormSession::SenderResetWatermark()
void NormSession::SenderCancelWatermark()
{
watermark_pending = false;
} // end NormSession::SenderCancelWatermark()
NormAckingNode* NormSession::SenderAddAckingNode(NormNodeId nodeId, const ProtoAddress* srcAddress)
{
NormAckingNode* theNode = static_cast<NormAckingNode*>(acking_node_tree.FindNodeById(nodeId));
if (NULL == theNode)
{
theNode = new NormAckingNode(*this, nodeId);
if (NULL != theNode)
{
theNode->Reset(GetTxRobustFactor());
acking_node_tree.AttachNode(theNode);
acking_node_count++;
}
else
{
PLOG(PL_ERROR, "NormSession::SenderAddAckingNode() new NormAckingNode error: %s\n", GetErrorString());
return NULL;
}
}
else
{
PLOG(PL_WARN, "NormSession::SenderAddAckingNode() warning: node already in list!?\n");
}
if (NULL != srcAddress) theNode->SetAddress(*srcAddress);
return theNode;
} // end NormSession::AddAckingNode(NormNodeId nodeId)
void NormSession::SenderRemoveAckingNode(NormNodeId nodeId)
{
NormAckingNode* theNode =
static_cast<NormAckingNode*>(acking_node_tree.FindNodeById(nodeId));
if (NULL != theNode)
{
acking_node_tree.DetachNode(theNode);
theNode->Release();
// TBD - if a watermark was pending and this is the only
// non-pending acker, can we immediately issue WATERMARK_COMPLETED?
acking_node_count--;
}
} // end NormSession::RemoveAckingNode()
NormSession::AckingStatus NormSession::SenderGetAckingStatus(NormNodeId nodeId)
{
if (NORM_NODE_ANY == nodeId)
{
// Return result based on overall success of acking process
if (watermark_pending)
{
return ACK_PENDING;
}
else
{
if (acking_success_count < acking_node_count)
return ACK_FAILURE;
else
return ACK_SUCCESS;
}
}
else
{
NormAckingNode* theNode =
static_cast<NormAckingNode*>(acking_node_tree.FindNodeById(nodeId));
if (NULL != theNode)
{
if (theNode->IsPending())
return ACK_PENDING;
else if (NORM_NODE_NONE == theNode->GetId())
return ACK_SUCCESS;
else if (theNode->AckReceived())
return ACK_SUCCESS;
else
return ACK_FAILURE;
}
else
{
return ACK_INVALID;
}
}
} // end NormSession::SenderGetAckingStatus()
bool NormSession::SenderGetNextAckingNode(NormNodeId& prevNodeId, AckingStatus* ackingStatus)
{
NormNode* prevNode = NULL;
if (NORM_NODE_NONE != prevNodeId)
prevNode = acking_node_tree.FindNodeById(prevNodeId);
NormNodeTreeIterator iterator(acking_node_tree, prevNode);
NormAckingNode* nextNode = static_cast<NormAckingNode*>(iterator.GetNextNode());
// Note we skip NORM_NODE_NONE even though it may be in the tree
// (This method only returns the id / status of _actual_ nodes)
// TBD - we could return NORM_NODE_ANY as a proxy id for a NORM_NODE_NONE entry
if ((NULL != nextNode) && (NORM_NODE_NONE == nextNode->GetId()))
nextNode = static_cast<NormAckingNode*>(iterator.GetNextNode());
if (NULL != nextNode)
{
prevNodeId = nextNode->GetId();
if (NULL != ackingStatus)
{
if (nextNode->IsPending())
*ackingStatus = ACK_PENDING;
else if (NORM_NODE_NONE == nextNode->GetId())
*ackingStatus = ACK_SUCCESS;
else if (nextNode->AckReceived())
*ackingStatus = ACK_SUCCESS;
else
*ackingStatus = ACK_FAILURE;
}
return true;
}
else
{
prevNodeId = NORM_NODE_NONE;
if (NULL != ackingStatus)
*ackingStatus = ACK_INVALID;
return false;
}
} // end NormSession::SenderGetNextAckingNode()
bool NormSession::SenderGetAckEx(NormNodeId nodeId, char* buffer, unsigned int* buflen)
{
NormAckingNode* theNode =
static_cast<NormAckingNode*>(acking_node_tree.FindNodeById(nodeId));
if (NULL != theNode)
{
return theNode->GetAckEx(buffer, buflen);
}
else
{
if (NULL != buflen) *buflen = 0;
return false;
}
} // end NormSession::SenderGetAckEx()
bool NormSession::SenderQueueWatermarkFlush()
{
if (flush_timer.IsActive()) return false;
NormCmdFlushMsg* flush = static_cast<NormCmdFlushMsg*>(GetMessageFromPool());
if (flush)
{
flush->Init();
flush->SetDestination(address);
flush->SetGrtt(grtt_quantized);
flush->SetBackoffFactor((unsigned char)backoff_factor);
flush->SetGroupSize(gsize_quantized);
flush->SetObjectId(watermark_object_id);
// _Attempt_ to set the fec_payload_id source block length field appropriately
UINT16 blockLen;
NormObject* obj = tx_table.Find(watermark_object_id);
if (NULL != obj)
blockLen = obj->GetBlockSize(watermark_block_id);
else if (watermark_segment_id < ndata)
blockLen = ndata;
else
blockLen = watermark_segment_id;
flush->SetFecPayloadId(fec_id, watermark_block_id.GetValue(), watermark_segment_id, blockLen, fec_m);
if (0 != ack_ex_length)
{
NormAppAckExtension ext;
flush->AttachExtension(ext);
ext.SetContent(ack_ex_buffer, ack_ex_length);
flush->PackExtension(ext);
}
NormNodeTreeIterator iterator(acking_node_tree);
NormAckingNode* next;
watermark_pending = false;
NormAckingNode* nodeNone = NULL;
acking_success_count = 0;
while (NULL != (next = static_cast<NormAckingNode*>(iterator.GetNextNode())))
{
// Save NORM_NODE_NONE for last
if (NORM_NODE_NONE == next->GetId())
{
if (next->IsPending())
nodeNone = next;
else
acking_success_count++; // implicit success for NORM_NODE_NONE
continue;
}
if (next->AckReceived())
{
acking_success_count++; // ACK was received for this node
}
else if (next->IsPending())
{
// Add node to list
if (flush->AppendAckingNode(next->GetId(), segment_size))
{
next->DecrementReqCount();
watermark_pending = true;
}
else
{
PLOG(PL_FATAL, "NormSession::ServeQueueWatermarkFlush() full cmd ...\n");
nodeNone = NULL;
break;
}
}
}
if (NULL != nodeNone)
{
if (flush->AppendAckingNode(NORM_NODE_NONE, segment_size))
{
nodeNone->DecrementReqCount();
watermark_pending = true;
}
else
{
PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() full cmd ...\n");
}
}
if (watermark_pending)
{
// (TBD) we should increment the "flush_count" here only iff the watermark
// corresponds to our "last_tx_object_id", etc
//if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor()))
// flush_count++;
QueueMessage(flush);
PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu cmd queued ...\n",
(unsigned long)LocalNodeId());
}
else if (NULL != acking_node_tree.GetRoot())
{
ReturnMessageToPool(flush);
PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu watermark ack finished.\n",
(unsigned long)LocalNodeId());
Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL);
return false;
}
else
{
ReturnMessageToPool(flush);
PLOG(PL_INFO, "NormSession::ServeQueueWatermarkFlush() node>%lu no acking nodes specified?!\n");
return false;
}
}
else
{
PLOG(PL_ERROR, "NormSession::SenderQueueWatermarkRequest() node>%lu message_pool exhausted! (couldn't req)\n",
(unsigned long)LocalNodeId());
}
PLOG(PL_DEBUG, "NormSession::SenderQueueWatermarkFlush() starting flush timeout: %lf sec ....\n", 2*grtt_advertised);
flush_timer.SetInterval(2*grtt_advertised);
ActivateTimer(flush_timer);
return true;
} // end NormSession::SenderQueueWatermarkFlush()
void NormSession::SenderQueueFlush()
{
// (TBD) Don't enqueue a new flush if there is already one in our tx_queue!
if (flush_timer.IsActive()) return;
NormObject* obj = tx_table.Find(tx_table.RangeHi());
NormObjectId objectId;
NormBlockId blockId;
NormSegmentId segmentId;
if (obj)
{
if (obj->IsStream())
{
NormStreamObject* stream = (NormStreamObject*)obj;
objectId = stream->GetId();
blockId = stream->FlushBlockId();
segmentId = stream->FlushSegmentId();
}
else
{
objectId = obj->GetId();
blockId = obj->GetFinalBlockId();
segmentId = obj->GetBlockSize(blockId) - 1;
}
NormCmdFlushMsg* flush = (NormCmdFlushMsg*)GetMessageFromPool();
if (flush)
{
flush->Init();
flush->SetDestination(address);
flush->SetGrtt(grtt_quantized);
flush->SetBackoffFactor((unsigned char)backoff_factor);
flush->SetGroupSize(gsize_quantized);
flush->SetObjectId(objectId);
flush->SetFecPayloadId(fec_id, blockId.GetValue(), segmentId, obj->GetBlockSize(blockId), fec_m);
QueueMessage(flush);
if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor()))
flush_count++;
PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() node>%lu, flush queued (flush_count:%u)...\n",
(unsigned long)LocalNodeId(), flush_count);
}
else
{
PLOG(PL_ERROR, "NormSession::SenderQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n",
(unsigned long)LocalNodeId());
}
}
else
{
// Why did I do this? - Brian // Because a squelch keeps the receivers from NACKing in futility
// (TBD) send NORM_CMD(EOT) instead? - no
// Perhaps I should send a flush anyway w/ (next_tx_object_id - 1) and squelch accordingly?
// This condition shouldn't occur if we have state on the most recent object ... we should
// unless the app does bad things like "cancel" all of its tx objects ...
// Maybe we shouldn't send anything if we have no pending tx objects? No need to flush, etc
// if all tx object state is gone ...
if (SenderQueueSquelch(next_tx_object_id))
{
if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor()))
flush_count++;
PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() node>%lu squelch queued (flush_count:%u)...\n",
(unsigned long)LocalNodeId(), flush_count);
}
else
{
PLOG(PL_ERROR, "NormSession::SenderQueueFlush() warning: node>%lu unable to queue squelch\n",
(unsigned long)LocalNodeId());
}
}
PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() starting flush timeout: %lf sec ....\n", 2*grtt_advertised);
flush_timer.SetInterval(2*grtt_advertised);
ActivateTimer(flush_timer);
} // end NormSession::SenderQueueFlush()
bool NormSession::OnFlushTimeout(ProtoTimer& /*theTimer*/)
{
PLOG(PL_DEBUG, "NormSession::OnFlushTimeout() deactivating flush_timer ....\n");
flush_timer.Deactivate();
PromptSender();
return false;
} // NormSession::OnFlushTimeout()
void NormSession::QueueMessage(NormMsg* msg)
{
/* A little test jig
static struct timeval lastTime = {0,0};
struct timeval currentTime;
ProtoSystemTime(currentTime);
if (0 != lastTime.tv_sec)
{
double delta = currentTime.tv_sec - lastTime.tv_sec;
delta += (((double)currentTime.tv_usec)*1.0e-06 -
((double)lastTime.tv_usec)*1.0e-06);
DMSG(0, "NormSession::QueueMessage() deltaT:%lf\n", delta);
}
lastTime = currentTime;
*/
// (TBD) if (0.0 == tx_rate), should we just dump the
// message rather than queueing it?
if (!tx_timer.IsActive() && (tx_rate > 0.0))
{
tx_timer.SetInterval(0.0);
ActivateTimer(tx_timer);
}
if (NULL != msg) message_queue.Append(msg);
} // end NormSesssion::QueueMessage(NormMsg& msg)
NormFileObject* NormSession::QueueTxFile(const char* path,
const char* infoPtr,
UINT16 infoLen)
{
if (!IsSender())
{
PLOG(PL_FATAL, "NormSession::QueueTxFile() Error: sender is closed\n");
return NULL;
}
NormFileObject* file = new NormFileObject(*this, (NormSenderNode*)NULL, next_tx_object_id);
if (NULL == file)
{
PLOG(PL_FATAL, "NormSession::QueueTxFile() new file object error: %s\n",
GetErrorString());
return NULL;
}
if (!file->Open(path, infoPtr, infoLen))
{
PLOG(PL_FATAL, "NormSession::QueueTxFile() file open error\n");
file->Release();
return NULL;
}
if (QueueTxObject(file))
{
return file;
}
else
{
file->Close();
file->Release();
return NULL;
}
} // end NormSession::QueueTxFile()
NormDataObject* NormSession::QueueTxData(const char* dataPtr,
UINT32 dataLen,
const char* infoPtr,
UINT16 infoLen)
{
if (!IsSender())
{
PLOG(PL_FATAL, "NormSession::QueueTxData() Error: sender is closed\n");
return NULL;
}
NormDataObject* obj = new NormDataObject(*this, (NormSenderNode*)NULL, next_tx_object_id, session_mgr.GetDataFreeFunction());
if (!obj)
{
PLOG(PL_FATAL, "NormSession::QueueTxData() new data object error: %s\n",
GetErrorString());
return NULL;
}
if (!obj->Open((char*)dataPtr, dataLen, false, infoPtr, infoLen))
{
PLOG(PL_FATAL, "NormSession::QueueTxData() object open error\n");
obj->Release();
return NULL;
}
if (QueueTxObject(obj))
{
return obj;
}
else
{
obj->Close();
obj->Release();
return NULL;
}
} // end NormSession::QueueTxData()
NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize,
bool doubleBuffer,
const char* infoPtr,
UINT16 infoLen)
{
if (!IsSender())
{
PLOG(PL_FATAL, "NormSession::QueueTxStream() Error: sender is closed\n");
return NULL;
}
NormStreamObject* stream = new NormStreamObject(*this, (NormSenderNode*)NULL, next_tx_object_id);
if (!stream)
{
PLOG(PL_FATAL, "NormSession::QueueTxStream() new stream object error: %s\n",
GetErrorString());
return NULL;
}
if (!stream->Open(bufferSize, doubleBuffer, infoPtr, infoLen))
{
PLOG(PL_FATAL, "NormSession::QueueTxStream() stream open error\n");
stream->Release();
return NULL;
}
if (QueueTxObject(stream))
{
// (???: stream has nothing pending until user writes to it???)
//stream->Reset();
return stream;
}
else
{
stream->Close();
stream->Release();
return NULL;
}
} // end NormSession::QueueTxStream()
#ifdef SIMULATE
NormSimObject* NormSession::QueueTxSim(unsigned long objectSize)
{
if (!IsSender())
{
PLOG(PL_FATAL, "NormSession::QueueTxSim() Error: sender is closed\n");
return NULL;
}
NormSimObject* simObject = new NormSimObject(*this, NULL, next_tx_object_id);
if (!simObject)
{
PLOG(PL_FATAL, "NormSession::QueueTxSim() new sim object error: %s\n",
GetErrorString());
return NULL;
}
if (!simObject->Open(objectSize))
{
PLOG(PL_FATAL, "NormSession::QueueTxSim() open error\n");
simObject->Release();
return NULL;
}
if (QueueTxObject(simObject))
{
return simObject;
}
else
{
simObject->Release();
return NULL;
}
} // end NormSession::QueueTxSim()
#endif // SIMULATE
bool NormSession::QueueTxObject(NormObject* obj)
{
if (!IsSender())
{
PLOG(PL_FATAL, "NormSession::QueueTxObject() non-sender session error!?\n");
return false;
}
if (preset_fti.IsValid() && (obj->GetSize() != preset_fti.GetObjectSize()))
{
PLOG(PL_FATAL, "NormSession::QueueTxObject() preset object info mismatch!\n");
return false;
}
// Manage tx_table min/max count and max size bounds
// Depending on tx cache bounds _and_ what has been
// enqueued/dequeued, we may need to prune the
// "tx_table" a little
// The cases when pruning is needed include:
//
// 1) When the cache bounds dictate:
// i.e., ((count > count_min) && ((count > count_max) || (size > size_max))), or
// 2) When the "tx_table" state (from insert/remove history) doesn't allow
// i.e., !tx_table.CanInsert(obj)
unsigned long newCount = tx_table.GetCount() + 1;
while (!tx_table.CanInsert(obj->GetId()) ||
((newCount > tx_cache_count_min) &&
((newCount > tx_cache_count_max) ||
((tx_table.GetSize() + obj->GetSize()) > tx_cache_size_max))))
{
// Remove oldest non-pending
NormObject* oldest = tx_table.Find(tx_table.RangeLo());
if (oldest->IsRepairPending() || oldest->IsPending())
{
PLOG(PL_ALWAYS, "NormSession::QueueTxObject() all held objects repair pending:%d (repair active:%d) pending:%d\n",
oldest->IsRepairPending(), repair_timer.IsActive(), oldest->IsPending());
posted_tx_queue_empty = false;
return false;
}
else
{
double delay = GetFlowControlDelay() - oldest->GetNackAge();
if (delay < 1.0e-06)
{
if (FlowControlIsActive()) DeactivateFlowControl();
DeleteTxObject(oldest, true);
}
else
{
PLOG(PL_ALWAYS, "NormSession::QueueTxObject() asserting flow control for object delay:%lf sec\n", delay);
// TBD - flow control as should allow for TX_QUEUE_VACANCY posting for session
ActivateFlowControl(delay, oldest->GetId(), NormController::TX_QUEUE_EMPTY);
posted_tx_queue_empty = false;
return false;
}
}
newCount = tx_table.GetCount() + 1;
}
// Attempt to queue the object (note it gets "retained" by the tx_table)
if (!tx_table.Insert(obj))
{
PLOG(PL_FATAL, "NormSession::QueueTxObject() tx_table insert error\n");
ASSERT(0);
return false;
}
tx_pending_mask.Set(obj->GetId());
ASSERT(tx_pending_mask.Test(obj->GetId()));
next_tx_object_id++;
TouchSender();
return true;
} // end NormSession::QueueTxObject()
bool NormSession::RequeueTxObject(NormObject* obj)
{
ASSERT(NULL != obj);
if (obj->IsStream())
{
// (TBD) allow buffered stream to be reset?
PLOG(PL_FATAL, "NormSession::RequeueTxObject() error: can't requeue NORM_OBJECT_STREAM\n");
return false;
}
NormObjectId objectId = obj->GetId();
if (tx_table.Find(objectId) == obj)
{
if (tx_pending_mask.Set(objectId))
{
obj->TxReset(0, true);
TouchSender();
return true;
}
else
{
PLOG(PL_FATAL, "NormSession::RequeueTxObject() error: couldn't set object as pending\n");
return false;
}
}
else
{
PLOG(PL_FATAL, "NormSession::RequeueTxObject() error: couldn't find object\n");
return false;
}
} // end NormSession::RequeueTxObject()
void NormSession::DeleteTxObject(NormObject* obj, bool notify)
{
ASSERT(NULL != obj);
if (tx_table.Remove(obj))
{
Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, obj);
NormObjectId objectId = obj->GetId();
tx_pending_mask.Unset(objectId);
tx_repair_mask.Unset(objectId);
obj->Close();
obj->Release();
}
} // end NormSession::DeleteTxObject()
bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax,
unsigned long countMin,
unsigned long countMax)
{
bool result = true;
tx_cache_size_max = sizeMax;
tx_cache_count_min = (countMin < countMax) ? countMin : countMax;
if (tx_cache_count_min < 1) tx_cache_count_min = 1;
tx_cache_count_max = (countMax > countMin) ? countMax : countMin;
if (tx_cache_count_max < 1) tx_cache_count_max = 1;
tx_cache_count_min &= 0x00007fff; // limited to one-half of 16-bit NormObjectId space
tx_cache_count_max &= 0x00007fff;
if (IsSender())
{
// Trim/resize the tx_table and tx masks as needed
unsigned long count = tx_table.GetCount();
while ((count >= tx_cache_count_min) &&
((count > tx_cache_count_max) ||
(tx_table.GetSize() > tx_cache_size_max)))
{
// Remove oldest (hopefully non-pending ) object
NormObject* oldest = tx_table.Find(tx_table.RangeLo());
ASSERT(NULL != oldest);
DeleteTxObject(oldest, true);
count = tx_table.GetCount();
}
if (tx_cache_count_max < DEFAULT_TX_CACHE_MAX)
countMax = DEFAULT_TX_CACHE_MAX;
else
countMax = tx_cache_count_max;
if (countMax != tx_table.GetRangeMax())
{
tx_table.SetRangeMax((UINT16)countMax);
result = tx_pending_mask.Resize(countMax);
result &= tx_repair_mask.Resize(countMax);
if (!result)
{
countMax = tx_pending_mask.GetSize();
if (tx_repair_mask.GetSize() < countMax)
countMax = tx_repair_mask.GetSize();
if (tx_cache_count_max > countMax)
tx_cache_count_max = countMax;
if (tx_cache_count_min > tx_cache_count_max)
tx_cache_count_min = tx_cache_count_max;
}
}
}
return result;
} // end NormSession::SetTxCacheBounds()
NormBlock* NormSession::SenderGetFreeBlock(NormObjectId objectId,
NormBlockId blockId)
{
// First, try to get one from our block pool
NormBlock* b = block_pool.Get();
// Second, try to steal oldest non-pending block
if (!b)
{
NormObjectTable::Iterator iterator(tx_table);
NormObject* obj;
while ((obj = iterator.GetNextObject()))
{
if (obj->GetId() == objectId)
b = obj->StealNonPendingBlock(true, blockId);
else
b = obj->StealNonPendingBlock(false);
if (b)
{
b->EmptyToPool(segment_pool);
break;
}
}
}
// Finally, try to steal newer pending block
if (!b)
{
// reverse iteration to find newest object with resources
NormObjectTable::Iterator iterator(tx_table);
NormObject* obj;
while ((obj = iterator.GetPrevObject()))
{
if (obj->GetId() < objectId)
{
break;
}
else
{
if (obj->GetId() > objectId)
b = obj->StealNewestBlock(false);
else
b = obj->StealNewestBlock(true, blockId);
if (b)
{
b->EmptyToPool(segment_pool);
break;
}
}
}
}
return b;
} // end NormSession::SenderGetFreeBlock()
char* NormSession::SenderGetFreeSegment(NormObjectId objectId,
NormBlockId blockId)
{
while (segment_pool.IsEmpty())
{
NormBlock* b = SenderGetFreeBlock(objectId, blockId);
if (b)
block_pool.Put(b);
else
return NULL;
}
return segment_pool.Get();
} // end NormSession::SenderGetFreeSegment()
void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket,
ProtoSocket::Event theEvent)
{
if (ProtoSocket::RECV == theEvent)
{
NormMsg msg;
unsigned int msgLength = NormMsg::MAX_SIZE;
while (true)
{
if (theSocket.RecvFrom(msg.AccessBuffer(),
msgLength,
msg.AccessAddress()))
{
if (0 == msgLength) break; // no more data to read
if (msg.InitFromBuffer(msgLength))
{
// Since it arrived on the tx_socket, we know it was unicast
HandleReceiveMessage(msg, true);
msgLength = NormMsg::MAX_SIZE;
}
else
{
PLOG(PL_ERROR, "NormSession::TxSocketRecvHandler() warning: received bad message\n");
}
}
else
{
// Probably an ICMP "port unreachable" error
if (Address().IsUnicast())
Notify(NormController::SEND_ERROR, NULL, NULL);
break;
}
}
}
else if (ProtoSocket::SEND == theEvent)
{
// This is a little cheesy, but ...
theSocket.StopOutputNotification();
if (tx_timer.IsActive()) tx_timer.Deactivate();
if (OnTxTimeout(tx_timer))
{
if (!tx_timer.IsActive()) ActivateTimer(tx_timer);
}
}
} // end NormSession::TxSocketRecvHandler()
//#define RX_MEASURE_ONLY
#ifdef RX_MEASURE_ONLY
static bool rxMeasureInit = true;
struct timeval rxMeasureRefTime;
unsigned int rxMeasurePktCount = 0;
unsigned int rxMeasurePktTotal = 0;
unsigned int rxMeasureByteTotal = 0;
UINT16 rxMeasureSeqPrev = 0;
int rxMeasureGapMax = 0;
#endif // RX_MEASURE_ONLY
void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket,
ProtoSocket::Event theEvent)
{
if (ProtoSocket::RECV == theEvent)
{
unsigned int recvCount = 0;
NormMsg msg;
unsigned int msgLength = NormMsg::MAX_SIZE;
while (true)
{
ProtoAddress destAddr; // we get the pkt destAddr to determine unicast/multicast
if (theSocket.RecvFrom(msg.AccessBuffer(),
msgLength,
msg.AccessAddress(),
destAddr))
{
if (0 == msgLength) break;
if (msg.InitFromBuffer(msgLength))
{
#ifdef RX_MEASURE_ONLY
// Measure rx rate / loss stats only
struct timeval currentTime;
ProtoSystemTime(currentTime);
UINT16 seq = msg.GetSequence();
if (rxMeasureInit)
{
rxMeasureRefTime = currentTime;
rxMeasureSeqPrev = seq;
rxMeasurePktCount = rxMeasurePktTotal = 1;
rxMeasureByteTotal = msgLength;
rxMeasureInit = false;
return;
}
int seqDelta = (int)seq - (int)rxMeasureSeqPrev;
ASSERT(seqDelta > 0);
rxMeasurePktTotal += seqDelta; // total should have received.
rxMeasurePktCount++; // total actually received
rxMeasureByteTotal += msgLength;
if (seqDelta > rxMeasureGapMax)
rxMeasureGapMax = seqDelta;
int deltaSec = currentTime.tv_sec - rxMeasureRefTime.tv_sec;
if (deltaSec >= 10)
{
if (currentTime.tv_usec > rxMeasureRefTime.tv_usec)
deltaSec += 1.0e-06 * (double)(currentTime.tv_usec - rxMeasureRefTime.tv_usec);
else
deltaSec -= 1.0e-06 * (double)(rxMeasureRefTime.tv_usec - currentTime.tv_usec);
double rxRate = (8.0/1000.0) * (double)rxMeasureByteTotal / (double)deltaSec;
double rxLoss = 100.0 * (1.0 - (double)rxMeasurePktCount / (double)rxMeasurePktTotal);
rxMeasureRefTime = currentTime;
rxMeasureByteTotal = rxMeasurePktCount = rxMeasurePktTotal = rxMeasureGapMax = 0;
}
rxMeasureSeqPrev = seq;
return;
#endif // RX_MEASURE_ONLY
bool ecnStatus = false;
#ifdef SIMULATE
ecnStatus = theSocket.GetEcnStatus();
//if (ecnStatus) TRACE("NORM RECEIVED PACKET W/ ECN BIT SET!!!!!\n");
#endif // SIMULATE
bool wasUnicast;
if (destAddr.IsValid())
wasUnicast = destAddr.IsUnicast();
else
wasUnicast = false;
HandleReceiveMessage(msg, wasUnicast, ecnStatus);
msgLength = NormMsg::MAX_SIZE;
}
else
{
PLOG(PL_ERROR, "NormSession::RxSocketRecvHandler() warning: received bad message\n");
}
// If our system gets very busy reading sockets, we should occasionally
// execute any timeouts to keep protocol operation smooth (i.e., sending feedback)
// TBD - perhaps this should be time based
if (++recvCount >= 100)
{
break;
//session_mgr.DoSystemTimeout();
//recvCount = 0;
}
}
else
{
// Probably an ICMP "port unreachable" error
if (Address().IsUnicast())
Notify(NormController::SEND_ERROR, NULL, NULL);
break;
}
}
}
else if (ProtoSocket::SEND == theEvent)
{
// This is a little cheesy, but ...
theSocket.StopOutputNotification();
if (tx_timer.IsActive()) tx_timer.Deactivate();
if (OnTxTimeout(tx_timer))
{
if (!tx_timer.IsActive()) ActivateTimer(tx_timer);
}
} // end if/else (theEvent == RECV/SEND)
} // end NormSession::RxSocketRecvHandler()
#ifndef SIMULATE
void NormSession::OnPktCapture(ProtoChannel& theChannel,
ProtoChannel::Notification notifyType)
{
// We only care about NOTIFY_INPUT events (all we should get anyway)
if (ProtoChannel::NOTIFY_INPUT != notifyType) return;
while(1)
{
ProtoCap::Direction direction;
// Note: We offset the buffer by 2 bytes since Ethernet header is 14 bytes
// (i.e. not a multiple of 4 (sizeof(UINT32))
// This gives us a properly aligned buffer for 32-bit aligned IP packets
// (The 256*sizeof(UINT32) bytes are for potential "smfPkt" message header use)
const int BUFFER_MAX = 4096;
UINT32 alignedBuffer[BUFFER_MAX/sizeof(UINT32)];
UINT16* ethBuffer = ((UINT16*)alignedBuffer) + 1; // offset by 2-bytes so IP content is 32-bit aligned
UINT32* ipBuffer = alignedBuffer + 4; // offset by ETHER header size + 2 bytes
unsigned int numBytes = (sizeof(UINT32) * (BUFFER_MAX/sizeof(UINT32))) - 2;
ProtoCap& cap = static_cast<ProtoCap&>(theChannel);
if (!cap.Recv((char*)ethBuffer, numBytes, &direction))
{
PLOG(PL_ERROR, "NormSession::OnPktCapture() ProtoCap::Recv() error\n");
break;
}
if (numBytes == 0) break; // no more packets to receive
// Map ProtoPktETH instance into buffer and init for processing
ProtoPktETH ethPkt((UINT32*)((void*)ethBuffer), BUFFER_MAX - 2);
if (!ethPkt.InitFromBuffer(numBytes))
{
PLOG(PL_ERROR, "NormSession::OnPktCapture() error: bad Ether frame\n");
continue;
}
// Only process IP packets (skip others)
UINT16 ethType = ethPkt.GetType();
if ((ethType != 0x0800) && (ethType != 0x86dd))
continue; // go read next packet
// Map ProtoPktIP instance into buffer and init for processing.
ProtoPktIP ipPkt(ipBuffer, BUFFER_MAX - 16);
if (!ipPkt.InitFromBuffer(ethPkt.GetPayloadLength()))
{
PLOG(PL_ERROR, "NormSession::OnPktCapture() error: bad IP packet\n");
continue;
}
// Does this packet match any of our valid destination addrs?
ProtoAddress dstIp;
ProtoAddress srcIp;
ProtoSocket::EcnStatus ecnStatus = ProtoSocket::ECN_NONE;
switch (ipPkt.GetVersion())
{
case 4:
{
ProtoPktIPv4 ip4Pkt(ipPkt);
ip4Pkt.GetDstAddr(dstIp);
ip4Pkt.GetSrcAddr(srcIp);
ecnStatus = (ProtoSocket::EcnStatus)(ip4Pkt.GetTOS() & ProtoSocket::ECN_CE);
break;
}
case 6:
{
ProtoPktIPv6 ip6Pkt(ipPkt);
ip6Pkt.GetDstAddr(dstIp);
ip6Pkt.GetSrcAddr(srcIp);
ecnStatus = (ProtoSocket::EcnStatus)(ip6Pkt.GetTrafficClass() & ProtoSocket::ECN_CE);
break;
}
default:
PLOG(PL_ERROR, "NormSession::OnPktCapture() error: recvd IP packet w/ bad version number\n");
continue; // go read next packet
}
if (!dst_addr_list.Contains(dstIp))
continue; // not a matching dst addr, go read next packet
// Is this a UDP packet for our session dst port?
int dstPort = -1;
ProtoPktUDP udpPkt;
if (udpPkt.InitFromPacket(ipPkt))
dstPort = udpPkt.GetDstPort();
//if (dstPort != address.GetPort())
if (dstPort != rx_socket.GetPort())
continue; // not a UDP packet for our session, go read next packet
// If our rx_socket is "connected", make sure source addr/port matches
srcIp.SetPort(udpPkt.GetSrcPort());
// if socket is connected, validate that the packet's from the specified source addr
if (rx_connect_addr.IsValid())
{
if (0 != rx_connect_addr.GetPort())
{
// check host addr component only for match
if (!rx_connect_addr.HostIsEqual(srcIp)) continue;
}
else
{
// check for addr _and_ port match
if (!rx_connect_addr.IsEqual(srcIp)) continue;
}
}
// if we are using SSM multicast make sure it's the right source addr
if (ssm_source_addr.IsValid() && !ssm_source_addr.HostIsEqual(srcIp)) continue;
// IMPORTANT NOTE: We ignore the checksum for OUTBOUND packets since these
// are often computed by the Ethernet hardware these days
if ((ProtoCap::INBOUND == direction) && !udpPkt.ChecksumIsValid(ipPkt))
{
PLOG(PL_WARN, "NormSession::OnPktCapture() error: recvd UDP packet w/ bad checksum: %04x (computed: %04x)\n",
(UINT16)udpPkt.GetChecksum(), udpPkt.ComputeChecksum(ipPkt));
continue; // go read next packet
}
// TBD - we can avoid this copy
NormMsg msg;
if (msg.CopyFromBuffer((const char*)udpPkt.GetPayload(), udpPkt.GetPayloadLength()))
{
msg.AccessAddress() = srcIp;
HandleReceiveMessage(msg, dstIp.IsUnicast(), (ProtoSocket::ECN_CE == ecnStatus));
}
else
{
PLOG(PL_WARN, "NormSession::OnPktCapture() error: recvd bad NORM packet?!\n");
}
} // end while(1)
} // end NormSession::OnPktCapture()
#endif // !SIMULATE
// TBD - move this to its own cpp file???
void NormTrace(const struct timeval& currentTime,
NormNodeId localId,
const NormMsg& msg,
bool sent,
UINT8 fecM,
UINT16 instId)
{
static const char* MSG_NAME[] =
{
"INVALID",
"INFO",
"DATA",
"CMD",
"NACK",
"ACK",
"REPORT"
};
static const char* CMD_NAME[] =
{
"CMD(INVALID)",
"CMD(FLUSH)",
"CMD(EOT)",
"CMD(SQUELCH)",
"CMD(CC)",
"CMD(REPAIR_ADV)",
"CMD(ACK_REQ)",
"CMD(APP)"
};
static const char* REQ_NAME[] =
{
"INVALID",
"WATERMARK",
"RTT",
"APP"
};
NormMsg::Type msgType = msg.GetType();
UINT16 length = msg.GetLength();
const char* status = sent ? "dst" : "src";
const ProtoAddress& addr = sent ? msg.GetDestination() : msg.GetSource();
UINT16 seq = msg.GetSequence();
#ifdef _WIN32_WCE
struct tm timeStruct;
timeStruct.tm_hour = currentTime.tv_sec / 3600;
unsigned long hourSecs = 3600 * timeStruct.tm_hour;
timeStruct.tm_min = (currentTime.tv_sec - (hourSecs)) / 60;
timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60*timeStruct.tm_min);
timeStruct.tm_hour = timeStruct.tm_hour % 24;
struct tm* ct = &timeStruct;
#else
time_t secs = (time_t)currentTime.tv_sec;
struct tm* ct = gmtime(&secs);
#endif // if/else _WIN32_WCE
PLOG(PL_ALWAYS, "trace>%02d:%02d:%02d.%06lu ",
(int)ct->tm_hour, (int)ct->tm_min, (int)ct->tm_sec, (unsigned int)currentTime.tv_usec);
PLOG(PL_ALWAYS, "node>%lu %s>%s/%hu ", (unsigned long)localId, status, addr.GetHostString(), addr.GetPort());
bool clrFlag = false;
switch (msgType)
{
case NormMsg::INFO:
{
const NormInfoMsg& info = (const NormInfoMsg&)msg;
PLOG(PL_ALWAYS, "inst>%hu seq>%hu INFO obj>%hu ",
info.GetInstanceId(), seq, (UINT16)info.GetObjectId());
break;
}
case NormMsg::DATA:
{
const NormDataMsg& data = (const NormDataMsg&)msg;
PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%lu seg>%hu ",
data.GetInstanceId(),
seq,
//data.IsData() ? "DATA" : "PRTY",
(UINT16)data.GetObjectId(),
(unsigned long)data.GetFecBlockId(fecM).GetValue(),
(UINT16)data.GetFecSymbolId(fecM));
if (data.IsStream())
{
UINT32 offset = NormDataMsg::ReadStreamPayloadOffset(data.GetPayload());
PLOG(PL_ALWAYS, "offset>%lu ", (unsigned long)offset);
/*if (data.GetFecSymbolId(fecM) < 32)
{
//if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_MSG_START))
UINT16 msgStartOffset = NormDataMsg::ReadStreamPayloadMsgStart(data.GetPayload());
if (0 != msgStartOffset)
{
PLOG(PL_ALWAYS, "start word>%hu ", msgStartOffset - 1);
}
//if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_STREAM_END))
if (0 == NormDataMsg::ReadStreamPayloadLength(data.GetPayload()))
PLOG(PL_ALWAYS, "(stream end) ");
}
*/
}
break;
}
case NormMsg::CMD:
{
const NormCmdMsg& cmd = static_cast<const NormCmdMsg&>(msg);
NormCmdMsg::Flavor flavor = cmd.GetFlavor();
PLOG(PL_ALWAYS, "inst>%hu seq>%hu %s ", cmd.GetInstanceId(), seq, CMD_NAME[flavor]);
switch (flavor)
{
case NormCmdMsg::ACK_REQ:
{
int index = ((const NormCmdAckReqMsg&)msg).GetAckType();
index = MIN(index, 3);
PLOG(PL_ALWAYS, "(%s) ", REQ_NAME[index]);
break;
}
case NormCmdMsg::SQUELCH:
{
const NormCmdSquelchMsg& squelch =
static_cast<const NormCmdSquelchMsg&>(msg);
PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ",
(UINT16)squelch.GetObjectId(),
(unsigned long)squelch.GetFecBlockId(fecM).GetValue(),
(UINT16)squelch.GetFecSymbolId(fecM));
break;
}
case NormCmdMsg::FLUSH:
{
const NormCmdFlushMsg& flush =
static_cast<const NormCmdFlushMsg&>(msg);
PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ",
(UINT16)flush.GetObjectId(),
(unsigned long)flush.GetFecBlockId(fecM).GetValue(),
(UINT16)flush.GetFecSymbolId(fecM));
if (0 != flush.GetAckingNodeCount())
PLOG(PL_ALWAYS, "(WATERMARK) "); // ACK requested
break;
}
case NormCmdMsg::CC:
{
const NormCmdCCMsg& cc = static_cast<const NormCmdCCMsg&>(msg);
PLOG(PL_ALWAYS, " seq>%u ", cc.GetCCSequence());
NormHeaderExtension ext;
while (cc.GetNextExtension(ext))
{
if (NormHeaderExtension::CC_RATE == ext.GetType())
{
UINT16 sendRate = ((NormCCRateExtension&)ext).GetSendRate();
PLOG(PL_ALWAYS, " rate>%f ", 8.0e-03 * NormUnquantizeRate(sendRate));
break;
}
}
break;
}
default:
break;
}
break;
}
case NormMsg::ACK:
case NormMsg::NACK:
{
PLOG(PL_ALWAYS, "inst>%hu ", instId);
// look for NormCCFeedback extension
NormHeaderExtension ext;
while (msg.GetNextExtension(ext))
{
if (NormHeaderExtension::CC_FEEDBACK == ext.GetType())
{
clrFlag = ((NormCCFeedbackExtension&)ext).CCFlagIsSet(NormCC::CLR);
break;
}
}
if (NormMsg::ACK == msgType)
{
const NormAckMsg& ack = static_cast<const NormAckMsg&>(msg);
if (NormAck::FLUSH == ack.GetAckType())
{
const NormAckFlushMsg& flushAck = static_cast<const NormAckFlushMsg&>(ack);
PLOG(PL_ALWAYS, "ACK(FLUSH) obj>%hu blk>%lu seg>%hu ",
(UINT16)flushAck.GetObjectId(),
(unsigned long)flushAck.GetFecBlockId(fecM).GetValue(),
(UINT16)flushAck.GetFecSymbolId(fecM));
}
else if (NormAck::CC == ack.GetAckType())
{
PLOG(PL_ALWAYS, "ACK(CC) ");
}
else
{
PLOG(PL_ALWAYS, "ACK(ZZZ) ");
}
}
else
{
PLOG(PL_ALWAYS, "NACK ");
// TBD - provide deeper NACK inspection?
}
break;
}
default:
PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]);
break;
} // end switch (msgType)
PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : "");
} // end NormTrace();
void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnStatus)
{
// Ignore messages from ourself unless "loopback" is enabled
if ((msg.GetSourceId() == LocalNodeId()) && !loopback)
return;
// Drop some rx messages for testing
if ((rx_loss_rate > 0) && (UniformRand(100.0) < rx_loss_rate))
return;
struct timeval currentTime;
::ProtoSystemTime(currentTime);
if (trace)
{
// Initially assume it's a message we generated (or similarly configured sender)
UINT8 fecM = fec_m;
UINT16 instId = instance_id;
NormNodeId senderId;
switch (msg.GetType())
{
case NormMsg::ACK:
senderId = static_cast<NormAckMsg&>(msg).GetSenderId();
break;
case NormMsg::NACK:
senderId = static_cast<NormAckMsg&>(msg).GetSenderId();
break;
default:
senderId = msg.GetSourceId();
break;
}
if (IsReceiver() && (senderId != LocalNodeId()))
{
// Use our receiver state to look up sender if possible
NormSenderNode* sender;
if (IsServerListener())
sender = client_tree.FindNodeByAddress(msg.GetSource());
else
sender = static_cast<NormSenderNode*>(sender_tree.FindNodeById(senderId));
if (NULL != sender)
{
fecM = sender->GetFecFieldSize();
instId = sender->GetInstanceId();
}
else
{
fecM = 16; // reasonable assumption
instId = 0;
}
}
NormTrace(currentTime, LocalNodeId(), msg, false, fecM, instId); // TBD don't assume m == 16 (i.e. for fec_id == 2)
} // end if (trace)
NormMsg::Type msgType = msg.GetType();
if (IsServerListener())
{
// Only pay attention to packets with FLAG_SYN set
// (NORM_CMD(CC), NORM_INFO, or NORM_DATA messages
// (Note FLAG_SYN is not part of RFC 5740)
bool syn = false;
bool senderMsg = true;
switch (msgType)
{
case NormMsg::CMD:
{
NormCmdMsg& cmd = static_cast<NormCmdMsg&>(msg);
if ((NormCmdMsg::CC == cmd.GetFlavor()) &&
static_cast<NormCmdCCMsg&>(cmd).SynIsSet())
{
syn = true;
}
break;
}
case NormMsg::INFO:
case NormMsg::DATA:
if (static_cast<NormObjectMsg&>(msg).FlagIsSet(NormObjectMsg::FLAG_SYN))
{
syn = true;
}
break;
default:
senderMsg = false;
// Receiver messages are ignored by unicast server-listener,
// but multicast server needs to process ACKS/NACKS from client receivers
if (!Address().IsMulticast()) return;
}
if (senderMsg)
{
if (!syn)
{
// Send "reject" command to source
char buffer[2];
buffer[0] = NORM_SOCKET_VERSION;
buffer[1] = NORM_SOCKET_CMD_REJECT;
SenderSendAppCmd(buffer, 2, msg.GetSource());
return;
}
}
}
// Add newly detected nodes to acking list _before_ processing message
if (IsSender() && (TRACK_NONE != acking_auto_populate))
{
bool addNode = false;
switch (acking_auto_populate)
{
case TRACK_ALL:
addNode = true;
break;
case TRACK_RECEIVERS:
addNode = (NormMsg::NACK == msgType) || (NormMsg::ACK == msgType);
break;
case TRACK_SENDERS:
addNode = (NormMsg::NACK != msgType) && (NormMsg::ACK != msgType);
break;
default:
break;
}
if (addNode)
{
NormNodeId sourceId = msg.GetSourceId();
if (NULL == acking_node_tree.FindNodeById(sourceId))
{
if (!SenderAddAckingNode(msg.GetSourceId(), &msg.GetSource()))
PLOG(PL_ERROR, "NormSession::HandleReceiveMessage() error: unable to add acking node!\n");
NormAckingNode* acker = (NormAckingNode*)acking_node_tree.FindNodeById(sourceId);
Notify(NormController::ACKING_NODE_NEW, acker, NULL);
}
}
}
switch (msg.GetType())
{
case NormMsg::INFO:
//DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::INFO)\n");
if (IsReceiver()) ReceiverHandleObjectMessage(currentTime, (NormObjectMsg&)msg, ecnStatus);
break;
case NormMsg::DATA:
//DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::DATA) ...\n");
if (IsReceiver()) ReceiverHandleObjectMessage(currentTime, (NormObjectMsg&)msg, ecnStatus);
break;
case NormMsg::CMD:
//DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::CMD) ...\n");
if (IsReceiver()) ReceiverHandleCommand(currentTime, (NormCmdMsg&)msg, ecnStatus);
break;
case NormMsg::NACK:
if (IsSender() && (((NormNackMsg&)msg).GetSenderId() == LocalNodeId()))
{
SenderHandleNackMessage(currentTime, (NormNackMsg&)msg);
if (wasUnicast && (backoff_factor > 0.5) && Address().IsMulticast())
{
// for suppression of unicast nack feedback
advertise_repairs = true;
QueueMessage(NULL); // to prompt transmit timeout
}
}
if (IsReceiver()) ReceiverHandleNackMessage((NormNackMsg&)msg);
break;
case NormMsg::ACK:
if (IsSender() && (((NormAckMsg&)msg).GetSenderId() == LocalNodeId()))
SenderHandleAckMessage(currentTime, (NormAckMsg&)msg, wasUnicast);
if (IsReceiver()) ReceiverHandleAckMessage((NormAckMsg&)msg);
break;
case NormMsg::REPORT:
case NormMsg::INVALID:
PLOG(PL_ERROR, "NormSession::HandleReceiveMessage(NormMsg::INVALID)\n");
break;
}
} // end NormSession::HandleReceiveMessage()
void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTime,
const NormObjectMsg& msg,
bool ecnStatus)
{
// Do common updates for senders we already know.
NormNodeId sourceId = msg.GetSourceId();
NormSenderNode* theSender;
if (IsServerListener())
theSender = client_tree.FindNodeByAddress(msg.GetSource());
else
theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId);
if (theSender)
{
if (msg.GetInstanceId() != theSender->GetInstanceId())
{
PLOG(PL_INFO, "NormSession::ReceiverHandleObjectMessage() node>%lu sender>%lu instanceId change - resyncing.\n",
(unsigned long)LocalNodeId(), (unsigned long)theSender->GetId());
theSender->Close();
Notify(NormController::REMOTE_SENDER_RESET, theSender, NULL);
if (!theSender->Open(msg.GetInstanceId()))
{
PLOG(PL_ERROR, "NormSession::ReceiverHandleObjectMessage() node>%lu error re-opening NormSenderNode\n",
(unsigned long)LocalNodeId());
// (TBD) notify application of error
return;
}
}
}
else
{
if (NULL != preset_sender)
{
theSender = preset_sender;
preset_sender = NULL;
theSender->SetId(msg.GetSourceId());
theSender->SetInstanceId(msg.GetInstanceId());
theSender->SetAddress(msg.GetSource());
if (IsServerListener())
client_tree.InsertNode(*theSender);
else
sender_tree.AttachNode(theSender);
PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n",
(unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId());
Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL);
}
else if (NULL != (theSender = new NormSenderNode(*this, msg.GetSourceId())))
{
theSender->SetAddress(msg.GetSource());
if (theSender->Open(msg.GetInstanceId()))
{
if (IsServerListener())
client_tree.InsertNode(*theSender);
else
sender_tree.AttachNode(theSender);
PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n",
(unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId());
}
else
{
PLOG(PL_FATAL, "NormSession::ReceiverHandleObjectMessage() node>%lu error opening NormSenderNode\n",
(unsigned long)LocalNodeId());
// (TBD) notify application of error
return;
}
Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL);
}
else
{
PLOG(PL_ERROR, "NormSession::ReceiverHandleObjectMessage() new NormSenderNode error: %s\n",
GetErrorString());
// (TBD) notify application of error
return;
}
}
theSender->Activate(true);
if (!theSender->GetAddress().IsEqual(msg.GetSource()))
{
// sender source address has changed
theSender->SetAddress(msg.GetSource());
Notify(NormController::REMOTE_SENDER_ADDRESS, theSender, NULL);
}
theSender->UpdateRecvRate(currentTime, msg.GetLength());
//if (ecnStatus) TRACE("UPDATING LOSS EST w/ ECN pkt\n");
theSender->UpdateLossEstimate(currentTime, msg.GetSequence(), ecnStatus);
theSender->IncrementRecvTotal(msg.GetLength()); // for statistics only (TBD) #ifdef NORM_DEBUG
theSender->HandleObjectMessage(msg);
theSender->CheckCCFeedback(); // this cues immediate CLR cc feedback if loss was detected
// and cc feedback was not provided in response otherwise
} // end NormSession::ReceiverHandleObjectMessage()
void NormSession::ReceiverHandleCommand(const struct timeval& currentTime,
const NormCmdMsg& cmd,
bool ecnStatus)
{
// Do common updates for senders we already know.
NormNodeId sourceId = cmd.GetSourceId();
NormSenderNode* theSender;
if (IsServerListener())
theSender = client_tree.FindNodeByAddress(cmd.GetSource());
else
theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId);
if (NULL != theSender)
{
if (cmd.GetInstanceId() != theSender->GetInstanceId())
{
PLOG(PL_INFO, "NormSession::ReceiverHandleCommand() node>%lu sender>%lu instanceId change - resyncing.\n",
(unsigned long)LocalNodeId(), theSender->GetId());
theSender->Close();
Notify(NormController::REMOTE_SENDER_RESET, theSender, NULL);
if (!theSender->Open(cmd.GetInstanceId()))
{
PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() node>%lu error re-opening NormSenderNode\n",
(unsigned long)LocalNodeId());
// (TBD) notify application of error
return;
}
}
}
else
{
//DMSG(0, "NormSession::ReceiverHandleCommand() node>%lu recvd command from unknown sender ...\n",
// (unsigned long)LocalNodeId());
if (NULL != preset_sender)
{
theSender = preset_sender;
preset_sender = NULL;
theSender->SetId(cmd.GetSourceId());
theSender->SetInstanceId(cmd.GetInstanceId());
theSender->SetAddress(cmd.GetSource());
if (IsServerListener())
client_tree.InsertNode(*theSender);
else
sender_tree.AttachNode(theSender);
PLOG(PL_DEBUG, "NormSession::ReceiverHandleCommand() node>%lu new remote sender:%lu ...\n",
(unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId());
Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL);
}
else if ((theSender = new NormSenderNode(*this, cmd.GetSourceId())))
{
theSender->SetAddress(cmd.GetSource());
if (theSender->Open(cmd.GetInstanceId()))
{
if (IsServerListener())
client_tree.InsertNode(*theSender);
else
sender_tree.AttachNode(theSender);
PLOG(PL_DEBUG, "NormSession::ReceiverHandleCommand() node>%lu new remote sender:%lu ...\n",
(unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId());
}
else
{
PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() node>%lu error opening NormSenderNode\n");
// (TBD) notify application of error
return;
}
Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL);
}
else
{
PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() new NormSenderNode node>%lu error: %s\n",
(unsigned long)LocalNodeId(), GetErrorString());
// (TBD) notify application of error
return;
}
}
// We should "re-activate" senders on NORM_CMD(FLUSH)
if (NormCmdMsg::FLUSH == cmd.GetFlavor())
theSender->Activate(true);
else
theSender->Activate(false);
if (!theSender->GetAddress().IsEqual(cmd.GetSource()))
{
// sender source address has changed
theSender->SetAddress(cmd.GetSource());
Notify(NormController::REMOTE_SENDER_ADDRESS, theSender, NULL);
}
theSender->UpdateRecvRate(currentTime, cmd.GetLength());
theSender->UpdateLossEstimate(currentTime, cmd.GetSequence(), ecnStatus);
theSender->IncrementRecvTotal(cmd.GetLength()); // for statistics only (TBD) #ifdef NORM_DEBUG
theSender->HandleCommand(currentTime, cmd);
theSender->CheckCCFeedback(); // this cues immediate CLR cc feedback if loss was detected
// and cc feedback was not provided in response otherwise
} // end NormSession::ReceiverHandleCommand()
bool NormSession::InsertRemoteSender(NormSenderNode& sender)
{
// Build a NORM_CMD(CC) message with information from
// a "sender" being inserted from another NormSession
// (supports NormSocket server operations)
if (!IsReceiver()) return false;
NormCmdCCMsg cmd;
cmd.Init();
cmd.SetSequence(sender.GetCurrentSequence());
cmd.SetSourceId(sender.GetId());
cmd.SetDestination(sender.GetAddress());
cmd.SetInstanceId(sender.GetInstanceId());
cmd.SetGrtt(sender.GetGrttQuantized());
cmd.SetBackoffFactor(sender.GetBackoffFactor());
cmd.SetGroupSize(sender.GetGroupSizeQuantized());
cmd.SetCCSequence(sender.GetCCSequence());
// Adjust send time for any current hold time
// since it will be "rehandled"
struct timeval adjustedSendTime;
struct timeval currentTime;
::ProtoSystemTime(currentTime);
sender.CalculateGrttResponse(currentTime, adjustedSendTime);
cmd.SetSendTime(adjustedSendTime);
// Insert NORM-CC header extension, if applicable
// (Note we set the extension "rate" _after_ AdjustRate() done below)
NormCCRateExtension ext;
cmd.AttachExtension(ext);
ext.SetSendRate(NormQuantizeRate(sender.GetSendRate()));
HandleReceiveMessage(cmd, false);
return true; // TBD - confirm the node was added
} // end NormSession::InsertRemoteSender()
double NormSession::CalculateRtt(const struct timeval& currentTime,
const struct timeval& grttResponse)
{
if (grttResponse.tv_sec || grttResponse.tv_usec)
{
double rcvrRtt;
// Calculate rtt estimate for this receiver and process the response
if (currentTime.tv_usec < grttResponse.tv_usec)
{
rcvrRtt =
(double)(currentTime.tv_sec - grttResponse.tv_sec - 1);
rcvrRtt +=
((double)(1000000 - (grttResponse.tv_usec - currentTime.tv_usec))) / 1.0e06;
}
else
{
rcvrRtt =
(double)(currentTime.tv_sec - grttResponse.tv_sec);
rcvrRtt +=
((double)(currentTime.tv_usec - grttResponse.tv_usec)) / 1.0e06;
}
// Lower limit on RTT (because of coarse timer resolution on some systems,
// this can sometimes actually end up a negative value!)
// (TBD) this should be system clock granularity?
return (rcvrRtt < 1.0e-06) ? 1.0e-06 : rcvrRtt;
}
else
{
return -1.0;
}
} // end NormSession::CalculateRtt()
void NormSession::SenderUpdateGrttEstimate(double receiverRtt)
{
grtt_response = true;
if ((receiverRtt > grtt_measured) || !address.IsMulticast())
{
// Immediately incorporate bigger RTT's
grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY;
grtt_measured = 0.25 * grtt_measured + 0.75 * receiverRtt;
//grtt_measured = 0.9 * grtt_measured + 0.1 * receiverRtt;
if (grtt_measured > grtt_max) grtt_measured = grtt_max;
UINT8 grttQuantizedOld = grtt_quantized;
double pktInterval = ((double)(44+segment_size))/tx_rate;
if (grtt_measured < pktInterval)
grtt_quantized = NormQuantizeRtt(pktInterval);
else
grtt_quantized = NormQuantizeRtt(grtt_measured);
// Calculate grtt_advertised since quantization rounds upward
grtt_advertised = NormUnquantizeRtt(grtt_quantized);
if (grtt_advertised > grtt_max)
{
grtt_quantized = NormQuantizeRtt(grtt_max);
grtt_advertised = NormUnquantizeRtt(grtt_quantized);
}
grtt_current_peak = grtt_measured;
if (grttQuantizedOld != grtt_quantized)
{
if (notify_on_grtt_update)
{
notify_on_grtt_update = false;
Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL);
}
Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL);
PLOG(PL_DEBUG, "NormSession::SenderUpdateGrttEstimate() node>%lu increased to new grtt>%lf sec\n",
(unsigned long)LocalNodeId(), grtt_advertised);
}
}
else if (receiverRtt > grtt_current_peak)
{
grtt_current_peak = receiverRtt;
}
} // end NormSession::SenderUpdateGrttEstimate()
double NormSession::CalculateRate(double size, double rtt, double loss)
{
// size
// rate = -------------------------------------------------------------
// rtt * (sqrt(2*loss/3) + 12*loss*(1 + 32*loss*loss)*sqrt(3*loss/8))
//
// notes: "b" = 1 and "t_RTO" = 4*rtt where "b" is number of TCP pkts/ACK
double denom = rtt * (sqrt((2.0/3.0)*loss) +
(12.0 * sqrt((3.0/8.0)*loss) * loss *
(1.0 + 32.0*loss*loss)));
return (size / denom);
} // end NormSession::CalculateRate()
void NormSession::SenderHandleCCFeedback(struct timeval currentTime,
NormNodeId nodeId,
UINT8 ccFlags,
double ccRtt,
double ccLoss,
double ccRate,
UINT16 ccSequence)
{
PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() cc feedback recvd at time %lu.%lf ccRate:%9.3lf ccRtt:%lf ccLoss:%lf ccFlags:%02x\n",
(unsigned long)currentTime.tv_sec, ((double)currentTime.tv_usec)*1.0e-06,
ccRate*8.0/1000.0, ccRtt, ccLoss, ccFlags);
// Keep track of current suppressing feedback
// (non-CLR, lowest rate, unconfirmed RTT)
if (0 == (ccFlags & NormCC::CLR))
{
if (suppress_rate < 0.0)
{
suppress_rate = ccRate;
suppress_rtt = ccRtt;
suppress_nonconfirmed = (0 == (ccFlags & NormCC::RTT));
}
else
{
if (ccRate < suppress_rate) suppress_rate = ccRate;
if (ccRtt > suppress_rtt) suppress_rtt = ccRtt;
if (0 == (ccFlags & NormCC::RTT)) suppress_nonconfirmed = true;
}
}
if (!cc_enable) return;
// Adjust ccRtt if we already have state on this nodeId
NormCCNode* node = (NormCCNode*)cc_node_list.FindNodeById(nodeId);
if (node) ccRtt = node->UpdateRtt(ccRtt);
bool ccSlowStart = (0 != (ccFlags & NormCC::START));
if (!ccSlowStart)
{
double calcRate = CalculateRate(nominal_packet_size, ccRtt, ccLoss);
#ifdef LIMIT_CC_RATE
// Experimental modification to NORM-CC where congestion control rate is limited
// to MIN(2.0*measured recv rate, calculated rate). This might prevent large rate
// overshoot in conditions where the loss measurement (perhaps initial loss) is
// very low due to big network packet buffers, etc
// Note that when the NORM_CC_FLAG_LIMIT is set, this indicates the receiver
// has set the rate field to 2.0 * measured recv rate instead of calculated rate.
if (0 != (ccFlags & NormCC::LIMIT))
{
// receiver set a limited rate instead of calculated rate
// so let's confirm which is the lower rate
if (calcRate < ccRate)
ccRate = calcRate;
}
else
#endif // LIMIT_CC_RATE
{
ccRate = calcRate;
}
}
PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() node>%lu rate>%lf (rtt>%lf loss>%lf slow_start>%d limit>%d)\n",
(unsigned long)nodeId, ccRate * 8.0 / 1000.0, ccRtt, ccLoss, (0 != (ccFlags & NormCC::START)),
(0 != (ccFlags & NormCC::LIMIT)));
// Keep the active CLR (if there is one) at the head of the list
NormNodeListIterator iterator(cc_node_list);
NormCCNode* next = (NormCCNode*)iterator.GetNextNode();
// 1) Does this response replace the active CLR?
if (next && next->IsActive())
{
// First, make sure this is _new_ non-duplicative
// feedback for the given "nodeId"
if (next->GetId() == nodeId)
{
INT16 ccDelta = ccSequence - next->GetCCSequence();
if (ccDelta <= 0) return;
}
if ((nodeId == next->GetId()) ||
(ccRate < next->GetRate()) ||
((ccRate < (next->GetRate() * 1.1)) &&
(ccRtt > next->GetRtt()))) // use Rtt as tie-breaker if close
{
NormNodeId savedId = next->GetId();
bool savedRttStatus = next->HasRtt();
double savedRtt = next->GetRtt();
double savedLoss = next->GetLoss();
double savedRate = next->GetRate();
UINT16 savedSequence = next->GetCCSequence();
struct timeval savedTime = next->GetFeedbackTime();
next->SetId(nodeId);
next->SetClrStatus(true);
next->SetRttStatus(0 != (ccFlags & NormCC::RTT));
next->SetLoss(ccLoss);
next->SetRate(ccRate);
next->SetCCSequence(ccSequence);
next->SetActive(true);
next->SetFeedbackTime(currentTime);
cc_slow_start = ccSlowStart; // use CLR status for our slow_start state
if (savedId == nodeId)
{
// This was feedback from the current CLR
AdjustRate(true);
return;
}
else
{
next->SetRtt(ccRtt);
AdjustRate(true);
}
ccFlags = 0;
nodeId = savedId;
if (savedRttStatus)
ccFlags = NormCC::RTT;
ccRtt = savedRtt;
ccLoss = savedLoss;
ccRate = savedRate,
ccSequence = savedSequence;
currentTime = savedTime;
}
}
else
{
// There was no active CLR
if (!next)
{
if ((next = new NormCCNode(*this, nodeId)))
{
cc_node_list.Append(next);
}
else
{
PLOG(PL_FATAL, "NormSession::SenderHandleCCFeedback() memory allocation error: %s\n",
GetErrorString());
return;
}
}
next->SetId(nodeId);
next->SetClrStatus(true);
//next->SetPlrStatus(false);
next->SetRttStatus(0 != (ccFlags & NormCC::RTT));
next->SetRtt(ccRtt);
next->SetLoss(ccLoss);
next->SetRate(ccRate);
next->SetCCSequence(ccSequence);
next->SetActive(true);
next->SetFeedbackTime(currentTime);
AdjustRate(true);
return;
}
// 2) Go through cc_node_list and find lowest priority candidate
NormCCNode* candidate = NULL;
if (cc_node_list.GetCount() < 5)
{
if ((candidate = new NormCCNode(*this, nodeId)))
{
cc_node_list.Append(candidate);
}
else
{
PLOG(PL_FATAL, "NormSession::SenderHandleCCFeedback() memory allocation error: %s\n",
GetErrorString());
}
}
else
{
while ((next = (NormCCNode*)iterator.GetNextNode()))
{
if (next->GetId() == nodeId)
{
candidate = next;
break;
}
else if (candidate)
{
if (candidate->IsActive() && !next->IsActive())
{
candidate = next;
continue;
}
if (!next->HasRtt() && candidate->HasRtt())
continue;
else if (!candidate->HasRtt() && next->HasRtt())
candidate = next;
else if (candidate->GetRate() < next->GetRate())
candidate = next;
}
else
{
candidate = next;
continue;
}
}
}
// 3) Replace candidate if this response is higher precedence
if (candidate)
{
bool haveRtt = (0 != (ccFlags & NormCC::RTT));
bool replace;
if (candidate->GetId() == nodeId)
replace = true;
else if (!candidate->IsActive())
replace = true;
else if (!haveRtt && candidate->HasRtt())
replace = true;
else if (haveRtt && !candidate->HasRtt())
replace = false;
else if (ccRate < candidate->GetRate())
replace = true;
else
replace = false;
if (replace)
{
candidate->SetId(nodeId);
candidate->SetClrStatus(false);
//candidate->SetPlrStatus(true); // do this only
candidate->SetRttStatus(0 != (ccFlags & NormCC::RTT));
candidate->SetRtt(ccRtt);
candidate->SetLoss(ccLoss);
candidate->SetRate(ccRate);
candidate->SetCCSequence(ccSequence);
candidate->SetActive(true);
}
}
} // end NormSession::SenderHandleCCFeedback()
void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, const NormAckMsg& ack, bool wasUnicast)
{
// Update GRTT estimate
struct timeval grttResponse;
ack.GetGrttResponse(grttResponse);
double receiverRtt = CalculateRtt(currentTime, grttResponse);
PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu sender received ACK from node>%lu rtt>%lf\n",
(unsigned long)LocalNodeId(), (unsigned long)ack.GetSourceId(), receiverRtt);
if (receiverRtt >= 0.0) SenderUpdateGrttEstimate(receiverRtt);
// Look for NORM-CC Feedback header extension
NormCCFeedbackExtension ext;
while (ack.GetNextExtension(ext))
{
if (NormHeaderExtension::CC_FEEDBACK == ext.GetType())
{
SenderHandleCCFeedback(currentTime,
ack.GetSourceId(),
ext.GetCCFlags(),
receiverRtt >= 0.0 ?
receiverRtt : NormUnquantizeRtt(ext.GetCCRtt()),
NormUnquantizeLoss32(ext.GetCCLoss32()),
NormUnquantizeRate(ext.GetCCRate()),
ext.GetCCSequence());
if (wasUnicast && probe_proactive && Address().IsMulticast())
{
// if it's the CLR, it doesn't suppress anyone, don't advertise
if (!ext.CCFlagIsSet(NormCC::CLR))
{
// for suppression of unicast cc feedback
advertise_repairs = true;
QueueMessage(NULL);
}
}
break;
}
}
switch (ack.GetAckType())
{
case NormAck::CC:
// Everything is in the ACK header or extension for this one
break;
case NormAck::FLUSH:
if (watermark_pending)
{
NormAckingNode* acker =
static_cast<NormAckingNode*>(acking_node_tree.FindNodeById(ack.GetSourceId()));
if (NULL != acker)
{
if (!acker->AckReceived())
{
const NormAckFlushMsg& flushAck = static_cast<const NormAckFlushMsg&>(ack);
if (flushAck.GetFecId() != fec_id)
{
PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received watermark ACK with wrong fec_id?!\n");
}
else if ((watermark_object_id == flushAck.GetObjectId()) &&
(watermark_block_id == flushAck.GetFecBlockId(fec_m)) &&
(watermark_segment_id == flushAck.GetFecSymbolId(fec_m)))
{
// Cache any application-defined extended ACK content for this acker
NormAppAckExtension ext;
while (ack.GetNextExtension(ext))
{
if (NormHeaderExtension::APP_ACK == ext.GetType())
{
if (!acker->SetAckEx(ext.GetContent(), ext.GetContentLength()))
{
// TBD - notify app of error
PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() error: unable to cache application-defined ACK content!\n");
}
}
}
acker->MarkAckReceived();
/* This code was an attempt to expedite delivery of the TX_WATERMARK_COMPLETED
notification to the application, but breaks some other desired behavior.
watermark_pending = false;
acking_success_count = 0;
NormNodeTreeIterator iterator(acking_node_tree);
NormAckingNode* next;
while (NULL != (next = static_cast<NormAckingNode*>(iterator.GetNextNode())))
{
if (next->IsPending())
watermark_pending = true;
else if (next->AckReceived() || (NORM_NODE_NONE == next->GetId()))
acking_success_count++;
}
if (!watermark_pending)
{
PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu watermark ack finished.\n",
(unsigned long)LocalNodeId());
Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL);
}
*/
}
else
{
// This can happen when new watermarks are set when an old watermark is still
// pending (i.e. receivers may still be in the process of replying)
PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received old/wrong watermark ACK?!\n");
}
}
else
{
PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received redundant watermark ACK?!\n");
}
}
else
{
PLOG(PL_WARN, "NormSession::SenderHandleAckMessage() received watermark ACK from unknown acker?!\n");
}
}
else
{
PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received unsolicited watermark ACK?!\n");
}
break;
// (TBD) Handle other acknowledgement types
default:
PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() node>%lu received unsupported ack type:%d\n",
(unsigned long)LocalNodeId(), ack.GetAckType());
break;
}
} // end SenderHandleAckMessage()
void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, NormNackMsg& nack)
{
struct timeval grttResponse;
nack.GetGrttResponse(grttResponse);
double receiverRtt = CalculateRtt(currentTime, grttResponse);
if (GetDebugLevel() >= PL_DEBUG)
{
PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu sender received NACK message from node>%lu rtt>%lf (tactive>%d) with content:\n",
(unsigned long)LocalNodeId(), (unsigned long)nack.GetSourceId(), receiverRtt, repair_timer.IsActive());
LogRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength(), fec_id, fec_m);
PLOG(PL_ALWAYS, "\n");
}
// (TBD) maintain average of "numErasures" for SEGMENT repair requests
// to use as input to a future automatic "auto parity" adjustor???
// Update GRTT estimate
if (receiverRtt >= 0.0) SenderUpdateGrttEstimate(receiverRtt);
// Look for NORM-CC Feedback header extension
NormCCFeedbackExtension ext;
while (nack.GetNextExtension(ext))
{
if (NormHeaderExtension::CC_FEEDBACK == ext.GetType())
{
SenderHandleCCFeedback(currentTime,
nack.GetSourceId(),
ext.GetCCFlags(),
receiverRtt >= 0.0 ?
receiverRtt : NormUnquantizeRtt(ext.GetCCRtt()),
NormUnquantizeLoss32(ext.GetCCLoss32()), // note using extended precision loss value here
NormUnquantizeRate(ext.GetCCRate()),
ext.GetCCSequence());
}
break;
}
// Parse and process NACK
UINT16 requestOffset = 0;
UINT16 requestLength = 0;
NormRepairRequest req;
NormObject* object = NULL;
bool freshObject = true;
NormObjectId prevObjectId = 0;
NormBlock* block = NULL;
bool freshBlock = true;
NormBlockId prevBlockId = 0;
bool startTimer = false;
UINT16 numErasures = extra_parity;
bool squelchQueued = false;
// Get the index of our next pending NORM_DATA transmission
NormObjectId txObjectIndex;
NormBlockId txBlockIndex;
if (SenderGetFirstPending(txObjectIndex))
{
NormObject* obj = tx_table.Find(txObjectIndex);
ASSERT(NULL != obj);
if (obj->IsPendingInfo())
{
txBlockIndex = 0;
}
else if (obj->GetFirstPending(txBlockIndex))
{
Increment(txBlockIndex);
}
else
{
txObjectIndex = next_tx_object_id;
txBlockIndex = 0;
}
}
else
{
txObjectIndex = next_tx_object_id;
txBlockIndex = 0;
}
bool holdoff = (repair_timer.IsActive() && !repair_timer.GetRepeatCount());
enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT};
while (0 != (requestLength = nack.UnpackRepairRequest(req, requestOffset)))
{
NormRepairRequest::Form requestForm = req.GetForm();
requestOffset += requestLength;
NormRequestLevel requestLevel;
if (req.FlagIsSet(NormRepairRequest::SEGMENT))
{
requestLevel = SEGMENT;
}
else if (req.FlagIsSet(NormRepairRequest::BLOCK))
{
requestLevel = BLOCK;
}
else if (req.FlagIsSet(NormRepairRequest::OBJECT))
{
requestLevel = OBJECT;
}
else if (req.FlagIsSet(NormRepairRequest::INFO))
{
requestLevel = INFO;
}
else
{
PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu recvd repair request w/ invalid repair level\n",
(unsigned long)LocalNodeId());
continue;
}
NormRepairRequest::Iterator iterator(req, fec_id, fec_m);
NormObjectId nextObjectId, lastObjectId;
NormBlockId nextBlockId, lastBlockId;
UINT16 nextBlockLen, lastBlockLen;
NormSegmentId nextSegmentId, lastSegmentId;
while (iterator.NextRepairItem(&nextObjectId, &nextBlockId,
&nextBlockLen, &nextSegmentId))
{
if (NormRepairRequest::RANGES == requestForm)
{
if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId,
&lastBlockLen, &lastSegmentId))
{
PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu recvd incomplete RANGE request!\n",
(unsigned long)LocalNodeId());
continue; // (TBD) break/return instead???
}
// (TBD) test for valid range form/level
}
else
{
lastObjectId = nextObjectId;
lastBlockId = nextBlockId;
lastBlockLen = nextBlockLen;
lastSegmentId = nextSegmentId;
}
bool inRange = true;
while (inRange)
{
if (nextObjectId != prevObjectId) freshObject = true;
if (freshObject)
{
freshBlock = true;
if (!(object = tx_table.Find(nextObjectId)))
{
PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu recvd repair request "
"for unknown object ...\n", (unsigned long)LocalNodeId());
if (!squelchQueued)
{
SenderQueueSquelch(nextObjectId);
squelchQueued = true;
}
if ((OBJECT == requestLevel) || (INFO == requestLevel))
{
nextObjectId++;
if (nextObjectId > lastObjectId)
inRange = false;
}
else
{
inRange = false;
}
continue;
}
prevObjectId = nextObjectId;
freshObject = false;
// Deal with INFO request if applicable
if (req.FlagIsSet(NormRepairRequest::INFO))
{
if (holdoff)
{
if (nextObjectId > txObjectIndex)
object->HandleInfoRequest(true);
}
else
{
// Update our minimum tx repair index as needed
if (tx_repair_pending)
{
if (nextObjectId <= tx_repair_object_min)
{
tx_repair_object_min = nextObjectId;
tx_repair_block_min = 0;
tx_repair_segment_min = 0;
}
}
else
{
tx_repair_pending = true;
tx_repair_object_min = nextObjectId;
tx_repair_block_min = 0;
tx_repair_segment_min = 0;
}
object->HandleInfoRequest(false);
startTimer = true;
}
}
} // end if (freshObject)
ASSERT(NULL != object);
object->SetLastNackTime(ProtoTime(currentTime));
switch (requestLevel)
{
case OBJECT:
PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(OBJECT) objs>%hu:%hu\n",
(UINT16)nextObjectId, (UINT16)lastObjectId);
if (holdoff)
{
if (nextObjectId > txObjectIndex)
{
if (object->IsStream())
object->TxReset(((NormStreamObject*)object)->StreamBufferLo());
else
object->TxReset();
if (!tx_pending_mask.Set(nextObjectId))
PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (1)\n",
(UINT16)nextObjectId);
}
}
else
{
// Update our minimum tx repair index as needed
if (tx_repair_pending)
{
if (nextObjectId <= tx_repair_object_min)
{
tx_repair_object_min = nextObjectId;
tx_repair_block_min = 0;
tx_repair_segment_min = 0;
}
}
else
{
tx_repair_pending = true;
tx_repair_object_min = nextObjectId;
tx_repair_block_min = 0;
tx_repair_segment_min = 0;
}
tx_repair_mask.Set(nextObjectId);
startTimer = true;
}
nextObjectId++;
if (nextObjectId > lastObjectId) inRange = false;
break;
case BLOCK:
PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(BLOCK) obj>%hu blks>%lu:%lu\n",
(UINT16)nextObjectId,
(unsigned long)nextBlockId.GetValue(),
(unsigned long)lastBlockId.GetValue());
inRange = false; // BLOCK requests are processed in one pass
// (TBD) if entire object is TxReset(), continue
if (object->IsStream())
{
// mark nack time for potential flow control
static_cast<NormStreamObject*>(object)->SetLastNackTime(nextBlockId, ProtoTime(currentTime));
bool attemptLock = true;
NormBlockId firstLockId = nextBlockId;
if (holdoff)
{
// Only lock blocks for which we're going to accept the repair request
if (nextObjectId == txObjectIndex)
{
//if (lastBlockId < txBlockIndex)
if (Compare(lastBlockId, txBlockIndex) < 0)
attemptLock = false;
//else if (nextBlockId < txBlockIndex)
else if (Compare(nextBlockId, txBlockIndex) < 0)
firstLockId = txBlockIndex;
}
else if (nextObjectId < txObjectIndex)
{
attemptLock = false; // NACK arrived too late to be useful
}
}
// Make sure the stream' pending_mask can be set as needed
// (TBD)
// Lock stream_buffer pending for block data retransmissions
if (attemptLock)
{
if (!((NormStreamObject*)object)->LockBlocks(firstLockId, lastBlockId, currentTime))
{
PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu LockBlocks() failure\n",
(unsigned long)LocalNodeId());
if (!squelchQueued)
{
SenderQueueSquelch(nextObjectId);
squelchQueued = true;
}
break;
}
}
else
{
break; // ignore late arriving NACK
}
} // end if (object->IsStream()
if (holdoff)
{
if (nextObjectId == txObjectIndex)
{
//if (nextBlockId >= txBlockIndex)
if (Compare(nextBlockId, txBlockIndex) >= 0)
object->TxResetBlocks(nextBlockId, lastBlockId);
//else if (lastBlockId >= txBlockIndex)
else if (Compare(lastBlockId, txBlockIndex) >= 0)
object->TxResetBlocks(txBlockIndex, lastBlockId);
}
else if (nextObjectId > txObjectIndex)
{
if (object->TxResetBlocks(nextBlockId, lastBlockId))
{
if (!tx_pending_mask.Set(nextObjectId))
PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (2)\n",
(UINT16)nextObjectId);
}
}
}
else
{
// Update our minimum tx repair index as needed
if (tx_repair_pending)
{
if (nextObjectId < tx_repair_object_min)
{
tx_repair_object_min = nextObjectId;
tx_repair_block_min = nextBlockId;
tx_repair_segment_min = 0;
}
else if (nextObjectId == tx_repair_object_min)
{
//if (nextBlockId <= tx_repair_block_min)
if (Compare(nextBlockId, tx_repair_block_min) <= 0)
{
tx_repair_block_min = nextBlockId;
tx_repair_segment_min = 0;
}
}
}
else
{
tx_repair_pending = true;
tx_repair_object_min = nextObjectId;
tx_repair_block_min = nextBlockId;
tx_repair_segment_min = 0;
}
if (!object->HandleBlockRequest(nextBlockId, lastBlockId))
{
if (!squelchQueued)
{
SenderQueueSquelch(nextObjectId);
squelchQueued = true;
}
}
startTimer = true;
}
break;
case SEGMENT:
PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(SEGMENT) obj>%hu blk>%lu segs>%hu:%hu\n",
(UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(),
(UINT16)nextSegmentId, (UINT16)lastSegmentId);
inRange = false; // SEGMENT repairs are also handled in one pass
if (nextBlockId != prevBlockId) freshBlock = true;
if (freshBlock)
{
// Is this entire block already repair pending?
if (object->IsRepairSet(nextBlockId))
continue;
if (NULL == (block = object->FindBlock(nextBlockId)))
{
// Is this entire block already tx pending?
if (object->IsPendingSet(nextBlockId))
{
// Entire block already tx pending, don't worry about individual segments
PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu "
"recvd SEGMENT repair request for pending block.\n",
(unsigned long)LocalNodeId());
continue;
}
else
{
// Try to recover block including parity calculation
if (NULL == (block = object->SenderRecoverBlock(nextBlockId)))
{
if (NormObject::STREAM == object->GetType())
{
PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu "
"recvd repair request for old stream block(%lu) ...\n",
(unsigned long)LocalNodeId(),
(unsigned long)nextBlockId.GetValue());
if (!squelchQueued)
{
SenderQueueSquelch(nextObjectId);
squelchQueued = true;
}
}
else
{
// Resource constrained, move on to next repair request
PLOG(PL_INFO, "NormSession::SenderHandleNackMessage() node>%lu "
"Warning - sender is resource constrained ...\n",
(unsigned long)LocalNodeId());
}
continue;
}
}
}
freshBlock = false;
numErasures = extra_parity;
prevBlockId = nextBlockId;
} // end if (freshBlock)
ASSERT(NULL != block);
// If stream && explicit data repair, lock the data for retransmission
// (TBD) this use of "ndata" needs to be replaced for dynamically shortened blocks
if (object->IsStream())
{
// mark nack time for potential flow control
static_cast<NormStreamObject*>(object)->SetLastNackTime(nextBlockId, ProtoTime(currentTime));
if (nextSegmentId < ndata)
{
bool attemptLock = true;
NormSegmentId firstLockId = nextSegmentId;
NormSegmentId lastLockId = ndata - 1;
lastLockId = MIN(lastLockId, lastSegmentId);
if (holdoff)
{
if (nextObjectId == txObjectIndex)
{
//if (nextBlockId < txBlockIndex)
if (Compare(nextBlockId, txBlockIndex) < 0)
{
//if (1 == (txBlockIndex - nextBlockId))
if (1 == (UINT32)Difference(txBlockIndex, nextBlockId))
{
// We're currently sending this block
if (block->IsPending())
{
NormSegmentId firstPending = 0;
block->GetFirstPending(firstPending);
if (lastLockId <= firstPending)
attemptLock = false;
else if (nextSegmentId < firstPending)
firstLockId = firstPending;
}
else
{
// block was just recovered
}
}
else
{
attemptLock = false; // NACK arrived way too late
}
}
}
else if (nextObjectId < txObjectIndex)
{
attemptLock = false; // NACK arrived too late
}
} // end if (holdoff)
if (attemptLock)
{
if (!((NormStreamObject*)object)->LockSegments(nextBlockId, firstLockId, lastLockId))
{
PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu "
"LockSegments() failure\n", (unsigned long)LocalNodeId());
if (!squelchQueued)
{
SenderQueueSquelch(nextObjectId);
squelchQueued = true;
}
break;
}
}
else
{
break; // ignore late arriving NACK
}
} // end if (nextSegmentId < ndata)
} // end if (object->IsStream())
// With a series of SEGMENT repair requests for a block, "numErasures" will
// eventually total the number of missing segments in the block.
numErasures += (lastSegmentId - nextSegmentId + 1);
if (holdoff)
{
if (nextObjectId > txObjectIndex)
{
if (object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures))
{
if (!tx_pending_mask.Set(nextObjectId))
PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() tx_pending_mask.Set(%hu) error (3)\n",
(UINT16)nextObjectId);
}
}
else if (nextObjectId == txObjectIndex)
{
//if (nextBlockId >= txBlockIndex)
if (Compare(nextBlockId, txBlockIndex) >= 0)
{
object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures);
}
//else if (1 == (txBlockIndex - nextBlockId))
else if (1 == (UINT32)Difference(txBlockIndex, nextBlockId))
{
NormSegmentId firstPending = 0;
if (block->GetFirstPending(firstPending))
{
if (nextSegmentId > firstPending)
object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures);
else if (lastSegmentId > firstPending)
object->TxUpdateBlock(block, firstPending, lastSegmentId, numErasures);
else if (numErasures > block->ParityCount())
object->TxUpdateBlock(block, firstPending, firstPending, numErasures);
}
else
{
// This block was just recovered, so do full update
object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures);
}
}
}
}
else // !holdoff
{
// Update our minimum tx repair index as needed
ASSERT(nextBlockId == block->GetId());
UINT16 nextBlockSize = object->GetBlockSize(nextBlockId);
if (tx_repair_pending)
{
if (nextObjectId < tx_repair_object_min)
{
tx_repair_block_min = nextBlockId;
tx_repair_segment_min = (nextSegmentId < nextBlockSize) ?
nextSegmentId : (nextBlockSize - 1);
}
else if (nextObjectId == tx_repair_object_min)
{
//if (nextBlockId < tx_repair_block_min)
if (Compare(nextBlockId, tx_repair_block_min) < 0)
{
tx_repair_block_min = nextBlockId;
tx_repair_segment_min = (nextSegmentId < nextBlockSize) ?
nextSegmentId : (nextBlockSize - 1);
}
else if (nextBlockId == tx_repair_block_min)
{
if (nextSegmentId < tx_repair_segment_min)
tx_repair_segment_min = nextSegmentId;
}
}
}
else
{
tx_repair_pending = true;
tx_repair_object_min = nextObjectId;
tx_repair_block_min = nextBlockId;
tx_repair_segment_min = (nextSegmentId < nextBlockSize) ?
nextSegmentId : (nextBlockSize - 1);
}
block->HandleSegmentRequest(nextSegmentId, lastSegmentId,
nextBlockSize, nparity,
numErasures);
startTimer = true;
} // end if/else (holdoff)
break;
case INFO:
// We already dealt with INFO request above with respect to initiating repair
nextObjectId++;
if (nextObjectId > lastObjectId) inRange = false;
break;
} // end switch(requestLevel)
} // end while(inRange)
} // end while(NextRepairItem())
} // end while(UnpackRepairRequest())
if (startTimer && !repair_timer.IsActive())
{
// BACKOFF related code
double aggregateInterval = address.IsMulticast() ?
grtt_advertised * (backoff_factor + 1.0) : 0.0;
// Uncommenting the line below treats ((0 == ndata) && 0.0 == backoff_factor)
// as a special case (sets zero sender aggregateInterval)
aggregateInterval = ((0 != nparity) || (backoff_factor > 0.0)) ? aggregateInterval : 0.0;
// TBD - why did we do this thing here to limit the min aggregateInterval???
// (I think to allow "11th hour NACKs to be incorporated .. so this should be
// for mcast only)
if (tx_timer.IsActive() && address.IsMulticast())
{
double txTimeout = tx_timer.GetTimeRemaining() - 1.0e-06;
aggregateInterval = MAX(txTimeout, aggregateInterval);
}
repair_timer.SetInterval(aggregateInterval);
PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu starting sender "
"NACK aggregation timer (%lf sec)...\n",
(unsigned long)LocalNodeId(), aggregateInterval);
ActivateTimer(repair_timer);
}
} // end NormSession::SenderHandleNackMessage()
void NormSession::ReceiverHandleAckMessage(const NormAckMsg& ack)
{
NormSenderNode* theSender = (NormSenderNode*)sender_tree.FindNodeById(ack.GetSenderId());
if (theSender)
{
theSender->HandleAckMessage(ack);
}
else if (ack.GetSenderId() != LocalNodeId())
{
PLOG(PL_DEBUG, "NormSession::ReceiverHandleAckMessage() node>%lu heard ACK for unknown sender>%lu\n",
(unsigned long)LocalNodeId(), (unsigned long)ack.GetSenderId(), IsServerListener());
}
} // end NormSession::ReceiverHandleAckMessage()
void NormSession::ReceiverHandleNackMessage(const NormNackMsg& nack)
{
NormSenderNode* theSender = (NormSenderNode*)sender_tree.FindNodeById(nack.GetSenderId());
if (theSender)
{
theSender->HandleNackMessage(nack);
}
else if (nack.GetSenderId() != LocalNodeId())
{
PLOG(PL_DEBUG, "NormSession::ReceiverHandleNackMessage() node>%lu heard NACK for unknown sender\n",
(unsigned long)LocalNodeId());
}
} // end NormSession::ReceiverHandleNackMessage()
bool NormSession::SenderQueueSquelch(NormObjectId objectId)
{
// If a squelch is already queued, update it if (objectId < squelch->objectId)
bool doEnqueue = true;
NormCmdSquelchMsg* squelch = NULL;
NormMsg* msg = message_queue.GetHead();
while (NULL != msg)
{
// (TBD) we need to depreceate the whole "message_pool" idea and
// have messages be built on demand in NormSession::Serve() according
// to some state variables (i.e. that dictate when to send a command
// instead of data, etc). This will simplify alot of stuff and
// probably improve performance some too.
if (NormMsg::CMD == msg->GetType())
{
if (NormCmdMsg::SQUELCH == static_cast<NormCmdMsg*>(msg)->GetFlavor())
{
squelch = static_cast<NormCmdSquelchMsg*>(msg);
break;
}
}
msg = msg->GetNext();
}
if (NULL != squelch)
{
if (objectId >= squelch->GetObjectId())
return false; // no need to update squelch
doEnqueue = false;
}
else
{
squelch = (NormCmdSquelchMsg*)GetMessageFromPool();
}
if (squelch)
{
squelch->Init(fec_id);
squelch->SetDestination(address);
squelch->SetGrtt(grtt_quantized);
squelch->SetBackoffFactor((unsigned char)backoff_factor);
squelch->SetGroupSize(gsize_quantized);
NormObject* obj = tx_table.Find(objectId);
NormObjectTable::Iterator iterator(tx_table);
NormObjectId nextId;
if (NULL != obj)
{
ASSERT(NormObject::STREAM == obj->GetType());
squelch->SetObjectId(objectId);
//NormBlockId blockId = static_cast<NormStreamObject*>(obj)->StreamBufferLo();
NormBlockId blockId = static_cast<NormStreamObject*>(obj)->RepairWindowLo();
squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m);
while ((obj = iterator.GetNextObject()))
if (objectId == obj->GetId()) break;
nextId = objectId + 1;
}
else
{
obj = iterator.GetNextObject();
if (NULL != obj)
{
squelch->SetObjectId(obj->GetId());
NormBlockId blockId;
if (obj->IsStream())
//blockId =static_cast<NormStreamObject*>(obj)->StreamBufferLo();
blockId = static_cast<NormStreamObject*>(obj)->RepairWindowLo();
else
blockId = NormBlockId(0);
squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m);
nextId = obj->GetId() + 1;
}
else
{
// Squelch to point to future object
squelch->SetObjectId(next_tx_object_id);
// (TBD) should the "blockLen" here be "ndata" instead? but we can't be sure
squelch->SetFecPayloadId(fec_id, 0, 0, 0, fec_m);
nextId = next_tx_object_id;
}
}
bool buildingList = true;
while (buildingList && (obj = iterator.GetNextObject()))
{
while (nextId != obj->GetId())
{
if (!squelch->AppendInvalidObject(nextId, segment_size))
{
buildingList = false;
break;
}
nextId++;
}
nextId++;
}
if (doEnqueue)
{
QueueMessage(squelch);
PLOG(PL_DEBUG, "NormSession::SenderQueueSquelch() node>%lu sender queued squelch ...\n",
(unsigned long)LocalNodeId());
}
else
{
PLOG(PL_DEBUG, "NormSession::SenderQueueSquelch() node>%lu sender updated squelch ...\n",
(unsigned long)LocalNodeId());
}
return true;
}
else
{
PLOG(PL_FATAL, "NormSession::SenderQueueSquelch() node>%lu message_pool exhausted! (couldn't squelch)\n",
(unsigned long)LocalNodeId());
return false;
}
} // end NormSession::SenderQueueSquelch()
bool NormSession::SenderSendCmd(const char* cmdBuffer, unsigned int cmdLength, bool robust)
{
if (!is_sender)
{
PLOG(PL_ERROR, "NormSession::SenderSendCmd() error: non-sender session!\n");
return false;
}
if (0 != cmd_count)
{
PLOG(PL_INFO, "NormSession::SenderSendCmd() error: command already pending!\n");
return false;
}
else if (cmdLength > segment_size)
{
PLOG(PL_INFO, "NormSession::SenderSendCmd() error: command length greater than segment_size!\n");
return false;
}
memcpy(cmd_buffer, cmdBuffer, cmdLength);
cmd_length = cmdLength;
cmd_count = robust ? tx_robust_factor : 1;
if (!tx_timer.IsActive()) PromptSender();
return true;
} // end NormSession::SenderSendCmd()
void NormSession::SenderCancelCmd()
{
if (0 != cmd_count)
{
if (cmd_timer.IsActive()) cmd_timer.Deactivate();
cmd_count = 0;
cmd_length = 0;
}
} // end NormSession::SenderCancelCmd()
bool NormSession::SenderQueueAppCmd()
{
if (0 == cmd_count) return false;
ASSERT(!cmd_timer.IsActive());
// 1) Build a NORM_CMD(APPLICATION) message
NormCmdAppMsg* appMsg = static_cast<NormCmdAppMsg*>(GetMessageFromPool());
if (NULL != appMsg)
{
appMsg->Init();
appMsg->SetDestination(address);
appMsg->SetGrtt(grtt_quantized);
appMsg->SetBackoffFactor((unsigned char)backoff_factor);
appMsg->SetGroupSize(gsize_quantized);
appMsg->SetContent(cmd_buffer, cmd_length, segment_size);
QueueMessage(appMsg);
PLOG(PL_DEBUG, "NormSession::SenderQueueAppCmd() node>%lu sender queued app-defined cmd ...\n",
(unsigned long)LocalNodeId());
cmd_count--;
if (0 != cmd_count)
{
cmd_timer.SetInterval(grtt_advertised * 2);
ActivateTimer(cmd_timer);
}
else
{
PLOG(PL_DEBUG, "NormSession::SenderQueueAppCmd() node>%lu cmd transmission completed ...\n",
(unsigned long)LocalNodeId());
Notify(NormController::TX_CMD_SENT, NULL, NULL);
}
return true;
}
else
{
PLOG(PL_FATAL, "NormSession::SenderQueueAppCmd() node>%lu message_pool exhausted!\n",
(unsigned long)LocalNodeId());
return false;
}
} // end NormSession::SenderQueueAppCmd()
bool NormSession::OnCmdTimeout(ProtoTimer& theTimer)
{
if (!tx_timer.IsActive()) PromptSender();
return true;
} // end NormSession::OnCmdTimeout()
bool NormSession::SenderSendAppCmd(const char* buffer, unsigned int length, const ProtoAddress& dst)
{
// Build/immediately send a NORM_CMD(APPLICATION) message
NormCmdAppMsg appMsg;
appMsg.Init();
appMsg.SetDestination(address);
appMsg.SetGrtt(grtt_quantized);
appMsg.SetBackoffFactor((unsigned char)backoff_factor);
appMsg.SetGroupSize(gsize_quantized);
// We use a surrogate segment_size in case sender not configured (e.g. server-listener)
appMsg.SetContent(buffer, length, segment_size ? segment_size : 64);
appMsg.SetDestination(dst);
if (MSG_SEND_OK != SendMessage(appMsg))
PLOG(PL_ERROR, "NormSession::SenderSendAppCmd() node>%lu sender unable to send app-defined cmd ...\n",
(unsigned long)LocalNodeId());
else
PLOG(PL_DEBUG, "NormSession::SenderSendAppCmd() node>%lu sender sending app-defined cmd len:%u...\n",
(unsigned long)LocalNodeId(), appMsg.GetLength());
return true;
} // end NormSession::SenderSendAppCmd()
void NormSession::ActivateFlowControl(double delay, NormObjectId objectId, NormController::Event event)
{
flow_control_object = objectId;
flow_control_event = event;
flow_control_timer.SetInterval(delay);
if (flow_control_timer.IsActive())
flow_control_timer.Reschedule();
else
ActivateTimer(flow_control_timer);
} // end NormSession::ActivateFlowControl()
bool NormSession::OnFlowControlTimeout(ProtoTimer& theTimer)
{
NormObject* object = tx_table.Find(flow_control_object);
if (NULL == object)
{
PLOG(PL_WARN, "NormSession::OnFlowControlTimeout() flow_control_object removed?!\n");
// Throw a TX_QUEUE_EMPTY just in case ???
//Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, (NormObject*)NULL);
return true;
}
double deltaTime;
if (object->IsStream())
{
// A stream was flow-controlled, so check its stream nack age, etc
NormBlock* block = static_cast<NormStreamObject*>(object)->StreamBlockLo();
if (NULL == block)
{
// No blocks in stream buffer, thus it is actually empty
// Notify(NormController::TX_QUEUE_VACANCY, (NormSenderNode*)NULL, object);
posted_tx_queue_empty = true;
Notify(NormController::TX_QUEUE_EMPTY, (NormSenderNode*)NULL, object);//(NormObject*)NULL);
return true;
}
deltaTime = GetFlowControlDelay() - block->GetNackAge();
if (deltaTime < 1.0e-06)
{
// no recent NACKing for "oldest block", so post EMPTY/VACANCY if non-pending
// TBD - retest this with mixed stream/object sender sessions
if (!block->IsPending())
{
posted_tx_queue_empty = (NormController::TX_QUEUE_EMPTY == flow_control_event);
Notify(flow_control_event, (NormSenderNode*)NULL, object);
}
return true;
}
}
else
{
// The tx cache (queue) is being flow controlled ...
deltaTime = GetFlowControlDelay() - object->GetNackAge();
if (deltaTime < 1.0e-06)
{
// no recent NACKing, so if non-pending, dispatch queue EMPTY or VACANCY
if (!object->IsRepairPending() && !object->IsPending())
{
posted_tx_queue_empty = (NormController::TX_QUEUE_EMPTY == flow_control_event);
Notify(flow_control_event, (NormSenderNode*)NULL, (NormObject*)NULL);
}
return true;
}
}
// Extend flow control timeout due to recent activity
// NOTE that above we limited the minimum "deltaTime" to 1.0-06 ... otherwise
// ProtoTime 1usec precision limitation put us into an infinite loop in
// _simulation_ environments where perfect scheduling occurs w/ zero processing time
theTimer.SetInterval(deltaTime);
theTimer.Reschedule();
return false;
} // end NormSession::OnFlowControlTimeout()
bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd)
{
// Build a NORM_CMD(REPAIR_ADV) message with current pending repair state.
NormRepairRequest req;
req.SetFlag(NormRepairRequest::OBJECT);
NormRepairRequest::Form prevForm = NormRepairRequest::INVALID;
NormObjectId firstId;
UINT16 objectCount = 0;
NormObjectTable::Iterator iterator(tx_table);
NormObject* nextObject = iterator.GetNextObject();
while (NULL != nextObject)
{
NormObject* currentObject = nextObject;
nextObject = iterator.GetNextObject();
NormObjectId currentId = currentObject->GetId();
bool repairEntireObject = tx_repair_mask.Test(currentId);
if (repairEntireObject)
{
if (!objectCount) firstId = currentId; // set first OBJECT level repair id
objectCount++; // increment consecutive OBJECT level repair count.
}
// Check for non-OBJECT level request or end
if (objectCount && (!repairEntireObject || (NULL != nextObject)))
{
NormRepairRequest::Form form;
switch (objectCount)
{
case 0:
form = NormRepairRequest::INVALID;
break;
case 1:
case 2:
form = NormRepairRequest::ITEMS;
break;
default:
form = NormRepairRequest::RANGES;
break;
}
if (form != prevForm)
{
if (NormRepairRequest::INVALID != prevForm)
{
if (0 == cmd.PackRepairRequest(req))
{
prevForm = NormRepairRequest::INVALID;
PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n");
// (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case
break;
}
}
req.SetForm(form);
cmd.AttachRepairRequest(req, segment_size);
prevForm = form;
}
switch (form)
{
case 0:
ASSERT(0); // can't happen
break;
case 1:
case 2:
req.SetForm(NormRepairRequest::ITEMS);
req.AppendRepairItem(fec_id, fec_m, firstId, 0, ndata, 0); // (TBD) error check
if (2 == objectCount)
req.AppendRepairItem(fec_id, fec_m, currentId, 0, ndata, 0); // (TBD) error check
break;
default:
req.SetForm(NormRepairRequest::RANGES);
req.AppendRepairRange(fec_id, fec_m, firstId, 0, ndata, 0, // (TBD) error check
currentId, 0, ndata, 0);
break;
}
prevForm = NormRepairRequest::INVALID;
if (0 == cmd.PackRepairRequest(req))
{
PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n");
// (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case
break;
}
objectCount = 0;
}
if (!repairEntireObject)
{
if (currentObject->IsRepairPending())
{
if (NormRepairRequest::INVALID != prevForm)// && currentObject->IsRepairPending())
{
prevForm = NormRepairRequest::INVALID;
if (0 == cmd.PackRepairRequest(req))
{
PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n");
// (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case
break;
}
}
if (!currentObject->AppendRepairAdv(cmd)) break;
}
objectCount = 0; // this is probably redundant
}
} // end while (nextObject)
if (NormRepairRequest::INVALID != prevForm)
{
if (0 == cmd.PackRepairRequest(req))
PLOG(PL_ERROR, "NormSession::SenderBuildRepairAdv() warning: full msg\n");
// (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case
}
return true;
} // end NormSession::SenderBuildRepairAdv()
bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/)
{
tx_repair_pending = false;
if (0 != repair_timer.GetRepeatCount())
{
// NACK aggregation period has ended. (incorporate accumulated repair requests)
PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu sender NACK aggregation time ended.\n",
(unsigned long)LocalNodeId());
NormObjectTable::Iterator iterator(tx_table);
NormObject* obj;
while ((obj = iterator.GetNextObject()))
{
NormObjectId objectId = obj->GetId();
if (tx_repair_mask.Test(objectId))
{
PLOG(PL_TRACE, "NormSession::OnRepairTimeout() node>%lu tx reset obj>%hu ...\n",
(unsigned long)LocalNodeId(), (UINT16)objectId);
if (obj->IsStream())
obj->TxReset(((NormStreamObject*)obj)->RepairWindowLo());
else
obj->TxReset();
tx_repair_mask.Unset(objectId);
if (!tx_pending_mask.Set(objectId))
{
PLOG(PL_ERROR, "NormSession::OnRepairTimeout() tx_pending_mask.Set(%hu) error (1)\n",
(UINT16)objectId);
}
}
else
{
PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu activating obj>%hu repairs ...\n",
(unsigned long)LocalNodeId(), (UINT16)objectId);
if (obj->ActivateRepairs())
{
PLOG(PL_TRACE, "NormSession::OnRepairTimeout() node>%lu activated obj>%hu repairs ...\n",
(unsigned long)LocalNodeId(), (UINT16)objectId);
if (!tx_pending_mask.Set(objectId))
PLOG(PL_ERROR, "NormSession::OnRepairTimeout() node>%lu tx_pending_mask.Set(%hu) error (2)\n",
(unsigned long)LocalNodeId(), (UINT16)objectId);
}
}
} // end while (iterator.GetNextObject())
PromptSender();
// BACKOFF related code
// Holdoff initiation of new repair cycle for one GRTT
// (TBD) for unicast sessions, use CLR RTT ???
//double holdoffInterval = backoff_factor > 0.0 ? grtt_advertised : 0.0;
double holdoffInterval = grtt_advertised;
repair_timer.SetInterval(holdoffInterval); // repair holdoff interval = 1*GRTT
PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu starting sender "
"NACK holdoff timer (%lf sec)...\n",
(unsigned long)LocalNodeId(), holdoffInterval);
}
else
{
// REPAIR holdoff interval has now ended.
PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu sender holdoff time ended.\n",
(unsigned long)LocalNodeId());
}
return true;
} // end NormSession::OnRepairTimeout()
// (TBD) Should pass current system time to ProtoTimer timeout handlers
// for more efficiency ...
bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/)
{
NormMsg* msg;
// Note: sometimes need RepairAdv even when cc_enable is false ...
NormCmdRepairAdvMsg adv;
if (advertise_repairs && (probe_proactive || (repair_timer.IsActive() &&
repair_timer.GetRepeatCount())))
{
// Build a NORM_CMD(NACK_ADV) in response to
// receipt of unicast NACK or CC update
adv.Init();
adv.SetGrtt(grtt_quantized);
adv.SetBackoffFactor((unsigned char)backoff_factor);
adv.SetGroupSize(gsize_quantized);
adv.SetDestination(address);
// Fill in congestion control header extension
NormCCFeedbackExtension ext;
adv.AttachExtension(ext);
if (suppress_rate < 0.0)
{
ext.SetCCFlag(NormCC::RTT);
ext.SetCCRtt(grtt_quantized);
ext.SetCCRate(NormQuantizeRate(tx_rate));
}
else
{
if (!suppress_nonconfirmed) ext.SetCCFlag(NormCC::RTT);
ext.SetCCRtt(NormQuantizeRtt(suppress_rtt));
ext.SetCCRate(NormQuantizeRate(suppress_rate));
}
SenderBuildRepairAdv(adv);
msg = (NormMsg*)&adv;
}
else
{
msg = message_queue.RemoveHead();
advertise_repairs = false;
}
if (NULL != msg)
{
// Do "packet pairing of NORM_CMD(CC) and subsequent message (usually NORM_DATA), if any
//unsigned int msgLength = msg->GetLength();
unsigned int msgLength = tx_residual;
/* Uncomment this section of code to instate CMD(CC) / NORM_DATA "packet pairing"
tx_residual = 0;
if ((NormMsg::CMD == msg->GetType()) &&
(NormCmdMsg::CC == static_cast<NormCmdMsg*>(msg)->GetFlavor()))
{
tx_residual = msg->GetLength();
}
else
*/
{
msgLength += msg->GetLength();
}
switch (SendMessage(*msg))
{
case MSG_SEND_OK:
if (tx_rate > 0.0)
tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate));
if (advertise_repairs)
{
advertise_repairs = false;
suppress_rate = -1.0; // reset cc feedback suppression rate
}
else
{
ReturnMessageToPool(msg);
}
// Pre-serve to allow pre-prompt for empty tx queue
// (TBD) do this in a better way ??? There is a slight chance
// that with this approach some new data may get pre-queued
// when an interim repair request should be serviced first
// instead ???
//if (message_queue.IsEmpty() && IsSender()) Serve();
return true; // reinstall tx_timer
case MSG_SEND_BLOCKED:
// Message was not sent due to to EWOULDBLOCK, so we invoke async i/o output notification
if (!advertise_repairs)
message_queue.Prepend(msg);
if (tx_timer.IsActive())
tx_timer.Deactivate();
//TRACE("starting output notification ...\n");
tx_socket->StartOutputNotification();
return false; // since timer was deactivated
case MSG_SEND_FAILED:
// Message was not sent due to socket error (no route, etc), so so just timeout and try again
// (TBD - is there something smarter we should do)
//TRACE("MSG_SEND_FAILED\n");
if (!advertise_repairs) message_queue.Prepend(msg);
if (tx_rate > 0.0)
tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate));
else if (0.0 == tx_timer.GetInterval())
tx_timer.SetInterval(0.001);
return true; // timer will be reactivated
}
}
else
{
// 1) Prompt for next sender message
if (IsSender()) Serve();
if (message_queue.IsEmpty())
{
if (tx_timer.IsActive())
tx_timer.Deactivate();
// Check that any possible notifications posted in
// the previous call to Serve() may have caused a
// change in sender state making it ready to send
//if (IsSender()) Serve();
return false;
}
else
{
// We have a new message as a result of serving, so send it immediately
return OnTxTimeout(tx_timer);
}
}
return true; // actually will never get here but compiler thinks it's needed
} // end NormSession::OnTxTimeout()
NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg)
{
//TRACE("sending message length %hu\n", msg.GetLength());
bool isReceiverMsg = false;
bool isProbe = false;
// Fill in any last minute timestamps
// (TBD) fill in InstanceId fields on all messages as needed
// We need "fec_m" for the message for NormTrace() purposes
UINT8 fecM = fec_m; // assume it's a sender message (will be overridden otherwise)
UINT16 instId = instance_id; // assume it's a sender message (will be overridden otherwise)
switch (msg.GetType())
{
case NormMsg::INFO:
case NormMsg::DATA:
{
NormObjectMsg& objMsg = static_cast<NormObjectMsg&>(msg);
objMsg.SetInstanceId(instId);
msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs
if (syn_status) objMsg.SetFlag(NormObjectMsg::FLAG_SYN);
break;
}
case NormMsg::CMD:
{
NormCmdMsg& cmd = static_cast<NormCmdMsg&>(msg);
((NormCmdMsg&)msg).SetInstanceId(instId);
switch (cmd.GetFlavor())
{
case NormCmdMsg::CC:
{
NormCmdCCMsg& ccMsg = static_cast<NormCmdCCMsg&>(cmd);
struct timeval currentTime;
ProtoSystemTime(currentTime);
ccMsg.SetSendTime(currentTime);
isProbe = true;
if (syn_status) ccMsg.SetSyn();
break;
}
case NormCmdMsg::SQUELCH:
break;
default:
break;
}
msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs
break;
}
case NormMsg::NACK:
{
msg.SetSequence(0); // TBD - set per destination
isReceiverMsg = true;
NormNackMsg& nack = (NormNackMsg&)msg;
NormSenderNode* theSender =
(NormSenderNode*)sender_tree.FindNodeById(nack.GetSenderId());
ASSERT(NULL != theSender);
fecM = theSender->GetFecFieldSize();
instId = theSender->GetInstanceId();
struct timeval currentTime;
ProtoSystemTime(currentTime);
struct timeval grttResponse;
theSender->CalculateGrttResponse(currentTime, grttResponse);
nack.SetGrttResponse(grttResponse);
break;
}
case NormMsg::ACK:
{
msg.SetSequence(0); // TBD - set per destination
isReceiverMsg = true;
NormAckMsg& ack = (NormAckMsg&)msg;
NormSenderNode* theSender;
if (IsServerListener())
theSender = client_tree.FindNodeByAddress(ack.GetDestination());
else
theSender = (NormSenderNode*)sender_tree.FindNodeById(ack.GetSenderId());
ASSERT(NULL != theSender);
fecM = theSender->GetFecFieldSize();
instId = theSender->GetInstanceId();
struct timeval grttResponse;
struct timeval currentTime;
ProtoSystemTime(currentTime);
theSender->CalculateGrttResponse(currentTime, grttResponse);
ack.SetGrttResponse(grttResponse);
break;
}
default:
break;
}
// Fill in common message fields
msg.SetSourceId(local_node_id);
UINT16 msgSize = msg.GetLength();
// Possibly drop some tx messages for testing purposes
bool drop = (tx_loss_rate > 0.0) ? (UniformRand(100.0) < tx_loss_rate) : false;
if (isReceiverMsg && receiver_silent)
{
// don't send receiver messages if "silent receiver"
// TBD - perhaps we should make sure silent receivers
// never enqueue any receiver messages. But we
// did this to make sure all integrity of timer
// state interdependencies wasn't messed up
return MSG_SEND_OK; // we lie as it wasn't sent but it wasn't supposed to
}
else if (drop)
{
//DMSG(0, "TX MESSAGE DROPPED! (tx_loss_rate:%lf\n", tx_loss_rate);
// "Pretend" like dropped message was sent for trace and timing purposes
if (trace)
{
struct timeval currentTime;
ProtoSystemTime(currentTime);
NormTrace(currentTime, LocalNodeId(), msg, true, fecM, instId);
}
// Update sent rate tracker even if dropped (for testing/debugging)
sent_accumulator.Increment(msgSize);
nominal_packet_size += 0.01 * (((double)msgSize) - nominal_packet_size);
}
else
{
unsigned int numBytes = msgSize;
bool result = tx_socket->SendTo(msg.GetBuffer(), numBytes, msg.GetDestination());
if (numBytes == msgSize)
{
// Separate send/recv tracing
if (trace)
{
struct timeval currentTime;
ProtoSystemTime(currentTime);
NormTrace(currentTime, LocalNodeId(), msg, true, fecM, instId);
}
// To keep track of _actual_ sent rate
sent_accumulator.Increment(msgSize);
// Update nominal packet size
nominal_packet_size += 0.01 * (((double)msgSize) - nominal_packet_size);
}
else
{
// packet not sent
tx_sequence--;
if (result)
{
// TBD - is PL_WARN too verbose here
PLOG(PL_WARN, "NormSession::SendMessage() sendto(%s/%hu) warning: %s\n",
msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString());
return MSG_SEND_BLOCKED;
}
else
{
PLOG(PL_WARN, "NormSession::SendMessage() sendto(%s/%hu) warning: %s\n",
msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString());
return MSG_SEND_FAILED;
}
}
}
if (isProbe)
{
probe_pending = false;
probe_data_check = true;
if (probe_reset)
{
probe_reset = false;
if (!probe_timer.IsActive())
ActivateTimer(probe_timer);
}
}
else if (!isReceiverMsg && IsSender())
{
probe_data_check = false;
if (!probe_pending && probe_reset)
{
probe_reset = false;
OnProbeTimeout(probe_timer);
if (!probe_timer.IsActive())
ActivateTimer(probe_timer);
}
}
return MSG_SEND_OK;
} // end NormSession::SendMessage()
void NormSession::SetGrttProbingInterval(double intervalMin, double intervalMax)
{
if ((intervalMin < 0.0) || (intervalMax < 0.0)) return;
double temp = intervalMin;
if (temp > intervalMax)
{
intervalMin = intervalMax;
intervalMax = temp;
}
if (intervalMin < NORM_TICK_MIN) intervalMin = NORM_TICK_MIN;
if (intervalMax < NORM_TICK_MIN) intervalMax = NORM_TICK_MIN;
grtt_interval_min = intervalMin;
grtt_interval_max = intervalMax;
if (grtt_interval < grtt_interval_min)
grtt_interval = grtt_interval_min;
if (grtt_interval > grtt_interval_max)
{
grtt_interval = grtt_interval_max;
if (probe_timer.IsActive() && !cc_enable)
{
double elapsed = probe_timer.GetInterval() - probe_timer.GetTimeRemaining();
if (elapsed < 0.0) elapsed = 0.0;
if (elapsed > grtt_interval)
probe_timer.SetInterval(0.0);
else
probe_timer.SetInterval(grtt_interval - elapsed);
probe_timer.Reschedule();
}
}
} // end NormSession::SetGrttProbingInterval()
void NormSession::SetGrttProbingMode(ProbingMode probingMode)
{
if (cc_enable) return; // can't change probing mode when cc is enabled!
// (cc _requires_ probing mode == PROBE_ACTIVE)
switch (probingMode)
{
case PROBE_NONE:
probe_reset = false;
if (probe_timer.IsActive())
probe_timer.Deactivate();
break;
case PROBE_PASSIVE:
probe_proactive = false;
if (IsSender())
{
if (!probe_timer.IsActive())
{
probe_timer.SetInterval(0.0);
ActivateTimer(probe_timer);
}
}
else
{
probe_reset = true;
}
break;
case PROBE_ACTIVE:
probe_proactive = true;
if (IsSender())
{
if (!probe_timer.IsActive())
{
probe_timer.SetInterval(0.0);
ActivateTimer(probe_timer);
}
}
else
{
probe_reset = true;
}
break;
}
} // end NormSession::SetGrttProbingMode()
bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/)
{
// 1) Temporarily kill probe_timer if CMD(CC) not yet tx'd
// (or if data has not been sent since last probe)
if (probe_pending || (data_active && probe_data_check) || (0.0 == tx_rate))
{
probe_reset = true;
if (probe_timer.IsActive())
probe_timer.Deactivate();
return false;
}
// 2) Update grtt_estimate _if_ sufficient time elapsed.
// This new code allows more liberal downward adjustment of
// of grtt when congestion control is enabled.
// We have to keep track of the _actual_ deltaTime instead
// of relying on the probe_timer interval because in real-
// world operating systems, they're aren't the same and
// sometimes not even close.
struct timeval currentTime;
ProtoSystemTime(currentTime);
if ((0 == probe_time_last.tv_sec) && (0 == probe_time_last.tv_usec))
{
grtt_age += probe_timer.GetInterval();
}
else
{
double deltaTime = currentTime.tv_sec - probe_time_last.tv_sec;
if (currentTime.tv_usec > probe_time_last.tv_usec)
deltaTime += 1.0e-06*((double)(currentTime.tv_usec - probe_time_last.tv_usec));
else
deltaTime -= 1.0e-06*((double)(probe_time_last.tv_usec - currentTime.tv_usec));
grtt_age += deltaTime;
}
probe_time_last = currentTime;
// (TBD) We need to revisit the whole set of issues surrounding dynamic
// estimation of grtt, particularly when congestion control is involved.
// The main issue is when the rate increases rapidly with respect to
// how the grtt estimate is descreasing ... this is most notable at
// startup and thus the hack here to allow the grtt estimate to more
// rapidly decrease during "slow start"
double ageMax = grtt_advertised;
if (!cc_enable && !cc_slow_start)
ageMax = ageMax > grtt_interval_min ? ageMax : grtt_interval_min;
if (grtt_age >= ageMax)
{
if (grtt_response)
{
// Update grtt estimate
if (grtt_current_peak < grtt_measured)
{
grtt_measured *= 0.9;
if (grtt_current_peak > grtt_measured)
grtt_measured = grtt_current_peak;
// (TBD) "grtt_decrease_delay_count" isn't needed any more ...
/*if (grtt_decrease_delay_count-- == 0)
{
grtt_measured = 0.5 * grtt_measured +
0.5 * grtt_current_peak;
grtt_current_peak = 0.0;
grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY;
}*/
}
else
{
// Increase already incorporated
grtt_current_peak = 0.0;
grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY;
}
if (grtt_measured < NORM_GRTT_MIN)
grtt_measured = NORM_GRTT_MIN;
else if (grtt_measured > grtt_max)
grtt_measured = grtt_max;
UINT8 grttQuantizedOld = grtt_quantized;
double pktInterval = (double)(44+segment_size)/tx_rate;
if (grtt_measured < pktInterval)
grtt_quantized = NormQuantizeRtt(pktInterval);
else
grtt_quantized = NormQuantizeRtt(grtt_measured);
// Recalculate grtt_advertise since quantization rounds upward
grtt_advertised = NormUnquantizeRtt(grtt_quantized);
if (grtt_advertised > grtt_max)
{
grtt_quantized = NormQuantizeRtt(grtt_max);
grtt_advertised = NormUnquantizeRtt(grtt_quantized);
}
if (grttQuantizedOld != grtt_quantized)
{
Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL);
PLOG(PL_DEBUG, "NormSession::OnProbeTimeout() node>%lu decreased to new grtt to: %lf sec\n",
(unsigned long)LocalNodeId(), grtt_advertised);
}
grtt_response = false; // reset
}
grtt_age = 0.0;
}
if (grtt_interval < grtt_interval_min)
grtt_interval = grtt_interval_min;
else
grtt_interval *= 1.5;
if (grtt_interval > grtt_interval_max)
grtt_interval = grtt_interval_max;
// 3) Build a NORM_CMD(CC) message
NormCmdCCMsg* cmd = (NormCmdCCMsg*)GetMessageFromPool();
if (!cmd)
{
PLOG(PL_FATAL, "NormSession::OnProbeTimeout() node>%lu message_pool empty! can't probe\n",
(unsigned long)LocalNodeId());
ASSERT(0);
return true;
}
cmd->Init();
cmd->SetDestination(address);
cmd->SetGrtt(grtt_quantized);
cmd->SetBackoffFactor((unsigned char)backoff_factor);
cmd->SetGroupSize(gsize_quantized);
// defer SetSendTime() to when message is being sent (in OnTxTimeout())
cmd->SetCCSequence(cc_sequence++);
// Insert NORM-CC header extension, if applicable
// (Note we set the extension "rate" _after_ AdjustRate() done below)
NormCCRateExtension ext;
if (probe_proactive)
cmd->AttachExtension(ext);
if (cc_enable)
{
// Iterate over cc_node_list and append cc_nodes ...
// (we also check cc_node "activity status here)
NormNodeListIterator iterator(cc_node_list);
NormCCNode* next;
while ((next = (NormCCNode*)iterator.GetNextNode()))
{
if (next->IsActive())
{
UINT8 ccFlags = 0;
if (next->IsClr())
{
ccFlags |= (UINT8)NormCC::CLR;
}
else if (next->IsPlr())
{
ccFlags |= (UINT8)NormCC::PLR;
}
ccFlags |= (UINT8)NormCC::RTT;
UINT8 rttQuantized = NormQuantizeRtt(next->GetRtt());
if (cc_slow_start) ccFlags |= (UINT8)NormCC::START;
UINT16 rateQuantized = NormQuantizeRate(next->GetRate());
// (TBD) check result
cmd->AppendCCNode(segment_size,
next->GetId(),
ccFlags,
rttQuantized,
rateQuantized);
//if (!next->IsClr()) next->SetActive(false);
// "Deactivate" any nodes who have stopped providing feedback
struct timeval feedbackTime = next->GetFeedbackTime();
double feedbackAge = currentTime.tv_sec - feedbackTime.tv_sec;
feedbackAge += 1.0e-06 * ((double)((currentTime.tv_usec - feedbackTime.tv_usec)));
/*if (currentTime.tv_usec > feedbackTime.tv_usec)
feedbackAge += 1.0e-06*((double)(currentTime.tv_usec - feedbackTime.tv_usec));
else
feedbackAge -= 1.0e-06*((double)(feedbackTime.tv_usec - currentTime.tv_usec));*/
double maxFeedbackAge = 20 * MAX(grtt_advertised, next->GetRtt());
// Safety bound to compensate for computer clock coarseness
// and possible sluggish feedback from slower machines
// at higher norm data rates (keeps rate from being
// prematurely reduced)
if (maxFeedbackAge <(10*NORM_TICK_MIN)) maxFeedbackAge = (10*NORM_TICK_MIN);
INT16 ccSeqDelta = cc_sequence - next->GetCCSequence();
if ((feedbackAge > maxFeedbackAge) && (ccSeqDelta > (INT16)(20 * probe_count)))
{
PLOG(PL_DEBUG, "Deactivating cc node feedbackAge:%lf sec maxAge:%lf sec ccSeqDelta:%u\n",
feedbackAge, maxFeedbackAge, ccSeqDelta);
next->SetActive(false);
}
}
}
AdjustRate(false);
} // end if (cc_enable)
if (probe_proactive)
ext.SetSendRate(NormQuantizeRate(tx_rate));
double probeInterval = GetProbeInterval();
/*// perhaps this instead of the commented out probe_reset case???
double nominalInterval = ((double)segment_size)/((double)tx_rate);
if (nominalInterval > grtt_max) nominalInterval = grtt_max;
if (nominalInterval > probeInterval) probeInterval = nominalInterval; */
// Set probe_timer interval for next probe
probe_timer.SetInterval(probeInterval);
QueueMessage(cmd);
probe_pending = true;
return true;
} // end NormSession::OnProbeTimeout()
double NormSession::GetProbeInterval()
{
if (cc_enable && data_active)
{
const NormCCNode* clr = static_cast<const NormCCNode*>(cc_node_list.Head());
if (NULL != clr)
{
double probeInterval = (clr->IsActive() ? MIN(grtt_advertised, clr->GetRtt()) : grtt_advertised);
// For "large" RTT (100 msec or bigger), we need to possibly probe more
// often depending on transmit rate, to make NORM-CC perform a little better
// The "probeCount" calculated here is based on sending no more probes per RTT
// than 0.25 * the number of data packets that would be sent per RTT.
// (although we floor the probeCount at 1 (i.e., the usual 1 probe per RTT)
// Note that no more than a few (e.g. 3) probes per RTT provides performance benefit
unsigned int probeCount = (unsigned int)(0.25 * tx_rate * probeInterval / (double)segment_size);
if (probeCount < 1) probeCount = 1;
if (clr->GetRtt() > 0.200)
{
if (probeCount > 3) probeCount = 3;
}
else if (clr->GetRtt() > 0.100)
{
if (probeCount > 2) probeCount = 2;
}
else
{
probeCount = 1;
}
if (1 != probe_count) probeCount = probe_count;
// Don't send more than one CLR probe per RTT during slow_start
return (cc_slow_start ? probeInterval : (probeInterval / (double)probeCount));
}
else
{
return grtt_advertised;
}
}
else
{
return grtt_interval;
}
} // end NormSession::GetProbeInterval()
void NormSession::AdjustRate(bool onResponse)
{
const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head();
double ccRtt = clr ? clr->GetRtt() : grtt_measured;
double ccLoss = clr ? clr->GetLoss() : 0.0;
double txRate = tx_rate;
if (onResponse)
{
if (!cc_active)
{
cc_active = true;
Notify(NormController::CC_ACTIVE, NULL, NULL);
}
if (data_active) // adjust only if actively transmitting
{
// Adjust rate based on CLR feedback and
// adjust probe schedule
ASSERT(NULL != clr);
// (TBD) check feedback age
if (cc_slow_start)
{
txRate = clr->GetRate();
if (GetDebugLevel() >= 6)
{
double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / (report_timer.GetInterval() - report_timer.GetTimeRemaining()));
PLOG(PL_DETAIL, "NormSession::AdjustRate(slow start) clr>%lu newRate>%lf (oldRate>%lf sentRate>%lf clrRate>%lf\n",
(unsigned long)clr->GetId(), 8.0e-03*txRate, 8.0e-03*tx_rate, sentRate, 8.0e-03*clr->GetRate());
}
}
else
{
double clrRate = clr->GetRate();
if (clrRate > txRate)
{
double maxRate = txRate*2;
txRate = MIN(clrRate, maxRate);
}
else
{
txRate = clrRate;
}
// Here, we use the most recent CLR rtt sample to "damp" oscillation
double damper = clr->GetRttSqMean() / sqrt(clr->GetRttSample());
if (damper < 0.5)
damper = 0.5;
else if (damper > 2.0)
damper = 2.0;
txRate *= damper;
PLOG(PL_DETAIL, "NormSession::AdjustRate(stdy state) clr>%lu newRate>%lf (rtt>%lf loss>%lf)\n",
(unsigned long)clr->GetId(), 8.0e-03*txRate, clr->GetRtt(), clr->GetLoss());
}
}
if (!address.IsMulticast())
{
// For unicast, adjust the probe timeout right away
double probeInterval = GetProbeInterval(); // based on CLR RTT, etc
if (probe_timer.GetInterval() > probeInterval)
{
// reduce to speed up rate increase
double elapsed = probe_timer.GetInterval() - probe_timer.GetTimeRemaining();
if (probeInterval > elapsed)
probeInterval -= elapsed;
else
probeInterval = 0.0;
probe_timer.SetInterval(probeInterval);
if (probe_timer.IsActive()) probe_timer.Reschedule();
}
}
}
else if (!data_active)
{
// reduce rate if no active data transmission
// (TBD) Perhaps we want to be less aggressive here someday
//TRACE("DATA INACTIVITY!\n");
txRate *= 0.5;
}
else if (clr && clr->IsActive())
{
// (TBD) fix CC feedback aging ...
/*int feedbackAge = abs((int)cc_sequence - (int)clr->GetCCSequence());
DMSG(0, "NormSession::AdjustRate() feedback age>%d (%d - %d\n",
feedbackAge, cc_sequence, clr->GetCCSequence());
if (feedbackAge > 50)
{
double linRate = txRate - segment_size;
linRate = MAX(linRate, 0.0);
double expRate = txRate * 0.5;
if (feedbackAge > 4)
txRate = MIN(linRate, expRate);
else
txRate = MAX(linRate, expRate);
}*/
}
else
{
// reduce rate by half if no active clr
txRate *= 0.5;
}
// Keep "tx_rate" within default or user set rate bounds (if any)
double minRate;
if(tx_rate_min > 0.0)
{
minRate = tx_rate_min;
}
else
{
// Don't let txRate below MIN(one segment per grtt, one segment per second)
if (grtt_measured > 1.0)
minRate = ((double)segment_size) / grtt_measured;
else
minRate = (double)(segment_size);
}
if (txRate <= minRate)
{
txRate = minRate;
if ((NULL == clr) || (!clr->IsActive()))
{
// Post notification that no cc feedback is being received
if (cc_active)
{
cc_active = false;
Notify(NormController::CC_INACTIVE, NULL, NULL);
}
}
}
if ((tx_rate_max >= 0.0) && (txRate > tx_rate_max))
txRate = tx_rate_max;
if (txRate != tx_rate)
{
if (cc_adjust) SetTxRateInternal(txRate);
if (!posted_tx_rate_changed)
{
// TBD - make API notification filtering more consistent
// (e.g., "notify_on_rate_update" like for grtt, etc
// putting API code in charge of resetting these API
// state variables).
posted_tx_rate_changed = true;
Notify(NormController::TX_RATE_CHANGED, (NormSenderNode*)NULL, (NormObject*)NULL);
}
}
struct timeval currentTime;
::ProtoSystemTime(currentTime);
double theTime = (double)currentTime.tv_sec + 1.0e-06 * ((double)currentTime.tv_usec);
PLOG(PL_DEBUG, "SenderRateTracking time>%lf rate>%lf rtt>%lf loss>%lf\n", theTime, 8.0e-03*txRate, ccRtt, ccLoss);
//TRACE("SenderRateTracking time>%lf rate>%lf rtt>%lf loss>%lf\n", theTime, 8.0e-03*txRate, ccRtt, ccLoss);
//double calcRate = NormSession::CalculateRate(nominal_packet_size, ccRtt, ccLoss);
//TRACE("SenderRateTracking time>%lf rate>%lf clrRate>%lf rtt>%lf loss>%lf calcRate>%lf size>%lf%s\n\n",
// theTime, 8.0e-03*txRate, clr ? 8.0e-03*clr->GetRate() : 0.0, ccRtt, ccLoss, calcRate*8.0e-03,
// nominal_packet_size, cc_slow_start ? " (slow start)" : "");
//ASSERT((NULL == clr) || (txRate <= clr->GetRate()));
} // end NormSession::AdjustRate()
bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/)
{
// Receiver reporting (just print out for now)
struct timeval currentTime;
ProtoSystemTime(currentTime);
#ifdef _WIN32_WCE
struct tm timeStruct;
timeStruct.tm_hour = currentTime.tv_sec / 3600;
unsigned long hourSecs = 3600 * timeStruct.tm_hour;
timeStruct.tm_min = (currentTime.tv_sec - (hourSecs)) / 60;
timeStruct.tm_sec = currentTime.tv_sec - (hourSecs) - (60*timeStruct.tm_min);
timeStruct.tm_hour = timeStruct.tm_hour % 24;
struct tm* ct = &timeStruct;
#else
time_t secs = (time_t)currentTime.tv_sec;
struct tm timeStruct;
#ifdef WIN32
gmtime_s(&timeStruct, &secs);
struct tm* ct = &timeStruct;
#else
struct tm* ct = gmtime_r(&secs, &timeStruct);
#endif
#endif // if/else _WIN32_WCE
ASSERT(NULL != ct);
ProtoDebugLevel reportDebugLevel = PL_INFO;
PLOG(reportDebugLevel, "REPORT time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n",
ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, (unsigned long)LocalNodeId());
if (IsSender())
{
PLOG(reportDebugLevel, "Local status:\n");
double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / report_timer.GetInterval()); // kbps
sent_accumulator.Reset();
PLOG(reportDebugLevel, " txRate>%9.3lf kbps sentRate>%9.3lf grtt>%lf\n",
8.0e-03*tx_rate, sentRate, grtt_advertised);
if (cc_enable)
{
const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head();
if (clr)
{
PLOG(reportDebugLevel, " clr>%lu rate>%9.3lf rtt>%lf loss>%lf %s\n",
(unsigned long)clr->GetId(), 8.0e-03*clr->GetRate(),
clr->GetRtt(), clr->GetLoss(), cc_slow_start ? "(slow_start)" : "");
}
}
}
if (IsReceiver())
{
NormNodeTreeIterator iterator(sender_tree);
NormSenderNode* next;
while ((next = (NormSenderNode*)iterator.GetNextNode()))
{
PLOG(reportDebugLevel, "Remote sender>%lu grtt>%lf sec loss>%lf\n", (unsigned long)next->GetId(),
next->GetGrttEstimate(), next->LossEstimate());
// TBD - Output sender congestion control status if cc is enabled
double rxRate = 8.0e-03*next->GetRecvRate(report_timer.GetInterval()); // kbps
double rxGoodput = 8.0e-03*next->GetRecvGoodput(report_timer.GetInterval()); // kbps
next->ResetRecvStats();
PLOG(reportDebugLevel, " rxRate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput);
PLOG(reportDebugLevel, " rxObjects> completed>%lu pending>%lu failed>%lu\n",
next->CompletionCount(), next->PendingCount(), next->FailureCount());
PLOG(reportDebugLevel, " fecBufferUsage> current>%lu peak>%lu overuns>%lu\n",
next->CurrentBufferUsage(), next->PeakBufferUsage(),
next->BufferOverunCount());
PLOG(reportDebugLevel, " strBufferUsage> current>%lu peak>%lu overuns>%lu\n",
next->CurrentStreamBufferUsage(), next->PeakStreamBufferUsage(),
next->StreamBufferOverunCount());
PLOG(reportDebugLevel, " resyncs>%lu nacks>%lu suppressed>%lu\n",
next->ResyncCount() ? next->ResyncCount() - 1 : 0, // "ResyncCount()" is really "SyncCount()"
next->NackCount(), next->SuppressCount());
// Some stream status for current receive stream (if applicable)
NormObject* obj = next->GetNextPendingObject();
if ((NULL != obj) && obj->IsStream())
{
NormStreamObject* stream = (NormStreamObject*)obj;
PLOG(reportDebugLevel, " stream_sync_id>%lu stream_next_id>%lu read_index:%lu.%hu\n",
(unsigned long)stream->GetSyncId().GetValue(),
(unsigned long)stream->GetNextId().GetValue(),
(unsigned long)stream->GetNextBlockId().GetValue(),
(UINT16)stream->GetNextSegmentId());
}
}
} // end if (IsReceiver())
PLOG(reportDebugLevel, "***************************************************************************\n");
return true;
} // end NormSession::OnReportTimeout()
bool NormSession::OnUserTimeout(ProtoTimer& /*theTimer*/)
{
Notify(NormController::USER_TIMEOUT, (NormSenderNode*)NULL, (NormObject*)NULL);
return true;
} // end NormSession::OnUserTimeout(
NormSessionMgr::NormSessionMgr(ProtoTimerMgr& timerMgr,
ProtoSocket::Notifier& socketNotifier,
ProtoChannel::Notifier* channelNotifier)
: timer_mgr(timerMgr), socket_notifier(socketNotifier), channel_notifier(channelNotifier),
controller(NULL), data_free_func(NULL), top_session(NULL)
{
}
NormSessionMgr::~NormSessionMgr()
{
Destroy();
}
void NormSessionMgr::Destroy()
{
NormSession* next;
while ((next = top_session))
{
top_session = next->next;
delete next;
}
} // end NormSessionMgr::Destroy()
NormSession* NormSessionMgr::NewSession(const char* sessionAddress,
UINT16 sessionPort,
NormNodeId localNodeId)
{
if ((NORM_NODE_ANY == localNodeId) || (NORM_NODE_NONE == localNodeId))
{
#ifndef SIMULATE
// Use local ip address to assign default localNodeId
ProtoAddress localAddr;
if (!localAddr.ResolveLocalAddress())
{
PLOG(PL_ERROR, "NormSessionMgr::NewSession() local address lookup error\n");
return ((NormSession*)NULL);
}
// (TBD) test IPv6 "EndIdentifier" ???
localNodeId = localAddr.EndIdentifier();
#else
localNodeId = NORM_NODE_ANY - 1;
#endif
}
ProtoAddress theAddress;
if (!theAddress.ResolveFromString(sessionAddress))
{
PLOG(PL_ERROR, "NormSessionMgr::NewSession() session address \"%s\" lookup error!\n", sessionAddress);
return ((NormSession*)NULL);
}
theAddress.SetPort(sessionPort);
NormSession* theSession = new NormSession(*this, localNodeId);
if (!theSession)
{
PLOG(PL_ERROR, "NormSessionMgr::NewSession() new session error: %s\n", GetErrorString());
return ((NormSession*)NULL);
}
theSession->SetAddress(theAddress);
// Add new session to our session list
theSession->next = top_session;
top_session = theSession;
return theSession;
} // end NormSessionMgr::NewSession()
void NormSessionMgr::DeleteSession(class NormSession* theSession)
{
NormSession* prev = NULL;
NormSession* next = top_session;
while (next && (next != theSession))
{
prev = next;
next = next->next;
}
if (next)
{
if (prev)
prev->next = theSession->next;
else
top_session = theSession->next;
delete theSession;
}
} // end NormSessionMgr::DeleteSession()
Reference in New Issue View Git Blame Copy Permalink