From d0417b042ae8ffedca44c143f01a644c4a9f13c9 Mon Sep 17 00:00:00 2001 From: Jeff Weston Date: Wed, 11 Sep 2019 11:25:00 -0400 Subject: [PATCH] v1.0b2 --- common/galois.h | 6 +- common/normApp.cpp | 611 +++++++++++--- common/normBitmask.cpp | 73 +- common/normBitmask.h | 3 + common/normEncoder.cpp | 35 +- common/normEncoder.h | 10 +- common/normFile.cpp | 15 +- common/normMessage.cpp | 253 ++++-- common/normMessage.h | 1443 +++++++++++++++++++++------------ common/normNode.cpp | 1456 ++++++++++++++++++++++++++++------ common/normNode.h | 253 +++++- common/normObject.cpp | 1043 +++++++++++++++++------- common/normObject.h | 100 ++- common/normPostProcess.cpp | 165 ++++ common/normPostProcess.h | 35 + common/normSegment.cpp | 181 ++++- common/normSegment.h | 33 +- common/normSession.cpp | 1534 ++++++++++++++++++++++++++---------- common/normSession.h | 125 ++- common/normSimAgent.cpp | 451 +++++++++-- common/normSimAgent.h | 39 +- common/version.h | 2 +- ns/README.txt | 5 +- ns/example.tcl | 8 +- ns/normcc.tcl | 253 ++++++ ns/ns-2.1b9-Makefile.in | 536 +++++++++++++ ns/nsNormAgent.cpp | 54 +- ns/nsNormAgent.h | 17 +- ns/simplenorm.tcl | 38 +- ns/suppress.tcl | 35 +- unix/Makefile.common | 14 +- unix/Makefile.freebsd | 2 + unix/Makefile.hpux | 2 + unix/Makefile.linux | 10 +- unix/Makefile.macosx | 43 +- unix/Makefile.mklinux | 2 + unix/Makefile.netbsd | 4 +- unix/Makefile.sgi | 39 +- unix/Makefile.solaris | 2 + unix/Makefile.solx86 | 2 + 40 files changed, 6936 insertions(+), 1996 deletions(-) create mode 100644 common/normPostProcess.cpp create mode 100644 common/normPostProcess.h create mode 100644 ns/normcc.tcl create mode 100644 ns/ns-2.1b9-Makefile.in diff --git a/common/galois.h b/common/galois.h index 076da09..8e008d2 100644 --- a/common/galois.h +++ b/common/galois.h @@ -35,9 +35,9 @@ namespace Norm { -extern const unsigned char GINV[256]; -extern const unsigned char GEXP[512]; -extern const unsigned char GMULT[256][256]; + extern const unsigned char GINV[256]; + extern const unsigned char GEXP[512]; + extern const unsigned char GMULT[256][256]; } inline unsigned char gexp(unsigned int x) {return Norm::GEXP[x];} inline unsigned char gmult(unsigned int x, unsigned int y) {return Norm::GMULT[x][y];} diff --git a/common/normApp.cpp b/common/normApp.cpp index 3ccb1dd..55be57b 100644 --- a/common/normApp.cpp +++ b/common/normApp.cpp @@ -3,12 +3,12 @@ #include "protoLib.h" #include "normSession.h" +#include "normPostProcess.h" #include // for stdout/stderr printouts #include // for SIGTERM/SIGINT handling #include - // Command-line application using Protolib EventDispatcher class NormApp : public NormController { @@ -23,7 +23,13 @@ class NormApp : public NormController bool ProcessCommand(const char* cmd, const char* val); bool ParseCommandLine(int argc, char* argv[]); + static void DoInputReady(EventDispatcher::Descriptor descriptor, + EventDispatcher::EventType theEvent, + const void* userData); + private: + void OnInputReady(); + enum CmdType {CMD_INVALID, CMD_NOARG, CMD_ARG}; CmdType CommandType(const char* cmd); @@ -40,41 +46,62 @@ class NormApp : public NormController static const char* const cmd_list[]; static void SignalHandler(int sigNum); + bool IsPostProcessing() {return post_processor.IsActive();} + void HandleSIGCHLD() {post_processor.HandleSIGCHLD();} EventDispatcher dispatcher; NormSessionMgr session_mgr; NormSession* session; - NormStreamObject* stream; + NormStreamObject* tx_stream; + NormStreamObject* rx_stream; // application parameters FILE* input; // input stream FILE* output; // output stream - char input_buffer[512]; + char input_buffer[1250]; unsigned int input_index; unsigned int input_length; + bool input_active; + bool push_stream; + bool input_messaging; // stream input mode + UINT16 input_msg_length; + UINT16 input_msg_index; + char output_buffer[65536]; + UINT16 output_index; + bool output_messaging; + UINT16 output_msg_length; + bool output_msg_sync; - // NormSession parameters + // NormSession common parameters char* address; // session address UINT16 port; // session port number UINT8 ttl; double tx_rate; // bits/sec + bool cc_enable; + + // NormSession server-only parameters UINT16 segment_size; UINT8 ndata; UINT8 nparity; UINT8 auto_parity; + UINT8 extra_parity; + double backoff_factor; unsigned long tx_buffer_size; // bytes - unsigned long rx_buffer_size; // bytes - NormFileList tx_file_list; double tx_object_interval; int tx_repeat_count; double tx_repeat_interval; + ProtocolTimer interval_timer; + + // NormSession client-only parameters + unsigned long rx_buffer_size; // bytes NormFileList rx_file_cache; char* rx_cache_path; + NormPostProcessor post_processor; + bool unicast_nacks; + bool silent_client; - ProtocolTimer interval_timer; - - // protocol debug parameters + // Debug parameters bool tracing; double tx_loss; double rx_loss; @@ -82,13 +109,16 @@ class NormApp : public NormController }; // end class NormApp NormApp::NormApp() - : session(NULL), stream(NULL), input(NULL), output(NULL), - input_index(0), input_length(0), - address(NULL), port(0), ttl(3), tx_rate(64000.0), - segment_size(1024), ndata(32), nparity(16), auto_parity(0), - tx_buffer_size(1024*1024), rx_buffer_size(1024*1024), + : session(NULL), tx_stream(NULL), rx_stream(NULL), input(NULL), output(NULL), + input_index(0), input_length(0), input_active(false), + push_stream(false), input_messaging(false), input_msg_length(0), input_msg_index(0), + output_index(0), output_messaging(false), output_msg_length(0), output_msg_sync(false), + address(NULL), port(0), ttl(3), tx_rate(64000.0), cc_enable(false), + segment_size(1024), ndata(32), nparity(16), auto_parity(0), extra_parity(0), + backoff_factor(NormSession::DEFAULT_BACKOFF_FACTOR), + tx_buffer_size(1024*1024), tx_object_interval(0.0), tx_repeat_count(0), tx_repeat_interval(0.0), - rx_cache_path(NULL), + rx_buffer_size(1024*1024), rx_cache_path(NULL), unicast_nacks(false), silent_client(false), tracing(false), tx_loss(0.0), rx_loss(0.0) { // Init tx_timer for 1.0 second interval, infinite repeats @@ -97,6 +127,10 @@ NormApp::NormApp() this); interval_timer.Init(0.0, 0, (ProtocolTimerOwner*)this, (ProtocolTimeoutFunc)&NormApp::OnIntervalTimeout); + + struct timeval currentTime; + GetSystemTime(¤tTime); + srand(currentTime.tv_usec); } NormApp::~NormApp() @@ -108,27 +142,36 @@ NormApp::~NormApp() const char* const NormApp::cmd_list[] = { - "+debug", - "+log", - "-trace", - "+txloss", - "+rxloss", - "+address", - "+ttl", - "+rate", - "+input", - "+output", - "+sendfile", - "+interval", - "+repeatcount", - "+rinterval", // repeat interval - "+rxcachedir", - "+segment", - "+block", - "+parity", - "+auto", - "+txbuffer", - "+rxbuffer", + "+debug", // debug level + "+log", // log file name + "-trace", // message tracing on + "+txloss", // tx packet loss percent (for testing) + "+rxloss", // rx packet loss percent (for testing) + "+address", // session destination address + "+ttl", // multicast hop count scope + "+cc", // congestion control on/off + "+rate", // tx date rate (bps) + "-push", // push stream writes for real-time messaging + "+input", // send stream input + "+output", // recv stream output + "+minput", // send message stream input + "+moutput", // recv message stream output + "+sendfile", // file/directory list to transmit + "+interval", // delay time (sec) between files (0.0 sec default) + "+repeatcount", // How many times to repeat the file/directory list + "+rinterval", // Interval (sec) between file/directory list repeats + "+rxcachedir", // recv file cache directory + "+segment", // payload segment size (bytes) + "+block", // User data packets per FEC coding block (blockSize) + "+parity", // FEC packets calculated per coding block (nparity) + "+auto", // Number of FEC packets to proactively send (<= nparity) + "+extra", // Number of extra FEC packets sent in response to repair requests + "+backoff", // Backoff factor to use + "+txbuffer", // Size of sender's buffer + "+rxbuffer", // Size receiver allocates for buffering each sender + "-unicastNacks", // unicast instead of multicast feedback messages + "-silentClient", // "silent" (non-nacking) client (EMCON mode) + "+processor", // receive file post processing command NULL }; @@ -236,23 +279,90 @@ bool NormApp::ProcessCommand(const char* cmd, const char* val) tx_rate = txRate; if (session) session->SetTxRate(txRate); } + else if (!strncmp("cc", cmd, len)) + { + if (!strcmp("on", val)) + { + cc_enable = true; + } + else if (!strcmp("off", val)) + { + cc_enable = false; + } + else + { + DMSG(0, "NormApp::ProcessCommand(cc) invalid option!\n"); + return false; + } + if (session) session->SetCongestionControl(cc_enable); + } else if (!strncmp("input", cmd, len)) { - if (!(input = fopen(val, "rb"))) + if (!strcmp(val, "STDIN")) + { + input = stdin; + } + else if (!(input = fopen(val, "rb"))) { DMSG(0, "NormApp::ProcessCommand(input) fopen() error: %s\n", strerror(errno)); return false; } + input_index = input_length = 0; + input_messaging = false; + // Set input non-blocking + if(-1 == fcntl(fileno(input), F_SETFL, fcntl(fileno(input), F_GETFL, 0) | O_NONBLOCK)) + perror("NormApp::ProcessCommand(input) fcntl(F_SETFL(O_NONBLOCK)) error"); } else if (!strncmp("output", cmd, len)) { - if (!(output = fopen(val, "wb"))) + if (!strcmp(val, "STDOUT")) + { + output = stdout; + } + else if (!(output = fopen(val, "wb"))) { DMSG(0, "NormApp::ProcessCommand(output) fopen() error: %s\n", strerror(errno)); return false; } + output_messaging = false; + output_index = 0; + output_msg_sync = true; + } + else if (!strncmp("minput", cmd, len)) + { + if (!strcmp(val, "STDIN")) + { + input = stdin; + } + else if (!(input = fopen(val, "rb"))) + { + DMSG(0, "NormApp::ProcessCommand(input) fopen() error: %s\n", + strerror(errno)); + return false; + } + input_index = input_length = 0; + input_messaging = true; + // Set input non-blocking + if(-1 == fcntl(fileno(input), F_SETFL, fcntl(fileno(input), F_GETFL, 0) | O_NONBLOCK)) + perror("NormApp::ProcessCommand(input) fcntl(F_SETFL(O_NONBLOCK)) error"); + } + else if (!strncmp("moutput", cmd, len)) + { + if (!strcmp(val, "STDOUT")) + { + output = stdout; + } + else if (!(output = fopen(val, "wb"))) + { + DMSG(0, "NormApp::ProcessCommand(output) fopen() error: %s\n", + strerror(errno)); + return false; + } + output_messaging = true; + output_msg_length = output_index = 0; + output_msg_sync = false; } else if (!strncmp("sendfile", cmd, len)) { @@ -350,6 +460,28 @@ bool NormApp::ProcessCommand(const char* cmd, const char* val) auto_parity = autoParity; if (session) session->ServerSetAutoParity(autoParity); } + else if (!strncmp("extra", cmd, len)) + { + int extraParity = atoi(val); + if ((extraParity < 0) || (extraParity > 254)) + { + DMSG(0, "NormApp::ProcessCommand(extra) invalid value!\n"); + return false; + } + extra_parity = extraParity; + if (session) session->ServerSetExtraParity(extraParity); + } + else if (!strncmp("backoff", cmd, len)) + { + double backoffFactor = atof(val); + if (backoffFactor < 0) + { + DMSG(0, "NormSimAgent::ProcessCommand(backoff) invalid txRate!\n"); + return false; + } + backoff_factor = backoffFactor; + if (session) session->SetBackoffFactor(backoffFactor); + } else if (!strncmp("txbuffer", cmd, len)) { if (1 != sscanf(val, "%lu", &tx_buffer_size)) @@ -366,6 +498,28 @@ bool NormApp::ProcessCommand(const char* cmd, const char* val) return false; } } + else if (!strncmp("unicastNacks", cmd, len)) + { + unicast_nacks = true; + if (session) session->SetUnicastNacks(true); + } + else if (!strncmp("silentClient", cmd, len)) + { + silent_client = true; + if (session) session->ClientSetSilent(true); + } + else if (!strncmp("push", cmd, len)) + { + push_stream = true; + } + else if (!strncmp("processor", cmd, len)) + { + if (!post_processor.SetCommand(val)) + { + DMSG(0, "NormApp::ProcessCommand(processor) error!\n"); + return false; + } + } return true; } // end NormApp::ProcessCommand() @@ -436,6 +590,159 @@ bool NormApp::ParseCommandLine(int argc, char* argv[]) return true; } // end NormApp::ParseCommandLine() +void NormApp::DoInputReady(EventDispatcher::Descriptor /*descriptor*/, + EventDispatcher::EventType /*theEvent*/, + const void* userData) +{ + ((NormApp*)userData)->OnInputReady(); +} // end NormApp::DoDataReady() + + +void NormApp::OnInputReady() +{ + + //TRACE("NormApp::OnInputReady() ...\n"); + bool flush = false; + // write to the stream while input is available _and_ + // the stream has buffer space for input + while (input) + { + if (0 == input_length) + { + // We need input ... + UINT16 readLength; + if (input_messaging) + { + if (input_msg_length) + { + UINT16 bufferSpace = 1024 - input_index; + UINT16 msgRemainder = input_msg_length - input_msg_index; + readLength = MIN(bufferSpace, msgRemainder); + } + else if (input_index < 2) + { + readLength = 2 - input_index; + } + else + { + input_msg_length = ntohs(*((UINT16*)input_buffer)); + ASSERT(input_msg_length >= 2); + UINT16 bufferSpace = 1024 - input_index; + UINT16 msgRemainder = input_msg_length - 2; + readLength = MIN(bufferSpace, msgRemainder); + } + } + else + { + readLength = 1024; + } + + // Read from "input" into our "input_buffer" + size_t result = fread(input_buffer+input_index, sizeof(char), readLength, input); + if (result > 0) + { + if (input_messaging) + { + if (input_msg_length) + { + input_length = input_index + result; + input_index = 0; + } + else + { + input_length = 0; + input_index += result; + } + } + else + { + input_length = input_index + result; + input_index = 0; + } + } + else + { + if (feof(input)) + { + DMSG(0, "norm: input end-of-file.\n"); + if (input_active) + { + dispatcher.RemoveGenericInput(fileno(input)); + input_active = false; + } + if (stdin != input) fclose(input); + input = NULL; + flush = true; + } + else if (ferror(input)) + { + switch (errno) + { + case EINTR: + case EAGAIN: + // Input not ready, will get notification when ready + break; + default: + DMSG(0, "norm: input error:%s\n", strerror(errno)); + break; + } + clearerr(input); + } + } + } // end if (0 == input_length) + + unsigned int writeLength = input_length;// ? input_length - input_index : 0; + + if (writeLength || flush) + { + unsigned int wroteLength = tx_stream->Write(input_buffer+input_index, writeLength, flush, false, push_stream); + input_length -= wroteLength; + if (0 == input_length) + input_index = 0; + else + input_index += wroteLength; + if (input_messaging) + { + input_msg_index += wroteLength; + if (input_msg_index == input_msg_length) + { + input_msg_index = 0; + input_msg_length = 0; + // Mark EOM + tx_stream->Write(NULL, 0, false, true, false); + } + } + if (wroteLength < writeLength) + { + // Stream buffer full, temporarily deactive "input" and + // wait for next TX_QUEUE_EMPTY notification + //TRACE("stream buffer full ...\n"); + if (input_active) + { + dispatcher.RemoveGenericInput(fileno(input)); + input_active = false; + } + break; + } + } + else + { + // Input not ready, wait for "input" to be ready + // (Activate/reactivate "input" as necessary + //TRACE("input starved ...\n"); + if (!input_active) + { + if (dispatcher.AddGenericInput(fileno(input), NormApp::DoInputReady, this)) + input_active = true; + else + DMSG(0, "NormApp::Notify(TX_QUEUE_EMPTY) error adding input notification!\n"); + } + break; + } + } // end while (1) +} // NormApp::OnInputReady() + + void NormApp::Notify(NormController::Event event, class NormSessionMgr* sessionMgr, class NormSession* session, @@ -447,45 +754,14 @@ void NormApp::Notify(NormController::Event event, case TX_QUEUE_EMPTY: // Write to stream as needed //DMSG(0, "NormApp::Notify(TX_QUEUE_EMPTY) ...\n"); - if (object && (object == stream)) + if (object && (object == tx_stream)) { - bool flush = false; - if (input) - { - if (input_index >= input_length) - { - size_t result = fread(input_buffer, sizeof(char), 512, input); - if (result) - { - input_index = 0; - input_length = result; - } - else - { - input_length = input_index = 0; - if (feof(input)) - { - DMSG(0, "norm: input end-of-file.\n"); - fclose(input); - input = NULL; - flush = true; - } - else if (ferror(input)) - { - DMSG(0, "norm: input error:%s\n", strerror(errno)); - clearerr(input); - break; - } - } - } - input_index += stream->Write(input_buffer+input_index, - input_length-input_index, - flush); - } // end if (input) + if (input) OnInputReady(); } else { // Can queue a new object for transmission + if (interval_timer.IsActive()) interval_timer.Deactivate(); if (interval_timer.Interval() > 0.0) { InstallTimer(&interval_timer); @@ -505,8 +781,23 @@ void NormApp::Notify(NormController::Event event, { case NormObject::STREAM: { - const NormObjectSize& size = object->Size(); - if (!((NormStreamObject*)object)->Accept(size.LSB())) + // Only have one rx_stream at a time for now. + // Reset stream i/o mgmt + output_msg_length = output_index = 0; + if (output_messaging) output_msg_sync = false; + + // object Size() has recommended buffering size + NormObjectSize size; + if (silent_client) + size = NormObjectSize(rx_buffer_size); + else + size = object->Size(); + + if (((NormStreamObject*)object)->Accept(size.LSB())) + { + rx_stream = (NormStreamObject*)object; + } + else { DMSG(0, "NormApp::Notify(RX_OBJECT_NEW) stream object accept error!\n"); } @@ -544,7 +835,7 @@ void NormApp::Notify(NormController::Event event, else { DMSG(0, "NormApp::Notify(RX_OBJECT_NEW) no rx cache for file\n"); - } + } } break; case NormObject::DATA: @@ -599,22 +890,95 @@ void NormApp::Notify(NormController::Event event, switch (object->GetType()) { case NormObject::FILE: - // (TBD) update progress + // (TBD) update reception progress display when applicable break; case NormObject::STREAM: { + if (object != rx_stream) + { + DMSG(0, "NormApp::Notify(RX_OBJECT_UPDATE) update for invalid stream\n"); + break; + } // Read the stream when it's updated ASSERT(output); - char buffer[256]; - unsigned int nBytes; - while ((nBytes = ((NormStreamObject*)object)->Read(buffer, 256))) + bool reading = true; + bool findMsgSync; + while (reading) { - unsigned int put = 0; - while (put < nBytes) + unsigned int readLength; + if (output_messaging) { - size_t result = fwrite(buffer, sizeof(char), nBytes, output); - fflush(output); + if (output_msg_length) + { + readLength = output_msg_length - output_index; + } + else if (output_index < 2) + { + readLength = 2 - output_index; + } + else + { + output_msg_length = ntohs(*((UINT16*)output_buffer)); + ASSERT(output_msg_length >= 2); + readLength = output_msg_length - output_index; + } + findMsgSync = output_msg_sync ? false : true; + } + else + { + output_index = 0; + readLength = 512; + findMsgSync = false; + } + + if(!((NormStreamObject*)object)->Read(output_buffer+output_index, + &readLength, findMsgSync)) + { + // The stream broke + if (output_messaging) + { + if (output_msg_sync) + DMSG(0, "NormApp::Notify() detected broken stream ...\n"); + output_msg_length = output_index = 0; + output_msg_sync = false; + continue; + } + } + else + { + output_msg_sync = true; + } + + if (readLength) + output_index += readLength; + else + reading = false; + + unsigned int writeLength; + if (output_messaging) + { + if (output_msg_length && (output_index >= output_msg_length)) + { + writeLength = output_msg_length; + output_msg_length = 0; + output_index = 0; + } + else + { + writeLength = 0; + } + } + else + { + writeLength = readLength; + output_index = 0; + } + + unsigned int put = 0; + while (put < writeLength) + { + size_t result = fwrite(output_buffer+put, sizeof(char), writeLength-put, output); if (result) { put += result; @@ -637,22 +1001,35 @@ void NormApp::Notify(NormController::Event event, } } } // end while(put < nBytes) - } + if (writeLength) memset(output_buffer, 0, writeLength); + } // end while (reading) + fflush(output); break; } case NormObject::DATA: - DMSG(0, "NormApp::Notify() FILE/DATA objects not _yet_ supported...\n"); + DMSG(0, "NormApp::Notify() DATA objects not _yet_ supported...\n"); break; } // end switch (object->GetType()) break; + case RX_OBJECT_COMPLETE: { switch(object->GetType()) { case NormObject::FILE: - DMSG(0, "norm: Completed rx file: %s\n", ((NormFileObject*)object)->Path()); + { + const char* filePath = ((NormFileObject*)object)->Path(); + //DMSG(0, "norm: Completed rx file: %s\n", filePath); + if (post_processor.IsEnabled()) + { + if (!post_processor.ProcessFile(filePath)) + { + DMSG(0, "norm: post processing error\n"); + } + } break; + } case NormObject::STREAM: ASSERT(0); @@ -694,12 +1071,15 @@ bool NormApp::OnIntervalTimeout() temp[len] = '\0'; if (!session->QueueTxFile(fileName, fileNameInfo, len)) { - DMSG(0, "NormApp::OnIntervalTimeout() Error opening tx file: %s\n", + DMSG(0, "NormApp::OnIntervalTimeout() Error queuing tx file: %s\n", fileName); - // Try the next file in the list - return OnIntervalTimeout(); + // Wait a second, then try the next file in the list + if (interval_timer.IsActive()) interval_timer.Deactivate(); + interval_timer.SetInterval(1.0); + InstallTimer(&interval_timer); + return false; } - DMSG(0, "norm: File \"%s\" queued for transmission.\n", fileName); + //DMSG(0, "norm: File \"%s\" queued for transmission.\n", fileName); interval_timer.SetInterval(tx_object_interval); } else if (tx_repeat_count) @@ -738,17 +1118,18 @@ bool NormApp::OnStartup() signal(SIGTERM, SignalHandler); signal(SIGINT, SignalHandler); + signal(SIGCHLD, SignalHandler); // Validate our application settings if (!address) { - DMSG(0, "NormApp::OnStartup() Error! no session address given."); + DMSG(0, "NormApp::OnStartup() Error! no session address given.\n"); return false; } if (!input && !output && tx_file_list.IsEmpty() && !rx_cache_path) { DMSG(0, "NormApp::OnStartup() Error! no \"input\", \"output\", " - "\"sendfile\", or \"rxcache\" given."); + "\"sendfile\", or \"rxcache\" given.\n"); return false; } @@ -761,10 +1142,14 @@ bool NormApp::OnStartup() session->SetTrace(tracing); session->SetTxLoss(tx_loss); session->SetRxLoss(rx_loss); - + session->SetBackoffFactor(backoff_factor); + if (input || !tx_file_list.IsEmpty()) { - // StartServer(bufferMax, segmentSize, fec_ndata, fec_nparity) + NormObjectId baseId = (unsigned short)(rand() * (65535.0/ (double)RAND_MAX)); + session->ServerSetBaseObjectId(baseId); + + session->SetCongestionControl(cc_enable); if (!session->StartServer(tx_buffer_size, segment_size, ndata, nparity)) { DMSG(0, "NormApp::OnStartup() start server error!\n"); @@ -772,12 +1157,12 @@ bool NormApp::OnStartup() return false; } session->ServerSetAutoParity(auto_parity); - + session->ServerSetExtraParity(extra_parity); if (input) { // Open a stream object to write to (QueueTxStream(stream bufferSize)) - stream = session->QueueTxStream(tx_buffer_size); - if (!stream) + tx_stream = session->QueueTxStream(tx_buffer_size); + if (!tx_stream) { DMSG(0, "NormApp::OnStartup() queue tx stream error!\n"); session_mgr.Destroy(); @@ -788,8 +1173,9 @@ bool NormApp::OnStartup() if (output || rx_cache_path) { - TRACE("starting client ...\n"); // StartClient(bufferMax (per-sender)) + session->SetUnicastNacks(unicast_nacks); + session->ClientSetSilent(silent_client); if (!session->StartClient(rx_buffer_size)) { DMSG(0, "NormApp::OnStartup() start client error!\n"); @@ -808,11 +1194,23 @@ bool NormApp::OnStartup() void NormApp::OnShutdown() { + //TRACE("NormApp::OnShutdown() ...\n"); session_mgr.Destroy(); - if (input) fclose(input); - input = NULL; - if (output) fclose(output); - output = NULL; + if (input) + { + if (input_active) + { + dispatcher.RemoveGenericInput(fileno(input)); + input_active = false; + } + if (stdin != input) fclose(input); + input = NULL; + } + if (output) + { + if (stdout != output) fclose(output); + output = NULL; + } } // end NormApp::OnShutdown() @@ -859,7 +1257,7 @@ int main(int argc, char* argv[]) { if (theApp.OnStartup()) { - exitCode = theApp.MainLoop(); + exitCode = theApp.MainLoop(); theApp.OnShutdown(); fprintf(stderr, "norm: Done.\n"); } @@ -886,6 +1284,8 @@ int main(int argc, char* argv[]) return exitCode; // exitCode contains "signum" causing exit } // end main(); + + void NormApp::SignalHandler(int sigNum) { switch(sigNum) @@ -895,6 +1295,11 @@ void NormApp::SignalHandler(int sigNum) theApp.Stop(sigNum); // causes theApp's main loop to exit break; + case SIGCHLD: + if (theApp.IsPostProcessing()) theApp.HandleSIGCHLD(); + signal(SIGCHLD, SignalHandler); + break; + default: fprintf(stderr, "norm: Unexpected signal: %d\n", sigNum); break; diff --git a/common/normBitmask.cpp b/common/normBitmask.cpp index 52b39bb..17c2f4e 100644 --- a/common/normBitmask.cpp +++ b/common/normBitmask.cpp @@ -3,6 +3,7 @@ #include "debug.h" #include "sysdefs.h" + // Hamming weights for given one-byte bit masks static unsigned char WEIGHT[256] = { @@ -191,7 +192,7 @@ void NormBitmask::Destroy() { if (mask) { - delete []mask; + delete[] mask; mask = (unsigned char*)NULL; num_bits = first_set = 0; } @@ -425,7 +426,7 @@ void NormBitmask::Display(FILE* stream) if (0x07 == (i & 0x07)) fprintf(stream, " "); if (0x3f == (i & 0x3f)) fprintf(stream, "\n"); } -} // end NormSlidingMask::NormBitmask() +} // end NormBitmask::Display() @@ -433,7 +434,7 @@ void NormBitmask::Display(FILE* stream) NormSlidingMask::NormSlidingMask() - : mask((unsigned char*)0), mask_len(0), num_bits(0), + : mask((unsigned char*)NULL), mask_len(0), num_bits(0), start(0), end(0), offset(0) { } @@ -446,6 +447,7 @@ NormSlidingMask::~NormSlidingMask() bool NormSlidingMask::Init(long numBits) { + if (mask) Destroy(); if (numBits <= 0) return false; unsigned long len = (numBits + 7) >> 3; @@ -519,6 +521,7 @@ bool NormSlidingMask::CanSet(unsigned long index) const bool NormSlidingMask::Set(unsigned long index) { + ASSERT(CanSet(index)); if (IsSet()) { // Determine position with respect to current start @@ -571,6 +574,8 @@ bool NormSlidingMask::Set(unsigned long index) if ((pos > end) || (pos < start)) end = pos; } } + ASSERT((pos >> 3) >= 0); + ASSERT((pos >> 3) < (long)mask_len); mask[(pos >> 3)] |= (0x80 >> (pos & 0x07)); } else @@ -584,6 +589,7 @@ bool NormSlidingMask::Set(unsigned long index) bool NormSlidingMask::Unset(unsigned long index) { + //ASSERT(CanSet(index)); if (IsSet()) { long pos = index - offset; @@ -602,6 +608,8 @@ bool NormSlidingMask::Unset(unsigned long index) if ((pos < start) || (pos > end)) return true; } // Unset the corresponding bit + ASSERT((pos >> 3) >= 0); + ASSERT((pos >> 3) < (long)mask_len); mask[(pos >> 3)] &= ~(0x80 >> (pos & 0x07)); if (start == end) { @@ -640,6 +648,8 @@ bool NormSlidingMask::Unset(unsigned long index) bool NormSlidingMask::SetBits(unsigned long index, long count) { + ASSERT(CanSet(index)); + ASSERT(CanSet(index+count-1)); if (count < 0) return false; if (0 == count) return true; long firstPos, lastPos; @@ -691,19 +701,28 @@ bool NormSlidingMask::SetBits(unsigned long index, long count) long maskIndex = firstPos >> 3; int bitIndex = firstPos & 0x07; int bitRemainder = 8 - bitIndex; + ASSERT(maskIndex >= 0); if (count <= bitRemainder) { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] |= (0x00ff >> bitIndex) & (0x00ff << (bitRemainder - count)); } else { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] |= 0x00ff >> bitIndex; count -= bitRemainder; - unsigned long nbytes = count >> 3; + long nbytes = count >> 3; + ASSERT((maskIndex+1+nbytes) <= (long)mask_len); memset(&mask[++maskIndex], 0xff, nbytes); count &= 0x07; - if (count) mask[maskIndex+nbytes] |= 0xff << (8-count); + if (count) + { + ASSERT((maskIndex+nbytes) >= 0); + ASSERT((maskIndex+nbytes) < (long)mask_len); + mask[maskIndex+nbytes] |= 0xff << (8-count); + } } firstPos = 0; } @@ -720,25 +739,36 @@ bool NormSlidingMask::SetBits(unsigned long index, long count) long maskIndex = firstPos >> 3; int bitIndex = firstPos & 0x07; int bitRemainder = 8 - bitIndex; + ASSERT(maskIndex >= 0); if (count <= bitRemainder) { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] |= (0x00ff >> bitIndex) & (0x00ff << (bitRemainder - count)); } else { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] |= 0x00ff >> bitIndex; count -= bitRemainder; - unsigned long nbytes = count >> 3; + long nbytes = count >> 3; + ASSERT((maskIndex+1+nbytes) <= (long)mask_len); memset(&mask[++maskIndex], 0xff, nbytes); count &= 0x07; - if (count) mask[maskIndex+nbytes] |= 0xff << (8-count); + if (count) + { + ASSERT((maskIndex+nbytes) >= 0); + ASSERT((maskIndex+nbytes) < (long)mask_len); + mask[maskIndex+nbytes] |= 0xff << (8-count); + } } return true; } // end NormSlidingMask::SetBits() bool NormSlidingMask::UnsetBits(unsigned long index, long count) { + //ASSERT(CanSet(index)); + //ASSERT(CanSet(index+count-1)); if (IsSet()) { // Trim to fit as needed. @@ -782,17 +812,24 @@ bool NormSlidingMask::UnsetBits(unsigned long index, long count) int bitRemainder = 8 - bitIndex; if (count <= bitRemainder) { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] &= (0x00ff << bitRemainder) | (0x00ff >> (bitIndex + count)); } else { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] &= 0x00ff << bitRemainder; count -= bitRemainder; unsigned long nbytes = count >> 3; + ASSERT((maskIndex+1+nbytes) <= mask_len); memset(&mask[++maskIndex], 0, nbytes); count &= 0x07; - if (count) mask[maskIndex+nbytes] &= 0xff >> count; + if (count) + { + ASSERT((maskIndex+nbytes) < mask_len); + mask[maskIndex+nbytes] &= 0xff >> count; + } } startPos = 0; } @@ -807,17 +844,24 @@ bool NormSlidingMask::UnsetBits(unsigned long index, long count) int bitRemainder = 8 - bitIndex; if (count <= bitRemainder) { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] &= (0x00ff << bitRemainder) | (0x00ff >> (bitIndex + count)); } else { + ASSERT(maskIndex < (long)mask_len); mask[maskIndex] &= 0x00ff << bitRemainder; count -= bitRemainder; - unsigned long nbytes = count >> 3; + unsigned long nbytes = count >> 3; + ASSERT((maskIndex+1+nbytes) <= mask_len); memset(&mask[++maskIndex], 0, nbytes); count &= 0x07; - if (count) mask[maskIndex+nbytes] &= 0xff >> count; + if (count) + { + ASSERT((maskIndex+nbytes) < mask_len); + mask[maskIndex+nbytes] &= 0xff >> count; + } } // Calling these will update the start/end state if (start == firstPos) @@ -1078,8 +1122,11 @@ bool NormSlidingMask::Copy(const NormSlidingMask& b) long endIndex = b.end >> 3; if (b.end < b.start) { + ASSERT((b.mask_len - startIndex) <= mask_len); memcpy(mask, b.mask+startIndex, b.mask_len - startIndex); - memcpy(mask+b.mask_len-startIndex, b.mask, endIndex+1); + ASSERT((b.mask_len - startIndex + endIndex) <= mask_len); + //memcpy(mask+b.mask_len-startIndex, b.mask, endIndex+1); // old + memcpy(mask+b.mask_len-startIndex, b.mask, endIndex); // new // Clear any possible start/end overlap if (mask_len > b.mask_len) { @@ -1087,11 +1134,15 @@ bool NormSlidingMask::Copy(const NormSlidingMask& b) if (remainder) mask[0] &= 0xff >> remainder; remainder = end & 0x7; if (remainder) + { + ASSERT((startIndex+endIndex) < (long)mask_len); mask[startIndex+endIndex] &= 0xff << (8 - remainder); + } } } else { + ASSERT((endIndex-startIndex+1) <= (long)mask_len); memcpy(mask, b.mask+startIndex, endIndex-startIndex+1); } return true; diff --git a/common/normBitmask.h b/common/normBitmask.h index 1426717..55fa646 100644 --- a/common/normBitmask.h +++ b/common/normBitmask.h @@ -117,6 +117,8 @@ class NormSlidingMask NormSlidingMask(); ~NormSlidingMask(); + const char* GetMask() const {return (const char*)mask;} + bool Init(long numBits); void Destroy(); long Size() const {return num_bits;} @@ -181,6 +183,7 @@ class NormSlidingMask long start; long end; unsigned long offset; + unsigned char* mask2; }; // end class NormSlidingMask #endif // _NORM_BITMASK_ diff --git a/common/normEncoder.cpp b/common/normEncoder.cpp index 7cc54ac..46ab715 100644 --- a/common/normEncoder.cpp +++ b/common/normEncoder.cpp @@ -39,8 +39,12 @@ #include "galois.h" // for Galois math routines #include "debug.h" +#ifdef SIMULATE +#include "normMessage.h" +#endif // SIMULATE + NormEncoder::NormEncoder() - : npar(0), vecSize(0), + : npar(0), vector_size(0), genPoly(NULL), scratch(NULL) { @@ -58,7 +62,7 @@ bool NormEncoder::Init(int numParity, int vecSizeMax) ASSERT(vecSizeMax >= 0); if (genPoly) Destroy(); npar = numParity; - vecSize = vecSizeMax; + vector_size = vecSizeMax; // Create generator polynomial if(!CreateGeneratorPolynomial()) { @@ -166,11 +170,6 @@ bool NormEncoder::CreateGeneratorPolynomial() // Parity data is written to list of parity vectors supplied by caller void NormEncoder::Encode(const char *data, char **pVec) { - -#if defined(NS2) || defined(OPNET) - return; // lobotomize FEC for faster simulations -#endif - int i, j; unsigned char *userData, *LSFR1, *LSFR2, *pVec0; int npar_minus_one = npar - 1; @@ -178,6 +177,14 @@ void NormEncoder::Encode(const char *data, char **pVec) ASSERT(scratch); // Make sure it's been init'd first // Assumes parity vectors are zero-filled at block start !!! // Copy pVec[0] for use in calculations + +#ifdef SIMULATE + UINT16 vecSize = MIN(SIM_PAYLOAD_MAX, vector_size); + vecSize = MAX(vecSize, NormDataMsg::PayloadHeaderLength()); +#else + UINT16 vecSize = vector_size; +#endif // if/else SIMULATE + memcpy(scratch, pVec[0], vecSize); if (npar > 1) { @@ -322,17 +329,20 @@ int NormDecoder::Decode(char** dVec, int ndata, UINT16 erasureCount, UINT16* era ASSERT(Lambda); ASSERT(erasureCount && (erasureCount<=npar)); -#if defined(NS2) || defined (OPNET) - return erasureCount; // lobotomize FEC for faster simulations -#endif - + // (A) Compute syndrome vectors // First zero out erasure vectors (MDP provides zero-filled vecs) // Then calculate syndrome (based on zero value erasures) int nvecs = npar + ndata; +#ifdef SIMULATE + int vecSize = MIN(SIM_PAYLOAD_MAX, vector_size); + vecSize = MAX(NormDataMsg::PayloadHeaderLength(), vecSize); +#else int vecSize = vector_size; +#endif // if/else SIMUATE + for (int i = 0; i < npar; i++) { int X = gexp(i+1); @@ -382,7 +392,7 @@ int NormDecoder::Decode(char** dVec, int ndata, UINT16 erasureCount, UINT16* era for (int i = 0; i < erasureCount; i++) { // Only fill _data_ erasures - if (erasureLocs[i] >= ndata) return erasureCount; + if (erasureLocs[i] >= ndata) break;//return erasureCount; // evaluate Lambda' (derivative) at alpha^(-i) // ( all odd powers disappear) @@ -413,6 +423,7 @@ int NormDecoder::Decode(char** dVec, int ndata, UINT16 erasureCount, UINT16* era data++; } } + return erasureCount; } // end NormDecoder::Decode() diff --git a/common/normEncoder.h b/common/normEncoder.h index 15bde5d..fb5c4f0 100644 --- a/common/normEncoder.h +++ b/common/normEncoder.h @@ -39,10 +39,10 @@ class NormEncoder { // Members private: - int npar; // No. of parity packets (N-k) - int vecSize; // Size of biggest vector to encode - unsigned char* genPoly; // Ptr to generator polynomial - unsigned char* scratch; // scratch space for encoding + int npar; // No. of parity packets (N-k) + int vector_size; // Size of biggest vector to encode + unsigned char* genPoly; // Ptr to generator polynomial + unsigned char* scratch; // scratch space for encoding // Methods public: @@ -53,7 +53,7 @@ class NormEncoder bool IsReady(){return (bool)(genPoly != NULL);} void Encode(const char *dataVector, char **parityVectorList); int NumParity() {return npar;} - int VectorSize() {return vecSize;} + int VectorSize() {return vector_size;} private: bool CreateGeneratorPolynomial(); diff --git a/common/normFile.cpp b/common/normFile.cpp index 085e326..e9911ad 100644 --- a/common/normFile.cpp +++ b/common/normFile.cpp @@ -121,15 +121,18 @@ bool NormFile::Lock() { #ifndef WIN32 // WIN32 files are automatically locked fchmod(fd, 0640 | S_ISGID); + #ifdef HAVE_FLOCK if (flock(fd, LOCK_EX | LOCK_NB)) -#else -//#ifdef HAVE_LOCKF - if (lockf(fd, F_LOCK, 0)) // assume lockf if not flock -//#endif // HAVE_LOCKF -#endif // if/else HAVE_FLOCK return false; else +#else +#ifdef HAVE_LOCKF + if (lockf(fd, F_LOCK, 0)) // assume lockf if not flock + return false; + else +#endif // HAVE_LOCKF +#endif // if/else HAVE_FLOCK #endif // !WIN32 return true; } // end NormFile::Lock() @@ -143,7 +146,7 @@ void NormFile::Unlock() #ifdef HAVE_LOCKF lockf(fd, F_ULOCK, 0); #endif // HAVE_LOCKF -#endif // HAVE_FLOCK +#endif // if/elseHAVE_FLOCK fchmod(fd, 0640); #endif // !WIN32 } // end NormFile::UnLock() diff --git a/common/normMessage.cpp b/common/normMessage.cpp index dec56d1..877ce04 100644 --- a/common/normMessage.cpp +++ b/common/normMessage.cpp @@ -1,5 +1,136 @@ #include "normMessage.h" + +NormHeaderExtension::NormHeaderExtension() + : buffer(NULL) +{ + +} + + +NormMsg::NormMsg() + : length(8), header_length(8), header_length_base(8) +{ + SetType(INVALID); + SetVersion(NORM_PROTOCOL_VERSION); +} + +bool NormMsg::InitFromBuffer(UINT16 msgLength) +{ + header_length = GetHeaderLength(); + // "header_length_base" is type dependent + switch (GetType()) + { + + // for INFO, DATA, and + case INFO: + header_length_base = 16; + break; + case DATA: + // (TBD) look at "fec_id" to determine "fec_payload_id" size + // (we _assume_ "fec_id" == 129 here + if ((unsigned char)buffer[NormDataMsg::FEC_ID_OFFSET] == 129) + { + header_length_base = 24; + } + else + { + DMSG(0, "NormMsg::InitFromBuffer(DATA) unknown fec_id value: %u\n", + buffer[NormDataMsg::FEC_ID_OFFSET]); + return false; + } + break; + case CMD: + switch (buffer[NormCmdMsg::FLAVOR_OFFSET]) + { + case NormCmdMsg::FLUSH: + case NormCmdMsg::SQUELCH: + // (TBD) look at "fec_id" to determine "fec_payload_id" size + // (we _assume_ "fec_id" == 129 here + if ((unsigned char)buffer[NormCmdFlushMsg::FEC_ID_OFFSET] == 129) + { + header_length_base = 24; + } + else + { + DMSG(0, "NormMsg::InitFromBuffer(FLUSH|SQUELCH) unknown fec_id value: %u\n", + buffer[NormDataMsg::FEC_ID_OFFSET]); + return false; + } + break; + case NormCmdMsg::EOT: + case NormCmdMsg::REPAIR_ADV: + case NormCmdMsg::ACK_REQ: + case NormCmdMsg::APPLICATION: + header_length_base = 16; + break; + case NormCmdMsg::CC: + header_length_base = 24; + break; + } + break; + case NACK: + case ACK: + header_length_base= 24; + break; + case REPORT: + header_length_base= 8; + break; + + default: + DMSG(0, "NormMsg::InitFromBuffer() invalid message type!\n"); + return false; + } + ASSERT(msgLength >= header_length); + length = msgLength; + return true; +} // end NormMsg::InitFromBuffer() + +bool NormCmdCCMsg::GetCCNode(NormNodeId nodeId, + UINT8& flags, + UINT8& rtt, + UINT16& rate) const +{ + UINT16 cmdLength = length; + UINT16 offset = header_length; + nodeId = htonl(nodeId); + while (offset < cmdLength) + { + if (nodeId == *((UINT32*)(buffer+offset))) + { + const char* ptr = buffer+offset; + flags = ptr[CC_FLAGS_OFFSET]; + rtt = ptr[CC_RTT_OFFSET]; + rate = ntohs(*((UINT16*)(ptr+CC_RATE_OFFSET))); + return true; + } + offset += CC_ITEM_SIZE; + } + return false; +} // end NormCmdCCMsg::GetCCNode() + +NormCmdCCMsg::Iterator::Iterator(const NormCmdCCMsg& msg) + : cc_cmd(msg), offset(0) +{ + +} + +bool NormCmdCCMsg::Iterator::GetNextNode(NormNodeId& nodeId, + UINT8& flags, + UINT8& rtt, + UINT16& rate) +{ + if ((offset+CC_ITEM_SIZE) > cc_cmd.GetLength()) return false; + const char* ptr = cc_cmd.buffer + cc_cmd.header_length; + nodeId = ntohl(*((UINT32*)(ptr+offset))); + flags = ptr[offset+CC_FLAGS_OFFSET]; + rtt = ptr[offset+CC_RTT_OFFSET]; + rate = ntohs(*((UINT16*)(ptr+CC_RATE_OFFSET))); + offset += CC_ITEM_SIZE; + return true; +} // end NormCmdCCMsg::Iterator::GetNextNode() + + NormRepairRequest::NormRepairRequest() : form(INVALID), flags(0), length(0), buffer(NULL), buffer_len(0) { @@ -7,22 +138,24 @@ NormRepairRequest::NormRepairRequest() bool NormRepairRequest::AppendRepairItem(const NormObjectId& objectId, const NormBlockId& blockId, + UINT16 blockLen, UINT16 symbolId) { if (RANGES == form) - DMSG(4, "NormRepairRequest::AppendRepairRange(obj>%hu blk>%lu seg>%hu) ...\n", + DMSG(4, "NormRepairRequest::AppendRepairItem-Range(obj>%hu blk>%lu seg>%hu) ...\n", (UINT16)objectId, (UINT32)blockId, (UINT32)symbolId); else DMSG(4, "NormRepairRequest::AppendRepairItem(obj>%hu blk>%lu seg>%hu) ...\n", (UINT16)objectId, (UINT32)blockId, (UINT32)symbolId); - if (buffer_len >= (length+CONTENT_OFFSET+RepairItemLength())) + if (buffer_len >= (length+ITEM_LIST_OFFSET+RepairItemLength())) { - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+CONTENT_OFFSET+length, &temp16, 2); - UINT32 temp32 =htonl((UINT32)blockId); - memcpy(buffer+length+CONTENT_OFFSET+2, &temp32, 4); - symbolId = htons(symbolId); - memcpy(buffer+length+CONTENT_OFFSET+6, &symbolId, 2); + char* ptr = buffer + length + ITEM_LIST_OFFSET; + ptr[FEC_ID_OFFSET] = 129; + ptr[RESERVED_OFFSET] = 0; + *((UINT16*)(ptr+OBJ_ID_OFFSET)) = htons((UINT16)objectId); + *((UINT32*)(ptr+BLOCK_ID_OFFSET)) = htonl((UINT32)blockId); + *((UINT16*)(ptr+BLOCK_LEN_OFFSET)) = htons((UINT16)blockLen); + *((UINT16*)(ptr+SYMBOL_ID_OFFSET)) = htons((UINT16)symbolId); length += RepairItemLength(); return true; } @@ -34,31 +167,34 @@ bool NormRepairRequest::AppendRepairItem(const NormObjectId& objectId, bool NormRepairRequest::AppendRepairRange(const NormObjectId& startObjectId, const NormBlockId& startBlockId, + UINT16 startBlockLen, UINT16 startSymbolId, const NormObjectId& endObjectId, const NormBlockId& endBlockId, + UINT16 endBlockLen, UINT16 endSymbolId) { DMSG(4, "NormRepairRequest::AppendRepairRange(%hu:%lu:%hu->%hu:%lu:%hu) ...\n", (UINT16)startObjectId, (UINT32)startBlockId, (UINT32)startSymbolId, (UINT16)endObjectId, (UINT32)endBlockId, (UINT32)endSymbolId); - if (buffer_len >= (length+CONTENT_OFFSET+RepairRangeLength())) + if (buffer_len >= (length+ITEM_LIST_OFFSET+RepairRangeLength())) { // range start - UINT16 temp16; - temp16 = htons((UINT16)startObjectId); - memcpy(buffer+CONTENT_OFFSET+length, &temp16, 2); - UINT32 temp32 =htonl((UINT32)startBlockId); - memcpy(buffer+length+CONTENT_OFFSET+2, &temp32, 4); - startSymbolId = htons(startSymbolId); - memcpy(buffer+length+CONTENT_OFFSET+6, &startSymbolId, 2); + char* ptr = buffer + length + ITEM_LIST_OFFSET; + ptr[FEC_ID_OFFSET] = 129; + ptr[RESERVED_OFFSET] = 0; + *((UINT16*)(ptr+OBJ_ID_OFFSET)) = htons((UINT16)startObjectId); + *((UINT32*)(ptr+BLOCK_ID_OFFSET)) = htonl((UINT32)startBlockId); + *((UINT16*)(ptr+BLOCK_LEN_OFFSET)) = htons((UINT16)startBlockLen); + *((UINT16*)(ptr+SYMBOL_ID_OFFSET)) = htons((UINT16)startSymbolId); + ptr += RepairItemLength(); // range end - temp16 = htons((UINT16)endObjectId); - memcpy(buffer+CONTENT_OFFSET+length+8, &temp16, 2); - temp32 =htonl((UINT32)endBlockId); - memcpy(buffer+length+CONTENT_OFFSET+10, &temp32, 4); - endSymbolId = htons(endSymbolId); - memcpy(buffer+length+CONTENT_OFFSET+14, &endSymbolId, 2); + ptr[FEC_ID_OFFSET] = 129; + ptr[RESERVED_OFFSET] = 0; + *((UINT16*)(ptr+OBJ_ID_OFFSET)) = htons((UINT16)endObjectId); + *((UINT32*)(ptr+BLOCK_ID_OFFSET)) = htonl((UINT32)endBlockId); + *((UINT16*)(ptr+BLOCK_LEN_OFFSET)) = htons((UINT16)endBlockLen); + *((UINT16*)(ptr+SYMBOL_ID_OFFSET)) = htons((UINT16)endSymbolId); length += RepairRangeLength(); return true; } @@ -66,20 +202,22 @@ bool NormRepairRequest::AppendRepairRange(const NormObjectId& startObjectId, { return false; } -} // end NormRepairRequest::AppendErasureCount() +} // end NormRepairRequest::AppendRepairRange() bool NormRepairRequest::AppendErasureCount(const NormObjectId& objectId, const NormBlockId& blockId, + UINT16 blockLen, UINT16 erasureCount) { - if (buffer_len >= (CONTENT_OFFSET+length+ErasureItemLength())) + if (buffer_len >= (ITEM_LIST_OFFSET+length+ErasureItemLength())) { - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+CONTENT_OFFSET+length, &temp16, 2); - UINT32 temp32 =htonl((UINT32)blockId); - memcpy(buffer+length+CONTENT_OFFSET+2, &temp32, 4); - erasureCount = htons(erasureCount); - memcpy(buffer+length+CONTENT_OFFSET+6, &erasureCount, 2); + char* ptr = buffer + length + ITEM_LIST_OFFSET; + ptr[FEC_ID_OFFSET] = 129; + ptr[RESERVED_OFFSET] = 0; + *((UINT16*)(ptr+OBJ_ID_OFFSET)) = htons((UINT16)objectId); + *((UINT32*)(ptr+BLOCK_ID_OFFSET)) = htonl((UINT32)blockId); + *((UINT16*)(ptr+BLOCK_LEN_OFFSET)) = htons((UINT16)blockLen); + *((UINT16*)(ptr+SYMBOL_ID_OFFSET)) = htons((UINT16)erasureCount); length += ErasureItemLength(); return true; } @@ -96,9 +234,8 @@ UINT16 NormRepairRequest::Pack() { buffer[FORM_OFFSET] = form; buffer[FLAGS_OFFSET] = (char)flags; - UINT16 temp16 = htons(length); - memcpy(buffer+LENGTH_OFFSET, &temp16, 2); - return (CONTENT_OFFSET + length); + *((UINT16*)(buffer+LENGTH_OFFSET)) = htons(length); + return (ITEM_LIST_OFFSET + length); } else { @@ -107,23 +244,26 @@ UINT16 NormRepairRequest::Pack() } // end NormRepairRequest::Pack() -UINT16 NormRepairRequest::Unpack() +UINT16 NormRepairRequest::Unpack(const char* bufferPtr, UINT16 bufferLen) { + buffer = (char*)bufferPtr; + buffer_len = bufferLen; + length = 0; + // Make sure there's at least a header - if (buffer_len >= CONTENT_OFFSET) + if (buffer_len >= ITEM_LIST_OFFSET) { form = (Form)buffer[FORM_OFFSET]; flags = (int)buffer[FLAGS_OFFSET]; - memcpy(&length, buffer+LENGTH_OFFSET, 2); - length = ntohs(length); - if (length > (buffer_len - CONTENT_OFFSET)) + length = ntohs(*((UINT16*)(buffer+LENGTH_OFFSET))); + if (length > (buffer_len - ITEM_LIST_OFFSET)) { // Badly formed message return 0; } else { - return (CONTENT_OFFSET+length); + return (ITEM_LIST_OFFSET+length); } } else @@ -135,18 +275,16 @@ UINT16 NormRepairRequest::Unpack() bool NormRepairRequest::RetrieveRepairItem(UINT16 offset, NormObjectId* objectId, NormBlockId* blockId, + UINT16* blockLen, UINT16* symbolId) const { if (length >= (offset + RepairItemLength())) { - UINT16 temp16; - memcpy(&temp16, buffer+CONTENT_OFFSET+offset, 2); - *objectId = ntohs(temp16); - UINT32 temp32; - memcpy(&temp32, buffer+CONTENT_OFFSET+offset+2, 4); - *blockId = ntohl(temp32); - memcpy(symbolId, buffer+CONTENT_OFFSET+offset+6, 2); - *symbolId = ntohs(*symbolId); + const char* ptr = buffer+ITEM_LIST_OFFSET+offset; + *objectId = ntohs(*((UINT16*)(ptr+OBJ_ID_OFFSET))); + *blockId = ntohl(*((UINT32*)(ptr+BLOCK_ID_OFFSET))); + *blockLen = ntohs(*((UINT16*)(ptr+BLOCK_LEN_OFFSET))); + *symbolId = ntohs(*((UINT16*)(ptr+SYMBOL_ID_OFFSET))); return true; } else @@ -163,9 +301,10 @@ NormRepairRequest::Iterator::Iterator(NormRepairRequest& theRequest) // For erasure requests, symbolId is loaded with erasureCount bool NormRepairRequest::Iterator::NextRepairItem(NormObjectId* objectId, NormBlockId* blockId, + UINT16* blockLen, UINT16* symbolId) { - if (request.RetrieveRepairItem(offset, objectId, blockId, symbolId)) + if (request.RetrieveRepairItem(offset, objectId, blockId, blockLen, symbolId)) { offset += NormRepairRequest::RepairItemLength(); return true; @@ -177,6 +316,8 @@ bool NormRepairRequest::Iterator::NextRepairItem(NormObjectId* objectId, } // end NormRepairRequest::Iterator::NextRepairItem() + + NormMessageQueue::NormMessageQueue() : head(NULL), tail(NULL) { @@ -189,7 +330,7 @@ NormMessageQueue::~NormMessageQueue() void NormMessageQueue::Destroy() { - NormMessage* next; + NormMsg* next; while ((next = head)) { head = next->next; @@ -198,7 +339,7 @@ void NormMessageQueue::Destroy() } // end NormMessageQueue::Destroy() -void NormMessageQueue::Prepend(NormMessage* msg) +void NormMessageQueue::Prepend(NormMsg* msg) { if ((msg->next = head)) head->prev = msg; @@ -208,7 +349,7 @@ void NormMessageQueue::Prepend(NormMessage* msg) head = msg; } // end NormMessageQueue::Prepend() -void NormMessageQueue::Append(NormMessage* msg) +void NormMessageQueue::Append(NormMsg* msg) { if ((msg->prev = tail)) tail->next = msg; @@ -218,7 +359,7 @@ void NormMessageQueue::Append(NormMessage* msg) tail = msg; } // end NormMessageQueue::Append() -void NormMessageQueue::Remove(NormMessage* msg) +void NormMessageQueue::Remove(NormMsg* msg) { if (msg->prev) msg->prev->next = msg->next; @@ -230,11 +371,11 @@ void NormMessageQueue::Remove(NormMessage* msg) tail = msg->prev; } // end NormMessageQueue::Remove() -NormMessage* NormMessageQueue::RemoveHead() +NormMsg* NormMessageQueue::RemoveHead() { if (head) { - NormMessage* msg = head; + NormMsg* msg = head; if ((head = msg->next)) msg->next->prev = NULL; else @@ -247,11 +388,11 @@ NormMessage* NormMessageQueue::RemoveHead() } } // end NormMessageQueue::RemoveHead() -NormMessage* NormMessageQueue::RemoveTail() +NormMsg* NormMessageQueue::RemoveTail() { if (tail) { - NormMessage* msg = tail; + NormMsg* msg = tail; if ((tail = msg->prev)) msg->prev->next = NULL; else diff --git a/common/normMessage.h b/common/normMessage.h index a53a80b..698dafd 100644 --- a/common/normMessage.h +++ b/common/normMessage.h @@ -10,10 +10,16 @@ #include // for memcpy(), etc #include + +#ifdef SIMULATE +#define SIM_PAYLOAD_MAX 36 // MGEN message size +#endif // SIMULATE + +const UINT8 NORM_PROTOCOL_VERSION = 1; + // (TBD) Alot of the "memcpy()" calls could be eliminated by // taking advantage of the alignment of NORM messsages -const UINT16 NORM_MSG_SIZE_MAX = 8192; const int NORM_ROBUST_FACTOR = 20; // default robust factor // These are the GRTT estimation bounds set for the current @@ -38,8 +44,34 @@ inline double NormUnquantizeGroupSize(unsigned char gsize) inline unsigned char NormQuantizeGroupSize(double gsize) { unsigned char exponent = (unsigned char)log10(gsize); - return ((((unsigned char)(gsize / pow(10.0, (double)exponent) + 0.5)) << 4) - | exponent); + exponent = MIN(exponent, 0x0f); + return ((((unsigned char)(gsize / pow(10.0, (double)exponent) + 0.5)) << 4) | exponent); +} + +inline unsigned short NormQuantizeLoss(double lossFraction) +{ + lossFraction = MAX(lossFraction, 0.0); + lossFraction = lossFraction*65535.0 + 0.5; + lossFraction = MIN(lossFraction, 65535.0); + return (unsigned short)lossFraction; +} // end NormQuantizeLossFraction() +inline double NormUnquantizeLoss(unsigned short lossQuantized) +{ + return (((double)lossQuantized) / 65535.0); +} // end NormUnquantizeLossFraction() + +inline unsigned short NormQuantizeRate(double rate) +{ + unsigned char exponent = (unsigned short)log10(rate); + exponent = MIN(exponent, 0xff); + unsigned short mantissa = (unsigned short)((256.0/10.0) * rate / pow(10.0, (double)exponent)); + return ((mantissa << 8) | exponent); +} +inline double NormUnquantizeRate(unsigned short rate) +{ + double mantissa = ((double)(rate >> 8)) * (10.0/256.0); + double exponent = (double)(rate & 0xff); + return mantissa * pow(10.0, exponent); } // This class is used to describe object "size" and/or "offset" @@ -92,6 +124,8 @@ class NormObjectSize }; // end class NormObjectSize typedef UINT32 NormNodeId; +const NormNodeId NORM_NODE_INVALID = 0x00000000; +const NormNodeId NORM_NODE_ANY = 0xffffffff; class NormObjectId { @@ -136,84 +170,169 @@ class NormBlockId typedef UINT16 NormSymbolId; typedef NormSymbolId NormSegmentId; -const NormNodeId NORM_NODE_INVALID = 0x0; -const NormNodeId NORM_NODE_ANY = 0xffffffff; - -enum NormMsgType +// Base class for NORM header extensions +class NormHeaderExtension { - NORM_MSG_INVALID, - NORM_MSG_INFO, - NORM_MSG_DATA, - NORM_MSG_CMD, - NORM_MSG_NACK, - NORM_MSG_ACK, - NORM_MSG_REPORT -}; + public: + enum Type + { + INVALID = 0, + FTI = 1, // FEC Object Transmission Information (FTI) extension + CC_FEEDBACK = 2, // NORM-CC Feedback extension + CC_RATE = 128 // NORM-CC Rate extension + }; + + NormHeaderExtension(); + virtual void Init(char* theBuffer) + { + buffer = theBuffer; + SetType(INVALID); + SetWords(0); + } + void SetType(Type type) {buffer[TYPE_OFFSET] = (UINT8)type;} + void SetWords(UINT8 words) {buffer[LENGTH_OFFSET] = words;} + + void AttachBuffer(const char* theBuffer) {buffer = (char*)theBuffer;} + const char* GetBuffer() {return buffer;} + + Type GetType() const + { + return buffer ? (Type)((UINT8)buffer[TYPE_OFFSET]) : INVALID; + } + UINT16 GetLength() const + { + return buffer ? ((GetType() < 128) ? (buffer[LENGTH_OFFSET] << 2) : 4) : 0; + } + + protected: + enum + { + TYPE_OFFSET = 0, + LENGTH_OFFSET = TYPE_OFFSET + 1 + }; + char* buffer; +}; // end class NormHeaderExtension + class NormMsg { + friend class NormMessageQueue; + public: + // (TBD) Use this "Type" enumeration instead of NormMsgType + enum Type + { + INVALID = 0, + INFO = 1, + DATA = 2, + CMD = 3, + NACK = 4, + ACK = 5, + REPORT = 6 + }; + enum {MAX_SIZE = 8192}; + + NormMsg(); + // Message building routines - void SetVersion(UINT8 version) {buffer[VERSION_OFFSET] = version;} - void SetType(NormMsgType type) {buffer[TYPE_OFFSET] = type;} + void SetVersion(UINT8 version) + { + buffer[VERSION_OFFSET] = (buffer[VERSION_OFFSET] & 0x0f) | (version << 4); + } + void SetType(NormMsg::Type type) + { + buffer[TYPE_OFFSET] = (buffer[VERSION_OFFSET] & 0xf0) | (type & 0x0f); + } void SetSequence(UINT16 sequence) { - UINT16 temp16 = htons(sequence); - memcpy(buffer+SEQUENCE_OFFSET, &temp16, 2); + *((UINT16*)(buffer+SEQUENCE_OFFSET)) = htons(sequence); } - void SetSender(NormNodeId sender) + void SetSourceId(NormNodeId sourceId) { - UINT32 temp32 = htonl(sender); - memcpy(buffer+SENDER_OFFSET, &temp32, 4); + *((UINT32*)(buffer+SOURCE_ID_OFFSET)) = htonl(sourceId); } void SetDestination(const NetworkAddress& dst) {addr = dst;} - void SetLength(UINT16 len) {length = len;} + + void AttachExtension(NormHeaderExtension& extension) + { + extension.Init(buffer+header_length); + ExtendHeaderLength(extension.GetLength()); + } // Message processing routines + bool InitFromBuffer(UINT16 msgLength); UINT8 GetVersion() const {return buffer[VERSION_OFFSET];} - NormMsgType GetType() const {return (NormMsgType)buffer[TYPE_OFFSET];} + NormMsg::Type GetType() const {return (Type)(buffer[TYPE_OFFSET] & 0x0f);} + UINT16 GetHeaderLength() {return buffer[HDR_LEN_OFFSET] << 2;} UINT16 GetSequence() const { - UINT16 temp16; - memcpy(&temp16, buffer+SEQUENCE_OFFSET, 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+SEQUENCE_OFFSET)))); } - NormNodeId GetSender() const + NormNodeId GetSourceId() const { - UINT32 temp32; - memcpy(&temp32, buffer+SENDER_OFFSET, 4); - return (ntohl(temp32)); + return (ntohl(*((UINT32*)(buffer+SOURCE_ID_OFFSET)))); } - const NetworkAddress& GetDestination() {return addr;} - const NetworkAddress& GetSource() {return addr;} - UINT16 GetLength() {return length;} - - // For message buffer transmission/reception and misc. - char* AccessBuffer() {return buffer;} - UINT16 AccessLength() const {return length;} - NetworkAddress& AccessAddress() {return addr;} + const NetworkAddress& GetDestination() const {return addr;} + const NetworkAddress& GetSource() const {return addr;} + const char* GetBuffer() {return buffer;} + UINT16 GetLength() const {return length;} + // To retrieve any attached header extensions + bool HasExtensions() const {return (header_length > header_length_base);} + bool GetNextExtension(NormHeaderExtension& ext) const + { + const char* currentBuffer = ext.GetBuffer(); + UINT16 nextOffset = currentBuffer ? (currentBuffer - buffer + ext.GetLength()) : header_length_base; + bool result = (nextOffset < header_length); + ext.AttachBuffer(result ? (buffer+nextOffset) : NULL); + return result; + } + + + // For message reception and misc. + char* AccessBuffer() {return buffer;} + NetworkAddress* AccessAddress() {return &addr;} protected: // Common message header offsets enum { VERSION_OFFSET = 0, - TYPE_OFFSET = VERSION_OFFSET + 1, - SEQUENCE_OFFSET = TYPE_OFFSET + 1, - SENDER_OFFSET = SEQUENCE_OFFSET + 2, - MSG_OFFSET = SENDER_OFFSET + 4 - }; - char buffer[NORM_MSG_SIZE_MAX]; + TYPE_OFFSET = VERSION_OFFSET, + HDR_LEN_OFFSET = VERSION_OFFSET + 1, + SEQUENCE_OFFSET = HDR_LEN_OFFSET + 1, + SOURCE_ID_OFFSET = SEQUENCE_OFFSET + 2, + MSG_OFFSET = SOURCE_ID_OFFSET + 4 + }; + + void SetBaseHeaderLength(UINT16 len) + { + buffer[HDR_LEN_OFFSET] = len >> 2; + length = header_length_base = header_length = len; + } + void ExtendHeaderLength(UINT16 len) + { + header_length += len; + length = header_length; + buffer[HDR_LEN_OFFSET] = header_length >> 2; + } + + char buffer[MAX_SIZE]; UINT16 length; - NetworkAddress addr; // src/dst (default dst is session address) + UINT16 header_length; + UINT16 header_length_base; + NetworkAddress addr; // src or dst address + + NormMsg* prev; + NormMsg* next; }; // end class NormMsg -// "NormObjectMsg" are comprised of NORM_INFO and NORM_DATA messages -// (NormInfoMsg and NormDataMsg will derive from this type) +// "NormObjectMsg" is a base class for the similar "NormInfoMsg" +// and "NormDataMsg" types class NormObjectMsg : public NormMsg { + friend class NormMsg; public: enum Flag { @@ -222,231 +341,256 @@ class NormObjectMsg : public NormMsg FLAG_INFO = 0x04, FLAG_UNRELIABLE = 0x08, FLAG_FILE = 0x10, - FLAG_STREAM = 0x20 + FLAG_STREAM = 0x20, + FLAG_MSG_START = 0x40 }; - bool FlagIsSet(NormObjectMsg::Flag flag) const - {return (0 != (flag & buffer[FLAGS_OFFSET]));} + UINT16 GetSessionId() const + { + return (ntohs(*((UINT16*)(buffer+SESSION_ID_OFFSET)))); + } UINT8 GetGrtt() const {return buffer[GRTT_OFFSET];} UINT8 GetGroupSize() const {return buffer[GSIZE_OFFSET];} + bool FlagIsSet(NormObjectMsg::Flag flag) const + {return (0 != (flag & buffer[FLAGS_OFFSET]));} UINT8 GetFecId() const {return buffer[FEC_ID_OFFSET];} - UINT16 GetSegmentSize() const - { - UINT16 segmentSize; - memcpy(&segmentSize, buffer+SEG_SIZE_OFFSET, 2); - return (ntohs(segmentSize)); - } - NormObjectSize GetObjectSize() const - { - UINT16 temp16; - memcpy(&temp16, buffer+OBJ_SIZE_OFFSET, 2); - UINT32 temp32; - memcpy(&temp32, buffer+OBJ_SIZE_OFFSET+2, 4); - return NormObjectSize(ntohs(temp16), ntohl(temp32)); - } - UINT16 GetFecEncodingName() const - { - UINT16 encodingName; - memcpy(&encodingName, buffer+FEC_NAME_OFFSET, 2); - return (ntohs(encodingName)); - } - UINT16 GetFecNumParity() const - { - UINT16 nparity; - memcpy(&nparity, buffer+FEC_NPARITY_OFFSET, 2); - return (ntohs(nparity)); - } - UINT16 GetFecBlockLen() const - { - UINT16 ndata; - memcpy(&ndata, buffer+FEC_NDATA_OFFSET, 2); - return (ntohs(ndata)); - } NormObjectId GetObjectId() const { - UINT16 temp16; - memcpy(&temp16, buffer+OBJ_ID_OFFSET, 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+OBJ_ID_OFFSET)))); } // Message building routines - void ResetFlags() {buffer[FLAGS_OFFSET] = 0;} - void SetFlag(NormObjectMsg::Flag flag) - {buffer[FLAGS_OFFSET] |= flag;} + void SetSessionId(UINT16 sessionId) + { + *((UINT16*)(buffer+SESSION_ID_OFFSET)) = htons(sessionId); + } void SetGrtt(UINT8 grtt) {buffer[GRTT_OFFSET] = grtt;} void SetGroupSize(UINT8 gsize) {buffer[GSIZE_OFFSET] = gsize;} - void SetFecId(UINT8 fecId) {buffer[FEC_ID_OFFSET] = fecId;} - void SetSegmentSize(UINT16 segmentSize) - { - segmentSize = htons(segmentSize); - memcpy(buffer+SEG_SIZE_OFFSET, &segmentSize, 2); - } - void SetObjectSize(const NormObjectSize& objectSize) - { - UINT16 temp16 = htons(objectSize.MSB()); - memcpy(buffer+OBJ_SIZE_OFFSET, &temp16, 2); - UINT32 temp32 = htonl(objectSize.LSB()); - memcpy(buffer+OBJ_SIZE_OFFSET+2, &temp32, 4); - } - void SetFecEncodingName(UINT16 encodingName) - { - encodingName = htons(encodingName); - memcpy(buffer+FEC_NAME_OFFSET, &encodingName, 2); - } - void SetFecNumParity(UINT16 nparity) - { - nparity = htons(nparity); - memcpy(buffer+FEC_NPARITY_OFFSET, &nparity, 2); - } - void SetFecBlockLen(UINT16 blockLen) - { - blockLen = htons(blockLen); - memcpy(buffer+FEC_NDATA_OFFSET, &blockLen, 2); - } + void ResetFlags() {buffer[FLAGS_OFFSET] = 0;} + void SetFlag(NormObjectMsg::Flag flag) {buffer[FLAGS_OFFSET] |= flag;} + void SetFecId(UINT8 fecId) {buffer[FEC_ID_OFFSET] = fecId;} void SetObjectId(const NormObjectId& objectId) { - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+OBJ_ID_OFFSET, &temp16, 2); + *((UINT16*)(buffer+OBJ_ID_OFFSET)) = htons((UINT16)objectId); } protected: enum { - FLAGS_OFFSET = MSG_OFFSET, - GRTT_OFFSET = FLAGS_OFFSET + 1, + SESSION_ID_OFFSET = MSG_OFFSET, + GRTT_OFFSET = SESSION_ID_OFFSET + 2, GSIZE_OFFSET = GRTT_OFFSET + 1, - FEC_ID_OFFSET = GSIZE_OFFSET + 1, - SEG_SIZE_OFFSET = FEC_ID_OFFSET + 1, - OBJ_SIZE_OFFSET = SEG_SIZE_OFFSET + 2, - RESV_OFFSET = OBJ_SIZE_OFFSET + 6, - FEC_NAME_OFFSET = RESV_OFFSET + 2, - FEC_NPARITY_OFFSET = FEC_NAME_OFFSET + 2, - FEC_NDATA_OFFSET = FEC_NPARITY_OFFSET + 2, - OBJ_ID_OFFSET = FEC_NDATA_OFFSET + 2 + FLAGS_OFFSET = GSIZE_OFFSET + 1, + FEC_ID_OFFSET = FLAGS_OFFSET + 1, + OBJ_ID_OFFSET = FEC_ID_OFFSET + 1, + OBJ_MSG_OFFSET = OBJ_ID_OFFSET + 2 }; }; // end class NormObjectMsg + + +// This FEC Object Transmission Information assumes "fec_id" == 129 +class NormFtiExtension : public NormHeaderExtension +{ + public: + // To build the FTI Header Extension + // (TBD) allow for different "fec_id" types in the future + virtual void Init(char* theBuffer) + { + AttachBuffer(theBuffer); + SetType(FTI); + SetWords(4); + } + void SetFecInstanceId(UINT16 instanceId) + { + *((UINT16*)(buffer+FEC_INSTANCE_OFFSET)) = htons(instanceId); + } + void SetFecMaxBlockLen(UINT16 ndata) + { + *((UINT16*)(buffer+FEC_NDATA_OFFSET)) = htons(ndata); + } + void SetFecNumParity(UINT16 nparity) + { + *((UINT16*)(buffer+FEC_NPARITY_OFFSET)) = htons(nparity); + } + void SetSegmentSize(UINT16 segmentSize) + { + *((UINT16*)(buffer+SEG_SIZE_OFFSET)) = htons(segmentSize); + } + void SetObjectSize(const NormObjectSize& objectSize) + { + *((UINT16*)(buffer+OBJ_SIZE_OFFSET)) = htons(objectSize.MSB()); + *((UINT32*)(buffer+OBJ_SIZE_OFFSET+2)) = htonl(objectSize.LSB()); + } + + // FTI Extension parsing methods + UINT16 GetFecInstanceId() const + { + return (ntohs(*((UINT16*)(buffer+FEC_INSTANCE_OFFSET)))); + } + UINT16 GetFecMaxBlockLen() const + { + return (ntohs(*((UINT16*)(buffer+FEC_NDATA_OFFSET)))); + } + UINT16 GetFecNumParity() const + { + return (ntohs(*((UINT16*)(buffer+FEC_NPARITY_OFFSET)))); + } + UINT16 GetSegmentSize() const + { + return (ntohs(*((UINT16*)(buffer+SEG_SIZE_OFFSET)))); + } + NormObjectSize GetObjectSize() const + { + return NormObjectSize(ntohs(*((UINT16*)(buffer+OBJ_SIZE_OFFSET))), + ntohl(*((UINT32*)(buffer+OBJ_SIZE_OFFSET+2)))); + } + + private: + enum + { + FEC_INSTANCE_OFFSET = LENGTH_OFFSET + 1, + FEC_NDATA_OFFSET = FEC_INSTANCE_OFFSET +2, + FEC_NPARITY_OFFSET = FEC_NDATA_OFFSET + 2, + SEG_SIZE_OFFSET = FEC_NPARITY_OFFSET+2, + OBJ_SIZE_OFFSET = SEG_SIZE_OFFSET + 2 + }; +}; // end class NormFtiExtension + + class NormInfoMsg : public NormObjectMsg { public: - UINT16 GetInfoLen() const {return (length - INFO_OFFSET);} - const char* GetInfo() const {return (buffer + INFO_OFFSET);} - void SetInfo(char* data, UINT16 len) + void Init() { - memcpy(buffer+INFO_OFFSET, data, len); - length = len + INFO_OFFSET; + SetType(INFO); + SetBaseHeaderLength(INFO_HEADER_LEN); + // Default "fec_id" = 129 + SetFecId(129); + ResetFlags(); } - - private: - enum + + UINT16 GetInfoLen() const {return (length - header_length);} + const char* GetInfo() const {return (buffer + header_length);} + + // Note: apply any header extensions first + void SetInfo(const char* data, UINT16 size) { - INFO_OFFSET = OBJ_ID_OFFSET + 2 - }; + memcpy(buffer+header_length, data, size); + length = size + header_length; + } + private: + enum {INFO_HEADER_LEN = OBJ_MSG_OFFSET}; }; // end class NormInfoMsg class NormDataMsg : public NormObjectMsg { public: - // Message building methods + void Init() + { + SetType(DATA); + // Default "fec_id" = 129 w/ "fec_payload_id" length = 8 bytes + SetBaseHeaderLength(DATA_HEADER_LEN); + SetFecId(129); + ResetFlags(); + } + // Message building methods (in addition to NormObjectMsg fields) void SetFecBlockId(const NormBlockId& blockId) { UINT32 temp32 = htonl((UINT32)blockId); memcpy(buffer+BLOCK_ID_OFFSET, &temp32, 4); } + void SetFecBlockLen(UINT16 blockLen) + { + blockLen = htons(blockLen); + memcpy(buffer+BLOCK_LEN_OFFSET, &blockLen, 2); + } void SetFecSymbolId(UINT16 symbolId) { symbolId = htons(symbolId); memcpy(buffer+SYMBOL_ID_OFFSET, &symbolId, 2); } - char* AccessData() {return (buffer+DATA_OFFSET);} - void SetData(const NormObjectSize& offset, char* data, UINT16 len) + + // Three ways to set payload content: + // 1) Directly access payload to copy segment, then set data message length + // (segment must include "payload_len", "payload_offset", and "payload_data" + char* AccessPayload() {return (buffer+header_length);} + void SetDataPayloadLength(UINT16 dataLength) { - WriteLength(buffer+LENGTH_OFFSET, len); - WriteOffset(buffer+LENGTH_OFFSET, offset); - memcpy(buffer+DATA_OFFSET, data, len); - length = DATA_OFFSET + len; - } - void SetDataOffset(const NormObjectSize& offset) - { - WriteOffset(buffer+LENGTH_OFFSET, offset); + length = header_length + PAYLOAD_DATA_OFFSET + dataLength; } - void SetDataLength(UINT16 len) + // 2) Set data segment payload with "offset", "data" ptr, and "len" + void SetDataPayload(const NormObjectSize& offset, char* data, UINT16 len) { - WriteLength(buffer+LENGTH_OFFSET, len); - length = DATA_OFFSET + len; - } - // (Note: "payload_len" and "offset" field spaces are in the FEC payload) - char* AccessPayload() {return (buffer+PAYLOAD_OFFSET);} - void SetPayload(const char* data, UINT16 len) + WritePayloadLength(AccessPayload(), len); + WritePayloadOffset(AccessPayload(), offset); + memcpy(buffer+header_length+PAYLOAD_DATA_OFFSET, data, len); + length = header_length + PAYLOAD_DATA_OFFSET + len; + } + // 3) Set "payload" directly (useful for FEC parity segments) + void SetPayload(char* payload, UINT16 payloadLength) { - memcpy(buffer+PAYLOAD_OFFSET, data, len); - length = PAYLOAD_OFFSET + len; + memcpy(buffer+header_length, payload, payloadLength); + length = header_length + payloadLength; } + // AccessPayloadData() for ZERO padding + char* AccessPayloadData() {return (buffer+header_length+PAYLOAD_DATA_OFFSET);} // Message processing methods NormBlockId GetFecBlockId() const { - UINT32 temp32; - memcpy(&temp32, buffer+BLOCK_ID_OFFSET, 4); - return (ntohl(temp32)); + return (ntohl(*((UINT32*)(buffer+BLOCK_ID_OFFSET)))); } + UINT16 GetFecBlockLen() const + { + return (ntohs(*((UINT16*)(buffer+BLOCK_LEN_OFFSET)))); + } + UINT16 GetFecSymbolId() const { - UINT16 temp16; - memcpy(&temp16, buffer+SYMBOL_ID_OFFSET, 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+SYMBOL_ID_OFFSET)))); } bool IsData() const {return (GetFecSymbolId() < GetFecBlockLen());} - const char* GetData() {return (buffer + DATA_OFFSET);} - UINT16 GetDataLength() const {return (length - DATA_OFFSET);} + const char* GetPayloadData() const {return (buffer + header_length + PAYLOAD_DATA_OFFSET);} + UINT16 GetPayloadDataLength() const {return (length - (header_length + PAYLOAD_DATA_OFFSET));} - const char* GetPayload() {return (buffer+LENGTH_OFFSET);} - UINT16 GetPayloadLength() const {return (length - LENGTH_OFFSET);} + // Note: "payload" includes "payload_len", "payload_offset", and "payload_data" fields + const char* GetPayload() const {return (buffer + header_length);} + UINT16 GetPayloadLength() const {return (length - header_length);} - bool IsParity() const {return (GetFecSymbolId() >= GetFecBlockLen());} - // (Note: "payload_len" and "offset" field spaces are in the FEC payload) - char* GetParity() {return buffer+LENGTH_OFFSET;} - // (Note: This should be the same as "segment_size") - UINT16 GetParityLen() const - {return (length - LENGTH_OFFSET);} - - // Some static helper routines for various purposes - static UINT16 PayloadHeaderLen() {return (DATA_OFFSET - LENGTH_OFFSET);} - static void WriteLength(char* payload, UINT16 len) + // Some static helper routines for reading/writing embedded payload length/offsets + static UINT16 PayloadHeaderLength() {return (PAYLOAD_DATA_OFFSET);} + static void WritePayloadLength(char* payload, UINT16 len) { - UINT16 temp16 = htons(len); - memcpy(payload+LENGTH_OFFSET-LENGTH_OFFSET, &temp16, 2); + *((UINT16*)(payload+PAYLOAD_LENGTH_OFFSET)) = htons(len); } - static void WriteOffset(char* payload, const NormObjectSize& offset) + static void WritePayloadOffset(char* payload, const NormObjectSize& offset) { - UINT16 temp16 = htons(offset.MSB()); - memcpy(payload+OFFSET_OFFSET-LENGTH_OFFSET, &temp16, 2); - UINT32 temp32 = htonl(offset.LSB()); - memcpy(payload+OFFSET_OFFSET-LENGTH_OFFSET+2, &temp32, 4); + *((UINT16*)(payload+PAYLOAD_OFFSET_OFFSET)) = htons(offset.MSB()); + *((UINT32*)(payload+PAYLOAD_OFFSET_OFFSET+2)) = htonl(offset.LSB()); } - static UINT16 ReadLength(const char* payload) + static UINT16 ReadPayloadLength(const char* payload) { - UINT16 temp16; - memcpy(&temp16, payload+LENGTH_OFFSET-LENGTH_OFFSET, 2); - return ntohs(temp16); + return (ntohs(*((UINT16*)(payload+PAYLOAD_LENGTH_OFFSET)))); } - static NormObjectSize ReadOffset(const char* payload) + static NormObjectSize ReadPayloadOffset(const char* payload) { - UINT16 temp16; - memcpy(&temp16, payload+OFFSET_OFFSET-LENGTH_OFFSET, 2); - UINT32 temp32; - memcpy(&temp32, payload+OFFSET_OFFSET-LENGTH_OFFSET+2, 4); - return NormObjectSize(ntohs(temp16), ntohl(temp32)); + return NormObjectSize(ntohs(*((UINT16*)(payload+PAYLOAD_OFFSET_OFFSET))), + ntohl(*((UINT32*)(payload+PAYLOAD_OFFSET_OFFSET+2)))); } private: enum { - BLOCK_ID_OFFSET = OBJ_ID_OFFSET + 2, - SYMBOL_ID_OFFSET = BLOCK_ID_OFFSET + 4, - PAYLOAD_OFFSET = SYMBOL_ID_OFFSET + 2, - LENGTH_OFFSET = PAYLOAD_OFFSET, - OFFSET_OFFSET = LENGTH_OFFSET + 2, - DATA_OFFSET = OFFSET_OFFSET + 6 + BLOCK_ID_OFFSET = OBJ_MSG_OFFSET, + BLOCK_LEN_OFFSET = BLOCK_ID_OFFSET + 4, + SYMBOL_ID_OFFSET = BLOCK_LEN_OFFSET + 2, + DATA_HEADER_LEN = SYMBOL_ID_OFFSET + 2 + }; + enum + { + PAYLOAD_LENGTH_OFFSET = 0, + PAYLOAD_OFFSET_OFFSET = PAYLOAD_LENGTH_OFFSET + 2, + PAYLOAD_DATA_OFFSET = PAYLOAD_OFFSET_OFFSET + 6 }; }; // end class NormDataMsg @@ -456,16 +600,21 @@ class NormCmdMsg : public NormMsg public: enum Flavor { - NORM_CMD_INVALID, - NORM_CMD_FLUSH, - NORM_CMD_SQUELCH, - NORM_CMD_ACK_REQ, - NORM_CMD_REPAIR_ADV, - NORM_CMD_CC, - NORM_CMD_APPLICATION + INVALID = 0, + FLUSH = 1, + EOT = 2, + SQUELCH = 3, + CC = 4, + REPAIR_ADV = 5, + ACK_REQ = 6, + APPLICATION = 7 }; // Message building + void SetSessionId(UINT16 sessionId) + { + *((UINT16*)(buffer+SESSION_ID_OFFSET)) = htons(sessionId); + } void SetGrtt(UINT8 quantizedGrtt) {buffer[GRTT_OFFSET] = quantizedGrtt;} void SetGroupSize(UINT8 quantizedGroupSize) @@ -474,6 +623,7 @@ class NormCmdMsg : public NormMsg {buffer[FLAVOR_OFFSET] = flavor;} // Message processing + UINT16 GetSessionId() const {return (ntohs(*((UINT16*)(buffer+SESSION_ID_OFFSET))));} UINT8 GetGrtt() const {return buffer[GRTT_OFFSET];} UINT8 GetGroupSize() const {return buffer[GSIZE_OFFSET];} NormCmdMsg::Flavor GetFlavor() const {return (Flavor)buffer[FLAVOR_OFFSET];} @@ -481,7 +631,8 @@ class NormCmdMsg : public NormMsg protected: enum { - GRTT_OFFSET = MSG_OFFSET, + SESSION_ID_OFFSET = MSG_OFFSET, + GRTT_OFFSET = SESSION_ID_OFFSET + 2, GSIZE_OFFSET = GRTT_OFFSET + 1, FLAVOR_OFFSET = GSIZE_OFFSET + 1 }; @@ -489,276 +640,312 @@ class NormCmdMsg : public NormMsg class NormCmdFlushMsg : public NormCmdMsg { - public: - enum Flag {NORM_FLUSH_FLAG_EOT = 0x01}; + friend class NormMsg; - // Message building - void Reset() + public: + void Init() { - buffer[FLAGS_OFFSET] = 0; - length = SYMBOL_ID_OFFSET + 2; - } - void SetFlag(NormCmdFlushMsg::Flag flag) - {buffer[FLAGS_OFFSET] |= flag;} - void UnsetFlag(NormCmdFlushMsg::Flag flag) - {buffer[FLAGS_OFFSET] &= ~flag;} + SetType(CMD); + SetFlavor(FLUSH); + SetBaseHeaderLength(FLUSH_HEADER_LEN); + SetFecId(129); // default "fec_id" + } + + void SetFecId(UINT8 fecId) {buffer[FEC_ID_OFFSET] = fecId;} void SetObjectId(const NormObjectId& objectId) { - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+OBJECT_ID_OFFSET, &temp16, 2); + *((UINT16*)(buffer+OBJ_ID_OFFSET)) = htons((UINT16)objectId); } + // "fec_payload_id" fields, assuming "fec_id" = 129 void SetFecBlockId(const NormBlockId& blockId) { - UINT32 temp32 = htonl((UINT32)blockId); - memcpy(buffer+BLOCK_ID_OFFSET, &temp32, 4); + *((UINT32*)(buffer+BLOCK_ID_OFFSET)) = htonl((UINT32)blockId); + } + void SetFecBlockLen(UINT16 blockLen) + { + *((UINT16*)(buffer+BLOCK_LEN_OFFSET)) = htons((UINT16)blockLen); } void SetFecSymbolId(UINT16 symbolId) { - symbolId = htons(symbolId); - memcpy(buffer+SYMBOL_ID_OFFSET, &symbolId, 2); + *((UINT16*)(buffer+SYMBOL_ID_OFFSET)) = htons((UINT16)symbolId); + } + void ResetAckingNodeList() {length = header_length;} + bool AppendAckingNode(NormNodeId nodeId, UINT16 segmentSize) + { + if ((length-header_length+ 4) > segmentSize) return false; + *((UINT32*)(buffer+length)) = htonl(nodeId); + length += 4; + return true; } // Message processing - bool FlagIsSet(NormCmdFlushMsg::Flag flag) - {return (0 != (buffer[FLAGS_OFFSET] & flag));} - NormObjectId GetObjectId() + UINT8 GetFecId() {return buffer[FEC_ID_OFFSET];} + NormObjectId GetObjectId() const { - UINT16 temp16; - memcpy(&temp16, buffer+OBJECT_ID_OFFSET, 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+OBJ_ID_OFFSET)))); } - NormBlockId GetFecBlockId() + NormBlockId GetFecBlockId() const { - UINT32 temp32; - memcpy(&temp32, buffer+BLOCK_ID_OFFSET, 4); - return (ntohl(temp32)); + return (ntohl(*((UINT32*)(buffer+BLOCK_ID_OFFSET)))); } - UINT16 GetFecSymbolId() + UINT16 GetFecBlockLen() const { - UINT16 temp16; - memcpy(&temp16, buffer+SYMBOL_ID_OFFSET, 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+BLOCK_LEN_OFFSET)))); + } + UINT16 GetFecSymbolId() const + { + return (ntohs(*((UINT16*)(buffer+SYMBOL_ID_OFFSET)))); + } + UINT16 GetAckingNodeCount() const {return ((length - header_length) >> 2);} + const UINT32* GetAckingNodeList() const {return (UINT32*)(buffer+header_length);} + NormNodeId GetAckingNodeId(UINT16 index) const + { + return (ntohl(*((UINT32*)(buffer+header_length+(index<<2))))); } - // Flush messages are fixed length - UINT16 GetLength() {return (SYMBOL_ID_OFFSET + 2);} private: enum { - FLAGS_OFFSET = FLAVOR_OFFSET + 1, - OBJECT_ID_OFFSET = FLAGS_OFFSET + 1, - BLOCK_ID_OFFSET = OBJECT_ID_OFFSET + 2, - SYMBOL_ID_OFFSET = BLOCK_ID_OFFSET + 4 + FEC_ID_OFFSET = FLAVOR_OFFSET + 1, + OBJ_ID_OFFSET = FEC_ID_OFFSET + 1, + BLOCK_ID_OFFSET = OBJ_ID_OFFSET + 2, + BLOCK_LEN_OFFSET = BLOCK_ID_OFFSET + 4, + SYMBOL_ID_OFFSET = BLOCK_LEN_OFFSET + 2, + FLUSH_HEADER_LEN = SYMBOL_ID_OFFSET + 2 }; }; // end class NormCmdFlushMsg +class NormCmdEotMsg : public NormCmdMsg +{ + public: + void Init() + { + SetType(CMD); + SetFlavor(EOT); + SetBaseHeaderLength(EOT_HEADER_LEN); + memset(buffer+RESERVED_OFFSET, 0, 3); + } + private: + enum + { + RESERVED_OFFSET = FLAVOR_OFFSET + 1, + EOT_HEADER_LEN = RESERVED_OFFSET + 3 + }; +}; // end class NormCmdEotMsg + class NormCmdSquelchMsg : public NormCmdMsg { public: // Message building - void SetObjectId(NormObjectId objectId) + void Init() { - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+OBJECT_ID_OFFSET, &temp16, 2); + SetType(CMD); + SetFlavor(SQUELCH); + SetBaseHeaderLength(SQUELCH_HEADER_LEN); + SetFecId(129); // default "fec_id" + } + void SetFecId(UINT8 fecId) {buffer[FEC_ID_OFFSET] = fecId;} + void SetObjectId(const NormObjectId& objectId) + { + *((UINT16*)(buffer+OBJ_ID_OFFSET)) = htons((UINT16)objectId); } - void SetFecBlockId(NormBlockId blockId) + // "fec_payload_id" fields, assuming "fec_id" = 129 + void SetFecBlockId(const NormBlockId& blockId) { - UINT32 temp32 = htonl((UINT32)blockId); - memcpy(buffer+BLOCK_ID_OFFSET, &temp32, 4); + *((UINT32*)(buffer+BLOCK_ID_OFFSET)) = htonl((UINT32)blockId); + } + void SetFecBlockLen(UINT16 blockLen) + { + *((UINT16*)(buffer+BLOCK_LEN_OFFSET)) = htons((UINT16)blockLen); } void SetFecSymbolId(UINT16 symbolId) { - symbolId = htons(symbolId); - memcpy(buffer+SYMBOL_ID_OFFSET, &symbolId, 2); + *((UINT16*)(buffer+SYMBOL_ID_OFFSET)) = htons((UINT16)symbolId); } - void ResetInvalidObjectList() {length = OBJECT_LIST_OFFSET;} + void ResetInvalidObjectList() {length = header_length;} bool AppendInvalidObject(NormObjectId objectId, UINT16 segmentSize) { - if ((length-OBJECT_LIST_OFFSET+2) > segmentSize) return false; - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+length, &temp16, 2); + if ((length-header_length+2) > segmentSize) return false; + *((UINT16*)(buffer+length)) = htons((UINT16)objectId); length += 2; return true; } // Message processing - NormObjectId GetObjectId() + UINT8 GetFecId() {return buffer[FEC_ID_OFFSET];} + NormObjectId GetObjectId() const { - UINT16 temp16; - memcpy(&temp16, buffer+OBJECT_ID_OFFSET, 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+OBJ_ID_OFFSET)))); } - NormBlockId GetFecBlockId() + NormBlockId GetFecBlockId() const { - UINT32 temp32; - memcpy(&temp32, buffer+BLOCK_ID_OFFSET, 4); - return (NormBlockId(ntohl(temp32))); + return (ntohl(*((UINT32*)(buffer+BLOCK_ID_OFFSET)))); } - UINT16 GetFecSymbolId() + UINT16 GetFecBlockLen() const { - UINT16 temp16; - memcpy(&temp16, buffer+SYMBOL_ID_OFFSET, 2); - return (ntohs(temp16)); - } + return (ntohs(*((UINT16*)(buffer+BLOCK_LEN_OFFSET)))); + } + UINT16 GetFecSymbolId() const + { + return (ntohs(*((UINT16*)(buffer+SYMBOL_ID_OFFSET)))); + } // Use these to parse invalid object list - UINT16 GetInvalidObjectCount() - {return ((length - OBJECT_LIST_OFFSET) / 2);} - NormObjectId GetInvalidObjectId(UINT16 index) + UINT16 GetInvalidObjectCount() const {return ((length - header_length) >> 1);} + UINT16* GetInvalidObjectList() const {return (UINT16*)(buffer+header_length);} + NormObjectId GetInvalidObjectId(UINT16 index) const { - UINT16 temp16; - memcpy(&temp16, buffer + (2*index+OBJECT_LIST_OFFSET), 2); - return (ntohs(temp16)); + return (ntohs(*((UINT16*)(buffer+header_length+(index << 1))))); } private: enum { - RESERVED_OFFSET = FLAVOR_OFFSET + 1, - OBJECT_ID_OFFSET = RESERVED_OFFSET + 1, - BLOCK_ID_OFFSET = OBJECT_ID_OFFSET + 2, - SYMBOL_ID_OFFSET = BLOCK_ID_OFFSET + 4, - OBJECT_LIST_OFFSET = SYMBOL_ID_OFFSET + 2, + FEC_ID_OFFSET = FLAVOR_OFFSET + 1, + OBJ_ID_OFFSET = FEC_ID_OFFSET + 1, + BLOCK_ID_OFFSET = OBJ_ID_OFFSET + 2, + BLOCK_LEN_OFFSET = BLOCK_ID_OFFSET + 4, + SYMBOL_ID_OFFSET = BLOCK_LEN_OFFSET + 2, + SQUELCH_HEADER_LEN = SYMBOL_ID_OFFSET + 2 }; }; // end class NormCmdSquelchMsg -class NormCmdAckReqMsg : public NormCmdMsg + +// These flag values are used "cc_flags" field of NORM_CMD(CC) CC_NODE_LIST +// items and NORM_NACK and NORM_ACK messages +class NormCC { public: - // Note: Support for application-defined AckFlavors (32-255) - // may be provided, but 0-31 are reserved values - enum AckFlavor + enum Flag { - INVALID = 0, - WATERMARK = 1, - RTT = 2, - APP_BASE = 32 + CLR = 0x01, + PLR = 0x02, + RTT = 0x04, + START = 0x08, + LEAVE = 0x10 }; - - // Message building - void SetAckFlavor(UINT8 ackFlavor) - {buffer[ACK_FLAVOR_OFFSET] = ackFlavor;} - // For setting generic content - void SetAckContent(char content[64]) - {memcpy(buffer+CONTENT_OFFSET, content, 64);} - - void ResetAckingNodeList() - {length = NODE_LIST_OFFSET;} - UINT16 AppendAckingNode(NormNodeId nodeId) - { - nodeId = htonl(nodeId); - memcpy(buffer+length, &nodeId, 4); - return length; - } - - void SetWatermark(NormObjectId objectId, - NormBlockId fecBlockId, - UINT16 fecSymbolId) - { - UINT16 temp16 = htons((UINT16)objectId); - memcpy(buffer+OBJECT_ID_OFFSET, &temp16, 2); - UINT32 temp32 = htonl((UINT32)fecBlockId); - memcpy(buffer+BLOCK_ID_OFFSET, &temp32, 4); - fecSymbolId = htons(fecSymbolId); - memcpy(buffer+SYMBOL_ID_OFFSET, &fecSymbolId, 2); - } - - void SetRttSendTime(const struct timeval& sendTime) - { - UINT32 temp32 = htonl(sendTime.tv_sec); - memcpy(buffer+SEND_TIME_OFFSET, &temp32, 4); - temp32 = htonl(sendTime.tv_usec); - memcpy(buffer+SEND_TIME_OFFSET+4, &temp32, 4); - } - - // Message processing - NormCmdAckReqMsg::AckFlavor GetAckFlavor() - {return (AckFlavor)buffer[ACK_FLAVOR_OFFSET];} - char* GetAckContent() {return (buffer+CONTENT_OFFSET);} - UINT16 GetAckingNodeCount() - {return ((length - NODE_LIST_OFFSET) / 4);} - NormNodeId GetAckingNodeId(UINT16 index) - { - UINT32 temp32; - memcpy(&temp32, buffer+NODE_LIST_OFFSET+(4*index), 4); - return (ntohl(temp32)); - } - - void GetWatermark(NormObjectId& objectId, - NormBlockId& fecBlockId, - UINT16 fecSymbolId) - { - UINT16 temp16; - memcpy(&temp16, buffer+OBJECT_ID_OFFSET, 2); - objectId = ntohs(temp16); - UINT32 temp32; - memcpy(&temp32, buffer+BLOCK_ID_OFFSET, 4); - fecBlockId = ntohl(temp32); - memcpy(&fecSymbolId, buffer+SYMBOL_ID_OFFSET, 2); - fecSymbolId = ntohs(fecSymbolId); - } +}; // end class NormCC - void GetRttSendTime(struct timeval& sendTime) - { - memcpy(&sendTime.tv_sec, buffer+SEND_TIME_OFFSET, 4); - sendTime.tv_sec = ntohl(sendTime.tv_sec); - memcpy(&sendTime.tv_usec, buffer+SEND_TIME_OFFSET+4, 4); - sendTime.tv_usec = ntohl(sendTime.tv_usec); - } - - private: - enum GenericOffsets - { - ACK_FLAVOR_OFFSET = FLAVOR_OFFSET + 1, - CONTENT_OFFSET = ACK_FLAVOR_OFFSET + 1, - NODE_LIST_OFFSET = CONTENT_OFFSET + 8 - }; - enum WatermarkOffsets - { - OBJECT_ID_OFFSET = CONTENT_OFFSET, - BLOCK_ID_OFFSET = OBJECT_ID_OFFSET + 2, - SYMBOL_ID_OFFSET = BLOCK_ID_OFFSET + 4 - }; - enum RttOffsets - { - SEND_TIME_OFFSET = CONTENT_OFFSET - }; -}; // end class NormCmdAckReqMsg - - -class NormCmdRepairAdvMsg : public NormCmdMsg -{ - public: - enum Flag {NORM_REPAIR_ADV_FLAG_LIMIT = 0x01}; - // TBD (Uses NormNack format) -}; // end class NormCmdRepairAdvMsg - - class NormCmdCCMsg : public NormCmdMsg { - // (TBD) Congestion control command message -}; // end class NormCmdCCMsg - -class NormCmdApplicationMsg : public NormCmdMsg -{ - // (TBD) Application-defined command (non-acked) -}; // end class NormCmdApplicationMsg - + public: + void Init() + { + SetType(CMD); + SetFlavor(CC); + SetBaseHeaderLength(CC_HEADER_LEN); + buffer[RESERVED_OFFSET] = 0; + } + void SetCCSequence(UINT16 ccSequence) + { + *((UINT16*)(buffer+CC_SEQUENCE_OFFSET)) = htons(ccSequence); + } + void SetSendTime(const struct timeval& sendTime) + { + *((UINT32*)(buffer+SEND_TIME_OFFSET)) = htonl(sendTime.tv_sec); + *((UINT32*)(buffer+SEND_TIME_OFFSET+4)) = htonl(sendTime.tv_usec); + } + + UINT16 GetCCSequence() const + { + return (ntohs(*((UINT16*)(buffer+CC_SEQUENCE_OFFSET)))); + } + void GetSendTime(struct timeval& sendTime) const + { + sendTime.tv_sec = ntohl(*((UINT32*)(buffer+SEND_TIME_OFFSET))); + sendTime.tv_usec = ntohl(*((UINT32*)(buffer+SEND_TIME_OFFSET+4))); + } + + bool AppendCCNode(UINT16 segMax, NormNodeId nodeId, UINT8 flags, + UINT8 rtt, UINT16 rate) + { + if ((length-header_length+CC_ITEM_SIZE)> segMax) return false; + char* ptr = buffer+length; + *((UINT32*)(ptr+CC_NODE_ID_OFFSET)) = htonl(nodeId); + ptr[CC_FLAGS_OFFSET] = flags; + ptr[CC_RTT_OFFSET] = rtt; + *((UINT16*)(ptr+CC_RATE_OFFSET)) = htons(rate); + length += CC_ITEM_SIZE; + return true; + } + bool GetCCNode(NormNodeId nodeId, UINT8& flags, UINT8& rtt, UINT16& rate) const; + + + class Iterator; + friend class Iterator; + + class Iterator + { + public: + Iterator(const NormCmdCCMsg& msg); + void Reset() {offset = 0;} + bool GetNextNode(NormNodeId& nodeId, UINT8& flags, UINT8& rtt, UINT16& rate); + + private: + const NormCmdCCMsg& cc_cmd; + UINT16 offset; + }; + + + private: + enum + { + RESERVED_OFFSET = FLAVOR_OFFSET + 1, + CC_SEQUENCE_OFFSET = RESERVED_OFFSET + 1, + SEND_TIME_OFFSET = CC_SEQUENCE_OFFSET + 2, + CC_HEADER_LEN = SEND_TIME_OFFSET + 8 + }; + + enum + { + CC_NODE_ID_OFFSET = 0, + CC_FLAGS_OFFSET = CC_NODE_ID_OFFSET + 4, + CC_RTT_OFFSET = CC_FLAGS_OFFSET + 1, + CC_RATE_OFFSET = CC_RTT_OFFSET + 1, + CC_ITEM_SIZE = CC_RATE_OFFSET + 2 + }; + +}; // end NormCmdCCMsg + +class NormCCRateExtension : public NormHeaderExtension +{ + public: + + virtual void Init(char* theBuffer) + { + AttachBuffer(theBuffer); + SetType(CC_RATE); + buffer[RESERVED_OFFSET] = 0; + } + void SetSendRate(UINT16 sendRate) + { + *((UINT16*)(buffer+SEND_RATE_OFFSET)) = htons(sendRate); + } + UINT16 GetSendRate() {return (ntohs(*((UINT16*)(buffer+SEND_RATE_OFFSET))));} + + private: + enum + { + RESERVED_OFFSET = TYPE_OFFSET + 1, + SEND_RATE_OFFSET = RESERVED_OFFSET + 1 + }; +}; // end class NormCCRateExtension + +// This implementation currently assumes "fec_id"= 129 class NormRepairRequest { public: - class Iterator; - friend class NormRepairRequest::Iterator; - + class Iterator; + friend class NormRepairRequest::Iterator; enum Form { - INVALID, - ITEMS, - RANGES, - ERASURES + INVALID = 0, + ITEMS = 1, + RANGES = 2, + ERASURES = 3 }; enum Flag @@ -771,45 +958,47 @@ class NormRepairRequest // Construction NormRepairRequest(); - void SetBuffer(char* bufferPtr, UINT16 bufferLen) + void Init(char* bufferPtr, UINT16 bufferLen) { - buffer = bufferPtr; + buffer = bufferPtr; buffer_len = bufferLen; length = 0; } - static UINT16 RepairItemLength() {return 8;} - static UINT16 RepairRangeLength() {return 16;} - static UINT16 ErasureItemLength() {return 8;} + // (TBD) these could be an enumeration for optimization + static UINT16 RepairItemLength() {return 12;} + static UINT16 RepairRangeLength() {return 24;} + static UINT16 ErasureItemLength() {return 12;} // Repair request building void SetForm(NormRepairRequest::Form theForm) {form = theForm;} void ResetFlags() {flags = 0;} - void SetFlag(NormRepairRequest::Flag theFlag) {flags |= theFlag;} - //void SetFlags(int theFlags) {flags |= theFlags;} - void UnsetFlag(NormRepairRequest::Flag theFlag) {flags &= ~theFlag;} - - + void SetFlag(NormRepairRequest::Flag theFlag) {flags |= theFlag;} + void UnsetFlag(NormRepairRequest::Flag theFlag) {flags &= ~theFlag;} // Returns length (each repair item requires 8 bytes of space) bool AppendRepairItem(const NormObjectId& objectId, const NormBlockId& blockId, + UINT16 blockLen, UINT16 symbolId); bool AppendRepairRange(const NormObjectId& startObjectId, const NormBlockId& startBlockId, - UINT16 startSymbolId, + UINT16 startBlockLen, + UINT16 startSymbolId, const NormObjectId& endObjectId, const NormBlockId& endBlockId, + UINT16 endBlockLen, UINT16 endSymbolId); bool AppendErasureCount(const NormObjectId& objectId, const NormBlockId& blockId, + UINT16 blockLen, UINT16 erasureCount); UINT16 Pack(); // Repair request processing - UINT16 Unpack(); + UINT16 Unpack(const char* bufferPtr, UINT16 bufferLen); NormRepairRequest::Form GetForm() const {return form;} bool FlagIsSet(NormRepairRequest::Flag theFlag) const {return (0 != (theFlag & flags));} @@ -822,6 +1011,7 @@ class NormRepairRequest void Reset() {offset = 0;} bool NextRepairItem(NormObjectId* objectId, NormBlockId* blockId, + UINT16* blockLen, UINT16* symbolId); private: NormRepairRequest& request; @@ -832,180 +1022,385 @@ class NormRepairRequest bool RetrieveRepairItem(UINT16 offset, NormObjectId* objectId, NormBlockId* blockId, - UINT16* erasureCount) const; + UINT16* blockLen, + UINT16* symbolId) const; enum { - FORM_OFFSET = 0, - FLAGS_OFFSET = FORM_OFFSET + 1, - LENGTH_OFFSET = FLAGS_OFFSET + 1, - CONTENT_OFFSET = LENGTH_OFFSET + 2 + FORM_OFFSET = 0, + FLAGS_OFFSET = FORM_OFFSET + 1, + LENGTH_OFFSET = FLAGS_OFFSET + 1, + ITEM_LIST_OFFSET = LENGTH_OFFSET + 2 }; - Form form; - int flags; - UINT16 length; - char* buffer; - UINT16 buffer_len; + + // These are the offsets for "fec_id" = 129 NormRepairRequest items + enum + { + FEC_ID_OFFSET = 0, + RESERVED_OFFSET = FEC_ID_OFFSET + 1, + OBJ_ID_OFFSET = RESERVED_OFFSET + 1, + BLOCK_ID_OFFSET = OBJ_ID_OFFSET + 2, + BLOCK_LEN_OFFSET = BLOCK_ID_OFFSET + 4, + SYMBOL_ID_OFFSET = BLOCK_LEN_OFFSET + 2 + }; + + Form form; + int flags; + UINT16 length; + char* buffer; + UINT16 buffer_len; }; // end class NormRepairRequest - -class NormNackMsg : public NormMsg +class NormCmdRepairAdvMsg : public NormCmdMsg { public: - // Message building - void SetServerId(NormNodeId serverId) + enum Flag {NORM_REPAIR_ADV_FLAG_LIMIT = 0x01}; + + void Init() { - serverId = htonl(serverId); - memcpy(buffer+SERVER_ID_OFFSET, &serverId, 4); - } - void SetGrttResponse(const struct timeval& grttResponse) - { - UINT32 temp32 = htonl(grttResponse.tv_sec); - memcpy(buffer+GRTT_RESPONSE_OFFSET, &temp32, 4); - temp32 = htonl(grttResponse.tv_usec); - memcpy(buffer+GRTT_RESPONSE_OFFSET+4, &temp32, 4); - } - void SetLossEstimate(UINT16 lossEstimate) - { - lossEstimate = htons(lossEstimate); - memcpy(buffer+LOSS_OFFSET, &lossEstimate, 2); - } - void SetGrttSequence(UINT16 sequence) - { - sequence = htons(sequence); - memcpy(buffer+GRTT_SEQUENCE_OFFSET, &sequence, 2); + SetType(CMD); + SetFlavor(REPAIR_ADV); + SetBaseHeaderLength(REPAIR_ADV_HEADER_LEN); + ResetFlags(); + *((UINT16*)(buffer+RESERVED_OFFSET)) = 0; } - void ResetNackContent() {length = CONTENT_OFFSET;} + // Message building + void ResetFlags() {buffer[FLAGS_OFFSET] = 0;} + void SetFlag(NormCmdRepairAdvMsg::Flag flag) + {buffer[FLAGS_OFFSET] |= (UINT8)flag;} void AttachRepairRequest(NormRepairRequest& request, UINT16 segmentMax) { - int buflen = segmentMax - (length - CONTENT_OFFSET); + int buflen = segmentMax - (length - header_length); buflen = (buflen>0) ? buflen : 0; - request.SetBuffer(buffer+length, buflen); + request.Init(buffer+length, buflen); } UINT16 PackRepairRequest(NormRepairRequest& request) { UINT16 requestLength = request.Pack(); - length += request.Pack(); + length += requestLength; return requestLength; } // Message processing - NormNodeId GetServerId() + bool FlagIsSet(NormCmdRepairAdvMsg::Flag flag) const { - UINT32 temp32; - memcpy(&temp32, buffer+SERVER_ID_OFFSET, 4); - return (ntohl(temp32)); + return (0 != ((UINT8)flag | buffer[FLAGS_OFFSET])); + } + //char* AccessRepairContent() {return (buffer + header_length);} + const char* GetRepairContent() const {return (buffer + header_length);} + UINT16 GetRepairContentLength() const {return (length - header_length);} + + private: + enum + { + FLAGS_OFFSET = FLAVOR_OFFSET + 1, + RESERVED_OFFSET = FLAGS_OFFSET + 1, + REPAIR_ADV_HEADER_LEN = RESERVED_OFFSET + 2 + }; + +}; // end class NormCmdRepairAdvMsg + + +class NormCCFeedbackExtension : public NormHeaderExtension +{ + public: + virtual void Init(char* theBuffer) + { + AttachBuffer(theBuffer); + SetType(CC_FEEDBACK); + SetWords(3); + buffer[CC_FLAGS_OFFSET] = 0; + *((UINT16*)(buffer+CC_RESERVED_OFFSET)) = 0; } - void GetGrttResponse(struct timeval& grttResponse) + void SetCCSequence(UINT16 ccSequence) { - memcpy(&grttResponse.tv_sec, buffer+GRTT_RESPONSE_OFFSET, 4); - grttResponse.tv_sec = ntohl(grttResponse.tv_sec); - memcpy(&grttResponse.tv_usec, buffer+GRTT_RESPONSE_OFFSET+4, 4); - grttResponse.tv_usec = ntohl(grttResponse.tv_usec); + *((UINT16*)(buffer+CC_SEQUENCE_OFFSET)) = htons(ccSequence); } - UINT16 GetLossEstimate() + void ResetCCFlags() {buffer[CC_FLAGS_OFFSET] = 0;} + void SetCCFlag(NormCC::Flag flag) {buffer[CC_FLAGS_OFFSET] |= (UINT8)flag;} + void SetCCRtt(UINT8 ccRtt) {buffer[CC_RTT_OFFSET] = ccRtt;} + void SetCCLoss(UINT16 ccLoss) {*((UINT16*)(buffer+CC_LOSS_OFFSET)) = htons(ccLoss);} + void SetCCRate(UINT16 ccRate) {*((UINT16*)(buffer+CC_RATE_OFFSET)) = htons(ccRate);} + + UINT16 GetCCSequence() const {return (ntohs(*((UINT16*)(buffer+CC_SEQUENCE_OFFSET))));} + UINT8 GetCCFlags() {return buffer[CC_FLAGS_OFFSET];} + bool CCFlagIsSet(NormCC::Flag flag) const { - UINT16 temp16; - memcpy(&temp16, buffer+LOSS_OFFSET, 2); - return (ntohs(temp16)); + return (0 != ((UINT8)flag & buffer[CC_FLAGS_OFFSET])); } - UINT16 GetGrttSequence() + UINT8 GetCCRtt() {return buffer[CC_RTT_OFFSET];} + UINT16 GetCCLoss() {return (ntohs(*((UINT16*)(buffer+CC_LOSS_OFFSET))));} + UINT16 GetCCRate() {return (ntohs(*((UINT16*)(buffer+CC_RATE_OFFSET))));} + + private: + enum { - UINT16 temp16; - memcpy(&temp16, buffer+GRTT_SEQUENCE_OFFSET, 2); - return (ntohs(temp16)); + CC_SEQUENCE_OFFSET = LENGTH_OFFSET + 1, + CC_FLAGS_OFFSET = CC_SEQUENCE_OFFSET + 2, + CC_RTT_OFFSET = CC_FLAGS_OFFSET + 1, + CC_LOSS_OFFSET = CC_RTT_OFFSET + 1, + CC_RATE_OFFSET = CC_LOSS_OFFSET + 2, + CC_RESERVED_OFFSET = CC_RATE_OFFSET + 2 + }; + +}; // end class NormCCFeedbackExtension + +// Note: Support for application-defined AckFlavors (32-255) +// may be provided, but 0-31 are reserved values +class NormAck +{ + public: + enum Type + { + INVALID = 0, + CC = 1, + FLUSH = 2, + APP_BASE = 16 + }; +}; + +class NormCmdAckReqMsg : public NormCmdMsg +{ + public: + + void Init() + { + SetType(CMD); + SetFlavor(ACK_REQ); + SetBaseHeaderLength(ACK_REQ_HEADER_LEN); + buffer[RESERVED_OFFSET] = 0; + } + + // Message building + void SetAckType(NormAck::Type ackType) {buffer[ACK_TYPE_OFFSET] = (UINT8)ackType;} + void SetAckId(UINT8 ackId) {buffer[ACK_ID_OFFSET] = ackId;} + void ResetAckingNodeList() {length = header_length;} + bool AppendAckingNode(NormNodeId nodeId, UINT16 segmentSize) + { + if ((length - header_length + 4) > segmentSize) return false; + *((UINT32*)(buffer+length)) = htonl(nodeId); + length += 4; + return true; } + // Message processing + NormAck::Type GetAckType() const {return (NormAck::Type)buffer[ACK_TYPE_OFFSET];} + UINT8 GetAckId() const {return buffer[ACK_ID_OFFSET];} + UINT16 GetAckingNodeCount() const {return ((length - header_length) >> 2);} + NormNodeId GetAckingNodeId(UINT16 index) const + { + return (ntohl(*((UINT32*)(buffer+header_length+(index<<2))))); + } + + + private: + enum + { + RESERVED_OFFSET = FLAVOR_OFFSET + 1, + ACK_TYPE_OFFSET = RESERVED_OFFSET + 1, + ACK_ID_OFFSET = ACK_TYPE_OFFSET + 1, + ACK_REQ_HEADER_LEN = ACK_ID_OFFSET + 1 + }; +}; // end class NormCmdAckReqMsg + + +class NormCmdApplicationMsg : public NormCmdMsg +{ + public: + void Init() + { + SetType(CMD); + SetFlavor(APPLICATION); + SetBaseHeaderLength(APPLICATION_HEADER_LEN); + memset(buffer+RESERVED_OFFSET, 0, 3); + } + + bool SetContent(const char* content, UINT16 contentLen, UINT16 segmentSize) + { + UINT16 len = MIN(contentLen, segmentSize); + memcpy(buffer+header_length, content, len); + return (contentLen <= segmentSize); + } + + UINT16 GetContentLength() {return (length - header_length);} + const char* GetContent() {return (buffer+header_length);} + + private: + enum + { + RESERVED_OFFSET = FLAVOR_OFFSET + 1, + APPLICATION_HEADER_LEN = RESERVED_OFFSET + 3 + }; +}; // end class NormCmdApplicationMsg + + +// Receiver Messages + +class NormNackMsg : public NormMsg +{ + public: + void Init() + { + SetType(NACK); + SetBaseHeaderLength(NACK_HEADER_LEN); + } + // Message building + void SetServerId(NormNodeId serverId) + { + *((UINT32*)(buffer+SERVER_ID_OFFSET)) = htonl(serverId); + } + void SetSessionId(UINT16 sessionId) + { + *((UINT16*)(buffer+SESSION_ID_OFFSET)) = htons(sessionId); + } + void SetGrttResponse(const struct timeval& grttResponse) + { + *((UINT32*)(buffer+GRTT_RESPONSE_OFFSET)) = htonl(grttResponse.tv_sec); + *((UINT32*)(buffer+GRTT_RESPONSE_OFFSET+4)) = htonl(grttResponse.tv_usec); + } + void AttachRepairRequest(NormRepairRequest& request, + UINT16 segmentMax) + { + int buflen = segmentMax - (length - header_length); + buflen = (buflen>0) ? buflen : 0; + request.Init(buffer+length, buflen); + } + UINT16 PackRepairRequest(NormRepairRequest& request) + { + UINT16 requestLength = request.Pack(); + length += requestLength; + return requestLength; + } + + // Message processing + NormNodeId GetServerId() const + { + return (ntohl(*((UINT32*)(buffer+SERVER_ID_OFFSET)))); + } + UINT16 GetSessionId() const + { + return (ntohs(*((UINT16*)(buffer+SESSION_ID_OFFSET)))); + } + void GetGrttResponse(struct timeval& grttResponse) const + { + grttResponse.tv_sec = ntohl(*((UINT32*)(buffer+GRTT_RESPONSE_OFFSET))); + grttResponse.tv_usec = ntohl(*((UINT32*)(buffer+GRTT_RESPONSE_OFFSET+4))); + } + //char* AccessRepairContent() {return (buffer + header_length);} + const char* GetRepairContent() const {return (buffer + header_length);} + UINT16 GetRepairContentLength() const + {return ((length > header_length) ? length - header_length : 0);} UINT16 UnpackRepairRequest(NormRepairRequest& request, UINT16 requestOffset) { - int buflen = length - CONTENT_OFFSET - requestOffset; + int buflen = length - header_length - requestOffset; buflen = (buflen > 0) ? buflen : 0; - request.SetBuffer(buffer + CONTENT_OFFSET + requestOffset, - buflen); - return request.Unpack(); + return request.Unpack(buffer + header_length + requestOffset, buflen); } private: enum { SERVER_ID_OFFSET = MSG_OFFSET, + SESSION_ID_OFFSET = SERVER_ID_OFFSET + 4, + RESERVED_OFFSET = SESSION_ID_OFFSET + 2, GRTT_RESPONSE_OFFSET = SERVER_ID_OFFSET + 4, - LOSS_OFFSET = GRTT_RESPONSE_OFFSET + 8, - GRTT_SEQUENCE_OFFSET = LOSS_OFFSET + 2, - CONTENT_OFFSET = GRTT_SEQUENCE_OFFSET + 2 + NACK_HEADER_LEN = GRTT_RESPONSE_OFFSET + 8 }; }; // end class NormNackMsg class NormAckMsg : public NormMsg { - // (TBD) + public: + + // Message building + void Init() + { + SetType(ACK); + SetBaseHeaderLength(ACK_HEADER_LEN); + SetAckType(NormAck::INVALID); + } + void SetServerId(NormNodeId serverId) + { + *((UINT32*)(buffer+SERVER_ID_OFFSET)) = htonl(serverId); + } + void SetSessionId(UINT16 sessionId) + { + *((UINT16*)(buffer+SESSION_ID_OFFSET)) = htons(sessionId); + } + void SetAckType(NormAck::Type ackType) {buffer[ACK_TYPE_OFFSET] = (UINT8)ackType;} + void SetAckId(UINT8 ackId) {buffer[ACK_ID_OFFSET] = ackId;} + void SetGrttResponse(const struct timeval& grttResponse) + { + *((UINT32*)(buffer+GRTT_RESPONSE_OFFSET)) = htonl(grttResponse.tv_sec); + *((UINT32*)(buffer+GRTT_RESPONSE_OFFSET+4)) = htonl(grttResponse.tv_usec); + } + bool SetAckPayload(const char* payload, UINT16 payloadLen, UINT16 segmentSize) + { + UINT16 len = MIN(payloadLen, segmentSize); + memcpy(buffer+header_length, payload, len); + return (payloadLen <= segmentSize); + } + + // Message processing + NormNodeId GetServerId() const + { + return (ntohl(*((UINT32*)(buffer+SERVER_ID_OFFSET)))); + } + UINT16 GetSessionId() const + { + return (ntohs(*((UINT16*)(buffer+SESSION_ID_OFFSET)))); + } + void GetGrttResponse(struct timeval& grttResponse) const + { + grttResponse.tv_sec = ntohl(*((UINT32*)(buffer+GRTT_RESPONSE_OFFSET))); + grttResponse.tv_usec = ntohl(*((UINT32*)(buffer+GRTT_RESPONSE_OFFSET+4))); + } + NormAck::Type GetAckType() const {return (NormAck::Type)buffer[ACK_TYPE_OFFSET];} + UINT8 GetAckId() const {return buffer[ACK_ID_OFFSET];} + UINT16 GetPayloadLength() const {return (length - header_length);} + const char* GetPayload() const {return (buffer + header_length);} + + private: + enum + { + SERVER_ID_OFFSET = MSG_OFFSET, + SESSION_ID_OFFSET = SERVER_ID_OFFSET + 4, + ACK_TYPE_OFFSET = SESSION_ID_OFFSET + 2, + ACK_ID_OFFSET = ACK_TYPE_OFFSET + 1, + GRTT_RESPONSE_OFFSET = ACK_ID_OFFSET + 1, + ACK_HEADER_LEN = GRTT_RESPONSE_OFFSET + 8 + }; }; // end class NormAckMsg class NormReportMsg : public NormMsg { // (TBD) + }; // end class NormReportMsg // One we've defined our basic message types, we // do some unions so we can easily use these // via casting or dereferencing the union members -class NormCommandMsg -{ - public: - union - { - NormCmdMsg generic; - NormCmdFlushMsg flush; - NormCmdSquelchMsg squelch; - NormCmdAckReqMsg ack_req; - NormCmdRepairAdvMsg repair_adv; - NormCmdCCMsg cc; - NormCmdApplicationMsg app; - }; -}; // end class NormCommandMsg - -class NormMessage -{ - friend class NormMessageQueue; - public: - union - { - NormMsg generic; - NormObjectMsg object; - NormInfoMsg info; - NormDataMsg data; - NormCommandMsg cmd; - NormNackMsg nack; - NormAckMsg ack; - NormReportMsg report; - }; - - private: - NormMessage* prev; - NormMessage* next; -}; // end class NormMessage - - class NormMessageQueue { public: NormMessageQueue(); ~NormMessageQueue(); void Destroy(); - void Prepend(NormMessage* msg); - void Append(NormMessage* msg); - void Remove(NormMessage* msg); - NormMessage* RemoveHead(); - NormMessage* RemoveTail(); - bool IsEmpty() {return ((NormMessage*)NULL == head);} + void Prepend(NormMsg* msg); + void Append(NormMsg* msg); + void Remove(NormMsg* msg); + NormMsg* RemoveHead(); + NormMsg* RemoveTail(); + bool IsEmpty() {return ((NormMsg*)NULL == head);} private: - NormMessage* head; - NormMessage* tail; + NormMsg* head; + NormMsg* tail; }; // end class NormMessageQueue #endif // _NORM_MESSAGE diff --git a/common/normNode.cpp b/common/normNode.cpp index 5ab1ad2..d100159 100644 --- a/common/normNode.cpp +++ b/common/normNode.cpp @@ -6,8 +6,7 @@ NormNode::NormNode(class NormSession* theSession, NormNodeId nodeId) : session(theSession), id(nodeId), - parent(NULL), right(NULL), left(NULL), - prev(NULL), next(NULL) + parent(NULL), right(NULL), left(NULL) { } @@ -16,23 +15,49 @@ NormNode::~NormNode() { } -const NormNodeId& NormNode::LocalNodeId() {return session->LocalNodeId();} +const NormNodeId& NormNode::LocalNodeId() const {return session->LocalNodeId();} + + +NormCCNode::NormCCNode(class NormSession* theSession, NormNodeId nodeId) + : NormNode(theSession, nodeId) +{ + +} + +NormCCNode::~NormCCNode() +{ +} NormServerNode::NormServerNode(class NormSession* theSession, NormNodeId nodeId) - : NormNode(theSession, nodeId), synchronized(false), + : NormNode(theSession, nodeId), session_id(0), synchronized(false), is_open(false), segment_size(0), ndata(0), nparity(0), erasure_loc(NULL), + cc_sequence(0), cc_enable(false), cc_rate(0.0), rtt_confirmed(false), is_clr(false), is_plr(false), + slow_start(true), send_rate(0.0), recv_rate(0.0), recv_accumulator(0), recv_total(0), recv_goodput(0), resync_count(0), nack_count(0), suppress_count(0), completion_count(0), failure_count(0) { repair_timer.Init(0.0, 1, (ProtocolTimerOwner*)this, (ProtocolTimeoutFunc)&NormServerNode::OnRepairTimeout); + activity_timer.Init(0.0, -1, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormServerNode::OnActivityTimeout); + cc_timer.Init(0.0, 1, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormServerNode::OnCCTimeout); grtt_send_time.tv_sec = 0; grtt_send_time.tv_usec = 0; grtt_quantized = NormQuantizeRtt(NormSession::DEFAULT_GRTT_ESTIMATE); grtt_estimate = NormUnquantizeRtt(grtt_quantized); gsize_quantized = NormQuantizeGroupSize(NormSession::DEFAULT_GSIZE_ESTIMATE); gsize_estimate = NormUnquantizeGroupSize(gsize_quantized); + + rtt_quantized = NormQuantizeRtt(NormSession::DEFAULT_GRTT_ESTIMATE); + rtt_estimate = NormUnquantizeRtt(rtt_quantized); + + loss_estimator.SetLossEventWindow(NormSession::DEFAULT_GRTT_ESTIMATE); + + prev_update_time.tv_sec = 0; + prev_update_time.tv_usec = 0; + } NormServerNode::~NormServerNode() @@ -40,8 +65,9 @@ NormServerNode::~NormServerNode() Close(); } -bool NormServerNode::Open(UINT16 segmentSize, UINT16 numData, UINT16 numParity) +bool NormServerNode::Open(UINT16 sessionId, UINT16 segmentSize, UINT16 numData, UINT16 numParity) { + session_id = sessionId; if (!rx_table.Init(256)) { DMSG(0, "NormServerNode::Open() rx_table init error\n"); @@ -67,7 +93,7 @@ bool NormServerNode::Open(UINT16 segmentSize, UINT16 numData, UINT16 numParity) unsigned long blockSpace = sizeof(NormBlock) + blockSize * sizeof(char*) + 2*maskSize + - numData * (segmentSize + NormDataMsg::PayloadHeaderLen()); + numData * (segmentSize + NormDataMsg::PayloadHeaderLength()); unsigned long bufferSpace = session->RemoteServerBufferSize(); unsigned long numBlocks = bufferSpace / blockSpace; if (bufferSpace > (numBlocks*blockSpace)) numBlocks++; @@ -81,14 +107,17 @@ bool NormServerNode::Open(UINT16 segmentSize, UINT16 numData, UINT16 numParity) return false; } - if (!segment_pool.Init(numSegments, segmentSize+NormDataMsg::PayloadHeaderLen())) + // The extra byte of segments is used for marking segments + // which are "start segments" for messages encapsulated in + // a NormStreamObject + if (!segment_pool.Init(numSegments, segmentSize+NormDataMsg::PayloadHeaderLength()+1)) { DMSG(0, "NormServerNode::Open() segment_pool init error\n"); Close(); return false; } - if (!decoder.Init(numParity, segmentSize+NormDataMsg::PayloadHeaderLen())) + if (!decoder.Init(numParity, segmentSize+NormDataMsg::PayloadHeaderLength())) { DMSG(0, "NormServerNode::Open() decoder init error: %s\n", strerror(errno)); @@ -103,9 +132,11 @@ bool NormServerNode::Open(UINT16 segmentSize, UINT16 numData, UINT16 numParity) return false; } segment_size = segmentSize; + nominal_packet_size = (double)segmentSize; ndata = numData; nparity = numParity; is_open= true; + Activate(); return true; } // end NormServerNode::Open() @@ -129,64 +160,181 @@ void NormServerNode::Close() } // end NormServerNode::Close() -void NormServerNode::HandleCommand(NormCommandMsg& cmd) +void NormServerNode::HandleCommand(const struct timeval& currentTime, + const NormCmdMsg& cmd) { - UINT8 grttQuantized = cmd.generic.GetGrtt(); + UINT8 grttQuantized = cmd.GetGrtt(); if (grttQuantized != grtt_quantized) { grtt_quantized = grttQuantized; grtt_estimate = NormUnquantizeRtt(grttQuantized); DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new grtt: %lf sec\n", LocalNodeId(), Id(), grtt_estimate); + activity_timer.SetInterval(grtt_estimate); + activity_timer.Reset(); } - UINT8 gsizeQuantized = cmd.generic.GetGroupSize(); + UINT8 gsizeQuantized = cmd.GetGroupSize(); if (gsizeQuantized != gsize_quantized) { gsize_quantized = gsizeQuantized; gsize_estimate = NormUnquantizeGroupSize(gsizeQuantized); - DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new group size: %lf\n", + DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new group size:%lf\n", LocalNodeId(), Id(), gsize_estimate); } - - NormCmdMsg::Flavor flavor = cmd.generic.GetFlavor(); + + NormCmdMsg::Flavor flavor = cmd.GetFlavor(); switch (flavor) { - case NormCmdMsg::NORM_CMD_SQUELCH: + case NormCmdMsg::SQUELCH: if (IsOpen()) { + const NormCmdSquelchMsg& squelch = (const NormCmdSquelchMsg&)cmd; // 1) Sync to squelch - NormObjectId objectId = cmd.squelch.GetObjectId(); + NormObjectId objectId = squelch.GetObjectId(); Sync(objectId); // 2) Prune stream object if applicable NormObject* obj = rx_table.Find(objectId); if (obj && (NormObject::STREAM == obj->GetType())) { - NormBlockId blockId = cmd.squelch.GetFecBlockId(); + NormBlockId blockId = squelch.GetFecBlockId(); ((NormStreamObject*)obj)->Prune(blockId); } // 3) (TBD) Discard any invalidated objects } break; - case NormCmdMsg::NORM_CMD_ACK_REQ: - GetSystemTime(&grtt_recv_time); - cmd.ack_req.GetRttSendTime(grtt_send_time); - switch(cmd.ack_req.GetAckFlavor()) + case NormCmdMsg::ACK_REQ: + // (TBD) handle ack requests + break; + + case NormCmdMsg::CC: + { + const NormCmdCCMsg& cc = (const NormCmdCCMsg&)cmd; + grtt_recv_time = currentTime; + cc.GetSendTime(grtt_send_time); + cc_sequence = cc.GetCCSequence(); + NormCCRateExtension ext; + while (cc.GetNextExtension(ext)) { - case NormCmdAckReqMsg::RTT: - break; - default: - break; + if (NormHeaderExtension::CC_RATE == ext.GetType()) + { + cc_enable = true; + send_rate = NormUnquantizeRate(ext.GetSendRate()); + // Are we in the cc_node_list? + UINT8 flags, rtt; + UINT16 loss; + if (cc.GetCCNode(LocalNodeId(), flags, rtt, loss)) + { + if (rtt != rtt_quantized) + { + rtt_quantized = rtt; + rtt_estimate = NormUnquantizeRtt(rtt); + loss_estimator.SetLossEventWindow(rtt_estimate); + } + rtt_confirmed = true; + if (0 != (flags & NormCC::CLR)) + { + is_clr = true; + is_plr = false; + } + else if (0 != (flags & NormCC::PLR)) + { + is_clr = false; + is_plr = true; + } + else + { + is_clr = is_plr = false; + } + } + else + { + is_clr = is_plr = false; + } + if (is_clr || is_plr) + { + // Respond immediately + if (cc_timer.IsActive()) cc_timer.Deactivate(); + cc_timer.ResetRepeats(); + OnCCTimeout(); + } + else if (!cc_timer.IsActive()) + { + double backoffFactor = session->BackoffFactor(); + backoffFactor = MAX(backoffFactor, 4.0); + double maxBackoff = grtt_estimate*backoffFactor; + // (TBD) don't backoff for unicast sessions + double backoffTime = (maxBackoff > 0.0) ? + ExponentialRand(maxBackoff, gsize_estimate) : 0.0; + // Bias backoff timeout based on our rate + double r; + if (slow_start) + { + r = recv_rate / send_rate; + cc_rate = 2.0 * recv_rate; + } + else + { + cc_rate = NormSession::CalculateRate(nominal_packet_size, + rtt_estimate, + LossEstimate()); + r = cc_rate / send_rate; + r = MIN(r, 0.9); + r = MAX(r, 0.5); + r = (r - 0.5) / 0.4; + } + //TRACE("NormServerNode::HandleCommand(CC) node>%lu bias:%lf recv_rate:%lf send_rate:%lf " + // "grtt:%lf gsize:%lf\n", + // LocalNodeId(), r, recv_rate*(8.0/1000.0), send_rate*(8.0/1000.0), + // + + backoffTime = 0.25 * r * maxBackoff + 0.75 * backoffTime; + cc_timer.SetInterval(backoffTime); + DMSG(4, "NormServerNode::HandleCommand() node>%lu begin CC back-off: %lf sec)...\n", + LocalNodeId(), backoffTime); + session->InstallTimer(&cc_timer); + } + } // end if (CC_RATE == ext.GetType()) + } // end while (GetNextExtension()) + break; + } + case NormCmdMsg::FLUSH: + // (TBD) should we force synchronize if we're expected + // to positively acknowledge the FLUSH + if (synchronized) + { + const NormCmdFlushMsg& flush = (const NormCmdFlushMsg&)cmd; + UpdateSyncStatus(flush.GetObjectId()); + RepairCheck(NormObject::THRU_SEGMENT, + flush.GetObjectId(), + flush.GetFecBlockId(), + flush.GetFecSymbolId()); } break; - case NormCmdMsg::NORM_CMD_FLUSH: - if (synchronized) UpdateSyncStatus(cmd.flush.GetObjectId()); - RepairCheck(NormObject::THRU_SEGMENT, - cmd.flush.GetObjectId(), - cmd.flush.GetFecBlockId(), - cmd.flush.GetFecSymbolId()); + case NormCmdMsg::REPAIR_ADV: + { + const NormCmdRepairAdvMsg& repairAdv = (const NormCmdRepairAdvMsg&)cmd; + // Does the CC feedback of this ACK suppress our CC feedback + if (!is_clr && !is_plr && cc_timer.IsActive() && cc_timer.RepeatCount()) + { + NormCCFeedbackExtension ext; + while (repairAdv.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) + { + HandleCCFeedback(ext.GetCCFlags(), NormUnquantizeRate(ext.GetCCRate())); + break; + } + } + } + if (repair_timer.IsActive() && repair_timer.RepeatCount()) + { + HandleRepairContent(repairAdv.GetRepairContent(), + repairAdv.GetRepairContentLength()); + } break; + } default: DMSG(0, "NormServerNode::HandleCommand() recv'd unimplemented command!\n"); @@ -195,127 +343,220 @@ void NormServerNode::HandleCommand(NormCommandMsg& cmd) } // end NormServerNode::HandleCommand() -void NormServerNode::HandleNackMessage(NormNackMsg& nack) +void NormServerNode::HandleCCFeedback(UINT8 ccFlags, double ccRate) { - // Clients only care about recvd NACKS for suppression - if (repair_timer.IsActive() && repair_timer.RepeatCount()) + ASSERT(cc_timer.IsActive() && cc_timer.RepeatCount()); + if (0 == (ccFlags & NormCC::CLR)) { - // Parse NACK and incorporate into repair state masks - NormRepairRequest req; - UINT16 requestOffset = 0; - UINT16 requestLength = 0; - bool freshObject = true; - NormObjectId prevObjectId; - NormObject* object = NULL; - bool freshBlock = true; - NormBlockId prevBlockId; - NormBlock* block = NULL; - while ((requestLength = nack.UnpackRepairRequest(req, requestOffset))) + // We're suppressed by non-CLR receivers with no RTT confirmed + // and/or lower rate + double localRate = slow_start ? + (2.0*recv_rate) : + NormSession::CalculateRate(nominal_packet_size, + rtt_estimate, + LossEstimate()); + localRate = MAX(localRate, cc_rate); + + bool hasRtt = (0 != (ccFlags & NormCC::RTT)); + bool suppressed; + + if (rtt_confirmed) { - enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; - NormRepairRequest::Form requestForm = req.GetForm(); - requestOffset += requestLength; - NormRequestLevel requestLevel; - if (req.FlagIsSet(NormRepairRequest::SEGMENT)) - requestLevel = SEGMENT; - else if (req.FlagIsSet(NormRepairRequest::BLOCK)) - requestLevel = BLOCK; - else if (req.FlagIsSet(NormRepairRequest::OBJECT)) - requestLevel = OBJECT; + // If we have confirmed our own RTT we + // are suppressed by _any_ receivers with + // lower rate than our own + if (localRate > (0.9 * ccRate)) + suppressed = true; else + suppressed = false; + } + else + { + // If we haven't confirmed our own RTT we + // are suppressed by only by other + // non-confirmed receivers + if (hasRtt) + suppressed = false; + else if (localRate > (0.9 * ccRate)) + suppressed = true; + else + suppressed = false; + } + if (suppressed) + { + if (cc_timer.IsActive()) cc_timer.Deactivate(); + cc_timer.SetInterval(grtt_estimate*session->BackoffFactor()); + session->InstallTimer(&cc_timer); + cc_timer.DecrementRepeatCount(); + } + } +} // end NormServerNode::HandleCCFeedback() + +void NormServerNode::HandleAckMessage(const NormAckMsg& ack) +{ + // Does the CC feedback of this ACK suppress our CC feedback + if (!is_clr && !is_plr && cc_timer.IsActive() && cc_timer.RepeatCount()) + { + NormCCFeedbackExtension ext; + while (ack.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) { - requestLevel = INFO; - ASSERT(req.FlagIsSet(NormRepairRequest::INFO)); + HandleCCFeedback(ext.GetCCFlags(), NormUnquantizeRate(ext.GetCCRate())); + break; } - bool repairInfo = req.FlagIsSet(NormRepairRequest::INFO); - - NormRepairRequest::Iterator iterator(req); - NormObjectId nextObjectId, lastObjectId; - NormBlockId nextBlockId, lastBlockId; - NormSegmentId nextSegmentId, lastSegmentId; - while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, &nextSegmentId)) + } + } +} // end NormServerNode::HandleAckMessage() + +void NormServerNode::HandleNackMessage(const NormNackMsg& nack) +{ + // Does the CC feedback of this NACK suppress our CC feedback + if (!is_clr && !is_plr && cc_timer.IsActive() && cc_timer.RepeatCount()) + { + NormCCFeedbackExtension ext; + while (nack.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) { - if (NormRepairRequest::RANGES == requestForm) - { - if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, &lastSegmentId)) - { - DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu recvd incomplete RANGE request!\n", - LocalNodeId()); - continue; // (TBD) break/return instead??? - } - // (TBD) test for valid range form/level - } - else - { - lastObjectId = nextObjectId; - lastBlockId = nextBlockId; - lastSegmentId = nextSegmentId; - } - switch(requestLevel) - { - case INFO: - { - while (nextObjectId <= lastObjectId) - { - NormObject* obj = rx_table.Find(nextObjectId); - if (obj) obj->SetRepairInfo(); - nextObjectId++; - } - break; - } - case OBJECT: - rx_repair_mask.SetBits(nextObjectId, lastObjectId - nextObjectId + 1); - break; - case BLOCK: - { - if (nextObjectId != prevObjectId) freshObject = true; - if (freshObject) - { - object = rx_table.Find(nextObjectId); - prevObjectId = nextObjectId; - } - if (object) - { - if (repairInfo) object->SetRepairInfo(); - object->SetRepairs(nextBlockId, lastBlockId); - } - break; - } - case SEGMENT: - { - if (nextObjectId != prevObjectId) freshObject = true; - if (freshObject) - { - object = rx_table.Find(nextObjectId); - prevObjectId = nextObjectId; - } - if (object) - { - if (repairInfo) object->SetRepairInfo(); - if (nextBlockId != prevBlockId) freshBlock = true; - if (freshBlock) - { - block = object->FindBlock(nextBlockId); - prevBlockId = nextBlockId; - } - if (block) block->SetRepairs(nextSegmentId,lastSegmentId); - } - break; - } - } // end switch(requestLevel) - } // end while (iterator.NextRepairItem()) - } // end while (nack.UnpackRepairRequest()) - } // end if (repair_timer.IsActive() && repair_timer.RepeatCount()) + HandleCCFeedback(ext.GetCCFlags(), NormUnquantizeRate(ext.GetCCRate())); + break; + } + } + } + // Clients also care about recvd NACKS for NACK suppression + if (repair_timer.IsActive() && repair_timer.RepeatCount()) + HandleRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength()); } // end NormServerNode::HandleNackMessage() +// Clients use this method to process NACK content overheard from other +// clients or via NORM_CMD(REPAIR_ADV) messages received from the server. +// Such content can "suppress" pending NACKs +void NormServerNode::HandleRepairContent(const char* buffer, UINT16 bufferLen) +{ + // Parse NACK and incorporate into repair state masks + NormRepairRequest req; + UINT16 requestLength = 0; + bool freshObject = true; + NormObjectId prevObjectId; + NormObject* object = NULL; + bool freshBlock = true; + NormBlockId prevBlockId; + NormBlock* block = NULL; + while ((requestLength = req.Unpack(buffer, bufferLen))) + { + // Point "buffer" to next request and adjust "bufferLen" + buffer += requestLength; + bufferLen -= requestLength; + // Process request + enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; + NormRepairRequest::Form requestForm = req.GetForm(); + NormRequestLevel requestLevel; + if (req.FlagIsSet(NormRepairRequest::SEGMENT)) + requestLevel = SEGMENT; + else if (req.FlagIsSet(NormRepairRequest::BLOCK)) + requestLevel = BLOCK; + else if (req.FlagIsSet(NormRepairRequest::OBJECT)) + requestLevel = OBJECT; + else + { + requestLevel = INFO; + ASSERT(req.FlagIsSet(NormRepairRequest::INFO)); + } + bool repairInfo = req.FlagIsSet(NormRepairRequest::INFO); -void NormServerNode::CalculateGrttResponse(struct timeval& grttResponse) + NormRepairRequest::Iterator iterator(req); + NormObjectId nextObjectId, lastObjectId; + NormBlockId nextBlockId, lastBlockId; + UINT16 nextBlockLen, lastBlockLen; + NormSegmentId nextSegmentId, lastSegmentId; + while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, + &nextBlockLen, &nextSegmentId)) + { + if (NormRepairRequest::RANGES == requestForm) + { + if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, + &lastBlockLen, &lastSegmentId)) + { + DMSG(0, "NormServerNode::HandleRepairContent() node>%lu recvd incomplete RANGE request!\n", + LocalNodeId()); + continue; // (TBD) break/return instead??? + } + // (TBD) test for valid range form/level + } + else + { + lastObjectId = nextObjectId; + lastBlockId = nextBlockId; + lastBlockLen = nextBlockLen; + lastSegmentId = nextSegmentId; + } + + switch(requestLevel) + { + case INFO: + { + while (nextObjectId <= lastObjectId) + { + NormObject* obj = rx_table.Find(nextObjectId); + if (obj) obj->SetRepairInfo(); + nextObjectId++; + } + break; + } + case OBJECT: + rx_repair_mask.SetBits(nextObjectId, lastObjectId - nextObjectId + 1); + break; + case BLOCK: + { + if (nextObjectId != prevObjectId) freshObject = true; + if (freshObject) + { + object = rx_table.Find(nextObjectId); + prevObjectId = nextObjectId; + } + if (object) + { + if (repairInfo) object->SetRepairInfo(); + object->SetRepairs(nextBlockId, lastBlockId); + } + break; + } + case SEGMENT: + { + if (nextObjectId != prevObjectId) freshObject = true; + if (freshObject) + { + object = rx_table.Find(nextObjectId); + prevObjectId = nextObjectId; + } + if (object) + { + if (repairInfo) object->SetRepairInfo(); + if (nextBlockId != prevBlockId) freshBlock = true; + if (freshBlock) + { + block = object->FindBlock(nextBlockId); + prevBlockId = nextBlockId; + } + if (block) block->SetRepairs(nextSegmentId,lastSegmentId); + } + break; + } + } // end switch(requestLevel) + } // end while (iterator.NextRepairItem()) + } // end while (nack.UnpackRepairRequest()) +} // end NormServerNode::HandleRepairContent() + + +void NormServerNode::CalculateGrttResponse(const struct timeval& currentTime, + struct timeval& grttResponse) const { grttResponse.tv_sec = grttResponse.tv_usec = 0; if (grtt_send_time.tv_sec || grtt_send_time.tv_usec) { // 1st - Get current time - ::GetSystemTime(&grttResponse); + grttResponse = currentTime; // 2nd - Calculate hold_time (current_time - recv_time) if (grttResponse.tv_usec < grtt_recv_time.tv_usec) { @@ -353,27 +594,56 @@ NormBlock* NormServerNode::GetFreeBlock(NormObjectId objectId, NormBlockId block NormBlock* b = block_pool.Get(); if (!b) { - // reverse iteration to find newer object with resources - NormObjectTable::Iterator iterator(rx_table); - NormObject* obj; - while ((obj = iterator.GetPrevObject())) + if (session->ClientIsSilent()) + //if (1) { - if (obj->Id() < objectId) - { - break; - } - else + // forward iteration to find oldest older object with resources + NormObjectTable::Iterator iterator(rx_table); + NormObject* obj; + while ((obj = iterator.GetNextObject())) { if (obj->Id() > objectId) - b = obj->StealNewestBlock(false); - else - b = obj->StealNewestBlock(true, blockId); - if (b) { - b->EmptyToPool(segment_pool); + break; + } + else + { + if (obj->Id() < objectId) + b = obj->StealOldestBlock(false); + else + b = obj->StealOldestBlock(true, blockId); + if (b) + { + b->EmptyToPool(segment_pool); + break; + } + } + } + } + else + { + // reverse iteration to find newest newer object with resources + NormObjectTable::Iterator iterator(rx_table); + NormObject* obj; + while ((obj = iterator.GetPrevObject())) + { + if (obj->Id() < objectId) + { break; } - } + else + { + if (obj->Id() > objectId) + b = obj->StealNewestBlock(false); + else + b = obj->StealNewestBlock(true, blockId); + if (b) + { + b->EmptyToPool(segment_pool); + break; + } + } + } } } return b; @@ -392,71 +662,87 @@ char* NormServerNode::GetFreeSegment(NormObjectId objectId, NormBlockId blockId) return segment_pool.Get(); } // end NormServerNode::GetFreeSegment() -void NormServerNode::HandleObjectMessage(NormMessage& msg) +void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) { - UINT8 grttQuantized = msg.object.GetGrtt(); + UINT8 grttQuantized = msg.GetGrtt(); if (grttQuantized != grtt_quantized) { grtt_quantized = grttQuantized; grtt_estimate = NormUnquantizeRtt(grttQuantized); - DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new grtt: %lf sec\n", + DMSG(4, "NormServerNode::HandleObjectMessage() node>%lu server>%lu new grtt: %lf sec\n", LocalNodeId(), Id(), grtt_estimate); + activity_timer.SetInterval(grtt_estimate); + activity_timer.Reset(); } - UINT8 gsizeQuantized = msg.object.GetGroupSize(); + UINT8 gsizeQuantized = msg.GetGroupSize(); if (gsizeQuantized != gsize_quantized) { gsize_quantized = gsizeQuantized; gsize_estimate = NormUnquantizeGroupSize(gsizeQuantized); - DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new group size: %lf\n", + DMSG(4, "NormServerNode::HandleObjectMessage() node>%lu server>%lu new group size: %lf\n", LocalNodeId(), Id(), gsize_estimate); } + + if (IsOpen()) { - // (TBD - also verify encoder name ...) - if ((msg.object.GetSegmentSize() != segment_size) && - (ndata != msg.object.GetFecBlockLen()) && - (nparity != msg.object.GetFecNumParity())) + if (msg.GetSessionId() != session_id) { - DMSG(2, "NormServerNode::HandleObjectMessage() node>%lu server>%lu parameter change - resyncing.\n", + DMSG(2, "NormServerNode::HandleObjectMessage() node>%lu server>%lu sessionId change - resyncing.\n", LocalNodeId(), Id()); Close(); - if (!Open(msg.object.GetSegmentSize(), - msg.object.GetFecBlockLen(), - msg.object.GetFecNumParity())) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu open error\n", - LocalNodeId(), Id()); - // (TBD) notify app of error ?? - return; - } resync_count++; } } - else + + if (!IsOpen()) { - if (!Open(msg.object.GetSegmentSize(), - msg.object.GetFecBlockLen(), - msg.object.GetFecNumParity())) + // Currently,, our implementation requires the FEC Object Transmission Information + // to properly allocate resources + NormFtiExtension fti; + while (msg.GetNextExtension(fti)) { - DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu open error\n", - LocalNodeId(), Id()); + if (NormHeaderExtension::FTI == fti.GetType()) + { + // (TBD) pass "fec_id" to Open() method too + if (!Open(msg.GetSessionId(), + fti.GetSegmentSize(), + fti.GetFecMaxBlockLen(), + fti.GetFecNumParity())) + { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu open error\n", + LocalNodeId(), Id()); + // (TBD) notify app of error ?? + return; + } + break; + } + } + if (!IsOpen()) + { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu - no FTI provided!\n", + LocalNodeId(), Id()); // (TBD) notify app of error ?? - return; - } + return; + } } - NormMsgType msgType = msg.generic.GetType(); - NormObjectId objectId = msg.object.GetObjectId(); + + + NormMsg::Type msgType = msg.GetType(); + NormObjectId objectId = msg.GetObjectId(); NormBlockId blockId; NormSegmentId segmentId; - if (NORM_MSG_INFO == msgType) + if (NormMsg::INFO == msgType) { blockId = 0; segmentId = 0; } else { - blockId = msg.data.GetFecBlockId(); - segmentId = msg.data.GetFecSymbolId(); + const NormDataMsg& data = (const NormDataMsg&)msg; + // (TBD) verify source block length per new spec + blockId = data.GetFecBlockId(); + segmentId = data.GetFecSymbolId(); } ObjectStatus status; @@ -487,7 +773,7 @@ void NormServerNode::HandleObjectMessage(NormMessage& msg) if ((obj = rx_table.Find(objectId))) break; case OBJ_NEW: { - if (msg.object.FlagIsSet(NormObjectMsg::FLAG_STREAM)) + if (msg.FlagIsSet(NormObjectMsg::FLAG_STREAM)) { if (!(obj = new NormStreamObject(session, this, objectId))) { @@ -495,7 +781,7 @@ void NormServerNode::HandleObjectMessage(NormMessage& msg) strerror(errno)); } } - else if (msg.object.FlagIsSet(NormObjectMsg::FLAG_FILE)) + else if (msg.FlagIsSet(NormObjectMsg::FLAG_FILE)) { #ifdef SIMULATE if (!(obj = new NormSimObject(session, this, objectId))) @@ -515,27 +801,43 @@ void NormServerNode::HandleObjectMessage(NormMessage& msg) if (obj) { - // Open receive object and notify app for accept. - NormObjectSize objectSize = msg.object.GetObjectSize(); - if (obj->Open(objectSize, msg.object.FlagIsSet(NormObjectMsg::FLAG_INFO))) + NormFtiExtension fti; + while (msg.GetNextExtension(fti)) { - session->Notify(NormController::RX_OBJECT_NEW, this, obj); - if (obj->Accepted()) + if (NormHeaderExtension::FTI == fti.GetType()) { - rx_table.Insert(obj); - DMSG(8, "NormServerNode::HandleObjectMessage() node>%lu server>%lu new obj>%hu\n", - LocalNodeId(), Id(), (UINT16)objectId); - } - else - { - DeleteObject(obj); - obj = NULL; + // Open receive object and notify app for accept. + if (obj->Open(fti.GetObjectSize(), msg.FlagIsSet(NormObjectMsg::FLAG_INFO))) + { + session->Notify(NormController::RX_OBJECT_NEW, this, obj); + if (obj->Accepted()) + { + if (obj->IsStream()) + ((NormStreamObject*)obj)->StreamUpdateStatus(blockId); + rx_table.Insert(obj); + DMSG(8, "NormServerNode::HandleObjectMessage() node>%lu server>%lu new obj>%hu\n", + LocalNodeId(), Id(), (UINT16)objectId); + } + else + { + DeleteObject(obj); + obj = NULL; + } + } + else + { + DeleteObject(obj); + obj = NULL; + } + break; } } - else + if (obj && !obj->IsOpen()) { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu " + "new obj>%hu - no FTI provided!\n", LocalNodeId(), Id(), (UINT16)objectId); DeleteObject(obj); - obj = NULL; + obj = NULL; } } break; @@ -560,22 +862,31 @@ void NormServerNode::HandleObjectMessage(NormMessage& msg) #else ((NormFileObject*)obj)->Close(); #endif // !SIMULATE - session->Notify(NormController::RX_OBJECT_COMPLETE, this, obj); - DeleteObject(obj); - completion_count++; + if (NormObject::STREAM != obj->GetType()) + { + // Streams neveer complete + session->Notify(NormController::RX_OBJECT_COMPLETE, this, obj); + DeleteObject(obj); + completion_count++; + } } } + RepairCheck(NormObject::TO_BLOCK, objectId, blockId, segmentId); } // end NormServerNode::HandleObjectMessage() -bool NormServerNode::SyncTest(const NormMessage& msg) const +bool NormServerNode::SyncTest(const NormObjectMsg& msg) const { - // (TBD) Additional sync policies + // Allow sync on stream at any time + bool result = msg.FlagIsSet(NormObjectMsg::FLAG_STREAM); + + // Allow sync on INFO or block zero DATA message + result = result || (NormMsg::INFO == msg.GetType()) ? + true : (0 == ((const NormDataMsg&)msg).GetFecBlockId()); + + // Never sync on repair messages + result = result && !msg.FlagIsSet(NormObjectMsg::FLAG_REPAIR); - // Sync if non-repair and (INFO or block zero message) - bool result = !msg.object.FlagIsSet(NormObjectMsg::FLAG_REPAIR) && - (NORM_MSG_INFO == msg.generic.GetType()) ? - true : (0 == msg.data.GetFecBlockId()); return result; } // end NormServerNode::SyncTest() @@ -587,7 +898,8 @@ void NormServerNode::Sync(NormObjectId objectId) if (rx_pending_mask.IsSet()) { NormObjectId firstSet = NormObjectId(rx_pending_mask.FirstSet()); - if (objectId > NormObjectId(rx_pending_mask.LastSet())) + if ((objectId > NormObjectId(rx_pending_mask.LastSet())) || + ((next_id - objectId) > 256)) { NormObject* obj; while ((obj = rx_table.Find(rx_table.RangeLo()))) @@ -610,14 +922,18 @@ void NormServerNode::Sync(NormObjectId objectId) rx_pending_mask.UnsetBits(firstSet, numBits); } } - if (next_id < objectId) next_id = objectId; + if ((next_id < objectId) || + ((next_id - objectId) > 256)) + { + max_pending_object = next_id = objectId; + } sync_id = objectId; ASSERT(OBJ_INVALID != GetObjectStatus(objectId)); } else { ASSERT(!rx_pending_mask.IsSet()); - sync_id = next_id = objectId; + sync_id = next_id = max_pending_object = objectId; synchronized = true; } } // end NormServerNode::Sync() @@ -671,35 +987,51 @@ NormServerNode::ObjectStatus NormServerNode::GetObjectStatus(const NormObjectId& if (objectId < sync_id) { if ((sync_id - objectId) > 256) + { return OBJ_INVALID; + } else + { return OBJ_COMPLETE; + } } else { if (objectId < next_id) { if (rx_pending_mask.Test(objectId)) + { return OBJ_PENDING; + } else + { return OBJ_COMPLETE; + } } else { if (rx_pending_mask.IsSet()) { if (rx_pending_mask.CanSet(objectId)) + { return OBJ_NEW; + } else + { return OBJ_INVALID; + } } else { NormObjectId delta = objectId - next_id + 1; if (delta > NormObjectId(rx_pending_mask.Size())) + { return OBJ_INVALID; + } else + { return OBJ_NEW; + } } } } @@ -717,6 +1049,7 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, NormSegmentId segmentId) { ASSERT(synchronized); + if (objectId > max_pending_object) max_pending_object = objectId; if (!repair_timer.IsActive()) { bool startTimer = false; @@ -749,24 +1082,21 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, current_object_id = objectId; if (startTimer) { - double backoffTime = - ExponentialRand(grtt_estimate*session->BackoffFactor(), - gsize_estimate); - repair_timer.SetInterval(backoffTime); - DMSG(4, "NormServerNode::RepairCheck() node>%lu begin NACK back-off: %lf sec)...\n", - LocalNodeId(), backoffTime); + // BACKOFF related code + double backoffInterval = + (session->Address().IsMulticast() && (session->BackoffFactor() > 0.0)) ? + ExponentialRand(grtt_estimate*session->BackoffFactor(), gsize_estimate) : + 0.0; + repair_timer.SetInterval(backoffInterval); + DMSG(3, "NormServerNode::RepairCheck() node>%lu begin NACK back-off: %lf sec)...\n", + LocalNodeId(), backoffInterval); session->InstallTimer(&repair_timer); } } - else - { - // No repairs needed - return; - } } else if (repair_timer.RepeatCount()) { - // Repair timer in back-off phase + // Repair timer in backoff phase // Trim server current transmit position reference NormObject* obj = rx_table.Find(objectId); if (obj) obj->ClientRepairCheck(checkLevel, blockId, segmentId, true); @@ -774,7 +1104,22 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, } else { - // Holding-off on repair cycle initiation + // Repair timer in holdoff phase + bool rewindDetected = objectId < current_object_id; + if (!rewindDetected) + { + NormObject* obj = rx_table.Find(objectId); + if (obj) + rewindDetected = obj->ClientRepairCheck(checkLevel, blockId, segmentId, true, true); + } + if (rewindDetected) + { + repair_timer.Deactivate(); + DMSG(0, "NormServerNode::RepairCheck() node>%lu server rewind detected, ending NACK hold-off ...\n", + LocalNodeId()); + + RepairCheck(checkLevel, objectId, blockId, segmentId); + } } } // end NormServerNode::RepairCheck() @@ -818,23 +1163,63 @@ bool NormServerNode::OnRepairTimeout() if (repairPending) { // We weren't completely suppressed, so build NACK - NormMessage* msg = session->GetMessageFromPool(); - if (!msg) + NormNackMsg* nack = (NormNackMsg*)session->GetMessageFromPool(); + if (!nack) { DMSG(0, "NormServerNode::OnRepairTimeout() node>%lu Warning! " "message pool empty ...\n", LocalNodeId()); repair_timer.Deactivate(); return false; } - NormNackMsg& nack = msg->nack; - nack.ResetNackContent(); + nack->Init(); + + if (cc_enable) + { + NormCCFeedbackExtension ext; + nack->AttachExtension(ext); + if (is_clr) + ext.SetCCFlag(NormCC::CLR); + else if (is_plr) + ext.SetCCFlag(NormCC::CLR); + if (rtt_confirmed) + ext.SetCCFlag(NormCC::RTT); + ext.SetCCRtt(rtt_quantized); + double ccLoss = LossEstimate(); + UINT16 lossQuantized = NormQuantizeLoss(ccLoss); + ext.SetCCLoss(lossQuantized); + if (slow_start) + { + ext.SetCCFlag(NormCC::START); + ext.SetCCRate(NormQuantizeRate(2.0 * recv_rate)); + } + else + { + double ccRate = NormSession::CalculateRate(nominal_packet_size, + rtt_estimate, + ccLoss); + ext.SetCCRate(NormQuantizeRate(ccRate)); + } + DMSG(6, "NormServerNode::OnRepairTimeout() node>%lu sending NACK rate:%lf kbps (rtt:%lf loss:%lf s:%hu) slow_start:%d\n", + LocalNodeId(), NormUnquantizeRate(ext.GetCCRate()) * (8.0/1000.0), + rtt_estimate, ccLoss, nominal_packet_size, slow_start); + ext.SetCCSequence(cc_sequence); + // Cancel potential pending NORM_ACK(RTT) + if (cc_timer.IsActive()) + { + cc_timer.Deactivate(); + cc_timer.SetInterval(grtt_estimate*session->BackoffFactor()); + session->InstallTimer(&cc_timer); + cc_timer.DecrementRepeatCount(); + } + } + NormRepairRequest req; NormObjectId prevId; UINT16 reqCount = 0; NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; nextId = rx_pending_mask.FirstSet(); lastId = rx_pending_mask.LastSet(); - if (current_object_id < lastId) lastId = current_object_id; + if (max_pending_object < lastId) lastId = max_pending_object; lastId++; // force loop to fully flush nack building. while ((nextId <= lastId) || (reqCount > 0)) { @@ -844,7 +1229,7 @@ bool NormServerNode::OnRepairTimeout() nextId++; // force break of possible ending consecutive series else obj = rx_table.Find(nextId); - if (obj) objPending = obj->IsPending(nextId != current_object_id); + if (obj) objPending = obj->IsPending(nextId != max_pending_object); if (!objPending && reqCount && (reqCount == (nextId - prevId))) { @@ -869,12 +1254,13 @@ bool NormServerNode::OnRepairTimeout() if (prevForm != nextForm) { if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check + nack->PackRepairRequest(req); // (TBD) error check if (NormRepairRequest::INVALID != nextForm) { - nack.AttachRepairRequest(req, segment_size); // (TBD) error check + nack->AttachRepairRequest(req, segment_size); // (TBD) error check req.SetForm(nextForm); req.ResetFlags(); + // (TBD) An "INFO-ONLY" repair policy would be enforced here req.SetFlag(NormRepairRequest::OBJECT); } prevForm = nextForm; @@ -884,13 +1270,13 @@ bool NormServerNode::OnRepairTimeout() switch (nextForm) { case NormRepairRequest::ITEMS: - req.AppendRepairItem(prevId, 0, 0); + req.AppendRepairItem(prevId, 0, ndata, 0); if (2 == reqCount) - req.AppendRepairItem(prevId+1, 0, 0); + req.AppendRepairItem(prevId+1, 0, ndata, 0); break; case NormRepairRequest::RANGES: - req.AppendRepairItem(prevId, 0, 0); - req.AppendRepairItem(prevId+reqCount-1, 0, 0); + req.AppendRepairItem(prevId, 0, ndata, 0); + req.AppendRepairItem(prevId+reqCount-1, 0, ndata, 0); break; default: break; @@ -904,23 +1290,29 @@ bool NormServerNode::OnRepairTimeout() if (objPending) { if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check + nack->PackRepairRequest(req); // (TBD) error check prevForm = NormRepairRequest::INVALID; reqCount = 0; - bool flush = (nextId != current_object_id); - obj->AppendRepairRequest(nack, flush); // (TBD) error check + bool flush = (nextId != max_pending_object); + obj->AppendRepairRequest(*nack, flush); // (TBD) error check } nextId++; if (nextId <= lastId) nextId = rx_pending_mask.NextSet(nextId); } // end while(nextId <= lastId) if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - // (TBD) Queue NACK for transmission - msg->generic.SetType(NORM_MSG_NACK); - msg->nack.SetServerId(Id()); - msg->generic.SetDestination(session->Address()); - session->QueueMessage(msg); + nack->PackRepairRequest(req); // (TBD) error check + // Queue NACK for transmission + nack->SetServerId(Id()); + nack->SetSessionId(session_id); + // GRTT response is deferred until transmit time + + // (TBD) support unicast nacking + if (session->UnicastNacks()) + nack->SetDestination(GetAddress()); + else + nack->SetDestination(session->Address()); + session->QueueMessage(nack); nack_count++; } else @@ -929,9 +1321,15 @@ bool NormServerNode::OnRepairTimeout() DMSG(6, "NormServerNode::OnRepairTimeout() node>%lu NACK SUPPRESSED ...\n", LocalNodeId()); } // end if/else(repairPending) - repair_timer.SetInterval(grtt_estimate*(session->BackoffFactor() + 2.0)); + // BACKOFF related code + double holdoffInterval = + session->Address().IsMulticast() ? grtt_estimate*(session->BackoffFactor() + 2.0) : + grtt_estimate; + holdoffInterval = (session->BackoffFactor() > 0.0) ? holdoffInterval : 1.01*grtt_estimate; + + repair_timer.SetInterval(holdoffInterval); DMSG(4, "NormServerNode::OnRepairTimeout() node>%lu begin NACK hold-off: %lf sec ...\n", - LocalNodeId(), repair_timer.Interval()); + LocalNodeId(), holdoffInterval); } else @@ -950,8 +1348,167 @@ bool NormServerNode::OnRepairTimeout() return true; } // end NormServerNode::OnRepairTimeout() - +void NormServerNode::UpdateRecvRate(const struct timeval& currentTime, unsigned short msgSize) +{ + double interval; + if (prev_update_time.tv_sec || prev_update_time.tv_usec) + { + interval = (double)(currentTime.tv_sec - prev_update_time.tv_sec); + if (currentTime.tv_usec > prev_update_time.tv_sec) + interval += 1.0e-06*(double)(currentTime.tv_usec - prev_update_time.tv_usec); + else + interval -= 1.0e-06*(double)(prev_update_time.tv_usec - currentTime.tv_usec); + } + else + { + recv_rate = ((double)msgSize) / grtt_estimate; + interval = 0.0; + prev_update_time = currentTime; + } + if (interval < grtt_estimate) + { + recv_accumulator += msgSize; + } + else + { + recv_rate = ((double)(recv_accumulator+msgSize)) / interval; + prev_update_time = currentTime; + recv_accumulator = 0; + } + nominal_packet_size += 0.05 * (((double)msgSize) - nominal_packet_size); +} // end NormServerNode::UpdateRecvRate() +void NormServerNode::Activate() +{ + if (activity_timer.IsActive()) + { + activity_timer.ResetRepeats(); + } + else + { + activity_timer.SetInterval(grtt_estimate); + session->InstallTimer(&activity_timer); + } +} // end NormServerNode::Activate() + +bool NormServerNode::OnActivityTimeout() +{ + if ((activity_timer.Repeats() - activity_timer.RepeatCount()) > 1) + { + DMSG(4, "NormServerNode::OnActivityTimeout() node>%lu for server>%lu\n", + LocalNodeId(), Id()); + struct timeval currentTime; + ::GetSystemTime(¤tTime); + UpdateRecvRate(currentTime, 0); + } + if (!activity_timer.RepeatCount()) + { + DMSG(0, "NormServerNode::OnActivityTimeout() node>%lu server>%lu gone inactive?\n", + LocalNodeId(), Id()); + } + return true; +} // end NormServerNode::OnActivityTimeout() + +bool NormServerNode::UpdateLossEstimate(const struct timeval& currentTime, + unsigned short seq, + bool ecn) +{ + bool result = loss_estimator.Update(currentTime, seq, ecn); + if (result && slow_start) + { + double lossInit = (recv_rate * rtt_estimate) / + (segment_size*sqrt(3.0/2.0)); + lossInit = (lossInit*lossInit); + double currentInterval = (double)loss_estimator.LastLossInterval(); + lossInit = 1.0 / MAX(lossInit, currentInterval); + loss_estimator.SetInitialLoss(lossInit); + slow_start = false; + } + return result; +} + +bool NormServerNode::OnCCTimeout() +{ + // Build and queue ACK() + switch (cc_timer.RepeatCount()) + { + case 0: + // "hold-off" time has ended + break; + + case 1: + { + // We weren't suppressed, so build an ACK(RTT) and send + NormAckMsg* ack = (NormAckMsg*)session->GetMessageFromPool(); + if (!ack) + { + DMSG(0, "NormServerNode::OnCCTimeout() node>%lu Warning! " + "message pool empty ...\n", LocalNodeId()); + if (cc_timer.IsActive()) cc_timer.Deactivate(); + return false; + } + ack->Init(); + ack->SetServerId(Id()); + ack->SetSessionId(session_id); + ack->SetAckType(NormAck::CC); + ack->SetAckId(0); + + // GRTT response is deferred until transmit time + NormCCFeedbackExtension ext; + ack->AttachExtension(ext); + if (is_clr) + { + ext.SetCCFlag(NormCC::CLR); + } + else if (is_plr) + ext.SetCCFlag(NormCC::PLR); + if (rtt_confirmed) + ext.SetCCFlag(NormCC::RTT); + ext.SetCCRtt(rtt_quantized); + double ccLoss = LossEstimate(); + UINT16 lossQuantized = NormQuantizeLoss(ccLoss); + ext.SetCCLoss(lossQuantized); + if (slow_start) + { + ext.SetCCFlag(NormCC::START); + ext.SetCCRate(NormQuantizeRate(2.0 * recv_rate)); + } + else + { + double ccRate = NormSession::CalculateRate(nominal_packet_size, + rtt_estimate, ccLoss); + ext.SetCCRate(NormQuantizeRate(ccRate)); + } + //TRACE("NormServerNode::OnCCTimeout() node>%lu sending ACK rate:%lf kbps (rtt:%lf loss:%lf s:%lf recvRate:%lf) slow_start:%d\n", + // LocalNodeId(), NormUnquantizeRate(ext.GetCCRate()) * (8.0/1000.0), + // rtt_estimate, ccLoss, nominal_packet_size, recv_rate*(8.0/1000.), slow_start); + ext.SetCCSequence(cc_sequence); + if (session->UnicastNacks()) + ack->SetDestination(GetAddress()); + else + ack->SetDestination(session->Address()); + if (is_clr || is_plr) + { + session->SendMessage(*ack); + session->ReturnMessageToPool(ack); + } + else + { + session->QueueMessage(ack); + } + + // Begin cc_timer "holdoff" phase + cc_timer.SetInterval(grtt_estimate*session->BackoffFactor()); + return true; + } + + default: + // Should never occur + ASSERT(0); + break; + } + return true; +} // end NormServerNode::OnCCTimeout() NormNodeTree::NormNodeTree() : root(NULL) @@ -1096,6 +1653,10 @@ NormNodeTreeIterator::NormNodeTreeIterator(const NormNodeTree& t) while (x->left) x = x->left; next = x; } + else + { + next = NULL; + } } void NormNodeTreeIterator::Reset() @@ -1106,6 +1667,10 @@ void NormNodeTreeIterator::Reset() while (x->left) x = x->left; next = x; } + else + { + next = NULL; + } } // end NormNodeTreeIterator::Reset() NormNode* NormNodeTreeIterator::GetNextNode() @@ -1135,7 +1700,7 @@ NormNode* NormNodeTreeIterator::GetNextNode() } // end NormNodeTreeIterator::GetNextNode() NormNodeList::NormNodeList() - : head(NULL), tail(NULL) + : head(NULL), tail(NULL), count(0) { } @@ -1167,6 +1732,7 @@ void NormNodeList::Append(NormNode *theNode) head = theNode; tail = theNode; theNode->right = NULL; + count++; } // end NormNodeList::Append() void NormNodeList::Remove(NormNode *theNode) @@ -1180,6 +1746,7 @@ void NormNodeList::Remove(NormNode *theNode) theNode->left->right = theNode->right; else head = theNode->right; + count--; } // end NormNodeList::Remove() void NormNodeList::Destroy() @@ -1192,3 +1759,390 @@ void NormNodeList::Destroy() } } // end NormNodeList::Destroy() + +////////////////////////////////////////////////////////// +// +// NormLossEstimator implementation +// + +NormLossEstimator::NormLossEstimator() + : synchronized(false), seeking_loss_event(true), event_window(0.0) +{ + memset(history, 0, (DEPTH+1)*sizeof(unsigned int)); +} + +const double NormLossEstimator::weight[DEPTH] = +{ + 1.0, 1.0, 1.0, 1.0, + 0.8, 0.6, 0.4, 0.2 +}; + +int NormLossEstimator::SequenceDelta(unsigned short a, unsigned short b) +{ + int delta = a - b; + if (delta < -0x8000) + return (delta + 0x10000); + else if (delta < 0x8000) + return delta; + else + return delta - 0x10000; +} // end NormLossEstimator::SequenceDelta() + +// Returns true when a loss event has occurred +bool NormLossEstimator::Update(const struct timeval& currentTime, + unsigned short seq, + bool ecn) +{ + if (!synchronized) + { + Sync(seq); + return false; + } + bool outage = false; + int delta = SequenceDelta(seq, index_seq); + if (abs(delta) > MAX_OUTAGE) // out-of-range packet + { + index_seq = seq; + return false; + } + else if (delta > 0) // new packet arrival + { + if (ecn || (delta > 1)) outage = true; + index_seq = seq; + } + else // (delta <= 0) // old misordered or duplicate packet + { + return false; + } + if (outage) + { + if (!seeking_loss_event) + { + double deltaTime = (double)(currentTime.tv_sec - event_time.tv_sec); + if (currentTime.tv_usec > event_time.tv_usec) + deltaTime += (double)(currentTime.tv_usec - event_time.tv_usec) * 1.0e-06; + else + deltaTime -= (double)(event_time.tv_usec - currentTime.tv_usec) * 1.0e-06; + if (deltaTime > event_window) seeking_loss_event = true; + } + if (seeking_loss_event) + { + memmove(history+1, history, DEPTH*sizeof(unsigned int)); + history[0] = 1; + seeking_loss_event = false; + event_time = currentTime; + return true; + } + else + { + // Only count one loss per loss event + history[0] = 1; + return false; + } + } + else + { + history[0]++; + return false; + } +} // end NormLossEstimator::Update() + +double NormLossEstimator::LossFraction() +{ + if (0 == history[1]) return 0.0; + double weightSum = 0.0; + double s0 = 0.0; + const double* wptr = weight; + const unsigned int* h = history; + for (unsigned int i = 0; i < DEPTH; i++) + { + if (0 == *h) break; + s0 += *wptr * *h++; + weightSum += *wptr++; + } + s0 /= weightSum; + + weightSum = 0.0; + double s1 = 0.0; + wptr = weight; + h = history + 1; + for (unsigned int i = 0; i < DEPTH; i++) + { + if (0 == *h) break; + s1 += *wptr * *h++; // ave loss interval w/out current interval + weightSum += *wptr++; // (TBD) this could be pre-computed + } + s1 /= weightSum; + return (1.0 / (MAX(s0,s1))); +} // end NormLossEstimator::LossFraction() + + + +NormLossEstimator2::NormLossEstimator2() + : lag_mask(0xffffffff), lag_depth(0), lag_test_bit(0x01), + event_window(0), event_index(0), + event_window_time(0.0), event_index_time(0.0), + seeking_loss_event(true), + no_loss(true), initial_loss(0.0), loss_interval(0.0), + current_discount(1.0) +{ + memset(history, 0, 9*sizeof(unsigned long)); + discount[0] = 1.0; +} + +const double NormLossEstimator2::weight[8] = +{ + 1.0, 1.0, 1.0, 1.0, + 0.8, 0.6, 0.4, 0.2 +}; + +int NormLossEstimator2::SequenceDelta(unsigned short a, unsigned short b) +{ + int delta = a - b; + if (delta < -0x8000) + return (delta + 0x10000); + else if (delta < 0x8000) + return delta; + else + return delta - 0x10000; +} // end NormLossEstimator2::SequenceDelta() + + +bool NormLossEstimator2::Update(const struct timeval& currentTime, + unsigned short theSequence, + bool ecnStatus) +{ + if (!init) + { + Init(theSequence); + return false; + } + unsigned int outageDepth = 0; + // Process packet through lag filter and check for loss + int delta = SequenceDelta(theSequence, lag_index); + if (delta > 100) // Very new packet arrived + { + Sync(theSequence); // resync + return false; + } + else if (delta > 0) // New packet arrived + { + if (lag_depth) + { + unsigned int outage = 0; + for (int i = 0; i < delta; i++) + { + if (i <= (int)lag_depth) + { + outage++; + if (lag_mask & lag_test_bit) + { + if (outage > 1) + outageDepth = MAX(outage, outageDepth); + outage = 0; + } + else + { + lag_mask |= lag_test_bit; + } + lag_mask <<= 1; + } + else + { + outage += delta - lag_depth - 1; + break; + } + } + outageDepth = MAX(outage, outageDepth); + lag_mask |= 0x01; + } + else + { + if (delta > 1) outageDepth = delta - 1; + } + lag_index = theSequence; + } + else if (delta < -100) // Very old packet arrived + { + Sync(theSequence); // resync + return false; + } + else if (delta < -((int)lag_depth)) // Old packet arrived + { + ChangeLagDepth(-delta); + } + else if (delta < 0) // Lagging packet arrived + { // (duplicates have no effect) + lag_mask |= (0x01 << (-delta)); + return false; + } + else // (delta == 0) + { + return false; // Duplicate packet arrived, ignore + } + + if (ecnStatus) outageDepth += 1; + + bool newLossEvent = false; + + if (!seeking_loss_event) + { + double theTime = (((double)currentTime.tv_sec) + + (((double)currentTime.tv_usec)/1.0e06)); + if (theTime > event_index_time) seeking_loss_event = true; + + // (TBD) Should we reset our history on + // outages within the event_window??? + } + + if (seeking_loss_event) + { + double scale; + if (history[0] > loss_interval) + scale = 0.125 / (1.0 + log((double)(event_window ? event_window : 1))); + else + scale = 0.125; + if (outageDepth) // non-zero outageDepth means pkt loss(es) + { + if (no_loss) // first loss + { + //fprintf(stderr, "First Loss: seq:%u init:%f history:%lu adjusted:", + // theSequence, initial_loss, history[0]); + if (initial_loss != 0.0) + { + unsigned long initialHistory = (unsigned long) ((1.0 / initial_loss) + 0.5); + history[0] = MAX(initialHistory, history[0]/2); + } + //fprintf(stderr, "%lu\n", history[0]); + no_loss = false; + } + + // Old method + if (loss_interval > 0.0) + loss_interval += scale*(((double)history[0]) - loss_interval); + else + loss_interval = (double) history[0]; + + // New method + // New loss event, shift loss interval history & discounts + memmove(&history[1], &history[0], 8*sizeof(unsigned long)); + history[0] = 0; + memmove(&discount[1], &discount[0], 8*sizeof(double)); + discount[0] = 1.0; + current_discount = 1.0; + + event_index = theSequence; + //if (event_window) + seeking_loss_event = false; + newLossEvent = true; + no_loss = false; + // (TBD) use fixed pt. math here ... + event_index_time = (((double)currentTime.tv_sec) + + (((double)currentTime.tv_usec)/1.0e06)); + event_index_time += event_window_time; + } + else + { + //if (no_loss) fprintf(stderr, "No loss (seq:%u) ...\n", theSequence); + if (loss_interval > 0.0) + { + double diff = ((double)history[0]) - loss_interval; + if (diff >= 1.0) + { + //scale *= (diff * diff) / (loss_interval * loss_interval); + loss_interval += scale*log(diff); + } + } + } + } + else + { + if (outageDepth) history[0] = 0; + } // end if/else (seeking_loss_event) + + if (history[0] < 100000) history[0]++; + + return newLossEvent; +} // end NormLossEstimator2::ProcessRecvPacket() + +double NormLossEstimator2::LossFraction() +{ +#if defined0 + if (use_ewma_loss_estimate) + return MdpLossFraction(); // MDP EWMA approach + else +#endif // SIMULATOR + return (TfrcLossFraction()); // ACIRI TFRC approach +} // end NormLossEstimator2::LossFraction() + + + +// TFRC Loss interval averaging with discounted, weighted averaging +double NormLossEstimator2::TfrcLossFraction() +{ + if (!history[1]) return 0.0; + // Compute older weighted average s1->s8 for discount determination + double average = 0.0; + double scaling = 0.0; + unsigned int i; + for (i = 1; i < 9; i++) + { + if (history[i]) + { + average += history[i] * weight[i-1] * discount[i]; + scaling += discount[i] * weight[i-1]; + } + else + { + break; + } + } + double s1 = average / scaling; + + // Compute discount if applicable + if (history[0] > (2.0*s1)) + { + current_discount = (2.0*s1) / (double) history[0]; + current_discount = MAX (current_discount, 0.5); + } + + // Re-compute older weighted average s1->s8 with discounting + if (current_discount < 1.0) + { + average = 0.0; + scaling = 0.0; + for (i = 1; i < 9; i++) + { + if (history[i]) + { + average += current_discount * history[i] * weight[i-1] * discount[i]; + scaling += current_discount * discount[i] * weight[i-1]; + } + else + { + break; + } + } + s1 = average / scaling; + } + + // Compute newer weighted average s0->s7 with discounting + average = 0.0; + scaling = 0.0; + for (i = 0; i < 8; i++) + { + if (history[i]) + { + double d = (i > 0) ? current_discount : 1.0; + average += d * history[i] * weight[i] * discount[i]; + scaling += d * discount[i] * weight[i]; + } + else + { + break; + } + } + double s0 = average / scaling; + // Use max of old/new averages + return (1.0 / MAX(s0, s1)); +} // end NormLossEstimator2::LossFraction() diff --git a/common/normNode.h b/common/normNode.h index 63e1176..6c9904e 100644 --- a/common/normNode.h +++ b/common/normNode.h @@ -16,12 +16,13 @@ class NormNode NormNode(class NormSession* theSession, NormNodeId nodeId); virtual ~NormNode(); - void SetAddress(const NetworkAddress& address) - {addr = address;} - const NetworkAddress Address() const {return addr;} + void SetAddress(const NetworkAddress& address) {addr = address;} + const NetworkAddress& GetAddress() const {return addr;} - const NormNodeId& Id() {return id;} - inline const NormNodeId& LocalNodeId(); + const NormNodeId& Id() const {return id;} + inline const NormNodeId& LocalNodeId() const; + + void SetId(const NormNodeId& nodeId) {id = nodeId;} protected: class NormSession* session; @@ -33,11 +34,154 @@ class NormNode NormNode* parent; NormNode* right; NormNode* left; - NormNode* prev; - NormNode* next; + + //NormNode* prev; + //NormNode* next; }; // end class NormNode +// Weighted-history loss event estimator +class NormLossEstimator +{ + public: + NormLossEstimator(); + double LossFraction(); + bool Update(const struct timeval& currentTime, + unsigned short seqNumber, + bool ecn = false); + void SetLossEventWindow(double lossWindow) {event_window = lossWindow;} + void SetInitialLoss(double lossFraction) + { + memset(history, 0, (DEPTH+1)*sizeof(unsigned int)); + history[1] = (unsigned int)((1.0 / lossFraction) + 0.5); + } + unsigned int LastLossInterval() {return history[1];} + + private: + enum {DEPTH = 8}; + enum {MAX_OUTAGE = 100}; + + void Sync(unsigned short seq) + { + index_seq = seq; + synchronized = true; + } + int SequenceDelta(unsigned short a, unsigned short b); + + static const double weight[8]; + + bool synchronized; + unsigned short index_seq; + bool seeking_loss_event; + double event_window; + struct timeval event_time; + unsigned int history[DEPTH+1]; +}; // end class NormLossEstimator + + +class NormLossEstimator2 +{ + public: + NormLossEstimator2(); + void SetEventWindow(unsigned short windowDepth) + {event_window = windowDepth;} + void SetLossEventWindow(double theTime) + {event_window_time = theTime;} + bool Update(const struct timeval& currentTime, + unsigned short seqNumber, + bool ecn = false); + double LossFraction(); + double MdpLossFraction() + {return ((loss_interval > 0.0) ? (1.0/loss_interval) : 0.0);} + double TfrcLossFraction(); + bool NoLoss() {return no_loss;} + void SetInitialLoss(double lossFraction) + { + memset(history, 0, (DEPTH+1)*sizeof(unsigned int)); + history[1] = (unsigned int)((1.0 / lossFraction) + 0.5); + } + unsigned long CurrentLossInterval() {return history[0];} + unsigned int LastLossInterval() {return history[1];} + + private: + enum {DEPTH = 8}; + // Members + bool init; + unsigned long lag_mask; + unsigned int lag_depth; + unsigned long lag_test_bit; + unsigned short lag_index; + + unsigned short event_window; + unsigned short event_index; + double event_window_time; + double event_index_time; + bool seeking_loss_event; + + bool no_loss; + double initial_loss; + + double loss_interval; // EWMA of loss event interval + + unsigned long history[9]; // loss interval history + double discount[9]; + double current_discount; + static const double weight[8]; + + void Init(unsigned short theSequence) + {init = true; Sync(theSequence);} + void Sync(unsigned short theSequence) + {lag_index = theSequence;} + void ChangeLagDepth(unsigned int theDepth) + { + theDepth = (theDepth > 20) ? 20 : theDepth; + lag_depth = theDepth; + lag_test_bit = 0x01 << theDepth; + } + int SequenceDelta(unsigned short a, unsigned short b); + +}; // end class NormLossEstimator2 + + +class NormCCNode : public NormNode +{ + public: + NormCCNode(class NormSession* theSession, NormNodeId nodeId); + ~NormCCNode(); + + bool IsClr() const {return is_clr;} + bool IsActive() const {return is_active;} + bool HasRtt() const {return rtt_confirmed;} + + double GetRtt() const {return rtt;} + double GetLoss() const {return loss;} + double GetRate() const {return rate;} + UINT8 GetCCSequence() const {return cc_sequence;} + + void SetActive(bool state) {is_active = state;} + void SetClrStatus(bool state) {is_clr = state;} + void SetRttStatus(bool state) {rtt_confirmed = state;} + void SetRtt(double value) {rtt = value;} + double UpdateRtt(double value) + { + rtt = 0.9*rtt + 0.1 * value; + return rtt; + } + void SetLoss(double value) {loss = value;} + void SetRate(double value) {rate = value;} + void SetCCSequence(UINT8 value) {cc_sequence = value;} + + private: + bool is_clr; // true if worst path representative + bool is_plr; // true if worst path candidate + bool is_active; + bool rtt_confirmed; + double rtt; // in seconds + double loss; // loss fraction + double rate; // in bytes per second + UINT8 cc_sequence; +}; // end class NormCCNode + class NormServerNode : public NormNode { public: @@ -46,15 +190,28 @@ class NormServerNode : public NormNode NormServerNode(class NormSession* theSession, NormNodeId nodeId); ~NormServerNode(); - void HandleCommand(NormCommandMsg& cmd); - void HandleObjectMessage(NormMessage& msg); - void HandleNackMessage(NormNackMsg& nack); - - bool Open(UINT16 segmentSize, UINT16 numData, UINT16 numParity); + bool UpdateLossEstimate(const struct timeval& currentTime, + unsigned short theSequence, + bool ecnStatus = false); + double LossEstimate() {return loss_estimator.LossFraction();} + + void UpdateRecvRate(const struct timeval& currentTime, + unsigned short msgSize); + + void HandleCommand(const struct timeval& currentTime, + const NormCmdMsg& cmd); + + void HandleObjectMessage(const NormObjectMsg& msg); + void HandleCCFeedback(UINT8 ccFlags, double ccRate); + void HandleNackMessage(const NormNackMsg& nack); + void HandleAckMessage(const NormAckMsg& ack); + + bool Open(UINT16 sessionId, UINT16 segmentSize, UINT16 numData, UINT16 numParity); + void Activate(); void Close(); bool IsOpen() const {return is_open;} - bool SyncTest(const NormMessage& msg) const; + bool SyncTest(const NormObjectMsg& msg) const; void Sync(NormObjectId objectId); ObjectStatus UpdateSyncStatus(const NormObjectId& objectId); void SetPending(NormObjectId objectId); @@ -65,8 +222,6 @@ class NormServerNode : public NormNode UINT16 SegmentSize() {return segment_size;} UINT16 BlockSize() {return ndata;} UINT16 NumParity() {return nparity;} - //NormBlockPool* BlockPool() {return &block_pool;} - //NormSegmentPool* SegmentPool() {return &segment_pool;} NormBlock* GetFreeBlock(NormObjectId objectId, NormBlockId blockId); void PutFreeBlock(NormBlock* block) @@ -90,27 +245,29 @@ class NormServerNode : public NormNode return decoder.Decode(segmentList, numData, erasureCount, erasure_loc); } - void CalculateGrttResponse(struct timeval& grttResponse); + void CalculateGrttResponse(const struct timeval& currentTime, + struct timeval& grttResponse) const; - unsigned long CurrentBufferUsage() + // Statistics kept on server + unsigned long CurrentBufferUsage() const {return (segment_size * segment_pool.CurrentUsage());} - unsigned long PeakBufferUsage() + unsigned long PeakBufferUsage() const {return (segment_size * segment_pool.PeakUsage());} - unsigned long BufferOverunCount() + unsigned long BufferOverunCount() const {return segment_pool.OverunCount() + block_pool.OverrunCount();} - - unsigned long RecvTotal() {return recv_total;} - unsigned long RecvGoodput() {return recv_goodput;} + unsigned long RecvTotal() const {return recv_total;} + unsigned long RecvGoodput() const {return recv_goodput;} void IncrementRecvTotal(unsigned long count) {recv_total += count;} void IncrementRecvGoodput(unsigned long count) {recv_goodput += count;} void ResetRecvStats() {recv_total = recv_goodput = 0;} void IncrementResyncCount() {resync_count++;} - unsigned long ResyncCount() {return resync_count;} - unsigned long NackCount() {return nack_count;} - unsigned long SuppressCount() {return suppress_count;} - unsigned long CompletionCount() {return completion_count;} - unsigned long PendingCount() {return rx_table.Count();} - unsigned long FailureCount() {return failure_count;} + unsigned long ResyncCount() const {return resync_count;} + unsigned long NackCount() const {return nack_count;} + unsigned long SuppressCount() const {return suppress_count;} + unsigned long CompletionCount() const {return completion_count;} + unsigned long PendingCount() const {return rx_table.Count();} + unsigned long FailureCount() const {return failure_count;} + private: void RepairCheck(NormObject::CheckLevel checkLevel, @@ -118,9 +275,12 @@ class NormServerNode : public NormNode NormBlockId blockId, NormSegmentId segmentId); + bool OnActivityTimeout(); bool OnRepairTimeout(); + bool OnCCTimeout(); + void HandleRepairContent(const char* buffer, UINT16 bufferLen); - + UINT16 session_id; bool synchronized; NormObjectId sync_id; // only valid if(synchronized) NormObjectId next_id; // only valid if(synchronized) @@ -138,8 +298,10 @@ class NormServerNode : public NormNode NormDecoder decoder; UINT16* erasure_loc; + ProtocolTimer activity_timer; ProtocolTimer repair_timer; - NormObjectId current_object_id; // index for repair + NormObjectId current_object_id; // index for suppression + NormObjectId max_pending_object; // index for NACK construction double grtt_estimate; UINT8 grtt_quantized; @@ -148,6 +310,25 @@ class NormServerNode : public NormNode double gsize_estimate; UINT8 gsize_quantized; + // Congestion control state + NormLossEstimator2 loss_estimator; + //NormLossEstimator loss_estimator; + UINT8 cc_sequence; + bool cc_enable; + double cc_rate; // ccRate at start of cc_timer + ProtocolTimer cc_timer; + double rtt_estimate; + UINT8 rtt_quantized; + bool rtt_confirmed; + bool is_clr; + bool is_plr; + bool slow_start; + double send_rate; // sender advertised rate + double recv_rate; // measured recv rate + struct timeval prev_update_time; // for recv_rate measurement + unsigned long recv_accumulator; // for recv_rate measurement + double nominal_packet_size; + // For statistics tracking unsigned long recv_total; // total recvd accumulator unsigned long recv_goodput; // goodput recvd accumulator @@ -157,7 +338,7 @@ class NormServerNode : public NormNode unsigned long completion_count; unsigned long failure_count; // due to re-syncs -}; // end class NodeServerNode +}; // end class NormServerNode // Used for binary trees of NormNodes @@ -179,9 +360,7 @@ class NormNodeTree void AttachNode(NormNode *theNode); void DetachNode(NormNode *theNode); void Destroy(); // delete all nodes in tree - - - + private: // Members NormNode* root; @@ -208,6 +387,7 @@ class NormNodeList // Construction NormNodeList(); ~NormNodeList(); + unsigned int GetCount() {return count;} NormNode* FindNodeById(NormNodeId nodeId) const; void Append(NormNode* theNode); void Remove(NormNode* theNode); @@ -218,13 +398,14 @@ class NormNodeList delete theNode; } void Destroy(); // delete all nodes in list - + const NormNode* Head() {return head;} // Members private: NormNode* head; NormNode* tail; + unsigned int count; }; // end class NormNodeList class NormNodeListIterator @@ -236,7 +417,7 @@ class NormNodeListIterator NormNode* GetNextNode() { NormNode* n = next; - next = n ? n->next : NULL; + next = n ? n->right : NULL; return n; } private: diff --git a/common/normObject.cpp b/common/normObject.cpp index 5bd33fc..797399f 100644 --- a/common/normObject.cpp +++ b/common/normObject.cpp @@ -8,9 +8,10 @@ NormObject::NormObject(NormObject::Type theType, class NormServerNode* theServer, const NormObjectId& objectId) : type(theType), session(theSession), server(theServer), - id(objectId), pending_info(false), repair_info(false), - current_block_id(0), next_segment_id(0), - info(NULL), info_len(0), accepted(false), stream_sync(false) + id(objectId), segment_size(0), pending_info(false), repair_info(false), + current_block_id(0), next_segment_id(0), + max_pending_block(0), max_pending_segment(0), + info(NULL), info_len(0), accepted(false) { } @@ -21,7 +22,11 @@ NormObject::~NormObject() else session->DeleteTxObject(true);*/ Close(); - if (info) delete info; + if (info) + { + delete info; + info = NULL; + } } // This is mainly used for debug messages @@ -102,6 +107,7 @@ bool NormObject::Open(const NormObjectSize& objectSize, Close(); return false; } + pending_mask.Clear(); pending_mask.SetBits(0, numBlocks.LSB()); // Init repair_mask @@ -117,7 +123,8 @@ bool NormObject::Open(const NormObjectSize& objectSize, { last_block_id = 0; // not applicable for STREAM last_block_size = numData; // assumed for STREAM - stream_next_id = numBlocks.LSB(); + NormStreamObject* stream = static_cast(this); + stream->StreamResync(numBlocks.LSB()); } else { @@ -126,7 +133,7 @@ bool NormObject::Open(const NormObjectSize& objectSize, lastBlockBytes = objectSize - lastBlockBytes; NormObjectSize lastBlockSize = lastBlockBytes / NormObjectSize(segmentSize); ASSERT(!lastBlockSize.MSB()); - ASSERT(lastBlockSize.LSB() < numData); + ASSERT(lastBlockSize.LSB() <= numData); last_block_size = lastBlockSize.LSB(); NormObjectSize lastSegmentSize = NormObjectSize(0,last_block_size-1) * NormObjectSize(segmentSize); @@ -142,6 +149,8 @@ bool NormObject::Open(const NormObjectSize& objectSize, nparity = numParity; current_block_id = 0; next_segment_id = 0; + max_pending_block = 0; + max_pending_segment = 0; return true; } // end NormObject::Open() @@ -159,6 +168,7 @@ void NormObject::Close() repair_mask.Destroy(); pending_mask.Destroy(); block_buffer.Destroy(); + segment_size = 0; } // end NormObject::Close(); bool NormObject::HandleInfoRequest() @@ -189,7 +199,7 @@ bool NormObject::HandleBlockRequest(NormBlockId nextId, NormBlockId lastId) if (!pending_mask.CanSet(nextId)) DMSG(0, "NormObject::HandleBlockRequest() pending_mask.CanSet(%lu) error\n", (UINT32)nextId); - if (!repair_mask.Set(nextId)) + if (!repair_mask.Set(nextId)) DMSG(0, "NormObject::HandleBlockRequest() repair_mask.Set(%lu) error\n", (UINT32)nextId); increasedRepair = true; @@ -304,6 +314,116 @@ bool NormObject::ActivateRepairs() return repairsActivated; } // end NormObject::ActivateRepairs() +// called by servers only +bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) +{ + // Determine range of blocks possibly pending repair + NormBlockId nextId = repair_mask.FirstSet(); + NormBlockId endId = repair_mask.LastSet(); + if (block_buffer.IsEmpty()) + { + if (repair_mask.IsSet()) endId++; + } + else + { + NormBlockId lo = block_buffer.RangeLo(); + NormBlockId hi = block_buffer.RangeHi(); + if (repair_mask.IsSet()) + { + nextId = (lo < nextId) ? lo : nextId; + endId = (hi > endId) ? hi : endId; + } + else + { + nextId = lo; + endId = hi; + } + endId++; + } + // Instantiate a repair request for BLOCK level repairs + NormRepairRequest req; + req.SetFlag(NormRepairRequest::BLOCK); + if (repair_info) req.SetFlag(NormRepairRequest::INFO); + NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; + // Iterate through the range of blocks possibly pending repair + NormBlockId firstId; + UINT32 blockCount = 0; + while (nextId < endId) + { + NormBlockId currentId = nextId; + nextId++; + //TRACE("NormObject::AppendRepairAdv() testing block:%lu nextId:%lu endId:%lu\n", + // (UINT32)currentId, (UINT32)nextId, (UINT32)endId); + bool repairEntireBlock = repair_mask.Test(currentId); + if (repairEntireBlock) + { + if (!blockCount) firstId = currentId; + blockCount++; + } + // Check for break in continuity or end + if (blockCount && (!repairEntireBlock || (nextId >= endId))) + { + NormRepairRequest::Form form; + switch (blockCount) + { + case 0: + form = NormRepairRequest::INVALID; + break; + case 1: + case 2: + form = NormRepairRequest::ITEMS; + break; + default: + form = NormRepairRequest::RANGES; + break; + } + if (form != prevForm) + { + if (NormRepairRequest::INVALID != prevForm) + cmd.PackRepairRequest(req); // (TBD) error check + cmd.AttachRepairRequest(req, segment_size); // (TBD) error check + req.SetForm(form); + prevForm = form; + } + switch (form) + { + case NormRepairRequest::INVALID: + ASSERT(0); // can't happen + break; + case NormRepairRequest::ITEMS: + req.AppendRepairItem(id, firstId, ndata, 0); + if (2 == blockCount) + req.AppendRepairItem(id, currentId, ndata, 0); + break; + case NormRepairRequest::RANGES: + req.AppendRepairRange(id, firstId, ndata, 0, + id, currentId, ndata, 0); + break; + case NormRepairRequest::ERASURES: + // erasure counts not used + break; + } + blockCount = 0; + } + if (!repairEntireBlock) + { + NormBlock* block = block_buffer.Find(currentId); + if (block && block->IsRepairPending()) + { + if (NormRepairRequest::INVALID != prevForm) + { + cmd.PackRepairRequest(req); // (TBD) error check + prevForm = NormRepairRequest::INVALID; + } + block->AppendRepairAdv(cmd, id, repair_info, ndata, segment_size); // (TBD) error check + } + } + } // end while(nextId < endId) + if (NormRepairRequest::INVALID != prevForm) + cmd.PackRepairRequest(req); // (TBD) error check + return true; +} // end NormObject::AppendRepairAdv() + // Called by server only bool NormObject::IsRepairPending() const { @@ -332,22 +452,22 @@ bool NormObject::IsPending(bool flush) const if (pending_mask.IsSet()) { NormBlockId firstId = pending_mask.FirstSet(); - if (firstId < current_block_id) + if (firstId < max_pending_block) { return true; } - else if (firstId > current_block_id) + else if (firstId > max_pending_block) { return false; } else { - if (next_segment_id > 0) + if (max_pending_segment > 0) { - NormBlock* block = block_buffer.Find(current_block_id); + NormBlock* block = block_buffer.Find(max_pending_block); if (block) { - if (block->FirstPending() < next_segment_id) + if (block->FirstPending() < max_pending_segment) return true; else return false; @@ -374,10 +494,51 @@ bool NormObject::IsPending(bool flush) const bool NormObject::ClientRepairCheck(CheckLevel level, NormBlockId blockId, NormSegmentId segmentId, - bool timerActive) + bool timerActive, + bool holdoffPhase) { + // (TBD) make sure it's OK for "max_pending_segment" + // and "next_segment_id" to be > blockSize + switch (level) + { + case THRU_INFO: + break; + case TO_BLOCK: + if (blockId >= max_pending_block) + { + max_pending_block = blockId; + max_pending_segment = 0; + } + break; + case THRU_SEGMENT: + if (blockId > max_pending_block) + { + max_pending_block = blockId; + max_pending_segment = segmentId + 1; + } + else if (blockId == max_pending_block) + { + if (segmentId >= max_pending_segment) + max_pending_segment = segmentId + 1; + } + break; + case THRU_BLOCK: + if (blockId > max_pending_block) + { + max_pending_block = blockId; + max_pending_segment = BlockSize(blockId); + } + break; + case THRU_OBJECT: + if (!IsStream()) max_pending_block = last_block_id; + max_pending_segment = BlockSize(max_pending_block); + default: + break; + } // end switch (level) + if (timerActive) { + if (holdoffPhase) return false; switch (level) { case THRU_INFO: @@ -475,8 +636,11 @@ bool NormObject::ClientRepairCheck(CheckLevel level, next_segment_id = BlockSize(blockId); break; case THRU_OBJECT: - current_block_id = last_block_id; - next_segment_id = BlockSize(blockId); + if (IsStream()) + current_block_id = max_pending_block; + else + current_block_id = last_block_id; + next_segment_id = BlockSize(current_block_id); default: break; } @@ -529,7 +693,7 @@ bool NormObject::IsRepairPending(bool flush) bool NormObject::AppendRepairRequest(NormNackMsg& nack, bool flush) { - // If !flush, we request only up to current_block_id::next_segment_id. + // If !flush, we request only up _to_ max_pending_block::max_pending_segment. NormRepairRequest req; NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; NormBlockId prevId; @@ -538,11 +702,13 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, { NormBlockId nextId = pending_mask.FirstSet(); NormBlockId lastId = pending_mask.LastSet(); - if (!flush && (current_block_id < lastId)) lastId = current_block_id; + if (!flush && (max_pending_block < lastId)) lastId = max_pending_block; lastId++; - //DMSG(6, "NormObject::AppendRepairRequest() node>%lu obj>%hu, blk>%lu->%lu (current:%lu)\n", - // LocalNodeId(), (UINT16)id, - // (UINT32)nextId, (UINT32)lastId, (UINT32)current_block_id); + DMSG(6, "NormObject::AppendRepairRequest() node>%lu obj>%hu, blk>%lu->%lu (maxPending:%lu)\n", + LocalNodeId(), (UINT16)id, + (UINT32)nextId, (UINT32)lastId, (UINT32)max_pending_block); + + // (TBD) simplify this loop code while ((nextId <= lastId) || (reqCount > 0)) { NormBlock* block = NULL; @@ -553,9 +719,9 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, block = block_buffer.Find(nextId); if (block) { - if (nextId == current_block_id) + if (nextId == max_pending_block) { - if (block->FirstPending() < next_segment_id) + if (block->FirstPending() < max_pending_segment) blockPending = true; } else @@ -603,13 +769,13 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, switch (nextForm) { case NormRepairRequest::ITEMS: - req.AppendRepairItem(id, prevId, 0); // (TBD) error check + req.AppendRepairItem(id, prevId, ndata, 0); // (TBD) error check if (2 == reqCount) - req.AppendRepairItem(id, prevId+1, 0); // (TBD) error check + req.AppendRepairItem(id, prevId+1, ndata, 0); // (TBD) error check break; case NormRepairRequest::RANGES: - req.AppendRepairItem(id, prevId, 0); // (TBD) error check - req.AppendRepairItem(id, prevId+reqCount-1, 0); // (TBD) error check + req.AppendRepairItem(id, prevId, ndata, 0); // (TBD) error check + req.AppendRepairItem(id, prevId+reqCount-1, ndata, 0); // (TBD) error check break; default: break; @@ -627,15 +793,15 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, nack.PackRepairRequest(req); // (TBD) error check reqCount = 0; prevForm = NormRepairRequest::INVALID; - if (flush || (nextId != current_block_id)) + if (flush || (nextId != max_pending_block)) { block->AppendRepairRequest(nack, numData, nparity, id, pending_info, segment_size); // (TBD) error check } else { - if (next_segment_id < numData) - block->AppendRepairRequest(nack, next_segment_id, 0, id, + if (max_pending_segment < numData) + block->AppendRepairRequest(nack, max_pending_segment, 0, id, pending_info, segment_size); // (TBD) error check else block->AppendRepairRequest(nack, numData, nparity, id, @@ -657,30 +823,32 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, req.SetForm(NormRepairRequest::ITEMS); req.ResetFlags(); req.SetFlag(NormRepairRequest::INFO); - req.AppendRepairItem(id, 0, 0); // (TBD) error check + req.AppendRepairItem(id, 0, ndata, 0); // (TBD) error check nack.PackRepairRequest(req); // (TBD) error check } // end if/else pending_mask.IsSet() + return true; } // end NormObject::AppendRepairRequest() -void NormObject::HandleObjectMessage(NormMessage& msg, - NormMsgType msgType, - NormBlockId blockId, - NormSegmentId segmentId) +void NormObject::HandleObjectMessage(const NormObjectMsg& msg, + NormMsg::Type msgType, + NormBlockId blockId, + NormSegmentId segmentId) { - if (NORM_MSG_INFO == msgType) + if (NormMsg::INFO == msgType) { if (pending_info) { - info_len = msg.info.GetInfoLen(); + const NormInfoMsg& infoMsg = (const NormInfoMsg&)msg; + info_len = infoMsg.GetInfoLen(); if (info_len > segment_size) { info_len = segment_size; DMSG(0, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "Warning! info too long.\n", LocalNodeId(), server->Id(), (UINT16)id); } - memcpy(info, msg.info.GetInfo(), info_len); + memcpy(info, infoMsg.GetInfo(), info_len); pending_info = false; session->Notify(NormController::RX_OBJECT_INFO, server, this); } @@ -694,36 +862,40 @@ void NormObject::HandleObjectMessage(NormMessage& msg, } else // NORM_MSG_DATA { + const NormDataMsg& data = (const NormDataMsg&)msg; // For stream objects, a little extra mgmt is required if (STREAM == type) { - if (!StreamUpdateStatus(blockId)) + NormStreamObject* stream = static_cast(this); + if (!stream->StreamUpdateStatus(blockId)) { - //DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj>%hu blk>%lu " - // "broken stream ...\n", LocalNodeId(), server->Id(), (UINT16)id, (UINT32)blockId); - + DMSG(4, "NormObject::HandleObjectMessage() node:%lu server:%lu obj>%hu blk>%lu " + "broken stream ...\n", LocalNodeId(), server->Id(), (UINT16)id, (UINT32)blockId); + + //ASSERT(0); // ??? Ignore this new packet and try to fix stream ??? - return; + //return; server->IncrementResyncCount(); - } - while (!StreamUpdateStatus(blockId)) - { - // Server is too far ahead of me ... - if (pending_mask.IsSet()) + + while (!stream->StreamUpdateStatus(blockId)) { - NormBlockId firstId = pending_mask.FirstSet(); - NormBlock* block = block_buffer.Find(firstId); - if (block) + // Server is too far ahead of me ... + if (pending_mask.IsSet()) { - block_buffer.Remove(block); - server->PutFreeBlock(block); + NormBlockId firstId = pending_mask.FirstSet(); + NormBlock* block = block_buffer.Find(firstId); + if (block) + { + block_buffer.Remove(block); + server->PutFreeBlock(block); + } + pending_mask.Unset(firstId); } - pending_mask.Unset(firstId); + else + { + stream->StreamResync(blockId);// - NormBlockId(pending_mask.Size())); + } } - else - { - stream_next_id = blockId - NormBlockId(pending_mask.Size()); - } } } UINT16 numData = BlockSize(blockId); @@ -744,7 +916,7 @@ void NormObject::HandleObjectMessage(NormMessage& msg, } if (block->IsPending(segmentId)) { - // 1) Store data in block buffer in case its needed for decoding + // 1) Cache segment in block buffer in case its needed for decoding char* segment = server->GetFreeSegment(id, blockId); if (!segment) { @@ -753,7 +925,7 @@ void NormObject::HandleObjectMessage(NormMessage& msg, (UINT16)id); return; } - UINT16 segmentLen = msg.data.GetDataLength(); + UINT16 segmentLen = data.GetPayloadDataLength(); if (segmentLen > segment_size) { DMSG(0, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " @@ -762,7 +934,28 @@ void NormObject::HandleObjectMessage(NormMessage& msg, server->PutFreeSegment(segment); return; } - memcpy(segment, msg.data.GetPayload(), segmentLen + NormDataMsg::PayloadHeaderLen()); +#ifdef SIMULATE + // For simulations, we just need the payload header (size & offset) + UINT16 simLen = MIN(segmentLen, SIM_PAYLOAD_MAX); + memcpy(segment, data.GetPayload(), simLen+NormDataMsg::PayloadHeaderLength()); + simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + if (segmentLen < simLen) + memset(segment+segmentLen+NormDataMsg::PayloadHeaderLength(), 0, simLen - segmentLen); + if (data.FlagIsSet(NormObjectMsg::FLAG_MSG_START)) + segment[simLen+NormDataMsg::PayloadHeaderLength()] = + NormObjectMsg::FLAG_MSG_START; + else + segment[simLen+NormDataMsg::PayloadHeaderLength()] = 0; +#else + memcpy(segment, data.GetPayload(), segmentLen+NormDataMsg::PayloadHeaderLength()); + if (segmentLen < segment_size) + memset(segment+segmentLen+NormDataMsg::PayloadHeaderLength(), 0, segment_size - segmentLen); + if (data.FlagIsSet(NormObjectMsg::FLAG_MSG_START)) + segment[segment_size+NormDataMsg::PayloadHeaderLength()] = + NormObjectMsg::FLAG_MSG_START; + else + segment[segment_size+NormDataMsg::PayloadHeaderLength()] = 0; +#endif // if/else SIMULATE block->AttachSegment(segmentId, segment); block->UnsetPending(segmentId); // 2) Write segment to object (if it's data) @@ -786,24 +979,32 @@ void NormObject::HandleObjectMessage(NormMessage& msg, (UINT16)id, (UINT32)block->Id()); UINT16 nextErasure = block->FirstPending(); UINT16 erasureCount = 0; - UINT16 blockLen = numData + nparity; - while (nextErasure < blockLen) + if (nextErasure < numData) { - server->SetErasureLoc(erasureCount++, nextErasure); - if (nextErasure < numData) + UINT16 blockLen = numData + nparity; + while (nextErasure < blockLen) { - if (!(segment = server->GetFreeSegment(id, blockId))) + server->SetErasureLoc(erasureCount++, nextErasure); + if (nextErasure < numData) { - DMSG(2, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " - "Warning! no free segments ...\n", LocalNodeId(), server->Id(), - (UINT16)id); - // (TBD) Dump the block ...??? - return; + if (!(segment = server->GetFreeSegment(id, blockId))) + { + DMSG(2, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " + "Warning! no free segments ...\n", LocalNodeId(), server->Id(), + (UINT16)id); + // (TBD) Dump the block ...??? + return; + } +#ifdef SIMULATE + UINT16 simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + memset(segment, 0, simLen + NormDataMsg::PayloadHeaderLength()+1); +#else + memset(segment, 0, segment_size+NormDataMsg::PayloadHeaderLength()+1); +#endif // !SIMULATE + block->SetSegment(nextErasure, segment); } - memset(segment, 0, segment_size+NormDataMsg::PayloadHeaderLen()); - block->SetSegment(nextErasure, segment); + nextErasure = block->NextPending(nextErasure+1); } - nextErasure = block->NextPending(nextErasure+1); } if (erasureCount) { @@ -816,7 +1017,7 @@ void NormObject::HandleObjectMessage(NormMessage& msg, if (WriteSegment(blockId, sid, block->Segment(sid))) { // For statistics only (TBD) #ifdef NORM_DEBUG - server->IncrementRecvGoodput(NormDataMsg::ReadLength(block->Segment(sid))); + server->IncrementRecvGoodput(NormDataMsg::ReadPayloadLength(block->Segment(sid))); } } else @@ -830,8 +1031,9 @@ void NormObject::HandleObjectMessage(NormMessage& msg, block_buffer.Remove(block); server->PutFreeBlock(block); } - // Notify application of new data available + // (TBD) this could be improved for stream objects + // so it's not called unnecessarily session->Notify(NormController::RX_OBJECT_UPDATE, server, this); } else @@ -850,77 +1052,7 @@ void NormObject::HandleObjectMessage(NormMessage& msg, } // end if/else (NORM_MSG_INFO) } // end NormObject::HandleObjectMessage() -bool NormObject::StreamUpdateStatus(NormBlockId blockId) -{ - if (stream_sync) - { - if (blockId < stream_sync_id) - { - // it's an old block or stream is _very_ broken - // (nothing to update) - return true; - } - else - { - if (blockId < stream_next_id) - { - // it's pending or complete (nothing to update) - return true; - } - else - { - if (pending_mask.IsSet()) - { - if (pending_mask.CanSet(blockId)) - { - pending_mask.SetBits(stream_next_id, blockId - stream_next_id + 1); - stream_next_id = blockId + 1; - // Handle potential sync block id wrap - NormBlockId delta = stream_next_id - stream_sync_id; - if (delta > NormBlockId(2*pending_mask.Size())) - stream_sync_id = NormBlockId(pending_mask.FirstSet()); - return true; - } - else - { - // Stream broken - return false; - } - } - else - { - NormBlockId delta = blockId - stream_next_id + 1; - if (delta > NormBlockId(pending_mask.Size())) - { - // Stream broken - return false; - } - else - { - pending_mask.SetBits(blockId, pending_mask.Size()); - stream_next_id = blockId + NormBlockId(pending_mask.Size()); - // Handle potential sync block id wrap - delta = stream_next_id - stream_sync_id; - if (delta > NormBlockId(2*pending_mask.Size())) - stream_sync_id = blockId; - return true; - } - - } - } - } - } - else - { - // For now, let stream begin anytime - pending_mask.Clear(); - pending_mask.SetBits(blockId, pending_mask.Size()); - stream_sync = true; - stream_sync_id = blockId; - stream_next_id = blockId + pending_mask.Size(); - return true; - } -} // end NormObject::StreamUpdateStatus() + // Steals non-pending block (oldest first) for server resource management // (optionally excludes block indicated by blockId) @@ -954,6 +1086,7 @@ NormBlock* NormObject::StealNonPendingBlock(bool excludeBlock, NormBlockId exclu return NULL; } // end NormObject::StealNonPendingBlock() + // For client resource management, steals newer block resources when // needing resources for ordinally older blocks. NormBlock* NormObject::StealNewestBlock(bool excludeBlock, NormBlockId excludeId) @@ -972,41 +1105,83 @@ NormBlock* NormObject::StealNewestBlock(bool excludeBlock, NormBlockId excludeId else { block_buffer.Remove(block); - return block; - } + return block; + } } } // end NormObject::StealNewestBlock() -bool NormObject::NextServerMsg(NormMessage* msg) + +// For silent client resource management, steals newer block resources when +// needing resources for ordinally older blocks. +NormBlock* NormObject::StealOldestBlock(bool excludeBlock, NormBlockId excludeId) { - msg->object.ResetFlags(); + if (block_buffer.IsEmpty()) + { + return NULL; + } + else + { + NormBlock* block = block_buffer.Find(block_buffer.RangeLo()); + if (excludeBlock && (excludeId == block->Id())) + { + return NULL; + } + else + { + block_buffer.Remove(block); + return block; + } + } +} // end NormObject::StealOldestBlock() + + + +bool NormObject::NextServerMsg(NormObjectMsg* msg) +{ + // Init() the message + if (pending_info) + ((NormInfoMsg*)msg)->Init(); + else + ((NormDataMsg*)msg)->Init(); + + // General object message flags + msg->ResetFlags(); switch(type) { case STREAM: - msg->object.SetFlag(NormObjectMsg::FLAG_STREAM); + msg->SetFlag(NormObjectMsg::FLAG_STREAM); break; case FILE: - msg->object.SetFlag(NormObjectMsg::FLAG_FILE); + msg->SetFlag(NormObjectMsg::FLAG_FILE); break; default: break; } - if (info) msg->object.SetFlag(NormObjectMsg::FLAG_INFO); - msg->object.SetSegmentSize(segment_size); - msg->object.SetObjectSize(object_size); - msg->object.SetFecNumParity(nparity); - msg->object.SetFecBlockLen(ndata); - msg->object.SetObjectId(id); - + if (info) msg->SetFlag(NormObjectMsg::FLAG_INFO); + msg->SetFecId(129); + msg->SetObjectId(id); + + // We currently always apply the FTI extension + NormFtiExtension fti; + msg->AttachExtension(fti); + + fti.SetSegmentSize(segment_size); + fti.SetObjectSize(object_size); + fti.SetFecMaxBlockLen(ndata); + fti.SetFecNumParity(nparity); + if (pending_info) { - msg->generic.SetType(NORM_MSG_INFO); - msg->info.SetInfo(info, info_len); + // (TBD) set REPAIR_FLAG for retransmitted info + NormInfoMsg* infoMsg = (NormInfoMsg*)msg; + infoMsg->SetInfo(info, info_len); pending_info = false; return true; } if (!pending_mask.IsSet()) return false; + NormDataMsg* data = (NormDataMsg*)msg; + NormBlockId blockId = pending_mask.FirstSet(); UINT16 numData = BlockSize(blockId); NormBlock* block = block_buffer.Find(blockId); @@ -1019,20 +1194,24 @@ bool NormObject::NextServerMsg(NormMessage* msg) return false; } // Load block with zero initialized parity segments - UINT16 totalBlockLen = numData + nparity; for (UINT16 i = numData; i < totalBlockLen; i++) { char* s = session->ServerGetFreeSegment(id, blockId); if (s) { - memset(s, 0, segment_size + NormDataMsg::PayloadHeaderLen()); +#ifdef SIMULATE + UINT16 simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + memset(s, 0, simLen + NormDataMsg::PayloadHeaderLength()+1); +#else + memset(s, 0, segment_size + NormDataMsg::PayloadHeaderLength()+1); +#endif // if/else SIMULATE block->AttachSegment(i, s); } else { DMSG(2, "NormObject::NextServerMsg() node>%lu Warning! server resource " - "constrained (no free segments).\n", LocalNodeId()); + "constrained (no free segments).\n", LocalNodeId()); session->ServerPutFreeBlock(block); return false; } @@ -1058,27 +1237,41 @@ bool NormObject::NextServerMsg(NormMessage* msg) if (segmentId < numData) { // Try to read data segment - char* buffer = msg->data.AccessPayload(); + char* buffer = data->AccessPayload(); if (!ReadSegment(blockId, segmentId, buffer)) { // (TBD) deal with read error - //(for streams, it currently means the stream is non-pending) - TRACE("NormObject::NextServerMsg() ReadSegment() error\n"); + //(for streams, it currently means the stream is non-pending) + if (!IsStream()) + DMSG(0, "NormObject::NextServerMsg() ReadSegment() error\n"); return false; } - UINT16 length = NormDataMsg::ReadLength(buffer); + UINT16 length = NormDataMsg::ReadPayloadLength(buffer); + data->SetDataPayloadLength(length); + + if (IsStream()) + { + // Look for FLAG_MSG_START +#ifdef SIMULATE + UINT16 simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + if (buffer[simLen+NormDataMsg::PayloadHeaderLength()]) + data->SetFlag(NormObjectMsg::FLAG_MSG_START); +#else + if (buffer[segment_size+NormDataMsg::PayloadHeaderLength()]) + data->SetFlag(NormObjectMsg::FLAG_MSG_START); +#endif // if/else SIMULATE + } - msg->data.SetDataLength(length); // Perform incremental FEC encoding as needed - if ((block->ParityReadiness() <= segmentId) && - nparity) // (TBD) && incrementalParity == true + if ((block->ParityReadiness() == segmentId) && nparity) + // (TBD) && incrementalParity == true { // (TBD) for non-stream objects, catch alternate "last block/segment len" + // ZERO pad any "runt" segments before encoding if (length < segment_size) - { - memset(msg->data.AccessData()+length, 0, segment_size-length); - } - session->ServerEncode(msg->data.AccessPayload(), block->SegmentList(numData)); + memset(buffer+NormDataMsg::PayloadHeaderLength()+length, 0, segment_size-length); + // (TBD) the encode routine could update the block's parity readiness + session->ServerEncode(data->AccessPayload(), block->SegmentList(numData)); block->IncreaseParityReadiness(); } } @@ -1092,13 +1285,14 @@ bool NormObject::NextServerMsg(NormMessage* msg) ASSERT(block->ParityReady(numData)); char* segment = block->Segment(segmentId); ASSERT(segment); - msg->data.SetPayload(segment, segment_size+NormDataMsg::PayloadHeaderLen()); + data->SetPayload(segment, segment_size+NormDataMsg::PayloadHeaderLength()); } - block->UnsetPending(segmentId); - msg->generic.SetType(NORM_MSG_DATA); - if (block->InRepair()) msg->object.SetFlag(NormObjectMsg::FLAG_REPAIR); - msg->data.SetFecBlockId(blockId); - msg->data.SetFecSymbolId(segmentId); + block->UnsetPending(segmentId); + if (block->InRepair()) + data->SetFlag(NormObjectMsg::FLAG_REPAIR); + data->SetFecBlockId(blockId); + data->SetFecBlockLen(numData); + data->SetFecSymbolId(segmentId); if (!block->IsPending()) { block->ResetParityCount(nparity); @@ -1107,57 +1301,58 @@ bool NormObject::NextServerMsg(NormMessage* msg) // This lets NORM_STREAM objects continue indefinitely if (IsStream() && !pending_mask.IsSet()) + static_cast(this)->StreamAdvance(); + return true; +} // end NormObject::NextServerMsg() + +void NormStreamObject::StreamAdvance() +{ + NormBlockId nextBlockId = stream_next_id; + // Make sure we won't prevent about any pending repairs + if (repair_mask.CanSet(nextBlockId)) { - NormBlockId nextBlockId = stream_next_id; - // Make sure we won't prevent about any pending repairs - if (repair_mask.CanSet(nextBlockId)) + if (block_buffer.CanInsert(nextBlockId)) { - if (block_buffer.CanInsert(nextBlockId)) + if (pending_mask.Set(nextBlockId)) + stream_next_id++; + else + DMSG(0, "NormStreamObject::StreamAdvance() error setting stream pending mask (1)\n"); + } + else + { + NormBlock* block = block_buffer.Find(block_buffer.RangeLo()); + ASSERT(block); + if (!block->IsTransmitPending()) { if (pending_mask.Set(nextBlockId)) stream_next_id++; else - DMSG(0, "NormObject::NextServerMsg() error setting stream pending mask (1)\n"); + DMSG(0, "NormStreamObject::StreamAdvance() error setting stream pending mask (2)\n"); } else { - block = block_buffer.Find(block_buffer.RangeLo()); - ASSERT(block); - if (!block->IsTransmitPending()) - { - if (pending_mask.Set(nextBlockId)) - stream_next_id++; - else - DMSG(0, "NormObject::NextServerMsg() error setting stream pending mask (2)\n"); - } - else - { - DMSG(4, "NormObject::NextServerMsg() node>%lu Pending segment repairs (blk>%lu) " - "delaying stream advance ...\n", LocalNodeId(), (UINT32)block->Id()); - } - } + DMSG(4, "NormStreamObject::StreamAdvance() node>%lu Pending segment repairs (blk>%lu) " + "delaying stream advance ...\n", LocalNodeId(), (UINT32)block->Id()); + } } - else - { - DMSG(0, "NormObject::NextServerMsg() Pending block repair delaying stream advance ...\n"); - } - } - return true; -} // end NormObject::NextServerMsg() + } + else + { + DMSG(0, "NormStreamObject::StreamAdvance() Pending block repair delaying stream advance ...\n"); + } +} // end NormStreamObject::StreamAdvance() bool NormObject::CalculateBlockParity(NormBlock* block) { - char buffer[NORM_MSG_SIZE_MAX]; + char buffer[NormMsg::MAX_SIZE]; UINT16 numData = BlockSize(block->Id()); for (UINT16 i = 0; i < numData; i++) { if (ReadSegment(block->Id(), i, buffer)) { - UINT16 length = NormDataMsg::ReadLength(buffer); + UINT16 length = NormDataMsg::ReadPayloadLength(buffer); if (length < segment_size) - { - memset(buffer+NormDataMsg::PayloadHeaderLen()+length, 0, segment_size-length); - } + memset(buffer+NormDataMsg::PayloadHeaderLength()+length, 0, segment_size-length); session->ServerEncode(buffer, block->SegmentList(numData)); } else @@ -1184,7 +1379,12 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) char* s = session->ServerGetFreeSegment(id, blockId); if (s) { - memset(s, 0, segment_size + NormDataMsg::PayloadHeaderLen()); +#ifdef SIMULATE + UINT16 simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + memset(s, 0, simLen + NormDataMsg::PayloadHeaderLength()); +#else + memset(s, 0, segment_size + NormDataMsg::PayloadHeaderLength()); +#endif // if/else SIMULATE block->AttachSegment(i, s); } else @@ -1198,6 +1398,12 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) // Attempt to re-generate parity if (CalculateBlockParity(block)) { + if (!block_buffer.Insert(block)) + { + session->ServerPutFreeBlock(block); + DMSG(4, "NormObject::ServerRecoverBlock() node>%lu couldn't buffer recovered block\n"); + return NULL; + } return block; } else @@ -1325,13 +1531,13 @@ bool NormFileObject::WriteSegment(NormBlockId blockId, len = last_segment_size; else len = segment_size; - NormObjectSize segmentOffset = NormDataMsg::ReadOffset(buffer); + NormObjectSize segmentOffset = NormDataMsg::ReadPayloadOffset(buffer); off_t offset = segmentOffset.LSB() + (segmentOffset.MSB() * 0x100000000LL); if (offset != file.GetOffset()) { if (!file.Seek(offset)) return false; } - UINT16 nbytes = file.Write(buffer+NormDataMsg::PayloadHeaderLen(), len); + UINT16 nbytes = file.Write(buffer+NormDataMsg::PayloadHeaderLength(), len); return (nbytes == len); } // end NormFileObject::WriteSegment() @@ -1350,14 +1556,14 @@ bool NormFileObject::ReadSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset = NormObjectSize(blockId) * block_size; segmentOffset = segmentOffset + (NormObjectSize(segmentId) * NormObjectSize(segment_size)); - NormDataMsg::WriteLength(buffer, len); - NormDataMsg::WriteOffset(buffer, segmentOffset); + NormDataMsg::WritePayloadLength(buffer, len); + NormDataMsg::WritePayloadOffset(buffer, segmentOffset); off_t offset = segmentOffset.LSB() + (segmentOffset.MSB() * 0x100000000LL); if (offset != file.GetOffset()) { if (!file.Seek(offset)) return false; } - UINT16 nbytes = file.Read(buffer+NormDataMsg::PayloadHeaderLen(), len); + UINT16 nbytes = file.Read(buffer+NormDataMsg::PayloadHeaderLength(), len); return (len == nbytes); } // end NormFileObject::ReadSegment() @@ -1370,7 +1576,8 @@ bool NormFileObject::ReadSegment(NormBlockId blockId, NormStreamObject::NormStreamObject(class NormSession* theSession, class NormServerNode* theServer, const NormObjectId& objectId) - : NormObject(STREAM, theSession, theServer, objectId), flush_pending(false) + : NormObject(STREAM, theSession, theServer, objectId), + stream_sync(false), flush_pending(false), msg_start(true) { } @@ -1389,30 +1596,25 @@ bool NormStreamObject::Open(unsigned long bufferSize, return false; } - UINT16 segmentSize, numData, numParity; + UINT16 segmentSize, numData; if (server) { segmentSize = server->SegmentSize(); numData = server->BlockSize(); - numParity = server->NumParity(); } else { segmentSize = session->ServerSegmentSize(); numData = session->ServerBlockSize(); - numParity = session->ServerNumParity(); } - unsigned long numSegments = bufferSize / segmentSize; - if ((numSegments*segmentSize) < bufferSize) numSegments++; NormObjectSize blockSize((UINT32)segmentSize * (UINT32)numData); NormObjectSize numBlocks = NormObjectSize(bufferSize) / blockSize; ASSERT(0 == numBlocks.MSB()); - - // Buffering requires at least 2 segments & 2 blocks - numSegments = MAX(2, numSegments); + // Buffering requires at least 2 blocks numBlocks = MAX(2, numBlocks.LSB()); - + unsigned long numSegments = numBlocks.LSB() * numData; + if (!block_pool.Init(numBlocks.LSB(), numData)) { DMSG(0, "NormStreamObject::Open() block_pool init error\n"); @@ -1420,7 +1622,7 @@ bool NormStreamObject::Open(unsigned long bufferSize, return false; } - if (!segment_pool.Init(numSegments, segmentSize+NormDataMsg::PayloadHeaderLen())) + if (!segment_pool.Init(numSegments, segmentSize+NormDataMsg::PayloadHeaderLength()+1)) { DMSG(0, "NormStreamObject::Open() segment_pool init error\n"); Close(); @@ -1447,6 +1649,8 @@ bool NormStreamObject::Open(unsigned long bufferSize, return false; } } + flush_pending = false; + msg_start = true; return true; } // end NormStreamObject::Open() @@ -1513,6 +1717,82 @@ bool NormStreamObject::LockSegments(NormBlockId blockId, NormSegmentId firstId, } } // end NormStreamObject::LockSegments() +bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) +{ + if (stream_sync) + { + if (blockId < stream_sync_id) + { + // it's an old block or stream is _very_ broken + // (nothing to update) + return true; + } + else + { + if (blockId < stream_next_id) + { + // it's pending or complete (nothing to update) + return true; + } + else + { + if (pending_mask.IsSet()) + { + if (pending_mask.CanSet(blockId)) + { + pending_mask.SetBits(stream_next_id, blockId - stream_next_id + 1); + stream_next_id = blockId + 1; + // Handle potential sync block id wrap + NormBlockId delta = stream_next_id - stream_sync_id; + if (delta > NormBlockId(2*pending_mask.Size())) + stream_sync_id = NormBlockId(pending_mask.FirstSet()); + return true; + } + else + { + // Stream broken + //TRACE("NormObject::StreamUpdateStatus() broken 1 ...\n"); + return false; + } + } + else + { + NormBlockId delta = blockId - stream_next_id + 1; + if (delta > NormBlockId(pending_mask.Size())) + { + // Stream broken + //TRACE("NormObject::StreamUpdateStatus() broken 2 ...\n"); + return false; + } + else + { + pending_mask.SetBits(blockId, pending_mask.Size()); + stream_next_id = blockId + NormBlockId(pending_mask.Size()); + // Handle potential sync block id wrap + delta = stream_next_id - stream_sync_id; + if (delta > NormBlockId(2*pending_mask.Size())) + stream_sync_id = blockId; + return true; + } + + } + } + } + } + else + { + // For now, let stream begin anytime + pending_mask.Clear(); + pending_mask.SetBits(blockId, pending_mask.Size()); + stream_sync = true; + stream_sync_id = blockId; + stream_next_id = blockId + pending_mask.Size(); + read_index.block = blockId; + return true; + } +} // end NormStreamObject::StreamUpdateStatus() + + bool NormStreamObject::ReadSegment(NormBlockId blockId, NormSegmentId segmentId, char* buffer) @@ -1532,36 +1812,90 @@ bool NormStreamObject::ReadSegment(NormBlockId blockId, } block->UnsetPending(segmentId); char* segment = block->Segment(segmentId); - ASSERT(segment); - UINT16 length = NormDataMsg::ReadLength(segment); + ASSERT(segment != NULL); + UINT16 length = NormDataMsg::ReadPayloadLength(segment); ASSERT(length <= segment_size); - memcpy(buffer, segment, length+NormDataMsg::PayloadHeaderLen()); +#ifdef SIMULATE + length = MIN(length, SIM_PAYLOAD_MAX); + UINT16 simLen = MIN(SIM_PAYLOAD_MAX, segment_size); + buffer[simLen+NormDataMsg::PayloadHeaderLength()] = + segment[simLen+NormDataMsg::PayloadHeaderLength()]; +#else + buffer[segment_size+NormDataMsg::PayloadHeaderLength()] = + segment[segment_size+NormDataMsg::PayloadHeaderLength()]; +#endif // if/else SIMULATE + memcpy(buffer, segment, length+NormDataMsg::PayloadHeaderLength()); return true; -} // end NormStreamObject::Read() +} // end NormStreamObject::ReadSegment() bool NormStreamObject::WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* segment) { - if ((blockId < read_index.block) || + /*if ((blockId < read_index.block) || ((blockId == read_index.block) && (segmentId < read_index.segment))) { //DMSG(0, "NormStreamObject::WriteSegment() block/segment < read_index!?\n"); return false; + }*/ + + + NormObjectSize segmentOffset = NormDataMsg::ReadPayloadOffset(segment); + if (read_offset > segmentOffset) + { + //DMSG(0, "NormStreamObject::WriteSegment() segmentOffset < read_offset!?\n"); + return false; } + NormBlock* block = stream_buffer.Find(blockId); if (!block) { bool broken = false; - // Prune (if necessary) stream_buffer (stream is broken) + // Prune (if necessary) stream_buffer (stream might be broken) while (!stream_buffer.CanInsert(blockId) || block_pool.IsEmpty()) { block = stream_buffer.Find(stream_buffer.RangeLo()); - stream_buffer.Remove(block); if (block->IsPending()) broken = true; - block->EmptyToPool(segment_pool); - block_pool.Put(block); + // This loop feeds any received user data segments to the application + // (if the application doesn't want the data, it's lost) + while (block->IsPending()) + { + // Notify app for stream data salvage + read_index.block = block->Id(); + read_index.segment = block->FirstPending(); + NormBlock* tempBlock = block; + NormObjectSize tempOffset = read_offset; + session->Notify(NormController::RX_OBJECT_UPDATE, server, this); + block = stream_buffer.Find(stream_buffer.RangeLo()); + if (tempBlock == block) + { + if (tempOffset == read_offset) + { + // App didn't want any data here, purge segment + TRACE("NormStreamObject::WriteSegment() app didn't want data ?!\n"); + ASSERT(0); + char* s = block->DetachSegment(read_index.segment); + segment_pool.Put(s); + block->UnsetPending(read_index.segment); + } + } + else + { + // App consumed block via Read() + block = NULL; + break; + } + } + if (block) + { + // if the app didn't consume the block, we must + // return it to the pool + ASSERT(!block->IsPending()); + stream_buffer.Remove(block); + block->EmptyToPool(segment_pool); + block_pool.Put(block); + } } block = block_pool.Get(); block->SetId(blockId); @@ -1570,23 +1904,33 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, ASSERT(success); if (broken) { - DMSG(2, "NormStreamObject::WriteSegment() node>%lu obj>%hu blk>%lu seg>%hu broken stream ...\n", + DMSG(4, "NormStreamObject::WriteSegment() node>%lu obj>%hu blk>%lu seg>%hu broken stream ...\n", LocalNodeId(), (UINT16)id, (UINT32)blockId, (UINT16)segmentId); NormBlock* first = stream_buffer.Find(stream_buffer.RangeLo()); read_index.block = first->Id(); read_index.segment = 0; } - } + // Make sure this segment hasn't already been written if(!block->Segment(segmentId)) { char* s = segment_pool.Get(); - ASSERT(s); // for now, this should always succeed - UINT16 length = NormDataMsg::ReadLength(segment); - memcpy(s, segment, length + NormDataMsg::PayloadHeaderLen()); + ASSERT(s != NULL); // for now, this should always succeed + UINT16 length = NormDataMsg::ReadPayloadLength(segment); +#ifdef SIMULATE + length = MIN(length, SIM_PAYLOAD_MAX); + UINT16 simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + s[simLen+NormDataMsg::PayloadHeaderLength()] = + segment[simLen+NormDataMsg::PayloadHeaderLength()]; +#else + s[segment_size+NormDataMsg::PayloadHeaderLength()] = + segment[segment_size+NormDataMsg::PayloadHeaderLength()]; +#endif // SIMULATE + memcpy(s, segment, length + NormDataMsg::PayloadHeaderLength()); block->AttachSegment(segmentId, s); block->SetPending(segmentId); + ASSERT(block->Segment(segmentId) == s); } return true; } // end NormStreamObject::WriteSegment() @@ -1624,43 +1968,102 @@ void NormStreamObject::Prune(NormBlockId blockId) } // end NormStreamObject::Prune() // Sequential (in order) read/write routines (TBD) Add a "Seek()" method -unsigned long NormStreamObject::Read(char* buffer, unsigned long len) +bool NormStreamObject::Read(char* buffer, unsigned int* buflen, bool findMsgStart) { - unsigned long nBytes = 0; - while (len > 0) + unsigned long bytesRead = 0; + unsigned long bytesToRead = *buflen; + while (bytesToRead > 0) { NormBlock* block = stream_buffer.Find(read_index.block); if (!block) { //DMSG(0, "NormStreamObject::Read() stream buffer empty (1)\n"); - return nBytes; + *buflen = bytesRead; + return true; } char* segment = block->Segment(read_index.segment); if (!segment) { //DMSG(0, "NormStreamObject::Read() stream buffer empty (2)\n"); - return nBytes; + *buflen = bytesRead; + return true; } - NormObjectSize segmentOffset = NormDataMsg::ReadOffset(segment); + NormObjectSize segmentOffset = NormDataMsg::ReadPayloadOffset(segment); if (segmentOffset > read_offset) { - DMSG(0, "NormStreamObject::Read() node>%lu broken stream!\n", LocalNodeId()); + DMSG(4, "NormStreamObject::Read() node>%lu broken stream!\n", LocalNodeId()); read_offset = segmentOffset; - return nBytes; + *buflen = bytesRead; + return false; } + NormObjectSize delta = read_offset - segmentOffset; + ASSERT(!delta.MSB()); UINT16 index = delta.LSB(); - UINT16 length = NormDataMsg::ReadLength(segment); - ASSERT(index < length); + UINT16 length = NormDataMsg::ReadPayloadLength(segment); + + if (index >= length) + { + DMSG(0, "NormStreamObject::Read() node>%lu mangled stream! index:%hu length:%hu\n", + LocalNodeId(), index, length); + read_offset = segmentOffset; + *buflen = bytesRead; + return false; + } + + UINT16 count = length - index; - count = MIN(count, len); - memcpy(buffer+nBytes, segment+index+NormDataMsg::PayloadHeaderLen(), count); + count = MIN(count, bytesToRead); + + if (findMsgStart) + { + bool msgStart; + if (0 != index) + { + msgStart = false; + } + else + { +#ifdef SIMULATE + UINT16 simLen = MIN(segment_size, SIM_PAYLOAD_MAX); + msgStart = (NormObjectMsg::FLAG_MSG_START == + segment[simLen+NormDataMsg::PayloadHeaderLength()]); +#else + msgStart = (NormObjectMsg::FLAG_MSG_START == + segment[segment_size+NormDataMsg::PayloadHeaderLength()]); +#endif // if/else SIMULATE + } + if (!msgStart) + { + block->DetachSegment(read_index.segment); + segment_pool.Put(segment); + block->UnsetPending(read_index.segment++); + if (read_index.segment >= ndata) + { + stream_buffer.Remove(block); + block->EmptyToPool(segment_pool); + block_pool.Put(block); + read_index.block++; + read_index.segment = 0; + } + continue; + } + } + +#ifdef SIMULATE + UINT16 simCount = ((index+count) < SIM_PAYLOAD_MAX) ? + count : ((index < SIM_PAYLOAD_MAX) ? + (SIM_PAYLOAD_MAX - index) : 0); + memcpy(buffer+bytesRead, segment+index+NormDataMsg::PayloadHeaderLength(), simCount); +#else + memcpy(buffer+bytesRead, segment+index+NormDataMsg::PayloadHeaderLength(), count); +#endif // if/else SIMULATE index += count; - nBytes += count; + bytesRead += count; read_offset += count; - len -= count; + bytesToRead -= count; if (index >= length) { block->DetachSegment(read_index.segment); @@ -1674,16 +2077,16 @@ unsigned long NormStreamObject::Read(char* buffer, unsigned long len) read_index.block++; read_index.segment = 0; } - } + } } // end while (len > 0) - return nBytes; + *buflen = bytesRead; + return true; } // end NormStreamObject::Read() -unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flush) +unsigned long NormStreamObject::Write(const char* buffer, unsigned long len, bool flush, bool eom, bool push) { - ASSERT(!server); unsigned long nBytes = 0; - do + while (nBytes < len) { NormBlock* block = stream_buffer.Find(write_index.block); if (!block) @@ -1692,16 +2095,25 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus { block = stream_buffer.Find(stream_buffer.RangeLo()); ASSERT(block); - if (block->IsPending()) + if (push) { - DMSG(0, "NormStreamObject::Write() stream buffer full (1)\n"); + NormBlockId blockId = block->Id(); + pending_mask.Unset(blockId); + repair_mask.Unset(blockId); + NormBlock* b = FindBlock(blockId); + if (b) + { + block_buffer.Remove(b); + session->ServerPutFreeBlock(b); + } + } + else if (block->IsPending()) + { + DMSG(0, "NormStreamObject::Write() stream buffer full (1) len:%d eom:%d\n", len, eom); break; } - else - { - stream_buffer.Remove(block); - block->EmptyToPool(segment_pool); - } + stream_buffer.Remove(block); + block->EmptyToPool(segment_pool); } block->SetId(write_index.block); block->ClearPending(); @@ -1715,7 +2127,19 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus { NormBlock* b = stream_buffer.Find(stream_buffer.RangeLo()); ASSERT(b != block); - if (b->IsPending()) + if (push) + { + NormBlockId blockId = block->Id(); + pending_mask.Unset(blockId); + repair_mask.Unset(blockId); + NormBlock* c = FindBlock(blockId); + if (c) + { + block_buffer.Remove(c); + session->ServerPutFreeBlock(c); + } + } + else if (b->IsPending()) { DMSG(0, "NormStreamObject::Write() stream buffer full (2)\n"); break; @@ -1726,22 +2150,52 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus segment = segment_pool.Get(); ASSERT(segment); } - NormDataMsg::WriteOffset(segment, write_offset); - NormDataMsg::WriteLength(segment, 0); - block->AttachSegment(write_index.segment, segment); + NormDataMsg::WritePayloadOffset(segment, write_offset); + NormDataMsg::WritePayloadLength(segment, 0); + block->AttachSegment(write_index.segment, segment); } - UINT16 index = NormDataMsg::ReadLength(segment); - ASSERT(write_offset == (NormDataMsg::ReadOffset(segment)+NormObjectSize(index))); + + UINT16 index = NormDataMsg::ReadPayloadLength(segment); + + if (0 == index) + { + // On first write to segment, mark MSG_START flag as appropriate + char msgStartValue; + if (msg_start) + { + msg_start = false; + msgStartValue = NormObjectMsg::FLAG_MSG_START; + } + else + { + msgStartValue = 0; + } +#ifdef SIMULATE + UINT16 simLen = MIN(SIM_PAYLOAD_MAX, segment_size); + segment[simLen+NormDataMsg::PayloadHeaderLength()] = msgStartValue; +#else + segment[segment_size+NormDataMsg::PayloadHeaderLength()] = msgStartValue; +#endif // if/else SIMULATE + } + + UINT16 count = len - nBytes; UINT16 space = segment_size - index; - UINT16 count = MIN(space, len); - memcpy(segment+index+NormDataMsg::PayloadHeaderLen(), buffer+nBytes, count); - NormDataMsg::WriteLength(segment, index+count); - len -= count; + count = MIN(count, space); +#ifdef SIMULATE + UINT16 simCount = ((index+count) < SIM_PAYLOAD_MAX) ? + count : ((index < SIM_PAYLOAD_MAX) ? + (SIM_PAYLOAD_MAX - index) : 0); + memcpy(segment+index+NormDataMsg::PayloadHeaderLength(), buffer+nBytes, simCount); +#else + memcpy(segment+index+NormDataMsg::PayloadHeaderLength(), buffer+nBytes, count); +#endif // if/else SIMULATE + NormDataMsg::WritePayloadLength(segment, index+count); nBytes += count; write_offset += count; - // Is the segment full? - if ((count == space) || (flush && (index > 0))) + // Is the segment full? or flushing, or end-of-message + if ((count == space) || ((flush || eom) && (index > 0))) { + if (eom) ASSERT(0); block->SetPending(write_index.segment); if (++write_index.segment >= ndata) { @@ -1749,12 +2203,23 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus write_index.segment = 0; } } - } while (len > 0); - if (flush) - flush_pending = true; + } + + // if this was end-of-message next Write() will be considered a new message + + if (nBytes == len) + { + if (eom) msg_start = true; + if (flush) + flush_pending = true; + else + flush_pending = false; + } else - flush_pending = false; - if (nBytes) session->TouchServer(); + { + flush = false; + } + if (nBytes || flush) session->TouchServer(); return nBytes; } // end NormStreamObject::Write() @@ -1789,7 +2254,7 @@ bool NormSimObject::ReadSegment(NormBlockId blockId, else len = segment_size; // the "len" is needed to build the correct size message - NormDataMsg::WriteLength(buffer, len); + NormDataMsg::WritePayloadLength(buffer, len); return true; } // end NormSimObject::ReadSegment() @@ -1842,7 +2307,7 @@ void NormObjectTable::Destroy() delete []table; table = (NormObject**)NULL; range = range_max = 0; - } + } } // end NormObjectTable::Destroy() NormObject* NormObjectTable::Find(const NormObjectId& objectId) const diff --git a/common/normObject.h b/common/normObject.h index 4fb9f9f..f1d62ce 100644 --- a/common/normObject.h +++ b/common/normObject.h @@ -28,6 +28,7 @@ class NormObject THRU_OBJECT }; + void Verify() const; virtual ~NormObject(); NormObject::Type GetType() const {return type;} @@ -39,14 +40,16 @@ class NormObject bool IsStream() const {return (STREAM == type);} NormNodeId LocalNodeId() const; + class NormServerNode* GetServer() {return server;} + bool IsOpen() {return (0 != segment_size);} // Opens (inits) object for tx operation bool Open(const NormObjectSize& objectSize, const char* infoPtr, UINT16 infoLen); // Opens (inits) object for rx operation bool Open(const NormObjectSize& objectSize, bool hasInfo) - {return Open(objectSize, NULL, hasInfo ? 1 : 0);} + {return Open(objectSize, (char*)NULL, hasInfo ? 1 : 0);} void Close(); virtual bool WriteSegment(NormBlockId blockId, @@ -68,7 +71,7 @@ class NormObject NormBlockId FirstPending() {return pending_mask.FirstSet();} // Methods available to server for transmission - bool NextServerMsg(NormMessage* msg); + bool NextServerMsg(NormObjectMsg* msg); NormBlock* ServerRecoverBlock(NormBlockId blockId); bool CalculateBlockParity(NormBlock* block); @@ -82,25 +85,18 @@ class NormObject NormBlockId blockId = theBlock->Id(); bool result = theBlock->TxUpdate(firstSegmentId, lastSegmentId, BlockSize(blockId), nparity, - segment_size, numErasures); - if (result) - { - result = pending_mask.Set(blockId); - ASSERT(result); - } + numErasures); + ASSERT(result ? pending_mask.Set(blockId) : true); + result = result ? pending_mask.Set(blockId) : false; return result; } // end NormObject::TxUpdateBlock() bool HandleInfoRequest(); bool HandleBlockRequest(NormBlockId nextId, NormBlockId lastId); - bool SetPending(NormBlockId blockId) - {return pending_mask.Set(blockId);} - NormBlock* FindBlock(NormBlockId blockId) - { - return block_buffer.Find(blockId); - } + bool SetPending(NormBlockId blockId) {return pending_mask.Set(blockId);} + NormBlock* FindBlock(NormBlockId blockId) {return block_buffer.Find(blockId);} bool ActivateRepairs(); - bool IsRepairSet(NormBlockId blockId) - {return repair_mask.Test(blockId);} + bool IsRepairSet(NormBlockId blockId) {return repair_mask.Test(blockId);} + bool AppendRepairAdv(NormCmdRepairAdvMsg& cmd); // Used by session server for resource management scheme NormBlock* StealNonPendingBlock(bool excludeBlock, NormBlockId excludeId = 0); @@ -108,29 +104,28 @@ class NormObject // Methods available to client for reception bool Accepted() {return accepted;} - void HandleObjectMessage(NormMessage& msg, - NormMsgType msgType, - NormBlockId blockId, - NormSegmentId segmentId); - bool StreamUpdateStatus(NormBlockId blockId); + void HandleObjectMessage(const NormObjectMsg& msg, + NormMsg::Type msgType, + NormBlockId blockId, + NormSegmentId segmentId); + // Used by remote server node for resource management scheme NormBlock* StealNewestBlock(bool excludeBlock, NormBlockId excludeId = 0); - + NormBlock* StealOldestBlock(bool excludeBlock, NormBlockId excludeId = 0); bool ClientRepairCheck(CheckLevel level, NormBlockId blockId, NormSegmentId segmentId, - bool timerActive); + bool timerActive, + bool holdoffPhase = false); bool IsRepairPending(bool flush); bool AppendRepairRequest(NormNackMsg& nack, bool flush); void SetRepairInfo() {repair_info = true;} bool SetRepairs(NormBlockId first, NormBlockId last) { - if (first == last) - return repair_mask.Set(first); - else - return repair_mask.SetBits(first, last-first+1); + return (first == last) ? repair_mask.Set(first) : + repair_mask.SetBits(first, last-first+1); } protected: @@ -155,8 +150,10 @@ class NormObject NormSlidingMask pending_mask; bool repair_info; NormSlidingMask repair_mask; - NormBlockId current_block_id; - NormSegmentId next_segment_id; + NormBlockId current_block_id; // for suppression + NormSegmentId next_segment_id; // for suppression + NormBlockId max_pending_block; // for NACK construction + NormSegmentId max_pending_segment; // for NACK construction NormBlockId last_block_id; NormSegmentId last_block_size; UINT16 last_segment_size; @@ -166,10 +163,7 @@ class NormObject bool accepted; - // Extra state for STREAM objects - bool stream_sync; - NormBlockId stream_sync_id; - NormBlockId stream_next_id; + NormObject* next; }; // end class NormObject @@ -191,15 +185,9 @@ class NormFileObject : public NormObject const char* Path() {return path;} bool Rename(const char* newPath) { - if (file.Rename(path, newPath)) - { - strncpy(path, newPath, PATH_MAX); - return true; - } - else - { - return false; - } + bool result = file.Rename(path, newPath); + result ? strncpy(path, newPath, PATH_MAX) : NULL; + return result; } virtual bool WriteSegment(NormBlockId blockId, @@ -231,7 +219,11 @@ class NormStreamObject : public NormObject void Close(); bool Accept(unsigned long bufferSize); - + bool StreamUpdateStatus(NormBlockId blockId); + void StreamResync(NormBlockId nextBlockId) + {stream_next_id = nextBlockId;} + void StreamAdvance(); + virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* buffer); @@ -239,8 +231,8 @@ class NormStreamObject : public NormObject NormSegmentId segmentId, char* buffer); - unsigned long Read(char* buffer, unsigned long len); - unsigned long Write(char* buffer, unsigned long len, bool flush = false); + bool Read(char* buffer, unsigned int* buflen, bool findMsgStart = false); + unsigned long Write(const char* buffer, unsigned long len, bool flush, bool eom, bool push); // For receive stream, we can rewind to earliest buffered offset void Rewind(); @@ -266,6 +258,11 @@ class NormStreamObject : public NormObject NormBlockId block; NormSegmentId segment; }; + // Extra state for STREAM objects + bool stream_sync; + NormBlockId stream_sync_id; + NormBlockId stream_next_id; + NormBlockPool block_pool; NormSegmentPool segment_pool; NormBlockBuffer stream_buffer; @@ -274,6 +271,7 @@ class NormStreamObject : public NormObject Index read_index; NormObjectSize read_offset; bool flush_pending; + bool msg_start; }; // end class NormStreamObject #ifdef SIMULATE @@ -317,17 +315,17 @@ class NormObjectTable bool Init(UINT16 rangeMax, UINT16 tableSize = 256); void Destroy(); - bool IsInited() {return (NULL != table);} + bool IsInited() const {return (NULL != table);} bool CanInsert(NormObjectId objectId) const; bool Insert(NormObject* theObject); bool Remove(const NormObject* theObject); NormObject* Find(const NormObjectId& objectId) const; - NormObjectId RangeLo() {return range_lo;} - NormObjectId RangeHi() {return range_hi;} - bool IsEmpty() {return (0 == range);} - unsigned long Count() {return count;} - const NormObjectSize& Size() {return size;} + NormObjectId RangeLo() const {return range_lo;} + NormObjectId RangeHi() const {return range_hi;} + bool IsEmpty() const {return (0 == range);} + unsigned long Count() const {return count;} + const NormObjectSize& Size() const {return size;} class Iterator { diff --git a/common/normPostProcess.cpp b/common/normPostProcess.cpp new file mode 100644 index 0000000..8ef3094 --- /dev/null +++ b/common/normPostProcess.cpp @@ -0,0 +1,165 @@ +#include "normPostProcess.h" +#include "protoLib.h" + +#include +#include +#include +#ifdef UNIX +#include +#include +#include +#endif // UNIX + +typedef void (*sighandler_t)(int); + +NormPostProcessor::NormPostProcessor() + : process_argv(NULL) +#ifdef UNIX + ,process_id(0) +#endif // UNIX +{ + +} + +NormPostProcessor::~NormPostProcessor() +{ + if (IsActive()) Kill(); + SetCommand(NULL); +} + +bool NormPostProcessor::SetCommand(const char* cmd) +{ + + // 1) Delete old command resources + if (process_argv) + { + const char** arg = process_argv; + while (*arg) + { + delete *arg; + arg++; + } + delete process_argv; + process_argv = NULL; + process_argc = 0; + } + + // 2) Count the number of tokens + if (!cmd) return true; // post processing disabled + if (!strcmp(cmd, "none")) return SetCommand(NULL); + const char* ptr = cmd; + unsigned int argCount = 0; + while (isspace(*ptr) && ('\0' != *ptr)) ptr++; + while ('\0' != *ptr) + { + argCount++; + while (!isspace(*ptr) && ('\0' != *ptr)) ptr++; + } + if (!argCount) return true; // post processing disabled + + + // 3) Allocate new process_argv array (2 extra slots, one for "target", + // and one for terminating NULL pointer. + if (!(process_argv = new const char*[argCount+2])) + { + DMSG(0, "NormPostProcessor::SetCommand new(process_argv) error: %s\n", + strerror(errno)); + return false; + } + memset((void*)process_argv, 0, (argCount+2)*sizeof(char**)); + process_argc = argCount; + + // 4) Fill in process_argv[] array with individual args + ptr = cmd; + while (isspace(*ptr) && ('\0' != *ptr)) ptr++; + unsigned int index = 0; + while ('\0' != *ptr) + { + const char* argStart = ptr; + while (!isspace(*ptr) && ('\0' != *ptr)) ptr++; + unsigned int argLength = ptr - argStart; + char* arg; + if (!(arg = new char[argLength+1])) + { + SetCommand(NULL); + DMSG(0, "NormPostProcessor::SetCommand new(process_arg) error: %s\n", + strerror(errno)); + return false; + } + strncpy(arg, argStart, argLength); + arg[argLength] = '\0'; + process_argv[index] = arg; + } + return true; +} // end NormPostProcessor::SetCommand() + +bool NormPostProcessor::ProcessFile(const char* path) +{ + + if (IsActive()) Kill(); + +#ifdef UNIX + // 1) temporarily disable signal handling + sighandler_t sigtermHandler = signal(SIGTERM, SIG_DFL); + sighandler_t sigintHandler = signal(SIGINT, SIG_DFL); + sighandler_t sigchldHandler = signal(SIGCHLD, SIG_DFL); + + process_argv[process_argc] = path; + + switch((process_id = fork())) + { + case -1: // error + DMSG(0, "NormPostProcessor::ProcessFile fork() error: %s\n", strerror(errno)); + process_id = 0; + process_argv[process_argc] = NULL; + return false; + + case 0: // child + if (execvp((char*)process_argv[0], (char**)process_argv) < 0) + { + DMSG(0, "NormPostProcessor::ProcessFile execvp() error: %s\n", strerror(errno)); + exit(-1); + } + break; + + default: // parent + process_argv[process_argc] = NULL; + // Restore signal handlers for parent + signal(SIGTERM, sigtermHandler); + signal(SIGINT, sigintHandler); + signal(SIGCHLD, sigchldHandler); + break; + } +#endif // UNIX + return true; +} // end NormPostProcessor::ProcessFile() + +void NormPostProcessor::Kill() +{ + if (!IsActive()) return; +#ifdef UNIX + int count = 0; + while((kill(process_id, SIGTERM) != 0) && count < 10) + { + if (errno == ESRCH) break; + count++; + DMSG(0, "NormPostProcessor::Kill kill() error: %s\n", strerror(errno)); + } + count = 0; + int status; + while((waitpid(process_id, &status, 0) != process_id) && count < 10) + { + if (errno == ECHILD) break; + count++; + DMSG(0, "NormPostProcessor::Kill waitpid() error: %s\n", strerror(errno)); + } + process_id = 0; +#endif // UNIX +} // end NormPostProcessor::Kill() + +void NormPostProcessor::HandleSIGCHLD() +{ + // See if processor exited itself + int status; + if (wait(&status) == process_id) process_id = 0; +} // end NormPostProcessor::HandleSIGCHLD() diff --git a/common/normPostProcess.h b/common/normPostProcess.h new file mode 100644 index 0000000..35d06c7 --- /dev/null +++ b/common/normPostProcess.h @@ -0,0 +1,35 @@ +#ifndef _NORM_POST_PROCESS +#define _NORM_POST_PROCESS + + +class NormPostProcessor +{ + public: + NormPostProcessor(); + ~NormPostProcessor(); + + bool IsEnabled() {return (0 != process_argv);} + bool SetCommand(const char* cmd); + bool ProcessFile(const char* path); + void Kill(); + bool IsActive() + { +#ifdef UNIX + return (0 != process_id); +#endif // UNIX + } + + void HandleSIGCHLD(); + + private: + const char** process_argv; + unsigned int process_argc; +#ifdef UNIX + int process_id; +#endif +#ifdef WIN32 + HANDLE process_handle; +#endif +}; // end class NormPostProcessor + +#endif // _NORM_POST_PROCESS diff --git a/common/normSegment.cpp b/common/normSegment.cpp index e2a6226..5531bc1 100644 --- a/common/normSegment.cpp +++ b/common/normSegment.cpp @@ -5,9 +5,10 @@ NormSegmentPool::NormSegmentPool() : seg_size(0), seg_count(0), seg_total(0), seg_list(NULL), peak_usage(0), overruns(0), overrun_flag(false) -{ +{ } + NormSegmentPool::~NormSegmentPool() { Destroy(); @@ -18,6 +19,12 @@ bool NormSegmentPool::Init(unsigned int count, unsigned int size) if (seg_list) Destroy(); peak_usage = 0; overruns = 0; + +#ifdef SIMULATE + // In simulations, don't really need big vectors for data + // since we don't actually read/write real data + size = MIN(size, (SIM_PAYLOAD_MAX+NormDataMsg::PayloadHeaderLength()+1)); +#endif // SIMULATE // This makes sure we get appropriate alignment unsigned int alloc_size = size / sizeof(char*); if ((alloc_size*sizeof(char*)) < size) alloc_size++; @@ -157,7 +164,7 @@ void NormBlock::EmptyToPool(NormSegmentPool& segmentPool) } } // end NormBlock::EmptyToPool() -bool NormBlock::IsEmpty() +bool NormBlock::IsEmpty() const { ASSERT(segment_table); for (unsigned int i = 0; i < size; i++) @@ -166,7 +173,7 @@ bool NormBlock::IsEmpty() } // end NormBlock::EmptyToPool() // Used by client side to determine if NACK should be sent -// Note: This clears the block's "repair_mask" state +// Note: This invalidates the block's "repair_mask" state bool NormBlock::IsRepairPending(UINT16 ndata, UINT16 nparity) { // Clients ask for a block of parity to fulfill their @@ -250,9 +257,10 @@ bool NormBlock::ActivateRepairs(UINT16 nparity) } // end NormBlock::ActivateRepairs() // For NACKs arriving during server repair_timer "holdoff" time +// (we directly update the "pending_mask" for blocks/segments +// greater than our current transmit index) bool NormBlock::TxUpdate(NormSegmentId nextId, NormSegmentId lastId, - UINT16 ndata, UINT16 nparity, - UINT16 segmentSize, UINT16 erasureCount) + UINT16 ndata, UINT16 nparity, UINT16 erasureCount) { bool increasedRepair = false; @@ -314,7 +322,7 @@ bool NormBlock::TxUpdate(NormSegmentId nextId, NormSegmentId lastId, bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, UINT16 ndata, UINT16 nparity, UINT16 erasureCount) { - DMSG(4, "NormBlock::HandleSegmentRequest() blk>%lu seg>%hu:%hu erasures:%hu\n", + DMSG(6, "NormBlock::HandleSegmentRequest() blk>%lu seg>%hu:%hu erasures:%hu\n", (UINT32)id, (UINT16)nextId, (UINT16)lastId, erasureCount); bool increasedRepair = false; if (nextId < ndata) @@ -373,6 +381,78 @@ bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, } // end NormBlock::HandleSegmentRequest() +bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, + NormObjectId objectId, + bool repairInfo, + UINT16 ndata, + UINT16 segmentSize) +{ + NormRepairRequest req; + req.SetFlag(NormRepairRequest::SEGMENT); + if (repairInfo) req.SetFlag(NormRepairRequest::INFO); + UINT16 nextId = repair_mask.FirstSet(); + UINT16 blockSize = size; + NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; + UINT16 segmentCount = 0; + UINT16 firstId = 0; + while (nextId < blockSize) + { + UINT16 currentId = nextId; + nextId = repair_mask.NextSet(++nextId); + if (!segmentCount) firstId = currentId; + + segmentCount++; + // Check for break in consecutive series or end + if (((nextId - currentId) > 1) || (nextId >= blockSize)) + { + NormRepairRequest::Form form; + switch (segmentCount) + { + case 0: + form = NormRepairRequest::INVALID; + break; + case 1: + case 2: + form = NormRepairRequest::ITEMS; + break; + default: + form = NormRepairRequest::RANGES; + break; + } + if (form != prevForm) + { + if (NormRepairRequest::INVALID != prevForm) + cmd.PackRepairRequest(req); // (TBD) error check + req.SetForm(form); + cmd.AttachRepairRequest(req, segmentSize); // (TBD) error check + prevForm = form; + } + switch(form) + { + case NormRepairRequest::INVALID: + ASSERT(0); // can't happen + break; + case NormRepairRequest::ITEMS: + req.AppendRepairItem(objectId, id, ndata, firstId); + if (2 == segmentCount) + req.AppendRepairItem(objectId, id, ndata, currentId); + break; + case NormRepairRequest::RANGES: + req.AppendRepairRange(objectId, id, ndata, firstId, + objectId, id, ndata, currentId); + break; + case NormRepairRequest::ERASURES: + // erasure counts not used + break; + } + segmentCount = 0; + } + } // end while (nextId < blockSize) + if (NormRepairRequest::INVALID != prevForm) + cmd.PackRepairRequest(req); // (TBD) error check + return true; +} // end NormBlock::AppendRepairAdv() + // Called by client bool NormBlock::AppendRepairRequest(NormNackMsg& nack, UINT16 ndata, @@ -382,11 +462,7 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, UINT16 segmentSize) { ASSERT(pending_mask.FirstSet() < ndata); - NormRepairRequest req; - nack.AttachRepairRequest(req, segmentSize); - req.SetFlag(NormRepairRequest::SEGMENT); - if (pendingInfo) req.SetFlag(NormRepairRequest::INFO); - NormSegmentId nextId, lastId; + NormSegmentId nextId, endId; if (erasure_count > nparity) { @@ -397,30 +473,92 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, while (i--) { nextId++; - nextId = pending_mask.NextSet(nextId); + endId = pending_mask.NextSet(nextId); } - lastId = ndata + nparity; + endId = ndata + nparity; } else { nextId = pending_mask.NextSet(ndata); - lastId = ndata + erasure_count; + endId = ndata + erasure_count; } + NormRepairRequest req; + req.SetFlag(NormRepairRequest::SEGMENT); + if (pendingInfo) req.SetFlag(NormRepairRequest::INFO); NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; - UINT16 reqCount = 0; + UINT16 segmentCount = 0; + // new code begins here + UINT16 firstId = 0; + while (nextId < endId) + { + UINT16 currentId = nextId; + nextId = pending_mask.NextSet(++nextId); + if (0 == segmentCount) firstId = currentId; + segmentCount++; + // Check for break in consecutive series or end + if (((nextId - currentId) > 1) || (nextId >= endId)) + { + NormRepairRequest::Form form; + switch (segmentCount) + { + case 0: + form = NormRepairRequest::INVALID; + break; + case 1: + case 2: + form = NormRepairRequest::ITEMS; + break; + default: + form = NormRepairRequest::RANGES; + break; + } + if (form != prevForm) + { + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + nack.AttachRepairRequest(req, segmentSize); // (TBD) error check + req.SetForm(form); + prevForm = form; + } + switch (form) + { + case NormRepairRequest::INVALID: + ASSERT(0); + break; + case NormRepairRequest::ITEMS: + req.AppendRepairItem(objectId, id, ndata, firstId); // (TBD) error check + if (2 == segmentCount) + req.AppendRepairItem(objectId, id, ndata, currentId); // (TBD) error check + break; + case NormRepairRequest::RANGES: + req.AppendRepairRange(objectId, id, ndata, firstId, // (TBD) error check + objectId, id, ndata, currentId); // (TBD) error check + break; + case NormRepairRequest::ERASURES: + // erasure counts not used + break; + } // end switch(form) + segmentCount = 0; + } + } // end while (nextId < lastId) + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + + /* + // old code begins here NormSegmentId prevId = nextId; - while ((nextId <= lastId) || (reqCount > 0)) + while ((nextId <= lastId) || (segmentCount > 0)) { // force break of possible ending consec. series if (nextId == lastId) nextId++; - if (reqCount && (reqCount == (nextId - prevId))) + if (segmentCount && (segmentCount == (nextId - prevId))) { - reqCount++; // consecutive series continues + segmentCount++; // consecutive series continues } else { NormRepairRequest::Form nextForm; - switch(reqCount) + switch(segmentCount) { case 0: nextForm = NormRepairRequest::INVALID; @@ -464,15 +602,16 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, } // end switch(nextForm) prevId = nextId; if (nextId < lastId) - reqCount = 1; + segmentCount = 1; else - reqCount = 0; + segmentCount = 0; } // end if/else (reqCount && (reqCount == (nextId - prevId))) nextId++; if (nextId <= lastId) nextId = pending_mask.NextSet(nextId); } // end while(nextId <= lastId) if (NormRepairRequest::INVALID != prevForm) nack.PackRepairRequest(req); // (TBD) error check + */ return true; } // end NormBlock::AppendRepairRequest() diff --git a/common/normSegment.h b/common/normSegment.h index 59703f4..1e227c2 100644 --- a/common/normSegment.h +++ b/common/normSegment.h @@ -24,11 +24,13 @@ class NormSegmentPool seg_list = segment; seg_count++; } - bool IsEmpty() {return (NULL == seg_list);} + bool IsEmpty() const {return (NULL == seg_list);} - unsigned long CurrentUsage() {return (seg_total - seg_count);} - unsigned long PeakUsage() {return peak_usage;} - unsigned long OverunCount() {return overruns;} + unsigned long CurrentUsage() const {return (seg_total - seg_count);} + unsigned long PeakUsage() const {return peak_usage;} + unsigned long OverunCount() const {return overruns;} + + unsigned int GetSegmentSize() {return seg_size;} private: unsigned int seg_size; @@ -117,8 +119,7 @@ class NormBlock bool TxReset(UINT16 ndata, UINT16 nparity, UINT16 autoParity, UINT16 segmentSize); bool TxUpdate(NormSegmentId nextId, NormSegmentId lastId, - UINT16 ndata, UINT16 nparity, - UINT16 segmentSize, UINT16 erasureCount); + UINT16 ndata, UINT16 nparity, UINT16 erasureCount); bool HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, UINT16 ndata, UINT16 nparity, @@ -130,6 +131,11 @@ class NormBlock parity_offset = MIN(parity_offset, nparity); parity_count = 0; } + bool AppendRepairAdv(NormCmdRepairAdvMsg& cmd, + NormObjectId objectId, + bool repairInfo, + UINT16 ndata, + UINT16 segmentSize); // Client routines void RxInit(NormBlockId& blockId, UINT16 ndata, UINT16 nparity) @@ -143,11 +149,12 @@ class NormBlock parity_offset = 0; flags = 0; } + // Note: This invalidates the repair_mask state. bool IsRepairPending(UINT16 ndata, UINT16 nparity); void DecrementErasureCount() {erasure_count--;} UINT16 ErasureCount() const {return erasure_count;} void IncrementParityCount() {parity_count++;} - UINT16 ParityCount() {return parity_count;} + UINT16 ParityCount() const {return parity_count;} NormSymbolId FirstPending() const {return pending_mask.FirstSet();} @@ -196,7 +203,7 @@ class NormBlock UINT16 segmentSize); //void DisplayPendingMask(FILE* f) {pending_mask.Display(f);} - bool IsEmpty(); + bool IsEmpty() const; void EmptyToPool(NormSegmentPool& segmentPool); private: @@ -222,7 +229,7 @@ class NormBlockPool ~NormBlockPool(); bool Init(UINT32 numBlocks, UINT16 blockSize); void Destroy(); - bool IsEmpty() {return (NULL == head);} + bool IsEmpty() const {return (NULL == head);} NormBlock* Get() { NormBlock* b = head; @@ -243,7 +250,7 @@ class NormBlockPool b->next = head; head = b; } - unsigned long OverrunCount() {return overruns;} + unsigned long OverrunCount() const {return overruns;} private: NormBlock* head; @@ -266,9 +273,9 @@ class NormBlockBuffer bool Remove(const NormBlock* theBlock); NormBlock* Find(const NormBlockId& blockId) const; - NormBlockId RangeLo() {return range_lo;} - NormBlockId RangeHi() {return range_hi;} - bool IsEmpty() {return (0 == range);} + NormBlockId RangeLo() const {return range_lo;} + NormBlockId RangeHi() const {return range_hi;} + bool IsEmpty() const {return (0 == range);} bool CanInsert(NormBlockId blockId) const; class Iterator diff --git a/common/normSession.cpp b/common/normSession.cpp index 05e8e33..4f5e6e4 100644 --- a/common/normSession.cpp +++ b/common/normSession.cpp @@ -2,30 +2,36 @@ #include #include // for gmtime() in NormTrace() - +const UINT8 NormSession::DEFAULT_TTL = 255; // bits/sec const double NormSession::DEFAULT_TRANSMIT_RATE = 64000.0; // bits/sec -const double NormSession::DEFAULT_PROBE_MIN = 1.0; // sec -const double NormSession::DEFAULT_PROBE_MAX = 10.0; // sec +const double NormSession::DEFAULT_GRTT_INTERVAL_MIN = 1.0; // sec +const double NormSession::DEFAULT_GRTT_INTERVAL_MAX = 30.0; // sec const double NormSession::DEFAULT_GRTT_ESTIMATE = 0.5; // sec const double NormSession::DEFAULT_GRTT_MAX = 10.0; // sec const unsigned int NormSession::DEFAULT_GRTT_DECREASE_DELAY = 3; const double NormSession::DEFAULT_BACKOFF_FACTOR = 4.0; -const double NormSession::DEFAULT_GSIZE_ESTIMATE = 1000.0; +const double NormSession::DEFAULT_GSIZE_ESTIMATE = 1000.0; +const UINT16 NormSession::DEFAULT_NDATA = 64; +const UINT16 NormSession::DEFAULT_NPARITY = 32; NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) : session_mgr(sessionMgr), notify_pending(false), local_node_id(localNodeId), - tx_rate(DEFAULT_TRANSMIT_RATE/8.0), backoff_factor(DEFAULT_BACKOFF_FACTOR), - is_server(false), - tx_cache_count_min(1), tx_cache_count_max(256), + ttl(DEFAULT_TTL), tx_rate(DEFAULT_TRANSMIT_RATE/8.0), + backoff_factor(DEFAULT_BACKOFF_FACTOR), is_server(false), session_id(0), + ndata(DEFAULT_NDATA), nparity(DEFAULT_NPARITY), auto_parity(0), extra_parity(0), + next_tx_object_id(0), tx_cache_count_min(1), tx_cache_count_max(256), tx_cache_size_max((unsigned long)20*1024*1024), flush_count(NORM_ROBUST_FACTOR+1), - posted_tx_queue_empty(false), - probe_interval(0.0), probe_interval_min(DEFAULT_PROBE_MIN), - probe_interval_max(DEFAULT_PROBE_MAX), + posted_tx_queue_empty(false), advertise_repairs(false), + suppress_nonconfirmed(false), suppress_rate(-1.0), suppress_rtt(-1.0), + probe_proactive(true), grtt_interval(0.5), + grtt_interval_min(DEFAULT_GRTT_INTERVAL_MIN), + grtt_interval_max(DEFAULT_GRTT_INTERVAL_MAX), grtt_max(DEFAULT_GRTT_MAX), grtt_decrease_delay_count(DEFAULT_GRTT_DECREASE_DELAY), - grtt_response(false), grtt_current_peak(0.0), - is_client(false), + grtt_response(false), grtt_current_peak(0.0), grtt_age(0.0), + cc_enable(false), cc_sequence(0), cc_slow_start(true), + is_client(false), unicast_nacks(false), client_silent(false), trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0), next(NULL) { @@ -44,20 +50,17 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) repair_timer.Init(0.0, 1, (ProtocolTimerOwner*)this, (ProtocolTimeoutFunc)&NormSession::OnRepairTimeout); flush_timer.Init(0.0, 0, (ProtocolTimerOwner*)this, - (ProtocolTimeoutFunc)&NormSession::OnCommandTimeout); + (ProtocolTimeoutFunc)&NormSession::OnFlushTimeout); probe_timer.Init(0.0, -1, (ProtocolTimerOwner*)this, (ProtocolTimeoutFunc)&NormSession::OnProbeTimeout); + grtt_quantized = NormQuantizeRtt(DEFAULT_GRTT_ESTIMATE); grtt_measured = grtt_advertised = NormUnquantizeRtt(grtt_quantized); gsize_measured = DEFAULT_GSIZE_ESTIMATE; gsize_quantized = NormQuantizeGroupSize(DEFAULT_GSIZE_ESTIMATE); gsize_advertised = NormUnquantizeGroupSize(gsize_quantized); - gsize_nack_ave = (1.2 / (2.0*DEFAULT_BACKOFF_FACTOR)) * - (log(DEFAULT_GSIZE_ESTIMATE) + 1.0); - gsize_nack_ave = exp(gsize_nack_ave); - gsize_correction_factor = 1.0; - gsize_nack_delta = 0.0; + // This timer is for printing out occasional status reports // (It may be used to trigger transmission of report messages @@ -92,20 +95,26 @@ bool NormSession::Open() Close(); return false; } - if (address.IsMulticast()) - { - if (UDP_SOCKET_ERROR_NONE != rx_socket.JoinGroup(&address, ttl)) - { - DMSG(0, "NormSession::Open() rx_socket join group error\n"); - Close(); - return false; - } - } } + if (address.IsMulticast()) + { + if (UDP_SOCKET_ERROR_NONE != rx_socket.JoinGroup(&address)) + { + DMSG(0, "NormSession::Open() rx_socket join group error\n"); + Close(); + return false; + } + if (UDP_SOCKET_ERROR_NONE != tx_socket.SetTTL(ttl)) + { + DMSG(0, "NormSession::Open() tx_socket set ttl error\n"); + Close(); + return false; + } + } for (unsigned int i = 0; i < DEFAULT_MESSAGE_POOL_DEPTH; i++) { - NormMessage* msg = new NormMessage(); + NormMsg* msg = new NormMsg(); if (msg) { message_pool.Append(msg); @@ -131,7 +140,11 @@ void NormSession::Close() message_queue.Destroy(); message_pool.Destroy(); if (tx_socket.IsOpen()) tx_socket.Close(); - if (rx_socket.IsOpen()) rx_socket.Close(); + if (rx_socket.IsOpen()) + { + if (address.IsMulticast()) rx_socket.LeaveGroup(&address); + rx_socket.Close(); + } } // end NormSession::Close() @@ -171,13 +184,14 @@ bool NormSession::StartServer(unsigned long bufferSpace, unsigned long blockSpace = sizeof(NormBlock) + blockSize * sizeof(char*) + 2*maskSize + - numParity * (segmentSize + NormDataMsg::PayloadHeaderLen()); + numParity * (segmentSize + NormDataMsg::PayloadHeaderLength()); unsigned long numBlocks = bufferSpace / blockSpace; if (bufferSpace > (numBlocks*blockSpace)) numBlocks++; if (numBlocks < 2) numBlocks = 2; unsigned long numSegments = numBlocks * numParity; + if (!block_pool.Init(numBlocks, blockSize)) { DMSG(0, "NormSession::StartServer() block_pool init error\n"); @@ -185,7 +199,7 @@ bool NormSession::StartServer(unsigned long bufferSpace, return false; } - if (!segment_pool.Init(numSegments, segmentSize + NormDataMsg::PayloadHeaderLen())) + if (!segment_pool.Init(numSegments, segmentSize + NormDataMsg::PayloadHeaderLength())) { DMSG(0, "NormSession::StartServer() segment_pool init error\n"); StopServer(); @@ -194,7 +208,7 @@ bool NormSession::StartServer(unsigned long bufferSpace, if (numParity) { - if (!encoder.Init(numParity, segmentSize + NormDataMsg::PayloadHeaderLen())) + if (!encoder.Init(numParity, segmentSize + NormDataMsg::PayloadHeaderLength())) { DMSG(0, "NormSession::StartServer() encoder init error\n"); StopServer(); @@ -203,15 +217,21 @@ bool NormSession::StartServer(unsigned long bufferSpace, } segment_size = segmentSize; + sent_rate = 0.0; + prev_update_time.tv_sec = prev_update_time.tv_usec = 0; + sent_accumulator = 0; + nominal_packet_size = (double)segmentSize; ndata = numData; nparity = numParity; is_server = true; flush_count = NORM_ROBUST_FACTOR+1; // (TBD) parameterize robust_factor - probe_timer.SetInterval(0.0); + //probe_timer.SetInterval(0.0); + OnProbeTimeout(); InstallTimer(&probe_timer); return true; } // end NormSession::StartServer() + void NormSession::StopServer() { if (probe_timer.IsActive()) probe_timer.Deactivate(); @@ -248,10 +268,8 @@ void NormSession::StopClient() void NormSession::Serve() { // Only send new data when no other messages are queued for transmission - //if (tx_timer.IsActive()) return; if (!message_queue.IsEmpty()) return; - - + NormObject* obj = NULL; // Queue next server message if (tx_pending_mask.IsSet()) @@ -269,21 +287,20 @@ void NormSession::Serve() (NormServerNode*)NULL, (NormObject*)NULL); // (TBD) Was session deleted? - Serve(); return; } } if (obj) { - NormMessage* msg = GetMessageFromPool(); + NormObjectMsg* msg = (NormObjectMsg*)GetMessageFromPool(); if (msg) { if (obj->NextServerMsg(msg)) { - msg->generic.SetDestination(address); - msg->object.SetGrtt(grtt_quantized); - msg->object.SetGroupSize(gsize_quantized); + msg->SetDestination(address); + msg->SetGrtt(grtt_quantized); + msg->SetGroupSize(gsize_quantized); QueueMessage(msg); flush_count = 0; if (flush_timer.IsActive()) flush_timer.Deactivate(); @@ -300,7 +317,8 @@ void NormSession::Serve() ReturnMessageToPool(msg); if (obj->IsStream()) { - if (((NormStreamObject*)obj)->IsFlushPending() && + NormStreamObject* stream = (NormStreamObject*)obj; + if (stream->IsFlushPending() && (flush_count < NORM_ROBUST_FACTOR)) { // Queue flush message @@ -309,17 +327,15 @@ void NormSession::Serve() if (!posted_tx_queue_empty) { posted_tx_queue_empty = true; - Notify(NormController::TX_QUEUE_EMPTY, - (NormServerNode*)NULL, obj); + Notify(NormController::TX_QUEUE_EMPTY, (NormServerNode*)NULL, obj); // (TBD) Was session deleted? - Serve(); return; } } else { - DMSG(0, "NormSession::Serve() pending obj, no message?.\n"); - ASSERT(0); + DMSG(0, "NormSession::Serve() pending non-stream obj, no message?.\n"); + //ASSERT(0); } } } @@ -341,64 +357,60 @@ void NormSession::Serve() flush_count++; } } // end NormSession::Serve() - + void NormSession::ServerQueueFlush() { // (TBD) Deal with EOT or pre-queued squelch on squelch case if (flush_timer.IsActive()) return; - NormMessage* msg = GetMessageFromPool(); - if (msg) + NormObject* obj = tx_table.Find(tx_table.RangeHi()); + NormObjectId objectId; + NormBlockId blockId; + NormSegmentId segmentId; + if (obj) { - msg->generic.SetType(NORM_MSG_CMD); - msg->generic.SetDestination(address); - msg->cmd.generic.SetGrtt(grtt_quantized); - msg->cmd.generic.SetGroupSize(gsize_quantized); - msg->cmd.generic.SetFlavor(NormCmdMsg::NORM_CMD_FLUSH); - NormCmdFlushMsg& flush = (NormCmdFlushMsg&)msg->cmd.flush; - flush.Reset(); // reset flags and length - NormObject* obj = tx_table.Find(tx_table.RangeHi()); - NormObjectId objectId; - NormBlockId blockId; - NormSegmentId segmentId; - if (obj) + if (obj->IsStream()) { - if (obj->IsStream()) - { - NormStreamObject* stream = (NormStreamObject*)obj; - objectId = stream->Id(); - blockId = stream->FlushBlockId(); - segmentId = stream->FlushSegmentId(); - } - else - { - objectId = obj->Id(); - blockId = obj->LastBlockId(); - segmentId = obj->LastBlockSize() - 1; - } + NormStreamObject* stream = (NormStreamObject*)obj; + objectId = stream->Id(); + blockId = stream->FlushBlockId(); + segmentId = stream->FlushSegmentId(); } else { - ReturnMessageToPool(msg); - if (ServerQueueSquelch(next_tx_object_id)) - { - flush_count++; - flush_timer.SetInterval(2*grtt_advertised); - InstallTimer(&flush_timer); - } - DMSG(8, "NormSession::ServerQueueFlush() node>%lu squelch queued (flush_count:%u)...\n", - LocalNodeId(), flush_count); - return; + objectId = obj->Id(); + blockId = obj->LastBlockId(); + segmentId = obj->LastBlockSize() - 1; } - flush.SetObjectId(objectId); - flush.SetFecBlockId(blockId); - flush.SetFecSymbolId(segmentId); - QueueMessage(msg); + } + else + { + // (TBD) send NORM_CMD(EOT) instead? + if (ServerQueueSquelch(next_tx_object_id)) + { + flush_count++; + flush_timer.SetInterval(2*grtt_advertised); + InstallTimer(&flush_timer); + } + DMSG(8, "NormSession::ServerQueueFlush() node>%lu squelch queued (flush_count:%u)...\n", + LocalNodeId(), flush_count); + return; + } + NormCmdFlushMsg* flush = (NormCmdFlushMsg*)GetMessageFromPool(); + if (flush) + { + flush->Init(); + flush->SetDestination(address); + flush->SetGrtt(grtt_quantized); + flush->SetGroupSize(gsize_quantized); + flush->SetObjectId(objectId); + flush->SetFecBlockId(blockId); + flush->SetFecSymbolId(segmentId); + QueueMessage(flush); flush_count++; flush_timer.SetInterval(2*grtt_advertised); InstallTimer(&flush_timer); DMSG(8, "NormSession::ServerQueueFlush() node>%lu, flush queued (flush_count:%u)...\n", LocalNodeId(), flush_count); - } else { @@ -407,14 +419,14 @@ void NormSession::ServerQueueFlush() } } // end NormSession::ServerQueueFlush() -bool NormSession::OnCommandTimeout() +bool NormSession::OnFlushTimeout() { flush_timer.Deactivate(); - Serve(); + Serve(); // (TBD) Change this to PromptServer() ?? return false; -} // NormSession::OnCommandTimeout() +} // NormSession::OnFlushTimeout() -void NormSession::QueueMessage(NormMessage* msg) +void NormSession::QueueMessage(NormMsg* msg) { /* A little test jig @@ -436,7 +448,7 @@ void NormSession::QueueMessage(NormMessage* msg) InstallTimer(&tx_timer); } message_queue.Append(msg); -} // end NormSesssion::QueueMessage(NormMessage& msg) +} // end NormSesssion::QueueMessage(NormMsg& msg) @@ -449,7 +461,7 @@ NormFileObject* NormSession::QueueTxFile(const char* path, DMSG(0, "NormSession::QueueTxFile() Error: server is closed\n"); return NULL; } - NormFileObject* file = new NormFileObject(this, NULL, next_tx_object_id); + NormFileObject* file = new NormFileObject(this, (NormServerNode*)NULL, next_tx_object_id); if (!file) { DMSG(0, "NormSession::QueueTxFile() new file object error: %s\n", @@ -483,7 +495,7 @@ NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, DMSG(0, "NormSession::QueueTxStream() Error: server is closed\n"); return NULL; } - NormStreamObject* stream = new NormStreamObject(this, NULL, next_tx_object_id); + NormStreamObject* stream = new NormStreamObject(this, (NormServerNode*)NULL, next_tx_object_id); if (!stream) { DMSG(0, "NormSession::QueueTxStream() new stream object error: %s\n", @@ -562,8 +574,7 @@ bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) { // Remove oldest non-pending NormObject* oldest = tx_table.Find(tx_table.RangeLo()); - ASSERT(!oldest->IsPending()); - if (oldest->IsRepairPending()) + if (oldest->IsRepairPending() || oldest->IsPending()) { DMSG(0, "NormSession::QueueTxObject() all held objects repair pending\n"); //posted_tx_queue_empty = false; @@ -588,8 +599,9 @@ bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) tx_pending_mask.Set(obj->Id()); ASSERT(tx_pending_mask.Test(obj->Id())); next_tx_object_id++; - posted_tx_queue_empty = false; - Serve(); + TouchServer(); + //posted_tx_queue_empty = false; + //Serve(); return true; } // end NormSession::QueueTxObject() @@ -612,7 +624,6 @@ NormBlock* NormSession::ServerGetFreeBlock(NormObjectId objectId, // Second, try to steal oldest non-pending block if (!b) { - NormObjectTable::Iterator iterator(tx_table); NormObject* obj; while ((obj = iterator.GetNextObject())) @@ -673,12 +684,13 @@ char* NormSession::ServerGetFreeSegment(NormObjectId objectId, bool NormSession::TxSocketRecvHandler(UdpSocket* /*theSocket*/) { - NormMessage msg; - unsigned int buflen = NORM_MSG_SIZE_MAX; - if (UDP_SOCKET_ERROR_NONE == tx_socket.RecvFrom(msg.generic.AccessBuffer(), &buflen, - &msg.generic.AccessAddress())) + NormMsg msg; + unsigned int msgLength = NormMsg::MAX_SIZE; + if (UDP_SOCKET_ERROR_NONE == tx_socket.RecvFrom(msg.AccessBuffer(), + &msgLength, + msg.AccessAddress())) { - msg.generic.SetLength(buflen); + msg.InitFromBuffer(msgLength); HandleReceiveMessage(msg, true); } else @@ -690,12 +702,13 @@ bool NormSession::TxSocketRecvHandler(UdpSocket* /*theSocket*/) bool NormSession::RxSocketRecvHandler(UdpSocket* /*theSocket*/) { - NormMessage msg; - unsigned int buflen = NORM_MSG_SIZE_MAX; - if (UDP_SOCKET_ERROR_NONE == rx_socket.RecvFrom(msg.generic.AccessBuffer(), &buflen, - &msg.generic.AccessAddress())) + NormMsg msg; + unsigned int msgLength = NormMsg::MAX_SIZE; + if (UDP_SOCKET_ERROR_NONE == rx_socket.RecvFrom(msg.AccessBuffer(), + &msgLength, + msg.AccessAddress())) { - msg.generic.SetLength(buflen); + msg.InitFromBuffer(msgLength); HandleReceiveMessage(msg, false); } else @@ -705,22 +718,10 @@ bool NormSession::RxSocketRecvHandler(UdpSocket* /*theSocket*/) return true; } // end NormSession::RxSocketRecvHandler() -#ifdef SIMULATE -bool NormSession::SimSocketRecvHandler(char* buffer, unsigned short buflen, - const NetworkAddress& src, bool unicast) -{ - NormMessage msg; - memcpy(msg.generic.AccessBuffer(), buffer, buflen); - msg.generic.SetLength(buflen); - msg.generic.AccessAddress() = src; - HandleReceiveMessage(msg, unicast); - return true; -} // end NormSession::SimSocketRecvHandler() -#endif // SIMULATE - -static double THE_TIME = 0.0; - -void NormTrace(NormNodeId localId, NormMessage& msg, bool sent) +void NormTrace(const struct timeval& currentTime, + NormNodeId localId, + const NormMsg& msg, + bool sent) { //if (DebugLevel() < 8) return; // (TBD) provide per-session trace on/off switch static const char* MSG_NAME[] = @@ -735,14 +736,18 @@ void NormTrace(NormNodeId localId, NormMessage& msg, bool sent) }; static const char* CMD_NAME[] = { - "CMD_INVALID", - "CMD_FLUSH", - "CMD_SQUELCH", - "CMD_ACK_REQ", - "CMD_REPAIR_ADV", - "CMD_CC", - "CMD_APP" - }; + "CMD(INVALID)", + "CMD(FLUSH)", + "CMD(EOT)", + "CMD(SQUELCH)", + "CMD(CC)", + "CMD(REPAIR_ADV)", + "CMD(ACK_REQ)", + "CMD(APP)" + }; + + + static const char* REQ_NAME[] = { "INVALID", @@ -751,63 +756,111 @@ void NormTrace(NormNodeId localId, NormMessage& msg, bool sent) "APP" }; - NormMsgType msgType = msg.generic.GetType(); - UINT16 length = msg.generic.AccessLength(); + NormMsg::Type msgType = msg.GetType(); + UINT16 length = msg.GetLength(); const char* status = sent ? "dst" : "src"; + const NetworkAddress& addr = sent ? msg.GetDestination() : msg.GetSource(); - struct timeval currentTime; - GetSystemTime(¤tTime); struct tm* ct = gmtime((time_t*)¤tTime.tv_sec); DMSG(0, "trace>%02d:%02d:%02d.%06lu node>%lu %s>%s ", ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, - (UINT32)localId, status, msg.generic.AccessAddress().HostAddressString()); - - THE_TIME = currentTime.tv_sec + 1.0e-06 * (double)currentTime.tv_usec; + (UINT32)localId, status, addr.HostAddressString()); + bool clrFlag = false; switch (msgType) { - case NORM_MSG_INFO: - DMSG(0, "INFO obj>%hu ", (UINT16)msg.object.GetObjectId()); + case NormMsg::INFO: + { + const NormInfoMsg& info = (const NormInfoMsg&)msg; + DMSG(0, "INFO obj>%hu ", (UINT16)info.GetObjectId()); break; - case NORM_MSG_DATA: - if (msg.data.GetFecSymbolId() < msg.object.GetFecBlockLen()) + } + case NormMsg::DATA: + { + const NormDataMsg& data = (const NormDataMsg&)msg; + if (data.IsData()) DMSG(0, "DATA "); else DMSG(0, "PRTY "); DMSG(0, "obj>%hu blk>%lu seg>%hu ", - (UINT16)msg.object.GetObjectId(), - (UINT32)msg.data.GetFecBlockId(), - (UINT16)msg.data.GetFecSymbolId()); - break; + (UINT16)data.GetObjectId(), + (UINT32)data.GetFecBlockId(), + (UINT16)data.GetFecSymbolId()); - case NORM_MSG_CMD: + if (data.IsData()) + { + const UINT16* x = (const UINT16*)data.GetPayloadData(); + if (data.FlagIsSet(NormObjectMsg::FLAG_MSG_START)) + DMSG(0, "start byte>%hu ", ntohs(*x)); + } + break; + } + case NormMsg::CMD: { - NormCmdMsg::Flavor flavor = msg.cmd.generic.GetFlavor(); - DMSG(0, "%s", CMD_NAME[flavor]); + NormCmdMsg::Flavor flavor = ((const NormCmdMsg&)msg).GetFlavor(); + DMSG(0, "%s ", CMD_NAME[flavor]); switch (flavor) { - case NormCmdMsg::NORM_CMD_ACK_REQ: + case NormCmdMsg::ACK_REQ: { - int index = msg.cmd.ack_req.GetAckFlavor(); + int index = ((const NormCmdAckReqMsg&)msg).GetAckType(); index = MIN(index, 3); - DMSG(0, "(%s)", REQ_NAME[index]); + DMSG(0, "(%s) ", REQ_NAME[index]); break; } - case NormCmdMsg::NORM_CMD_SQUELCH: - DMSG(0, " obj>%hu blk>%lu seg>%hu", - (UINT16)msg.cmd.squelch.GetObjectId(), - (UINT32)msg.cmd.squelch.GetFecBlockId(), - (UINT16)msg.cmd.squelch.GetFecSymbolId()); + case NormCmdMsg::SQUELCH: + { + const NormCmdSquelchMsg& squelch = (const NormCmdSquelchMsg&)msg; + DMSG(0, " obj>%hu blk>%lu seg>%hu ", + (UINT16)squelch.GetObjectId(), + (UINT32)squelch.GetFecBlockId(), + (UINT16)squelch.GetFecSymbolId()); break; - case NormCmdMsg::NORM_CMD_FLUSH: - DMSG(0, " obj>%hu blk>%lu seg>%hu", - (UINT16)msg.cmd.flush.GetObjectId(), - (UINT32)msg.cmd.flush.GetFecBlockId(), - (UINT16)msg.cmd.flush.GetFecSymbolId()); + } + case NormCmdMsg::FLUSH: + { + const NormCmdFlushMsg& flush = (const NormCmdFlushMsg&)msg; + DMSG(0, " obj>%hu blk>%lu seg>%hu ", + (UINT16)flush.GetObjectId(), + (UINT32)flush.GetFecBlockId(), + (UINT16)flush.GetFecSymbolId()); break; + } + case NormCmdMsg::CC: + { + const NormCmdCCMsg& cc = (const NormCmdCCMsg&)msg; + DMSG(0, " seq>%u ", cc.GetCCSequence()); + NormHeaderExtension ext; + while (cc.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_RATE == ext.GetType()) + { + UINT16 sendRate = ((NormCCRateExtension&)ext).GetSendRate(); + DMSG(0, " rate>%f ", (8.0/1000.0) * NormUnquantizeRate(sendRate)); + break; + } + } + break; + } default: break; } - DMSG(0, " "); + break; + } + + case NormMsg::ACK: + case NormMsg::NACK: + { + // look for NormCCFeedback extension + NormHeaderExtension ext; + while (msg.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) + { + clrFlag = ((NormCCFeedbackExtension&)ext).CCFlagIsSet(NormCC::CLR); + break; + } + } + DMSG(0, "%s ", MSG_NAME[msgType]); break; } @@ -815,59 +868,107 @@ void NormTrace(NormNodeId localId, NormMessage& msg, bool sent) DMSG(0, "%s ", MSG_NAME[msgType]); break; } - DMSG(0, "len>%hu\n", length); + DMSG(0, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); } // end NormTrace(); -void NormSession::HandleReceiveMessage(NormMessage& msg, bool wasUnicast) +void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast) { - ASSERT(this == session_mgr.top_session); // Drop some rx messages for testing - if (UniformRand(100.0) < rx_loss_rate) return; - - if (trace) NormTrace(LocalNodeId(), msg, false); - switch (msg.generic.GetType()) + if (UniformRand(100.0) < rx_loss_rate) { - case NORM_MSG_INFO: - //DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_INFO)\n"); - if (IsClient()) ClientHandleObjectMessage(msg); + return; + } + + + struct timeval currentTime; + ::GetSystemTime(¤tTime); + + if (trace) NormTrace(currentTime, LocalNodeId(), msg, false); + + // Do common updates for servers we already know. + NormNodeId sourceId = msg.GetSourceId(); + NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(sourceId); + if (theServer) + { + // (TBD) combine these as needed for efficiency + // (i.e. fewer function calls per message) + theServer->UpdateRecvRate(currentTime, msg.GetLength()); + theServer->UpdateLossEstimate(currentTime, msg.GetSequence()); + theServer->SetAddress(msg.GetSource()); + + // for statistics only (TBD) #ifdef NORM_DEBUG + theServer->IncrementRecvTotal(msg.GetLength()); + } + + switch (msg.GetType()) + { + case NormMsg::INFO: + //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::INFO)\n"); + if (IsClient()) ClientHandleObjectMessage(currentTime, (NormObjectMsg&)msg, theServer); break; - case NORM_MSG_DATA: - //DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_DATA) ...\n"); - if (IsClient()) ClientHandleObjectMessage(msg); + case NormMsg::DATA: + //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::DATA) ...\n"); + if (IsClient()) ClientHandleObjectMessage(currentTime, (NormObjectMsg&)msg, theServer); break; - case NORM_MSG_CMD: - if (IsClient()) ClientHandleCommand(msg); + case NormMsg::CMD: + //DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::CMD) ...\n"); + if (IsClient()) ClientHandleCommand(currentTime, (NormCmdMsg&)msg, theServer); break; - case NORM_MSG_NACK: - DMSG(4, "NormSession::HandleReceiveMessage(NORM_MSG_NACK) node>%lu ...\n", + case NormMsg::NACK: + DMSG(4, "NormSession::HandleReceiveMessage(NormMsg::NACK) node>%lu ...\n", LocalNodeId()); - if (IsServer() && (msg.nack.GetServerId() == LocalNodeId())) - ServerHandleNackMessage(msg.nack); - if (IsClient()) ClientHandleNackMessage(msg.nack); + if (IsServer() && (((NormNackMsg&)msg).GetServerId() == LocalNodeId())) + { + ServerHandleNackMessage(currentTime, (NormNackMsg&)msg); + if (wasUnicast && (backoff_factor > 0.5)) + { + // for suppression of unicast nack feedback + advertise_repairs = true; + if (!tx_timer.IsActive()) + { + tx_timer.SetInterval(0.0); + InstallTimer(&tx_timer); + } + } + } + if (IsClient()) ClientHandleNackMessage((NormNackMsg&)msg); break; - case NORM_MSG_ACK: - case NORM_MSG_REPORT: - case NORM_MSG_INVALID: - DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_INVALID)\n"); + case NormMsg::ACK: + DMSG(4, "NormSession::HandleReceiveMessage(NormMsg::ACK) node>%lu ...\n", + LocalNodeId()); + if (IsServer() && (((NormAckMsg&)msg).GetServerId() == LocalNodeId())) + ServerHandleAckMessage(currentTime, (NormAckMsg&)msg, wasUnicast); + if (IsClient()) + ClientHandleAckMessage((NormAckMsg&)msg); + break; + + case NormMsg::REPORT: + case NormMsg::INVALID: + DMSG(0, "NormSession::HandleReceiveMessage(NormMsg::INVALID)\n"); break; } } // end NormSession::HandleReceiveMessage() -void NormSession::ClientHandleObjectMessage(NormMessage& msg) +void NormSession::ClientHandleObjectMessage(const struct timeval& currentTime, + const NormObjectMsg& msg, + NormServerNode* theServer) { - NormNodeId serverId = msg.generic.GetSender(); - NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(serverId); if (!theServer) { - if ((theServer = new NormServerNode(this, serverId))) + if ((theServer = new NormServerNode(this, msg.GetSourceId()))) { server_tree.AttachNode(theServer); DMSG(4, "NormSession::ClientHandleObjectMessage() node>%lu new remote server:%lu ...\n", - LocalNodeId(), serverId); + LocalNodeId(), msg.GetSourceId()); + theServer->UpdateRecvRate(currentTime, msg.GetLength()); + theServer->UpdateLossEstimate(currentTime, msg.GetSequence()); + theServer->SetAddress(msg.GetSource()); + // for statistics only (TBD) #ifdef NORM_DEBUG + theServer->IncrementRecvTotal(msg.GetLength()); } else { @@ -877,26 +978,28 @@ void NormSession::ClientHandleObjectMessage(NormMessage& msg) return; } } - // for statistics only (TBD) #ifdef NORM_DEBUG - theServer->IncrementRecvTotal(msg.generic.GetLength()); - theServer->SetAddress(msg.generic.GetSource()); theServer->HandleObjectMessage(msg); - } // end NormSession::ClientHandleObjectMessage() -void NormSession::ClientHandleCommand(NormMessage& msg) +void NormSession::ClientHandleCommand(const struct timeval& currentTime, + const NormCmdMsg& cmd, + NormServerNode* theServer) { - NormNodeId serverId = msg.generic.GetSender(); - NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(serverId); if (!theServer) { //DMSG(0, "NormSession::ClientHandleCommand() node>%lu recvd command from unknown server ...\n", // LocalNodeId()); - if ((theServer = new NormServerNode(this, serverId))) + if ((theServer = new NormServerNode(this, cmd.GetSourceId()))) { + server_tree.AttachNode(theServer); DMSG(4, "NormSession::ClientHandleCommand() node>%lu new remote server:%lu ...\n", - LocalNodeId(), serverId); + LocalNodeId(), cmd.GetSourceId()); + theServer->UpdateRecvRate(currentTime, cmd.GetLength()); + theServer->UpdateLossEstimate(currentTime, cmd.GetSequence()); + theServer->SetAddress(cmd.GetSource()); + // for statistics only (TBD) #ifdef NORM_DEBUG + theServer->IncrementRecvTotal(cmd.GetLength()); } else { @@ -906,19 +1009,15 @@ void NormSession::ClientHandleCommand(NormMessage& msg) return; } } - // for statistics only (TBD) #ifdef NORM_DEBUG - theServer->IncrementRecvTotal(msg.generic.GetLength()); - theServer->SetAddress(msg.generic.GetSource()); - theServer->HandleCommand(msg.cmd); + theServer->HandleCommand(currentTime, cmd); } // end NormSession::ClientHandleCommand() -void NormSession::ServerUpdateGrttEstimate(const struct timeval& grttResponse) +double NormSession::CalculateRtt(const struct timeval& currentTime, + const struct timeval& grttResponse) { if (grttResponse.tv_sec || grttResponse.tv_usec) { - struct timeval currentTime; - GetSystemTime(¤tTime); double clientRtt; // Calculate rtt estimate for this client and process the response if (currentTime.tv_usec < grttResponse.tv_usec) @@ -937,39 +1036,309 @@ void NormSession::ServerUpdateGrttEstimate(const struct timeval& grttResponse) } // Lower limit on RTT (because of coarse timer resolution on some systems, // this can sometimes actually end up a negative value!) - if (clientRtt < 1.0e-06) clientRtt = 1.0e-06; - grtt_response = true; - if (clientRtt > grtt_current_peak) - { - // Immediately incorporate bigger RTT's - grtt_current_peak = clientRtt; - if (clientRtt > grtt_measured) - { - grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; - grtt_measured = 0.25 * grtt_measured + 0.75 * clientRtt; - if (grtt_measured > grtt_max) grtt_measured = grtt_max; - double pktInterval = ((double)(44+segment_size))/tx_rate; - grtt_advertised = MAX(pktInterval, grtt_measured); - UINT8 grttQuantizedOld = grtt_quantized; - grtt_quantized = NormQuantizeRtt(grtt_advertised); - // Recalculate grtt_advertise since quantization rounds upward - grtt_advertised = NormUnquantizeRtt(grtt_quantized); - if (grttQuantizedOld != grtt_quantized) - DMSG(4, "NormSession::ServerUpdateGrttEstimate() node>%lu new grtt: %lf sec.\n", - LocalNodeId(), grtt_advertised); - } - } + // (TBD) this should be system clock granularity? + return (clientRtt < 1.0e-06) ? 1.0e-06 : clientRtt; } + else + { + return -1.0; + } +} // end NormSession::CalculateRtt() + +void NormSession::ServerUpdateGrttEstimate(double clientRtt) +{ + + grtt_response = true; + if (clientRtt > grtt_current_peak) + { + // Immediately incorporate bigger RTT's + grtt_current_peak = clientRtt; + if (clientRtt > grtt_measured) + { + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + grtt_measured = 0.25 * grtt_measured + 0.75 * clientRtt; + if (grtt_measured > grtt_max) grtt_measured = grtt_max; + double pktInterval = 2.0 * ((double)(44+segment_size))/tx_rate; + UINT8 grttQuantizedOld = grtt_quantized; + grtt_quantized = NormQuantizeRtt(MAX(pktInterval, grtt_measured)); + // Calculate grtt_advertised since quantization rounds upward + grtt_advertised = NormUnquantizeRtt(grtt_quantized); + if (grttQuantizedOld != grtt_quantized) + DMSG(4, "NormSession::ServerUpdateGrttEstimate() node>%lu new grtt: %lf sec.\n", + LocalNodeId(), grtt_advertised); + } + } } // end NormSession::ServerUpdateGrttEstimate() +double NormSession::CalculateRate(double size, double rtt, double loss) +{ + double denom = rtt * (sqrt((2.0/3.0)*loss) + + (12.0 * sqrt((3.0/8.0)*loss) * loss * + (1.0 + 32.0*loss*loss))); + return (size / denom); +} // end NormSession::CalculateRate() -void NormSession::ServerHandleNackMessage(NormNackMsg& nack) +void NormSession::ServerHandleCCFeedback(NormNodeId nodeId, + UINT8 ccFlags, + double ccRtt, + double ccLoss, + double ccRate, + UINT8 ccSequence) +{ + // Keep track of current suppressing feedback + // (non-CLR, lowest rate, unconfirmed RTT) + if (0 == (ccFlags & NormCC::CLR)) + { + if (suppress_rate < 0.0) + { + suppress_rate = ccRate; + suppress_rtt = ccRtt; + suppress_nonconfirmed = (0 == (ccFlags & NormCC::RTT)); + } + else + { + if (ccRate < suppress_rate) suppress_rate = ccRate; + if (ccRtt > suppress_rtt) suppress_rtt = ccRtt; + if (0 == (ccFlags & NormCC::RTT)) suppress_nonconfirmed = true; + } + } + if (!cc_enable) return; + + // Adjust ccRtt if we have state on this nodeId + NormCCNode* node = (NormCCNode*)cc_node_list.FindNodeById(nodeId); + if (node) ccRtt = node->UpdateRtt(ccRtt); + if (0 == (ccFlags & NormCC::START)) + { + // slow start has ended + cc_slow_start = false; + // adjust rate using current rtt for node + ccRate = CalculateRate(nominal_packet_size, ccRtt, ccLoss); + } + //TRACE("NormSession::ServerHandleCCFeedback() node>%lu rate>%lf (rtt>%lf loss>%lf slow_start>%d)\n", + // nodeId, ccRate * 8.0 / 1000.0, ccRtt, ccLoss, (0 != (ccFlags & NormCC::START))); + + // Keep the active CLR (if there is one) at the head of the list + NormNodeListIterator iterator(cc_node_list); + NormCCNode* next = (NormCCNode*)iterator.GetNextNode(); + // 1) Does this response replace the active CLR? + if (next && next->IsActive()) + { + if (ccRate < next->GetRate() || (nodeId == next->Id())) + { + NormNodeId savedId = next->Id(); + bool savedRttStatus = next->HasRtt(); + double savedRtt = next->GetRtt(); + double savedLoss = next->GetLoss(); + double savedRate = next->GetRate(); + UINT8 savedSequence = next->GetCCSequence(); + + next->SetId(nodeId); + next->SetClrStatus(true); + next->SetRttStatus(0 != (ccFlags & NormCC::RTT)); + next->SetLoss(ccLoss); + next->SetRate(ccRate); + next->SetCCSequence(ccSequence); + next->SetActive(true); + if (next->Id() == nodeId) + { + // This was feedback from the current CLR + AdjustRate(true); + return; + } + else + { + next->SetRtt(ccRtt); + AdjustRate(true); + } + ccFlags = 0; + nodeId = savedId; + if (savedRttStatus) + ccFlags = NormCC::RTT; + ccRtt = savedRtt; + ccLoss = savedLoss; + ccRate = savedRate, + ccSequence = savedSequence; + } + } + else + { + // There was no active CLR + if (!next) + { + if ((next = new NormCCNode(this, nodeId))) + { + cc_node_list.Append(next); + } + else + { + DMSG(0, "NormSession::ServerHandleCCFeedback() memory allocation error: %s\n", + strerror(errno)); + return; + } + } + next->SetId(nodeId); + next->SetClrStatus(true); + next->SetRttStatus(0 != (ccFlags & NormCC::RTT)); + next->SetRtt(ccRtt); + next->SetLoss(ccLoss); + next->SetRate(ccRate); + next->SetCCSequence(ccSequence); + next->SetActive(true); + AdjustRate(true); + return; + } + + // 2) Go through cc_node_list and find lowest priority candidate + NormCCNode* candidate = NULL; + if (cc_node_list.GetCount() < 5) + { + if ((candidate = new NormCCNode(this, nodeId))) + { + cc_node_list.Append(candidate); + } + else + { + DMSG(0, "NormSession::ServerHandleCCFeedback() memory allocation error: %s\n", + strerror(errno)); + } + } + else + { + while ((next = (NormCCNode*)iterator.GetNextNode())) + { + if (next->Id() == nodeId) + { + candidate = next; + break; + } + else if (candidate) + { + if (candidate->IsActive() && !next->IsActive()) + { + candidate = next; + continue; + } + if (!next->HasRtt() && candidate->HasRtt()) + continue; + else if (!candidate->HasRtt() && next->HasRtt()) + candidate = next; + else if (candidate->GetRate() < next->GetRate()) + candidate = next; + } + else + { + candidate = next; + continue; + } + } + } + + // 3) Replace candidate if this response is higher precedence + if (candidate) + { + bool haveRtt = (0 != (ccFlags && NormCC::RTT)); + bool replace; + if (candidate->Id() == nodeId) + replace = true; + else if (!candidate->IsActive()) + replace = true; + else if (!haveRtt && candidate->HasRtt()) + replace = true; + else if (haveRtt && !candidate->HasRtt()) + replace = false; + else if (ccRate < candidate->GetRate()) + replace = true; + else + replace = false; + if (replace) + { + candidate->SetId(nodeId); + candidate->SetClrStatus(false); + candidate->SetRttStatus(0 != (ccFlags & NormCC::RTT)); + candidate->SetRtt(ccRtt); + candidate->SetLoss(ccLoss); + candidate->SetRate(ccRate); + candidate->SetCCSequence(ccSequence); + candidate->SetActive(true); + } + } +} // end NormSession::ServerHandleCCFeedback() + + +void NormSession::ServerHandleAckMessage(const struct timeval& currentTime, const NormAckMsg& ack, bool wasUnicast) +{ + // Update GRTT estimate + struct timeval grttResponse; + ack.GetGrttResponse(grttResponse); + double clientRtt = CalculateRtt(currentTime, grttResponse); + if (clientRtt >= 0.0) ServerUpdateGrttEstimate(clientRtt); + + // Look for NORM-CC Feedback header extension + NormCCFeedbackExtension ext; + while (ack.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) + { + ServerHandleCCFeedback(ack.GetSourceId(), + ext.GetCCFlags(), + clientRtt >= 0.0 ? + clientRtt : NormUnquantizeRtt(ext.GetCCRtt()), + NormUnquantizeLoss(ext.GetCCLoss()), + NormUnquantizeRate(ext.GetCCRate()), + ext.GetCCSequence()); + } + break; + } + + if (wasUnicast && probe_proactive) + { + // for suppression of unicast feedback + advertise_repairs = true; + if (!tx_timer.IsActive()) + { + tx_timer.SetInterval(0.0); + InstallTimer(&tx_timer); + } + } + + switch (ack.GetAckType()) + { + case NormAck::CC: + // Everything is in the ACK header for this one + break; + + // (TBD) Handle other acknowledgement types + + default: + DMSG(0, "NormSession::ServerHandleAckMessage() node>%lu received " + "unsupported ack type:%d\n", LocalNodeId(), ack.GetAckType()); + } +} // end ServerHandleAckMessage() + +void NormSession::ServerHandleNackMessage(const struct timeval& currentTime, NormNackMsg& nack) { // Update GRTT estimate struct timeval grttResponse; nack.GetGrttResponse(grttResponse); - ServerUpdateGrttEstimate(grttResponse); - gsize_nack_count++; + double clientRtt = CalculateRtt(currentTime, grttResponse); + if (clientRtt >= 0.0) ServerUpdateGrttEstimate(clientRtt); + + // Look for NORM-CC Feedback header extension + NormCCFeedbackExtension ext; + while (nack.GetNextExtension(ext)) + { + if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) + { + ServerHandleCCFeedback(nack.GetSourceId(), + ext.GetCCFlags(), + clientRtt >= 0.0 ? + clientRtt : NormUnquantizeRtt(ext.GetCCRtt()), + NormUnquantizeLoss(ext.GetCCLoss()), + NormUnquantizeRate(ext.GetCCRate()), + ext.GetCCSequence()); + } + break; + } // Parse and process NACK UINT16 requestOffset = 0; @@ -983,7 +1352,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) NormBlockId prevBlockId; bool startTimer = false; - UINT16 numErasures = 0; + UINT16 numErasures = extra_parity; bool squelchQueued = false; @@ -1013,8 +1382,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) txBlockIndex = 0; } - bool holdoff = (repair_timer.IsActive() && !repair_timer.RepeatCount()); - + bool holdoff = (repair_timer.IsActive() && !repair_timer.RepeatCount()); enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; while ((requestLength = nack.UnpackRepairRequest(req, requestOffset))) { @@ -1036,12 +1404,15 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) NormRepairRequest::Iterator iterator(req); NormObjectId nextObjectId, lastObjectId; NormBlockId nextBlockId, lastBlockId; + UINT16 nextBlockLen, lastBlockLen; NormSegmentId nextSegmentId, lastSegmentId; - while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, &nextSegmentId)) + while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, + &nextBlockLen, &nextSegmentId)) { if (NormRepairRequest::RANGES == requestForm) { - if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, &lastSegmentId)) + if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, + &lastBlockLen, &lastSegmentId)) { DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu recvd incomplete RANGE request!\n", LocalNodeId()); @@ -1053,6 +1424,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) { lastObjectId = nextObjectId; lastBlockId = nextBlockId; + lastBlockLen = nextBlockLen; lastSegmentId = nextSegmentId; } @@ -1218,7 +1590,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) { if (NormObject::STREAM == object->GetType()) { - DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu " + DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu " "recvd repair request for old stream block(%lu) ...\n", LocalNodeId(), (UINT32)nextBlockId); inRange = false; @@ -1232,7 +1604,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) else { // Resource constrained, move on. - DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu " + DMSG(2, "NormSession::ServerHandleNackMessage() node>%lu " "Warning - server is resource contrained ...\n"); inRange = false; continue; @@ -1240,7 +1612,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) } } freshBlock = false; - numErasures = 0; + numErasures = extra_parity; prevBlockId = nextBlockId; } // If stream && explicit data repair, lock the data for retransmission @@ -1297,7 +1669,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) inRange = false; continue; } - } + } // end (object->IsStream() && (nextSegmentId < ndata)) // With a series of SEGMENT repair requests for a block, "numErasures" will // eventually total the number of missing segments in the block. @@ -1306,8 +1678,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) { if (nextObjectId > txObjectIndex) { - if (object->TxUpdateBlock(block, nextSegmentId, - lastSegmentId, numErasures)) + if (object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures)) { if (!tx_pending_mask.Set(nextObjectId)) DMSG(0, "NormSession::ServerHandleNackMessage() tx_pending_mask.Set(%hu) error (3)\n", @@ -1318,21 +1689,17 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) { if (nextBlockId >= txBlockIndex) { - object->TxUpdateBlock(block, nextSegmentId, - lastSegmentId, numErasures); + object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); } else if (1 == (txBlockIndex - nextBlockId)) { NormSegmentId firstPending = block->FirstPending(); if (nextSegmentId > firstPending) - object->TxUpdateBlock(block, nextSegmentId, - lastSegmentId, numErasures); + object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); else if (lastSegmentId > firstPending) - object->TxUpdateBlock(block, firstPending, - lastSegmentId, numErasures); + object->TxUpdateBlock(block, firstPending, lastSegmentId, numErasures); else if (numErasures > block->ParityCount()) - object->TxUpdateBlock(block, firstPending, - firstPending, numErasures); + object->TxUpdateBlock(block, firstPending, firstPending, numErasures); } } } @@ -1340,7 +1707,7 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) { block->HandleSegmentRequest(nextSegmentId, lastSegmentId, ndata, nparity, numErasures); startTimer = true; - } + } // end if/else (holdoff) inRange = false; break; @@ -1354,15 +1721,40 @@ void NormSession::ServerHandleNackMessage(NormNackMsg& nack) } // end while(UnpackRepairRequest()) if (startTimer && !repair_timer.IsActive()) { - repair_timer.SetInterval(grtt_advertised * (backoff_factor + 1.0)); + // BACKOFF related code + double aggregateInterval = address.IsMulticast() ? grtt_advertised * (backoff_factor + 1.0) : + 0.0; + aggregateInterval = (backoff_factor > 0.0) ? aggregateInterval : 0.0;//grtt_advertised / 100.0; + + if (tx_timer.IsActive()) + { + double txTimeout = tx_timer.TimeRemaining() - 1.0e-06; + aggregateInterval = MAX(txTimeout, aggregateInterval); + } + + repair_timer.SetInterval(aggregateInterval); DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu starting server " - "NACK aggregation timer (%lf sec)...\n", LocalNodeId(), repair_timer.Interval()); - gsize_nack_count = 1; + "NACK aggregation timer (%lf sec)...\n", LocalNodeId(), aggregateInterval); InstallTimer(&repair_timer); } } // end NormSession::ServerHandleNackMessage() -void NormSession::ClientHandleNackMessage(NormNackMsg& nack) + +void NormSession::ClientHandleAckMessage(const NormAckMsg& ack) +{ + NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(ack.GetServerId()); + if (theServer) + { + theServer->HandleAckMessage(ack); + } + else + { + DMSG(4, "NormSession::ClientHandleAckMessage() node>%lu heard ACK for unknown server.\n", + LocalNodeId()); + } +} // end NormSession::ClientHandleAckMessage() + +void NormSession::ClientHandleNackMessage(const NormNackMsg& nack) { NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(nack.GetServerId()); if (theServer) @@ -1371,8 +1763,8 @@ void NormSession::ClientHandleNackMessage(NormNackMsg& nack) } else { - DMSG(4, "NormSession::ClientHandleNackMessage() node>%lu heard NACK for unkown server\n", - (UINT32)nack.GetServerId()); + DMSG(4, "NormSession::ClientHandleNackMessage() node>%lu heard NACK for unknown server\n", + LocalNodeId()); } } // end NormSession::ClientHandleNackMessage() @@ -1380,23 +1772,23 @@ void NormSession::ClientHandleNackMessage(NormNackMsg& nack) bool NormSession::ServerQueueSquelch(NormObjectId objectId) { // (TBD) if a squelch is already queued, update it if (objectId < squelch->objectId) - NormMessage* msg = GetMessageFromPool(); - if (msg) + NormCmdSquelchMsg* squelch = (NormCmdSquelchMsg*)GetMessageFromPool(); + if (squelch) { - msg->generic.SetType(NORM_MSG_CMD); - msg->generic.SetDestination(address); - msg->cmd.generic.SetFlavor(NormCmdMsg::NORM_CMD_SQUELCH); - NormCmdSquelchMsg& squelch = (NormCmdSquelchMsg&)msg->cmd.squelch; + squelch->Init(); + squelch->SetDestination(address); + squelch->SetGrtt(grtt_quantized); + squelch->SetGroupSize(gsize_quantized); NormObject* obj = tx_table.Find(objectId); NormObjectTable::Iterator iterator(tx_table); NormObjectId nextId; if (obj) { ASSERT(NormObject::STREAM == obj->GetType()); - squelch.SetObjectId(objectId); - squelch.SetFecBlockId(((NormStreamObject*)obj)->StreamBufferLo()); - squelch.SetFecSymbolId(0); - squelch.ResetInvalidObjectList(); + squelch->SetObjectId(objectId); + squelch->SetFecBlockId(((NormStreamObject*)obj)->StreamBufferLo()); + squelch->SetFecSymbolId(0); + squelch->ResetInvalidObjectList(); while ((obj = iterator.GetNextObject())) if (objectId == obj->Id()) break; nextId = objectId + 1; @@ -1406,20 +1798,20 @@ bool NormSession::ServerQueueSquelch(NormObjectId objectId) obj = iterator.GetNextObject(); if (obj) { - squelch.SetObjectId(obj->Id()); + squelch->SetObjectId(obj->Id()); if (obj->IsStream()) - squelch.SetFecBlockId(((NormStreamObject*)obj)->StreamBufferLo()); + squelch->SetFecBlockId(((NormStreamObject*)obj)->StreamBufferLo()); else - squelch.SetFecBlockId(0); - squelch.SetFecSymbolId(0); + squelch->SetFecBlockId(0); + squelch->SetFecSymbolId(0); nextId = obj->Id() + 1; } else { // Squelch to point to future object - squelch.SetObjectId(next_tx_object_id); - squelch.SetFecBlockId(0); - squelch.SetFecSymbolId(0); + squelch->SetObjectId(next_tx_object_id); + squelch->SetFecBlockId(0); + squelch->SetFecSymbolId(0); nextId = next_tx_object_id; } } @@ -1428,7 +1820,7 @@ bool NormSession::ServerQueueSquelch(NormObjectId objectId) { while (nextId != obj->Id()) { - if (!squelch.AppendInvalidObject(nextId, segment_size)) + if (!squelch->AppendInvalidObject(nextId, segment_size)) { buildingList = false; break; @@ -1436,8 +1828,8 @@ bool NormSession::ServerQueueSquelch(NormObjectId objectId) nextId++; } } - QueueMessage(msg); - DMSG(2, "NormSession::ServerQueueSquelch() node>%lu server queued squelch ...\n", + QueueMessage(squelch); + DMSG(4, "NormSession::ServerQueueSquelch() node>%lu server queued squelch ...\n", LocalNodeId()); return true; } @@ -1450,17 +1842,96 @@ bool NormSession::ServerQueueSquelch(NormObjectId objectId) } // end NormSession::ServerQueueSquelch() -void NormSession::ServerUpdateGroupSize() // (TBD) not doing anything yet +bool NormSession::ServerBuildRepairAdv(NormCmdRepairAdvMsg& cmd) { - // Possible approach ... use smoothed history of gsize_nack_count information - // to adjust a group size estimator (if needed to keep the nack count average - // with an acceptable volume) there may be some issue here since either - // underestimating _or_ overestimating the group size may reduce - // NACK suppression performance ... but it's somewhat safer to overestimate - // and a check could be provided to help guess if we're over or under estimating? - gsize_nack_ave = 0.9*gsize_nack_ave + 0.1*(double)gsize_nack_count; - -} // end NormSession::ServerUpdateGroupSize() + // Build a NORM_CMD(REPAIR_ADV) message with current pending repair state. + NormRepairRequest req; + req.SetFlag(NormRepairRequest::OBJECT); + NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; + NormObjectId firstId; + UINT16 objectCount = 0; + NormObjectTable::Iterator iterator(tx_table); + NormObject* nextObject = iterator.GetNextObject(); + while (nextObject) + { + NormObject* currentObject = nextObject; + nextObject = iterator.GetNextObject(); + NormObjectId currentId = currentObject->Id(); + bool repairEntireObject = tx_repair_mask.Test(currentId); + if (repairEntireObject) + { + if (!objectCount) firstId = currentId; // set first OBJECT level repair id + objectCount++; // increment consecutive OBJECT level repair count. + } + + // Check for non-OBJECT level request or end + if (objectCount && (!repairEntireObject || !nextObject)) + { + NormRepairRequest::Form form; + switch (objectCount) + { + case 0: + form = NormRepairRequest::INVALID; + break; + case 1: + case 2: + form = NormRepairRequest::ITEMS; + break; + default: + form = NormRepairRequest::RANGES; + break; + } + if (form != prevForm) + { + if (NormRepairRequest::INVALID != prevForm) + cmd.PackRepairRequest(req); // (TBD) error check; + req.SetForm(form); + cmd.AttachRepairRequest(req, segment_size); + prevForm = form; + } + switch (form) + { + case 0: + ASSERT(0); // can't happen + break; + case 1: + case 2: + req.SetForm(NormRepairRequest::ITEMS); + req.AppendRepairItem(firstId, 0, ndata, 0); // (TBD) error check + if (2 == objectCount) + req.AppendRepairItem(currentId, 0, ndata, 0); // (TBD) error check + break; + default: + req.SetForm(NormRepairRequest::RANGES); + req.AppendRepairRange(firstId, 0, ndata, 0, // (TBD) error check + currentId, 0, ndata, 0); + break; + } + cmd.PackRepairRequest(req); + objectCount = 0; + } + if (!repairEntireObject) + { + if ((NormRepairRequest::INVALID != prevForm) && currentObject->IsRepairPending()) + { + cmd.PackRepairRequest(req); // (TBD) error check; + prevForm = NormRepairRequest::INVALID; + currentObject->AppendRepairAdv(cmd); + } + else + { + currentObject->AppendRepairAdv(cmd); + } + objectCount = 0; + } + } // end while (nextObject) + if (NormRepairRequest::INVALID != prevForm) + { + cmd.PackRepairRequest(req); // (TBD) error check; + prevForm = NormRepairRequest::INVALID; + } + return true; +} // end NormSession::ServerBuildRepairAdv() bool NormSession::OnRepairTimeout() { @@ -1470,9 +1941,6 @@ bool NormSession::OnRepairTimeout() DMSG(4, "NormSession::OnRepairTimeout() node>%lu server NACK aggregation time ended.\n", LocalNodeId()); - // Update our group size estimate (we don't include nacks during repair hold-off interval) - ServerUpdateGroupSize(); - NormObjectTable::Iterator iterator(tx_table); NormObject* obj; while ((obj = iterator.GetNextObject())) @@ -1506,8 +1974,15 @@ bool NormSession::OnRepairTimeout() } } // end while (iterator.GetNextObject()) TouchServer(); + // BACKOFF related code // Holdoff initiation of new repair cycle for one GRTT - repair_timer.SetInterval(grtt_advertised); // repair holdoff interval = 1*GRTT + // (TBD) for unicast sessions, use CLR RTT ??? + //double holdoffInterval = backoff_factor > 0.0 ? grtt_advertised : 0.0; + double holdoffInterval = grtt_advertised; + repair_timer.SetInterval(holdoffInterval); // repair holdoff interval = 1*GRTT + DMSG(4, "NormSession::OnRepairTimeout() node>%lu starting server " + "NACK holdoff timer (%lf sec)...\n", LocalNodeId(), holdoffInterval); + } else { @@ -1519,95 +1994,75 @@ bool NormSession::OnRepairTimeout() } // end NormSession::OnRepairTimeout() - +// (TBD) Should pass current system time to ProtocolTimer timeout handlers +// for more efficiency ... bool NormSession::OnTxTimeout() { - NormMessage* msg = message_queue.RemoveHead(); - if (msg) + NormMsg* msg; + + // (TBD) sometimes need RepairAdv even when cc_enable is false ... + NormCmdRepairAdvMsg adv; + + if (advertise_repairs && (probe_proactive || (repair_timer.IsActive() && + repair_timer.RepeatCount()))) { - // Fill in any last minute timestamps - switch (msg->generic.GetType()) + // Build a NORM_CMD(NACK_ADV) in response to + // receipt of unicast NACK or CC update + adv.Init(); + adv.SetGrtt(grtt_quantized); + adv.SetGroupSize(gsize_quantized); + adv.SetDestination(address); + + // Fill in congestion control header extension + NormCCFeedbackExtension ext; + adv.AttachExtension(ext); + + if (suppress_rate < 0.0) { - case NORM_MSG_CMD: - switch (msg->cmd.generic.GetFlavor()) - { - case NormCmdMsg::NORM_CMD_ACK_REQ: - if (NormCmdAckReqMsg::RTT == msg->cmd.ack_req.GetAckFlavor()) - { - struct timeval currentTime; - GetSystemTime(¤tTime); - msg->cmd.ack_req.SetRttSendTime(currentTime); - } - break; - default: - break; - } - break; - - case NORM_MSG_NACK: - { - NormServerNode* theServer = - (NormServerNode*)server_tree.FindNodeById(msg->nack.GetServerId()); - ASSERT(theServer); - struct timeval grttResponse; - theServer->CalculateGrttResponse(grttResponse); - msg->nack.SetGrttResponse(grttResponse); - // (TBD) fill in cc_sequence - // (TBD) fill in loss estimate - break; - } - - case NORM_MSG_ACK: - { - NormServerNode* theServer = - (NormServerNode*)server_tree.FindNodeById(msg->nack.GetServerId()); - ASSERT(theServer); - struct timeval grttResponse; - theServer->CalculateGrttResponse(grttResponse); - //msg->ack.SetGrttResponse(grttResponse); - // (TBD) fill in cc_sequence - // (TBD) fill in loss estimate - break; - } - - default: - break; - } - // Fill in common message fields - msg->generic.SetVersion(1); - msg->generic.SetSequence(tx_sequence++); - msg->generic.SetSender(local_node_id); - UINT16 len = msg->generic.GetLength(); - // Drop some tx messages for testing purposes - bool drop = (UniformRand(100.0) < tx_loss_rate); - if (!drop) - { - if (UDP_SOCKET_ERROR_NONE != - tx_socket.SendTo(&msg->generic.GetDestination(), - msg->generic.AccessBuffer(), len)) - { - DMSG(0, "NormSession::OnTxTimeout() sendto() error\n"); - } - else - { - // Separate send/recv tracing - if (trace) NormTrace(LocalNodeId(), *msg, true); - } + ext.SetCCFlag(NormCC::RTT); + ext.SetCCRtt(grtt_quantized); + ext.SetCCRate(NormQuantizeRate(tx_rate)); } else { - //TRACE("TX MESSAGE DROPPED! (tx_loss_rate:%lf\n", tx_loss_rate); + if (!suppress_nonconfirmed) ext.SetCCFlag(NormCC::RTT); + ext.SetCCRtt(NormQuantizeRtt(suppress_rtt)); + ext.SetCCRate(NormQuantizeRate(suppress_rate)); } - tx_timer.SetInterval(((double)len) / tx_rate); - ReturnMessageToPool(msg); + + ServerBuildRepairAdv(adv); + + msg = (NormMsg*)&adv; + } + else + { + msg = message_queue.RemoveHead(); + advertise_repairs = false; + } + + suppress_rate = -1.0; // reset cc feedback suppression rate + + if (msg) + { + SendMessage(*msg); + if (advertise_repairs) + advertise_repairs = false; + else + ReturnMessageToPool(msg); return true; // reinstall tx_timer } else { + // 1) Prompt for next server message if (IsServer()) Serve(); + if (message_queue.IsEmpty()) { tx_timer.Deactivate(); + // Check that any possible notifications posted in + // the previous call to Serve() may have caused a + // change in server state making it ready to send + if (IsServer()) Serve(); return false; } else @@ -1619,74 +2074,305 @@ bool NormSession::OnTxTimeout() } } // end NormSession::OnTxTimeout() -bool NormSession::OnProbeTimeout() -{ - NormMessage* msg = GetMessageFromPool(); - if (msg) +void NormSession::SendMessage(NormMsg& msg) +{ + struct timeval currentTime; + GetSystemTime(¤tTime); + + bool clientMsg = false; + + // Fill in any last minute timestamps + // (TBD) fill in SessionId fields on all messages as needed + switch (msg.GetType()) { - msg->generic.SetType(NORM_MSG_CMD); - msg->generic.SetDestination(address); - msg->cmd.generic.SetGrtt(grtt_quantized); - msg->cmd.generic.SetGroupSize(gsize_quantized); - msg->cmd.generic.SetFlavor(NormCmdMsg::NORM_CMD_ACK_REQ); - NormCmdAckReqMsg& ackReq = msg->cmd.ack_req; - ackReq.SetAckFlavor(NormCmdAckReqMsg::RTT); - // SetRttSendTime() when message is being sent - ackReq.ResetAckingNodeList(); - QueueMessage(msg); + case NormMsg::INFO: + case NormMsg::DATA: + ((NormObjectMsg&)msg).SetSessionId(session_id); + break; + case NormMsg::CMD: + ((NormCmdMsg&)msg).SetSessionId(session_id); + switch (((NormCmdMsg&)msg).GetFlavor()) + { + case NormCmdMsg::CC: + ((NormCmdCCMsg&)msg).SetSendTime(currentTime); + break; + default: + break; + } + break; + + case NormMsg::NACK: + { + clientMsg = true; + NormNackMsg& nack = (NormNackMsg&)msg; + NormServerNode* theServer = + (NormServerNode*)server_tree.FindNodeById(nack.GetServerId()); + ASSERT(theServer); + struct timeval grttResponse; + theServer->CalculateGrttResponse(currentTime, grttResponse); + nack.SetGrttResponse(grttResponse); + break; + } + + case NormMsg::ACK: + { + clientMsg = true; + NormAckMsg& ack = (NormAckMsg&)msg; + NormServerNode* theServer = + (NormServerNode*)server_tree.FindNodeById(ack.GetServerId()); + ASSERT(theServer); + struct timeval grttResponse; + theServer->CalculateGrttResponse(currentTime, grttResponse); + ack.SetGrttResponse(grttResponse); + break; + } + + default: + break; } + // Fill in common message fields + msg.SetSequence(tx_sequence++); + msg.SetSourceId(local_node_id); + UINT16 msgSize = msg.GetLength(); + // Drop some tx messages for testing purposes + bool drop = (UniformRand(100.0) < tx_loss_rate); + + if (drop || (clientMsg && client_silent)) + { + //TRACE("TX MESSAGE DROPPED! (tx_loss_rate:%lf\n", tx_loss_rate); + } else { - DMSG(0, "NormSession::OnProbeTimeout() node>%lu message_pool empty! can't probe\n", - LocalNodeId()); - } - - if (grtt_response) - { - // Update grtt estimate - if (grtt_current_peak < grtt_measured) + if (UDP_SOCKET_ERROR_NONE != tx_socket.SendTo(&msg.GetDestination(), + msg.GetBuffer(), + msgSize)) { - if (grtt_decrease_delay_count-- == 0) - { - grtt_measured = 0.5 * grtt_measured + - 0.5 * grtt_current_peak; - grtt_current_peak = 0.0; - grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; - } + DMSG(0, "NormSession::SendMessage() sendto() error\n"); } else { - // Increase already incorporated - grtt_current_peak = 0.0; - grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + // Separate send/recv tracing + if (trace) NormTrace(currentTime, LocalNodeId(), msg, true); } - if (grtt_measured < NORM_GRTT_MIN) - grtt_measured = NORM_GRTT_MIN; - else if (grtt_measured > grtt_max) - grtt_measured = grtt_max; - double pktInterval = (double)(44+segment_size)/tx_rate; - grtt_advertised = MAX(pktInterval, grtt_measured); - double grttQuantizedOld = grtt_quantized; - grtt_quantized = NormQuantizeRtt(grtt_advertised); - // Recalculate grtt_advertise since quantization rounds upward - grtt_advertised = NormUnquantizeRtt(grtt_quantized); - grtt_response = false; - if (grttQuantizedOld != grtt_quantized) - DMSG(2, "NormSession::OnProbeTimeout() node>%lu new grtt: %lf\n", - LocalNodeId(), grtt_advertised); + // Keep track of _actual_ sent rate + double interval; + if (prev_update_time.tv_sec || prev_update_time.tv_usec) + { + interval = (double)(currentTime.tv_sec - prev_update_time.tv_sec); + if (currentTime.tv_usec > prev_update_time.tv_sec) + interval += 1.0e-06*(double)(currentTime.tv_usec - prev_update_time.tv_usec); + else + interval -= 1.0e-06*(double)(prev_update_time.tv_usec - currentTime.tv_usec); + } + else + { + sent_rate = ((double)msgSize) / grtt_advertised; + interval = -1.0; + prev_update_time = currentTime; + } + if (interval < grtt_advertised) + { + sent_accumulator += msgSize; + } + else + { + sent_rate = ((double)(sent_accumulator+msgSize)) / interval; + prev_update_time = currentTime; + sent_accumulator = 0; + } + } + nominal_packet_size += 0.05 * (((double)msgSize) - nominal_packet_size); + tx_timer.SetInterval(((double)msgSize) / tx_rate); +} // end NormSession::SendMessage() + +bool NormSession::OnProbeTimeout() +{ + // 1) Update grtt_estimate _if_ sufficient time elapsed. + grtt_age += probe_timer.Interval(); + if (grtt_age >= grtt_interval) + { + if (grtt_response) + { + // Update grtt estimate + if (grtt_current_peak < grtt_measured) + { + if (grtt_decrease_delay_count-- == 0) + { + grtt_measured = 0.5 * grtt_measured + + 0.5 * grtt_current_peak; + grtt_current_peak = 0.0; + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + } + } + else + { + // Increase already incorporated + grtt_current_peak = 0.0; + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + } + if (grtt_measured < NORM_GRTT_MIN) + grtt_measured = NORM_GRTT_MIN; + else if (grtt_measured > grtt_max) + grtt_measured = grtt_max; + double pktInterval = (double)(44+segment_size)/tx_rate; + grtt_advertised = MAX(pktInterval, grtt_measured); + double grttQuantizedOld = grtt_quantized; + grtt_quantized = NormQuantizeRtt(grtt_advertised); + // Recalculate grtt_advertise since quantization rounds upward + grtt_advertised = NormUnquantizeRtt(grtt_quantized); + grtt_response = false; + if (grttQuantizedOld != grtt_quantized) + DMSG(4, "NormSession::OnProbeTimeout() node>%lu new grtt: %lf\n", + LocalNodeId(), grtt_advertised); + } + grtt_age = 0.0; } - // Manage probe_timer interval - if (probe_interval < probe_interval_min) - probe_interval = probe_interval_min; + if (grtt_interval < grtt_interval_min) + grtt_interval = grtt_interval_min; else - probe_interval *= 2.0; - if (probe_interval > probe_interval_max) - probe_interval = probe_interval_max; - probe_timer.SetInterval(probe_interval); + grtt_interval *= 2.0; + if (grtt_interval > grtt_interval_max) + grtt_interval = grtt_interval_max; + + // 2) Build probe message and determine next probe interval + NormCmdCCMsg* cmd = (NormCmdCCMsg*)GetMessageFromPool(); + if (!cmd) + { + DMSG(0, "NormSession::OnProbeTimeout() node>%lu message_pool empty! can't probe\n", + LocalNodeId()); + return true; + } + // Build a NORM_CMD(CC) message + cmd->Init(); + cmd->SetDestination(address); + cmd->SetGrtt(grtt_quantized); + cmd->SetGroupSize(gsize_quantized); + // SetSendTime() when message is being sent (in OnTxTimeout()) + cmd->SetCCSequence(cc_sequence++); + + if (probe_proactive) + { + NormCCRateExtension ext; + cmd->AttachExtension(ext); + ext.SetSendRate(NormQuantizeRate(tx_rate)); + } + + double probeInterval; + if (cc_enable) + { + // Iterate over cc_node_list and append + NormNodeListIterator iterator(cc_node_list); + NormCCNode* next; + while ((next = (NormCCNode*)iterator.GetNextNode())) + { + if (next->IsActive()) + { + UINT8 ccFlags = 0; + if (next->IsClr()) ccFlags |= (UINT8)NormCC::CLR; + ccFlags |= (UINT8)NormCC::RTT; + UINT8 rttQuantized = NormQuantizeRtt(next->GetRtt()); + if (cc_slow_start) ccFlags |= (UINT8)NormCC::START; + UINT16 rateQuantized = NormQuantizeRate(next->GetRate()); + // (TBD) check result + cmd->AppendCCNode(segment_size, + next->Id(), + ccFlags, + rttQuantized, + rateQuantized); + if (!next->IsClr()) next->SetActive(false); + } + } + + AdjustRate(false); + + // Determine next probe_interval + const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head(); + probeInterval = clr ? MIN(grtt_advertised, clr->GetRtt()) : grtt_advertised; + double nominalRate = ((double)segment_size)/((double)tx_rate); + probeInterval = MAX(probeInterval, nominalRate); + } + else + { + // Determine next probe_interval + probeInterval = grtt_interval; + } + + QueueMessage(cmd); + + // 3) Set probe_timer interval + probe_timer.SetInterval(probeInterval); + return true; } // end NormSession::OnProbeTimeout() + +void NormSession::AdjustRate(bool onResponse) +{ + const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head(); + double ccRtt = clr ? clr->GetRtt() : grtt_measured; + double ccLoss = clr ? clr->GetLoss() : 0.0; + if (onResponse) + { + // Adjust rate based on CLR feedback and + // adjust probe schedule + ASSERT(clr); + // (TBD) check feedback age + if (cc_slow_start) + { + double scale = tx_rate / sent_rate; + scale = MAX(1.0, scale); + scale = 1.0; + //TRACE("NormSession::AdjustRate() slow start clr>%lu rate>%lf tx_rate>%lf sent_rate>%lf\n", + // clr->Id(), clr->GetRate() * 8.0/1000.0, tx_rate * 8.0/1000.0, sent_rate * 8.0/1000.0); + tx_rate = clr->GetRate() * scale; + } + else + { + double clrRate = clr->GetRate(); + if (clrRate > tx_rate) + { + + double linRate = tx_rate + segment_size; + tx_rate = MIN(clrRate, linRate); + } + else + { + tx_rate = clrRate; + } + } + //TRACE("NormSession::AdjustRate() regular rate adjust clr>%lu rate>%lf (rtt>%lf loss>%lf slow_start>%d)\n", + // clr->Id(), tx_rate*8.0/1000.0, clr->GetRtt(), clr->GetLoss(), cc_slow_start); + } + else if (clr) + { + // (TBD) fix CC feedback aging ... + /*int feedbackAge = abs((int)cc_sequence - (int)clr->GetCCSequence()); + TRACE("NormSession::AdjustRate() feedback age>%d (%d - %d\n", + feedbackAge, cc_sequence, clr->GetCCSequence()); + + if (feedbackAge > 50) + { + double linRate = tx_rate - segment_size; + linRate = MAX(linRate, 0.0); + double expRate = tx_rate * 0.5; + if (feedbackAge > 4) + tx_rate = MIN(linRate, expRate); + else + tx_rate = MAX(linRate, expRate); + + }*/ + } + double minRate = ((double)segment_size) / grtt_measured; + minRate = MIN((double)segment_size, minRate); + tx_rate = MAX(tx_rate, minRate); + + struct timeval currentTime; + ::GetSystemTime(¤tTime); + double theTime = (double)currentTime.tv_sec + 1.0e-06 * ((double)currentTime.tv_usec); + DMSG(6, "ServerRateTracking time>%lf rate>%lf rtt>%lf loss>%lf\n", theTime, tx_rate * (8.0/1000.0), ccRtt, ccLoss); +} // end NormSession::AdjustRate() + bool NormSession::OnReportTimeout() { // Client reporting (just print out for now) diff --git a/common/normSession.h b/common/normSession.h index fdffd3c..067862d 100644 --- a/common/normSession.h +++ b/common/normSession.h @@ -87,14 +87,17 @@ class NormSession public: enum {DEFAULT_MESSAGE_POOL_DEPTH = 16}; + static const UINT8 DEFAULT_TTL; static const double DEFAULT_TRANSMIT_RATE; // in bytes per second - static const double DEFAULT_PROBE_MIN; - static const double DEFAULT_PROBE_MAX; + static const double DEFAULT_GRTT_INTERVAL_MIN; + static const double DEFAULT_GRTT_INTERVAL_MAX; static const double DEFAULT_GRTT_ESTIMATE; static const double DEFAULT_GRTT_MAX; static const unsigned int DEFAULT_GRTT_DECREASE_DELAY; static const double DEFAULT_BACKOFF_FACTOR; // times GRTT = backoff max static const double DEFAULT_GSIZE_ESTIMATE; + static const UINT16 DEFAULT_NDATA; + static const UINT16 DEFAULT_NPARITY; // General methods const NormNodeId& LocalNodeId() {return local_node_id;} @@ -103,20 +106,22 @@ class NormSession bool IsOpen() {return (rx_socket.IsOpen() || tx_socket.IsOpen());} const NetworkAddress& Address() {return address;} void SetAddress(const NetworkAddress& addr) {address = addr;} + static double CalculateRate(double size, double rtt, double loss); // Session parameters double TxRate() {return (tx_rate * 8.0);} + // (TBD) watch timer scheduling and min/max bounds void SetTxRate(double txRate) {tx_rate = txRate / 8.0;} double BackoffFactor() {return backoff_factor;} void SetBackoffFactor(double value) {backoff_factor = value;} - void SetUnicastNacks(bool state) {unicast_nacks = state;} - bool UnicastNacks() {return unicast_nacks;} void SetLoopback(bool state) { rx_socket.SetLoopback(state); tx_socket.SetLoopback(state); } + bool CongestionControl() {return cc_enable;} + void SetCongestionControl(bool state) {cc_enable = state;} void Notify(NormController::Event event, class NormServerNode* server, @@ -127,11 +132,19 @@ class NormSession notify_pending = false; } - NormMessage* GetMessageFromPool() {return message_pool.RemoveHead();} - void ReturnMessageToPool(NormMessage* msg) {message_pool.Append(msg);} - void QueueMessage(NormMessage* msg); + NormMsg* GetMessageFromPool() {return message_pool.RemoveHead();} + void ReturnMessageToPool(NormMsg* msg) {message_pool.Append(msg);} + void QueueMessage(NormMsg* msg); + void SendMessage(NormMsg& msg); + void InstallTimer(ProtocolTimer* timer) + {session_mgr.InstallTimer(timer);} // Server methods + void ServerSetBaseObjectId(NormObjectId baseId) + { + next_tx_object_id = IsServer() ? next_tx_object_id : baseId; + session_id = IsServer() ? session_id : (UINT16)baseId; + } bool StartServer(unsigned long bufferSpace, UINT16 segmentSize, UINT16 numData, @@ -151,6 +164,9 @@ class NormSession UINT16 ServerAutoParity() {return auto_parity;} void ServerSetAutoParity(UINT16 autoParity) {ASSERT(autoParity <= nparity); auto_parity = autoParity;} + UINT16 ServerExtraParity() {return extra_parity;} + void ServerSetExtraParity(UINT16 extraParity) + {extra_parity = extraParity;} double ServerGroupSize() {return gsize_measured;} void ServerSetGroupSize(double gsize) @@ -172,10 +188,23 @@ class NormSession } char* ServerGetFreeSegment(NormObjectId objectId, NormBlockId blockId); void ServerPutFreeSegment(char* segment) {segment_pool.Put(segment);} + + + void PromptServer() + { + if (!tx_timer.IsActive()) + { + tx_timer.SetInterval(0.0); + InstallTimer(&tx_timer); + } + } + + void TouchServer() { posted_tx_queue_empty = false; - if (!notify_pending) Serve(); + PromptServer(); + //if (!notify_pending) Serve(); } // Client methods @@ -184,8 +213,10 @@ class NormSession bool IsClient() {return is_client;} unsigned long RemoteServerBufferSize() {return remote_server_buffer_size;} - void InstallTimer(ProtocolTimer* timer) - {session_mgr.InstallTimer(timer);} + void SetUnicastNacks(bool state) {unicast_nacks = state;} + bool UnicastNacks() {return unicast_nacks;} + void ClientSetSilent(bool state) {client_silent = state;} + bool ClientIsSilent() {return client_silent;} // Debug settings void SetTrace(bool state) {trace = state;} @@ -210,24 +241,45 @@ class NormSession bool OnTxTimeout(); bool OnRepairTimeout(); - bool OnCommandTimeout(); + bool OnFlushTimeout(); + bool OnWatermarkTimeout(); bool OnProbeTimeout(); bool OnReportTimeout(); bool TxSocketRecvHandler(UdpSocket* theSocket); - bool RxSocketRecvHandler(UdpSocket* theSocket); + bool RxSocketRecvHandler(UdpSocket* theSocket); + void HandleReceiveMessage(NormMsg& msg, bool wasUnicast); - void HandleReceiveMessage(NormMessage& msg, bool wasUnicast); - - void ServerHandleNackMessage(NormNackMsg& nack); - void ServerUpdateGrttEstimate(const struct timeval& grttResponse); + // Server message handling routines + void ServerHandleNackMessage(const struct timeval& currentTime, + NormNackMsg& nack); + void ServerHandleAckMessage(const struct timeval& currentTime, + const NormAckMsg& ack, + bool wasUnicast); + void ServerUpdateGrttEstimate(double clientRtt); + double CalculateRtt(const struct timeval& currentTime, + const struct timeval& grttResponse); + void ServerHandleCCFeedback(NormNodeId nodeId, + UINT8 ccFlags, + double ccRtt, + double ccLoss, + double ccRate, + UINT8 ccSequence); + void AdjustRate(bool onResponse); bool ServerQueueSquelch(NormObjectId objectId); void ServerQueueFlush(); + bool ServerBuildRepairAdv(NormCmdRepairAdvMsg& cmd); void ServerUpdateGroupSize(); - void ClientHandleObjectMessage(NormMessage& msg); - void ClientHandleCommand(NormMessage& msg); - void ClientHandleNackMessage(NormNackMsg& nack); + // Client message handling routines + void ClientHandleObjectMessage(const struct timeval& currentTime, + const NormObjectMsg& msg, + NormServerNode* theServer); + void ClientHandleCommand(const struct timeval& currentTime, + const NormCmdMsg& msg, + NormServerNode* theServer); + void ClientHandleNackMessage(const NormNackMsg& nack); + void ClientHandleAckMessage(const NormAckMsg& ack); NormSessionMgr& session_mgr; bool notify_pending; @@ -248,10 +300,12 @@ class NormSession // Server parameters and state bool is_server; + UINT16 session_id; UINT16 segment_size; UINT16 ndata; UINT16 nparity; UINT16 auto_parity; + UINT16 extra_parity; NormObjectTable tx_table; NormSlidingMask tx_pending_mask; @@ -268,32 +322,51 @@ class NormSession ProtocolTimer flush_timer; int flush_count; bool posted_tx_queue_empty; + ProtocolTimer watermark_timer; + int watermark_count; + // (TBD) watermark_object_id, watermark_block_id, watermark_symbol_id + + // for unicast nack/cc feedback suppression + bool advertise_repairs; + bool suppress_nonconfirmed; + double suppress_rate; + double suppress_rtt; ProtocolTimer probe_timer; // GRTT/congestion control probes - double probe_interval; - double probe_interval_min; - double probe_interval_max; + bool probe_proactive; + + double grtt_interval; // current GRTT update interval + double grtt_interval_min; // minimum GRTT update interval + double grtt_interval_max; // maximum GRTT update interval double grtt_max; unsigned int grtt_decrease_delay_count; bool grtt_response; double grtt_current_peak; double grtt_measured; + double grtt_age; double grtt_advertised; UINT8 grtt_quantized; double gsize_measured; double gsize_advertised; UINT8 gsize_quantized; - unsigned int gsize_nack_count; - double gsize_nack_ave; - double gsize_correction_factor; - double gsize_nack_delta; + + // Server congestion control parameters + bool cc_enable; + UINT8 cc_sequence; + NormNodeList cc_node_list; + bool cc_slow_start; + double sent_rate; // measured sent rate + struct timeval prev_update_time; // for sent_rate measurement + unsigned long sent_accumulator; // for sent_rate measurement + double nominal_packet_size; // Client parameters bool is_client; NormNodeTree server_tree; unsigned long remote_server_buffer_size; bool unicast_nacks; + bool client_silent; // Protocol test/debug parameters bool trace; diff --git a/common/normSimAgent.cpp b/common/normSimAgent.cpp index 7641197..c87b463 100644 --- a/common/normSimAgent.cpp +++ b/common/normSimAgent.cpp @@ -2,25 +2,36 @@ #include -NormSimAgent::NormSimAgent() - : session(NULL), stream(NULL), +// This NORM simulation agent includes support for providing transport +// of a message stream from the MGEN simulation agent with restrictions. The +// current restriction is the MGEN simulation agent + +NormSimAgent::NormSimAgent(ProtocolTimerInstallFunc* timerInstaller, + const void* timerInstallData, + UdpSocketInstallFunc* socketInstaller, + void* socketInstallData) + : session(NULL), address(NULL), port(0), ttl(3), - tx_rate(NormSession::DEFAULT_TRANSMIT_RATE), + tx_rate(NormSession::DEFAULT_TRANSMIT_RATE), + cc_enable(false), unicast_nacks(false), silent_client(false), backoff_factor(NormSession::DEFAULT_BACKOFF_FACTOR), - segment_size(1024), ndata(32), nparity(16), auto_parity(0), + segment_size(1024), ndata(32), nparity(16), auto_parity(0), extra_parity(0), group_size(NormSession::DEFAULT_GSIZE_ESTIMATE), tx_buffer_size(1024*1024), rx_buffer_size(1024*1024), tx_object_size(0), tx_object_interval(0.0), tx_repeat_count(0), tx_repeat_interval(0.0), + stream(NULL), auto_stream(false), push_stream(false), + mgen(NULL), msg_sync(false), mgen_bytes(0), mgen_pending_bytes(0), tracing(false), tx_loss(0.0), rx_loss(0.0) { // Bind NormSessionMgr to this agent and simulation environment - session_mgr.Init(ProtoSimAgent::TimerInstaller, this, - ProtoSimAgent::SocketInstaller, this, + session_mgr.Init(timerInstaller, timerInstallData, + socketInstaller, socketInstallData, static_cast(this)); interval_timer.Init(0.0, 0, (ProtocolTimerOwner*)this, (ProtocolTimeoutFunc)&NormSimAgent::OnIntervalTimeout); + memset(mgen_buffer, 0, 64); } NormSimAgent::~NormSimAgent() @@ -31,30 +42,38 @@ NormSimAgent::~NormSimAgent() const char* const NormSimAgent::cmd_list[] = { - "+debug", - "+log", - "-trace", - "+txloss", - "+rxloss", - "+address", - "+ttl", - "+rate", - "+backoff", - "+input", - "+output", - "+interval", - "+repeat", - "+rinterval", // repeat interval - "+segment", - "+block", - "+parity", - "+auto", - "+gsize", - "+txbuffer", - "+rxbuffer", - "+start", - "-stop", - "+sendFile", + "+debug", // debug level + "+log", // log file name + "-trace", // message tracing + "+txloss", // tx packet loss percent + "+rxloss", // rx packet loss percent + "+address", // session dest addres + "+ttl", // multicast ttl + "+rate", // tx rate + "+cc", // congestion control on/off + "+backoff", // backoff factor 'k' (maxBackoff = k * GRTT) + "+input", // stream input + "+output", // stream output + "+interval", // delay between tx objects + "+repeat", // number of times to repeat tx object set + "+rinterval", // repeat interval + "+segment", // server segment size + "+block", // server blocking size + "+parity", // server parity segments calculated per block + "+auto", // server auto parity count + "+extra", // server extra parity count + "+gsize", // group size estimate + "+txbuffer", // tx buffer size (bytes) + "+rxbuffer", // rx buffer size (bytes) + "+start", // open session and beging activity + "-stop", // cease activity and close session + "+sendFile", // queue a "sim" file for transmission + "+sendStream", // send a simulated NORM stream + "+openStream", // open a stream object for messaging + "+push", // "on" means real-time push stream advancement (non-blocking) + "-flushStream", // flush output stream + "+unicastNacks", // clients will unicast feedback + "+silentClient", // clients will not transmit NULL }; @@ -162,6 +181,22 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) tx_rate = txRate; if (session) session->SetTxRate(txRate); } + else if (!strncmp("cc", cmd, len)) + { + bool ccEnable; + if (!strcmp(val, "on")) + ccEnable = true; + else if (!strcmp(val, "off")) + ccEnable = false; + else + { + DMSG(0, "NormSimAgent::ProcessCommand(cc) invalid argument!\n"); + return false; + } + cc_enable = ccEnable; + if (session) session->SetCongestionControl(ccEnable); + return true; + } else if (!strncmp("backoff", cmd, len)) { double backoffFactor = atof(val); @@ -171,7 +206,7 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) return false; } backoff_factor = backoffFactor; - if (session) session->SetTxRate(backoffFactor); + if (session) session->SetBackoffFactor(backoffFactor); } else if (!strncmp("interval", cmd, len)) { @@ -230,6 +265,17 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) auto_parity = autoParity; if (session) session->ServerSetAutoParity(autoParity); } + else if (!strncmp("extra", cmd, len)) + { + int extraParity = atoi(val); + if ((extraParity < 0) || (extraParity > 254)) + { + DMSG(0, "NormSimAgent::ProcessCommand(extra) invalid value!\n"); + return false; + } + extra_parity = extraParity; + if (session) session->ServerSetExtraParity(extraParity); + } else if (!strncmp("gsize", cmd, len)) { if (1 != sscanf(val, "%lf", &group_size)) @@ -291,6 +337,120 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) DMSG(0, "NormSimAgent::ProcessCommand(sendFile) no session started!\n"); return false; } + } + else if (!strncmp("sendStream", cmd, len)) + { + if (session) + { + if (stream) + { + DMSG(0, "NormSimAgent::ProcessCommand(sendStream) stream already open!\n"); + return false; + } + if (1 != sscanf(val, "%lu", &tx_object_size)) + { + DMSG(0, "NormSimAgent::ProcessCommand(sendStream) invalid buffer size!\n"); + return false; + } + if (!(stream = session->QueueTxStream(tx_object_size))) + { + DMSG(0, "NormSimAgent::ProcessCommand(sendStream) error opening stream!\n"); + return false; + } + auto_stream = true; + } + else + { + DMSG(0, "NormSimAgent::ProcessCommand(sendStream) session not started!\n"); + return false; + } + } + else if (!strncmp("openStream", cmd, len)) + { + if (session) + { + if (stream) + { + DMSG(0, "NormSimAgent::ProcessCommand(openStream) stream already open!\n"); + return false; + } + if (1 != sscanf(val, "%lu", &tx_object_size)) + { + DMSG(0, "NormSimAgent::ProcessCommand(openStream) invalid buffer size!\n"); + return false; + } + if(!(tx_msg_buffer = new char[65536])) + { + DMSG(0, "NormSimAgent::ProcessCommand(openStream) error allocating tx_msg_buffer: %s\n", + strerror(errno)); + return false; + } + if (!(stream = session->QueueTxStream(tx_object_size))) + { + DMSG(0, "NormSimAgent::ProcessCommand(openStream) error opening stream!\n"); + return false; + } + auto_stream = false; + tx_msg_len = tx_msg_index = 0; + } + else + { + DMSG(0, "NormSimAgent::ProcessCommand(openStream) session not started!\n"); + return false; + } + } + else if (!strncmp("flushStream", cmd, len)) + { + if (session) + { + return FlushStream(); + } + else + { + DMSG(0, "NormSimAgent::ProcessCommand(flushStream) session not started!\n"); + return false; + } + } + else if (!strncmp("push", cmd, len)) + { + if (!strcmp(val, "on")) + push_stream = true; + else if (!strcmp(val, "off")) + push_stream = false; + else + { + DMSG(0, "NormSimAgent::ProcessCommand(push) invalid argument!\n"); + return false; + } + return true; + } + else if (!strncmp("unicastNacks", cmd, len)) + { + if (!strcmp(val, "on")) + unicast_nacks = true; + else if (!strcmp(val, "off")) + unicast_nacks = false; + else + { + DMSG(0, "NormSimAgent::ProcessCommand(unicastNacks) invalid argument!\n"); + return false; + } + if (session) session->SetUnicastNacks(unicast_nacks); + return true; + } + else if (!strncmp("silentClient", cmd, len)) + { + if (!strcmp(val, "on")) + silent_client = true; + else if (!strcmp(val, "off")) + silent_client = false; + else + { + DMSG(0, "NormSimAgent::ProcessCommand(silentClient) invalid argument!\n"); + return false; + } + if (session) session->ClientSetSilent(silent_client); + return true; } return true; } // end NormSimAgent::ProcessCommand() @@ -326,6 +486,81 @@ NormSimAgent::CmdType NormSimAgent::CommandType(const char* cmd) return type; } // end NormSimAgent::CommandType() +bool NormSimAgent::SendMessage(unsigned int len, const char* txBuffer) +{ + if (session) + { + if (!stream) + { + if(!(tx_msg_buffer = new char[65536])) + { + DMSG(0, "NormSimAgent::SendMessage() error allocating tx_msg_buffer: %s\n", + strerror(errno)); + return false; + } + if (!(stream = session->QueueTxStream(tx_object_size))) + { + DMSG(0, "NormSimAgent::SendMessage() error opening stream!\n"); + return false; + } + auto_stream = false; + tx_msg_len = tx_msg_index = 0; + } + + if (0 == tx_msg_len) + { + memcpy(tx_msg_buffer, txBuffer, len); + tx_msg_len = len; + OnInputReady(); + return true; + } + else + { + // Message still pending, can't send yet + DMSG(0, "NormSimAgent::SendMessage() input overflow!\n"); + ASSERT(0); + return false; + } + } + else + { + DMSG(0, "NormSimAgent::SendMessage() session not started!\n"); + return false; + } +} // end NormSimAgent::SendMessage() + +void NormSimAgent::OnInputReady() +{ + //TRACE("NormSimAgent::OnInputReady() index:%lu len:%lu\n", + // tx_msg_index, tx_msg_len); + if (tx_msg_index < tx_msg_len) + { + unsigned int bytesWrote = stream->Write(tx_msg_buffer+tx_msg_index, + tx_msg_len - tx_msg_index, + false, false, push_stream); + tx_msg_index += bytesWrote; + if (tx_msg_index == tx_msg_len) + { + // Provide EOM indication to norm stream + stream->Write(NULL, 0, false, true, false); + tx_msg_index = tx_msg_len = 0; + } + } +} // end NormSimAgent::InputReady() + +bool NormSimAgent::FlushStream() +{ + if (stream && session && session->IsServer()) + { + stream->Write(NULL, 0, true, false, false); + return true; + } + else + { + DMSG(0, "NormSimAgent::FlushStream() no output stream to flush\n"); + } +} // end NormSimAgent::FlushStream() + void NormSimAgent::Notify(NormController::Event event, class NormSessionMgr* sessionMgr, @@ -337,13 +572,31 @@ void NormSimAgent::Notify(NormController::Event event, { case TX_QUEUE_EMPTY: // Can queue a new object or write to stream for transmission - if (interval_timer.Interval() > 0.0) + if (object && (object == stream)) { - InstallTimer(interval_timer); + if (auto_stream) + { + // sending a dummy byte stream + char buffer[NormMsg::MAX_SIZE]; + unsigned int count = stream->Write(buffer, segment_size, false, false, false); + } + else + { + // Stream starved, ask for input from "source" ? + OnInputReady(); + } } else - { - OnIntervalTimeout(); + { + // Schedule or queue next "sim file" transmission + if (interval_timer.Interval() > 0.0) + { + InstallTimer(interval_timer); + } + else + { + OnIntervalTimeout(); + } } break; @@ -355,11 +608,19 @@ void NormSimAgent::Notify(NormController::Event event, { case NormObject::STREAM: { - const NormObjectSize& size = object->Size(); + NormObjectSize size; + if (silent_client) + size = NormObjectSize(rx_buffer_size); + else + size = object->Size(); if (!((NormStreamObject*)object)->Accept(size.LSB())) { DMSG(0, "NormSimAgent::Notify(RX_OBJECT_NEW) stream object accept error!\n"); } + if (!stream) + stream = (NormStreamObject*)object; + else + DMSG(0, "NormSimAgent::Notify(RX_OBJECT_NEW) warning! one stream already accepted.\n"); } break; case NormObject::FILE: @@ -399,11 +660,106 @@ void NormSimAgent::Notify(NormController::Event event, case NormObject::STREAM: { // Read the stream when it's updated - char buffer[256]; - unsigned int nBytes; - while ((nBytes = ((NormStreamObject*)object)->Read(buffer, 256))) + if (mgen) + { + bool dataReady = true; + while (dataReady) + { + if (!mgen_pending_bytes) + { + // Read 2 byte MGEN msg len header + unsigned int want = 2 - mgen_bytes; + unsigned int got = want; + bool findMsgSync = msg_sync ? false : true; + if (((NormStreamObject*)object)->Read(mgen_buffer + mgen_bytes, + &got, findMsgSync)) + { + mgen_bytes += got; + msg_sync = true; + if (got != want) dataReady = false; + } + else + { + DMSG(0, "NormSimAgent::Notify(1) detected stream break\n"); + mgen_bytes = mgen_pending_bytes = 0; + msg_sync = false; + continue; + } + if (2 == mgen_bytes) + { + UINT16 msgSize; + memcpy(&msgSize, mgen_buffer, sizeof(UINT16)); + mgen_pending_bytes = ntohs(msgSize) - 2; + } + } + if (mgen_pending_bytes) + { + // Save the first part for MGEN logging + if (mgen_bytes < 64) + { + unsigned int want = MIN(mgen_pending_bytes, 62); + unsigned int got = want; + if (((NormStreamObject*)object)->Read(mgen_buffer+mgen_bytes, + &got)) + { + mgen_pending_bytes -= got; + mgen_bytes += got; + if (got != want) dataReady = false; + } + else + { + DMSG(0, "NormSimAgent::Notify(2) detected stream break\n"); + mgen_bytes = mgen_pending_bytes = 0; + msg_sync = false; + continue; + } + } + while (dataReady && mgen_pending_bytes) + { + char buffer[256]; + unsigned int want = MIN(256, mgen_pending_bytes); + unsigned int got = want; + if (((NormStreamObject*)object)->Read(buffer, &got)) + { + mgen_pending_bytes -= got; + mgen_bytes += got; + if (got != want) dataReady = false; + } + else + { + DMSG(0, "NormSimAgent::Notify(3) detected stream break\n"); + mgen_bytes = mgen_pending_bytes = 0; + msg_sync = false; + break; + } + } + if (msg_sync && (0 == mgen_pending_bytes)) + { + const NetworkAddress& srcAddr = server->GetAddress(); + mgen->HandleMgenMessage(mgen_buffer, mgen_bytes, srcAddr); + mgen_bytes = 0; + } + } // end if (mgen_pending_bytes) + } // end while(dataReady) + } + else { - + char buffer[1024]; + unsigned int want = 1024; + unsigned int got = want; + while (1) + { + if (((NormStreamObject*)object)->Read(buffer, &got)) + { + // Break when data is no longer available + if (got != want) break; + } + else + { + DMSG(0, "NormSimAgent::Notify() detected stream break\n"); + } + got = want = 1024; + } } break; } @@ -440,7 +796,7 @@ bool NormSimAgent::OnIntervalTimeout() { if (stream) { - + // (TBD) } else { @@ -457,7 +813,7 @@ bool NormSimAgent::OnIntervalTimeout() else { // Done - interval_timer.Deactivate(); + if (interval_timer.IsActive()) interval_timer.Deactivate(); return false; } @@ -484,20 +840,21 @@ bool NormSimAgent::StartServer() { // Common session parameters session->SetTxRate(tx_rate); + session->SetCongestionControl(cc_enable); session->SetBackoffFactor(backoff_factor); session->SetTrace(tracing); session->SetTxLoss(tx_loss); session->SetRxLoss(rx_loss); - + session->ServerSetGroupSize(group_size); // StartServer(bufferMax, segmentSize, fec_ndata, fec_nparity) if (!session->StartServer(tx_buffer_size, segment_size, ndata, nparity)) { DMSG(0, "NormSimAgent::OnStartup() start server error!\n"); session_mgr.Destroy(); return false; - } + } session->ServerSetAutoParity(auto_parity); - session->ServerSetGroupSize(group_size); + session->ServerSetExtraParity(extra_parity); return true; } else @@ -534,6 +891,9 @@ bool NormSimAgent::StartClient() session->SetTxLoss(tx_loss); session->SetRxLoss(rx_loss); + session->SetUnicastNacks(unicast_nacks); + session->ClientSetSilent(silent_client); + // StartClient(bufferSize) if (!session->StartClient(rx_buffer_size)) { @@ -558,6 +918,7 @@ void NormSimAgent::Stop() if (session->IsClient()) session->StopClient(); session_mgr.DeleteSession(session); session = NULL; + stream = NULL; } } // end NormSimAgent::StopServer() diff --git a/common/normSimAgent.h b/common/normSimAgent.h index 1265767..7b799c6 100644 --- a/common/normSimAgent.h +++ b/common/normSimAgent.h @@ -2,38 +2,40 @@ // normSimAgent.h - Generic (base class) NORM simulation agent #include "protoLib.h" +#include "protoSim.h" #include "normSession.h" +#include "mgen.h" // for MGEN instance attachment + // Base class for Norm simulation agents (e.g. ns-2, OPNET, etc) class NormSimAgent : public NormController { public: - NormSimAgent(); virtual ~NormSimAgent(); - void Init(ProtocolTimerInstallFunc* timerInstaller, - const void* timerInstallData, - UdpSocketInstallFunc* socketInstaller, - void* socketInstallData) - { - session_mgr.Init(timerInstaller, timerInstallData, - socketInstaller, socketInstallData, - this); - } bool ProcessCommand(const char* cmd, const char* val); // Note: don't allow client _and_ server operation at same time bool StartServer(); bool StartClient(); + bool IsActive() {return (NULL != session);} void Stop(); + bool SendMessage(unsigned int len, const char* txBuffer); + void AttachMgen(Mgen* mgenInstance) {mgen = mgenInstance;} protected: + NormSimAgent(ProtocolTimerInstallFunc* timerInstaller, + const void* timerInstallData, + UdpSocketInstallFunc* socketInstaller, + void* socketInstallData); enum CmdType {CMD_INVALID, CMD_NOARG, CMD_ARG}; CmdType CommandType(const char* cmd); virtual unsigned long GetAgentId() = 0; private: + void OnInputReady(); + bool FlushStream(); virtual void Notify(NormController::Event event, class NormSessionMgr* sessionMgr, class NormSession* session, @@ -49,18 +51,21 @@ class NormSimAgent : public NormController NormSessionMgr session_mgr; NormSession* session; - NormStreamObject* stream; // session parameters char* address; // session address UINT16 port; // session port number UINT8 ttl; double tx_rate; // bits/sec + bool cc_enable; + bool unicast_nacks; + bool silent_client; double backoff_factor; UINT16 segment_size; UINT8 ndata; UINT8 nparity; UINT8 auto_parity; + UINT8 extra_parity; double group_size; unsigned long tx_buffer_size; // bytes unsigned long rx_buffer_size; // bytes @@ -70,6 +75,18 @@ class NormSimAgent : public NormController double tx_object_interval; int tx_repeat_count; double tx_repeat_interval; + + NormStreamObject* stream; + bool auto_stream; + bool push_stream; + char* tx_msg_buffer; + unsigned int tx_msg_len; + unsigned int tx_msg_index; + Mgen* mgen; + char mgen_buffer[64]; + bool msg_sync; + unsigned int mgen_bytes; + unsigned int mgen_pending_bytes; ProtocolTimer interval_timer; diff --git a/common/version.h b/common/version.h index 572cb6f..fb37e98 100644 --- a/common/version.h +++ b/common/version.h @@ -32,6 +32,6 @@ #ifndef _NORM_VERSION #define _NORM_VERSION -#define VERSION "1.0x1" +#define VERSION "1.0b2" #endif // _NORM_VERSION diff --git a/ns/README.txt b/ns/README.txt index 42b8f3c..7dc5c0d 100644 --- a/ns/README.txt +++ b/ns/README.txt @@ -25,9 +25,12 @@ suppress.tcl - Executable TCL script which iteratively invokes "ns" with the "simplenorm.tcl" script and "nc" (nackCount) to evaluate NORM NACK suppression performance. - +ns-2.1b9-Makefile.in - Patched version of ns-2.1b9 Makefile.in + with NORM, MDP & PROTOLIB stuff included. + (MDP stuff can be removed if desired) + ns-2.1b7a-Makefile.in - Patched version of ns-2.1b7a Makefile.in with NORM & PROTOLIB stuff included. diff --git a/ns/example.tcl b/ns/example.tcl index 3618dbe..d5c7579 100644 --- a/ns/example.tcl +++ b/ns/example.tcl @@ -33,7 +33,7 @@ for {set i 1} {$i <= $numNodes} {incr i} { puts "Creating topology ..." for {set i 1} {$i <= $numNodes} {incr i} { - $ns_ duplex-link $n(0) $n($i) 64kb 100ms DropTail + $ns_ duplex-link $n(0) $n($i) 1Mb 1ms DropTail $ns_ queue-limit $n(0) $n($i) 100 #$ns_ duplex-link-op $n(0) $n($i) orient right $ns_ duplex-link-op $n(0) $n($i) queuePos 0.5 @@ -53,7 +53,7 @@ puts "Configuring NORM agents ..." # 5) Configure global NORM agent parameters (debugging/logging is global) # (Uncomment the "log" command to direct NORM debug output to a file) -$norm(1) debug 2 +$norm(1) debug 1 #$norm(1) log normLog.txt # 6) Configure NORM server agent at node 1 @@ -63,7 +63,7 @@ $norm(1) block 1 $norm(1) parity 0 $norm(1) repeat -1 $norm(1) interval 0.0 -$norm(1) txloss 10.0 +$norm(1) txloss 0.0 # Enabl NORM message tracing at the server #$norm(1) trace @@ -71,7 +71,7 @@ $norm(1) txloss 10.0 for {set i 2} {$i <= $numNodes} {incr i} { $norm($i) address $group/5000 $norm($i) rxbuffer 100000 - $norm($i) txloss 10.0 + $norm($i) rxloss 0.0 #$norm($i) trace } diff --git a/ns/normcc.tcl b/ns/normcc.tcl new file mode 100644 index 0000000..3b44ab9 --- /dev/null +++ b/ns/normcc.tcl @@ -0,0 +1,253 @@ + +# This script exercises NORM-CC versus TCP flows over +# a single bottleneck topology. There is one NORM flow +# and possibly multiple TCP flows. The number of receivers +# in the NORM "group" can vary (default gsize:10) + +proc normcc {optionList} { + +# normcc usage +set usage {Usage: ns normcc.tcl [brate ][tcp ][gsize ]} +append usage {[rate ][duration ][backoff ][trace]} + +#Some default parameters for NORM +set groupSize "10" +set backoffFactor "4.0" +set sendRate "32kb" +set bottleneckRate "1Mb" +set duration "240.0" +set nsTracing 1 +set numTcp 1 +set queueType "DropTail" +set queueSize 100 + +#Parse optionList for parameters +set state flag +foreach option $optionList { + switch -- $state { + "flag" { + switch -glob -- $option { + "gsize" {set state "gsize"} + "rate" {set state "rate"} + "brate" {set state "brate"} + "backoff" {set state "backoff"} + "duration" {set state "duration"} + "tcp" {set state "tcp"} + "red" {set queueType "RED"} + "trace" {set nsTracing 1} + default { + puts "normcc: Bad option $option" + puts "$usage" + exit + } + } + + } + "gsize" { + set groupSize $option + set state "flag" + } + "backoff" { + set backoffFactor $option + set state "flag" + } + "brate" { + set bottleneckRate $option + set state "flag" + } + "rate" { + set sendRate $option + set state "flag" + } + "duration" { + set duration $option + set state "flag" + } + "tcp" { + set numTcp $option + set state "flag" + } + default { + error "normcc: Bad option parse state!" + } + } +} + + +# 1) An ns-2 simulator instance is created an configured +# for multicast operation with a dense mode (DM) multicast +# routing protocol: +set ns_ [new Simulator -multicast on] +$ns_ multicast + + +# 2) Trace files are opened and ns and nam +# tracing is enabled: +if {$nsTracing} { + set f [open normcc.tr w] + $ns_ trace-all $f + set nf [open normcc.nam w] + $ns_ namtrace-all $nf +} else { + set f 0 + set nf 0 +} + + +# 3) Create a single bottleneck +# Link 0 <-> 1 is our bottleneck +set n(0) [$ns_ node] +set n(1) [$ns_ node] +puts "normcc: Creating bottleneck link ..." +$ns_ duplex-link $n(0) $n(1) $bottleneckRate 10ms $queueType +$ns_ queue-limit $n(0) $n(1) $queueSize +$ns_ duplex-link-op $n(0) $n(1) queuePos 0.5 + +if {"RED" == $queueType} { + set redq [[$ns_ link $n(0) $n(1)] queue] + $redq set thresh_ [expr int($queueSize/10 + 0.9)] + $redq set maxthresh_ [expr int($queueSize/2 +0.9)] + $redq set linterm_ 10 +} + + +# Non-bottleneck link rate is 10 * bottleneckRate +set linkRate [expr 10 * [bw_parse $bottleneckRate]] + +# 4) Create a single NORM server and link to bottleneck +puts "normcc: Creating NORM server ..." +set n(2) [$ns_ node] +set norm_server [new Agent/NORM] +$ns_ attach-agent $n(2) $norm_server +# Link from NORM server to bottleneck +$ns_ duplex-link $n(2) $n(0) $linkRate 1ms DropTail +$ns_ queue-limit $n(2) $n(0) 100 +$ns_ duplex-link-op $n(2) $n(0) queuePos 0.5 + +# 5) Create nodes with TCP sources and links to bottleneck +if {$numTcp > 0} { + puts "normcc: Creating TCP sources ..." +} +for {set i 0} {$i < $numTcp} {incr i} { + set k [expr $i + 3] + set n($k) [$ns_ node] + set tcp_src($i) [new Agent/TCP/FullTcp] + $ns_ attach-agent $n($k) $tcp_src($i) + $tcp_src($i) set window_ 100 + $tcp_src($i) set packetSize_ 512 + # Links from TCP sources to bottleneck + $ns_ duplex-link $n($k) $n(0) $linkRate 1ms DropTail + $ns_ queue-limit $n($k) $n(0) 100 + $ns_ duplex-link-op $n($k) $n(0) queuePos 0.5 + # Attach FTP app to tcp sources + set ftp($i) [new Application/FTP] + $ftp($i) attach-agent $tcp_src($i) +} + +# 6) Create nodes with NORM clients and links to bottleneck +puts "normcc: Creating NORM clients ..." +for {set i 0} {$i < $groupSize} {incr i} { + set k [expr $i + 3 + $numTcp] + set n($k) [$ns_ node] + set norm_client($i) [new Agent/NORM] + $ns_ attach-agent $n($k) $norm_client($i) + # Links from bottleneck to NORM clients + $ns_ duplex-link $n(1) $n($k) $linkRate 1ms DropTail + $ns_ queue-limit $n(1) $n($k) 100 + $ns_ duplex-link-op $n(1) $n($k) queuePos 0.5 +} + +# 7) Create nodes with TCP sinks and links to bottleneck +puts "normcc: Creating TCP sinks..." +for {set i 0} {$i < $numTcp} {incr i} { + set k [expr $i + 4 + $numTcp + $groupSize] + set n($k) [$ns_ node] + set tcp_sink($i) [new Agent/TCP/FullTcp] + $ns_ attach-agent $n($k) $tcp_sink($i) + # Links from bottleneck to TCP sinks + $ns_ duplex-link $n(1) $n($k) $linkRate 1ms DropTail + $ns_ queue-limit $n(1) $n($k) 100 + $ns_ duplex-link-op $n(1) $n($k) queuePos 0.5 + # Connect tcp sources->sinks + $ns_ connect $tcp_src($i) $tcp_sink($i) + $tcp_sink($i) listen + +} + +# 8) Configure multicast routing for topology +set mproto DM +set mrthandle [$ns_ mrtproto $mproto {}] + if {$mrthandle != ""} { + $mrthandle set_c_rp [list $n(1)] +} + +# 9) Allocate a multicast address to use +set group [Node allocaddr] + +puts "normcc: Configuring NORM agents ..." + +# 10) Configure global NORM agent commands (using norm_server) +$norm_server debug 2 +#$norm_server log normLog.txt + +# 11) Configure NORM server parameters +$norm_server address $group/5000 +$norm_server rate [bw_parse $sendRate] +$norm_server cc on +$norm_server backoff $backoffFactor + +$norm_server segment 532 +$norm_server block 64 +$norm_server parity 32 + +$norm_server repeat -1 +$norm_server interval 0.0 +$norm_server txloss 0.0 +$norm_server gsize $groupSize +#$norm_server trace + +# 12) Configure NORM client parameters +for {set i 0} {$i < $groupSize} {incr i} { + $norm_client($i) address $group/5000 + $norm_client($i) backoff $backoffFactor + $norm_client($i) rxbuffer 1000000 + $norm_client($i) txloss 0.0 + $norm_client($i) gsize $groupSize + #$norm_client($i) trace +} + +# 13) Start NORM server and clients +$ns_ at 0.0 "$norm_server start server" +for {set i 0} {$i < $groupSize} {incr i} { + $ns_ at 0.0 "$norm_client($i) start client" +} +$ns_ at 0.0 "$norm_server sendStream 1000000" + +# 14) Start tcp flows +for {set i 0} {$i < $numTcp} {incr i} { + $ns_ at 0.0 "$ftp($i) start" +} + +# 15) Schedule finish time +$ns_ at $duration "finish $ns_ $f $nf $nsTracing" + +puts "normcc: Running simulation (tcp:$numTcp gsize:$groupSize rate:$sendRate backoff $backoffFactor duration:$duration) ..." +$ns_ run + +} + +proc finish {ns_ f nf nsTracing} { + puts "normcc: Done." + $ns_ flush-trace + if {$nsTracing} { + close $f + close $nf + } + $ns_ halt + delete $ns_ +} + +# Run a set of trials with optional command-line parameters +puts "Running normcc: $argv" +normcc $argv + diff --git a/ns/ns-2.1b9-Makefile.in b/ns/ns-2.1b9-Makefile.in new file mode 100644 index 0000000..153dfc2 --- /dev/null +++ b/ns/ns-2.1b9-Makefile.in @@ -0,0 +1,536 @@ +# Copyright (c) 1994, 1995, 1996 +# The Regents of the University of California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that: (1) source code distributions +# retain the above copyright notice and this paragraph in its entirety, (2) +# distributions including binary code include the above copyright notice and +# this paragraph in its entirety in the documentation or other materials +# provided with the distribution, and (3) all advertising materials mentioning +# features or use of this software display the following acknowledgement: +# ``This product includes software developed by the University of California, +# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of +# the University nor the names of its contributors may be used to endorse +# or promote products derived from this software without specific prior +# written permission. +# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# @(#) $Header: /cvsroot/norm/norm/ns/ns-2.1b9-Makefile.in,v 1.1 2002/11/18 16:14:07 adamson Exp $ (LBL) + +# +# Various configurable paths (remember to edit Makefile.in, not Makefile) +# + +# Top level hierarchy +prefix = @prefix@ +# Pathname of directory to install the binary +BINDEST = @prefix@/bin +# Pathname of directory to install the man page +MANDEST = @prefix@/man + +BLANK = # make a blank space. DO NOT add anything to this line + +# The following will be redefined under Windows (see WIN32 lable below) +CC = @CC@ +CPP = @CXX@ +LINK = $(CPP) +MKDEP = ./conf/mkdep +TCLSH = @V_TCLSH@ +TCL2C = @V_TCL2CPP@ +AR = ar rc $(BLANK) + +RANLIB = @V_RANLIB@ +INSTALL = @INSTALL@ +LN = ln +TEST = test +RM = rm -f +MV = mv +PERL = @PERL@ + +# for diffusion +#DIFF_INCLUDES = "./diffusion3/main ./diffusion3/lib ./diffusion3/nr ./diffusion3/ns" + +CCOPT = @V_CCOPT@ +STATIC = @V_STATIC@ +LDFLAGS = $(STATIC) +LDOUT = -o $(BLANK) + +DEFINE = -DTCP_DELAY_BIND_ALL -DNO_TK -DNIXVECTOR @V_DEFINE@ @V_DEFINES@ @DEFS@ -DPGM -DPGM_DEBUG -DNS_DIFFUSION + + +PROTOLIB_INCLUDES = -Iprotolib/common -Iprotolib/ns +MDP_INCLUDES = -Imdp/common -Imdp/ns -Imdp/unix +NORM_INCLUDES = -Inorm/common -Inorm/ns + +INCLUDES = \ + -I. @V_INCLUDE_X11@ \ + @V_INCLUDES@ \ + -I./tcp -I./common -I./link -I./queue \ + -I./adc -I./apps -I./mac -I./mobile -I./trace \ + -I./routing -I./tools -I./classifier -I./mcast \ + -I./diffusion3/main -I./diffusion3/lib \ + -I./diffusion3/nr -I./diffusion3/ns -I./asim/ \ + $(PROTOLIB_INCLUDES) $(MDP_INCLUDES) $(NORM_INCLUDES) + + +LIB = \ + @V_LIBS@ \ + @V_LIB_X11@ \ + @V_LIB@ \ + -lm @LIBS@ +# -L@libdir@ \ + +# These flags work on Linux +PROTOLIB_CFLAGS = -g -DUNIX -DNS2 -DPROTO_DEBUG -DSIMULATE -DHAVE_ASSERT -DHAVE_DIRFD + +CFLAGS = $(CCOPT) $(DEFINE) $(PROTOLIB_CFLAGS) + +# Explicitly define compilation rules since SunOS 4's make doesn't like gcc. +# Also, gcc does not remove the .o before forking 'as', which can be a +# problem if you don't own the file but can write to the directory. +# PROTOLIB/MDP/NORM code uses .cpp files so we added a suffix and rule +.SUFFIXES: .cc .cpp # $(.SUFFIXES) + +.cc.o: + @rm -f $@ + $(CPP) -c $(CFLAGS) $(INCLUDES) -o $@ $*.cc + +.c.o: + @rm -f $@ + $(CC) -c $(CFLAGS) $(INCLUDES) -o $@ $*.c + +.cpp.o: + @rm -f $@ + $(CC) -c $(CFLAGS) $(INCLUDES) -o $@ $*.cpp + + +GEN_DIR = gen/ +LIB_DIR = lib/ +NS = ns +NSX = nsx +NSE = nse + +# To allow conf/makefile.win overwrite this macro +# We will set these two macros to empty in conf/makefile.win since VC6.0 +# does not seem to support the STL in gcc 2.8 and up. +OBJ_STL = linkstate/ls.o linkstate/rtProtoLS.o \ + pgm/classifier-pgm.o pgm/pgm-agent.o pgm/pgm-sender.o \ + pgm/pgm-receiver.o pgm/rcvbuf.o \ + diffusion3/nr/nr.o diffusion3/lib/dr.o \ + diffusion3/ns/diffagent.o diffusion3/ns/diffrtg.o \ + diffusion3/ns/difftimer.o diffusion3/main/diffusion.o \ + diffusion3/main/attrs.o diffusion3/main/iodev.o \ + diffusion3/main/events.o diffusion3/main/message.o \ + diffusion3/main/hashutils.o diffusion3/main/stats.o \ + diffusion3/main/tools.o diffusion3/main/drivers/rpc_stats.o \ + diffusion3/apps/sysapps/gradient.o \ + diffusion3/apps/sysapps/log.o \ + diffusion3/lib/diffapp.o \ + diffusion3/apps/agents/ping_sender.o \ + diffusion3/apps/agents/ping_receiver.o \ + diffusion3/apps/agents/ping_common.o \ + diffusion3/apps/gear/geo-attr.o \ + diffusion3/apps/gear/geo-routing.o \ + diffusion3/apps/gear/geo-tools.o + +NS_TCL_LIB_STL = tcl/rtglib/ns-rtProtoLS.tcl \ + tcl/lib/ns-diffusion.tcl + +# WIN32: uncomment the following line to include specific make for VC++ +# !include + +OBJ_CC = \ + tools/random.o tools/rng.o tools/ranvar.o common/misc.o common/timer-handler.o \ + common/scheduler.o common/object.o common/packet.o \ + common/ip.o routing/route.o common/connector.o common/ttl.o \ + trace/trace.o trace/trace-ip.o \ + classifier/classifier.o classifier/classifier-addr.o \ + classifier/classifier-hash.o \ + classifier/classifier-virtual.o \ + classifier/classifier-mcast.o \ + classifier/classifier-bst.o \ + classifier/classifier-mpath.o mcast/replicator.o \ + classifier/classifier-mac.o \ + classifier/classifier-port.o src_rtg/classifier-sr.o \ + src_rtg/sragent.o src_rtg/hdr_src.o adc/ump.o \ + apps/app.o apps/telnet.o tcp/tcplib-telnet.o \ + tools/trafgen.o trace/traffictrace.o tools/pareto.o \ + tools/expoo.o tools/cbr_traffic.o \ + adc/tbf.o adc/resv.o adc/sa.o tcp/saack.o \ + tools/measuremod.o adc/estimator.o adc/adc.o adc/ms-adc.o \ + adc/timewindow-est.o adc/acto-adc.o \ + adc/pointsample-est.o adc/salink.o adc/actp-adc.o \ + adc/hb-adc.o adc/expavg-est.o\ + adc/param-adc.o adc/null-estimator.o \ + adc/adaptive-receiver.o apps/vatrcvr.o adc/consrcvr.o \ + common/agent.o common/message.o apps/udp.o \ + common/session-rtp.o apps/rtp.o tcp/rtcp.o \ + common/ivs.o \ + tcp/tcp.o tcp/tcp-sink.o tcp/tcp-reno.o \ + tcp/tcp-newreno.o \ + tcp/tcp-vegas.o tcp/tcp-rbp.o tcp/tcp-full.o tcp/rq.o \ + baytcp/tcp-full-bay.o baytcp/ftpc.o baytcp/ftps.o \ + tcp/scoreboard.o tcp/tcp-sack1.o tcp/tcp-fack.o \ + tcp/tcp-asym.o tcp/tcp-asym-sink.o tcp/tcp-fs.o \ + tcp/tcp-asym-fs.o \ + tcp/tcp-int.o tcp/chost.o tcp/tcp-session.o \ + tcp/nilist.o \ + tools/integrator.o tools/queue-monitor.o \ + tools/flowmon.o tools/loss-monitor.o \ + queue/queue.o queue/drop-tail.o \ + adc/simple-intserv-sched.o queue/red.o \ + queue/semantic-packetqueue.o queue/semantic-red.o \ + tcp/ack-recons.o \ + queue/sfq.o queue/fq.o queue/drr.o queue/srr.o queue/cbq.o \ + link/hackloss.o queue/errmodel.o queue/fec.o\ + link/delay.o tcp/snoop.o \ + gaf/gaf.o \ + link/dynalink.o routing/rtProtoDV.o common/net-interface.o \ + mcast/ctrMcast.o mcast/mcast_ctrl.o mcast/srm.o \ + common/sessionhelper.o queue/delaymodel.o \ + mcast/srm-ssm.o mcast/srm-topo.o \ + apps/mftp.o apps/mftp_snd.o apps/mftp_rcv.o \ + apps/codeword.o \ + routing/alloc-address.o routing/address.o \ + $(LIB_DIR)int.Vec.o $(LIB_DIR)int.RVec.o \ + $(LIB_DIR)dmalloc_support.o \ + webcache/http.o webcache/tcp-simple.o webcache/pagepool.o \ + webcache/inval-agent.o webcache/tcpapp.o webcache/http-aux.o \ + webcache/mcache.o webcache/webtraf.o \ + empweb/empweb.o \ + empweb/empftp.o \ + realaudio/realaudio.o \ + mac/lanRouter.o classifier/filter.o \ + common/pkt-counter.o \ + common/Decapsulator.o common/Encapsulator.o \ + common/encap.o \ + mac/channel.o mac/mac.o mac/ll.o mac/mac-802_11.o \ + mac/mac-802_3.o mac/mac-tdma.o \ + mobile/mip.o mobile/mip-reg.o mobile/gridkeeper.o \ + mobile/propagation.o mobile/tworayground.o \ + mobile/antenna.o mobile/omni-antenna.o \ + mobile/shadowing.o mobile/shadowing-vis.o \ + common/bi-connector.o common/node.o \ + common/mobilenode.o \ + mac/arp.o mobile/god.o mobile/dem.o \ + mobile/topography.o mobile/modulation.o \ + queue/priqueue.o \ + mac/phy.o mac/wired-phy.o mac/wireless-phy.o \ + mac/mac-timers.o trace/cmu-trace.o mac/varp.o \ + dsdv/dsdv.o dsdv/rtable.o queue/rtqueue.o \ + routing/rttable.o \ + imep/imep.o imep/dest_queue.o imep/imep_api.o \ + imep/imep_rt.o imep/rxmit_queue.o imep/imep_timers.o \ + imep/imep_util.o imep/imep_io.o \ + tora/tora.o tora/tora_api.o tora/tora_dest.o \ + tora/tora_io.o tora/tora_logs.o tora/tora_neighbor.o \ + dsr/dsragent.o dsr/hdr_sr.o dsr/mobicache.o dsr/path.o \ + dsr/requesttable.o dsr/routecache.o dsr/add_sr.o \ + dsr/dsr_proto.o dsr/flowstruct.o dsr/linkcache.o \ + dsr/simplecache.o dsr/sr_forwarder.o \ + aodv/aodv_logs.o aodv/aodv.o \ + aodv/aodv_rtable.o aodv/aodv_rqueue.o \ + common/ns-process.o \ + satellite/satgeometry.o satellite/sathandoff.o \ + satellite/satlink.o satellite/satnode.o \ + satellite/satposition.o satellite/satroute.o \ + satellite/sattrace.o \ + rap/raplist.o rap/rap.o rap/media-app.o rap/utilities.o \ + common/fsm.o tcp/tcp-abs.o \ + diffusion/diffusion.o diffusion/diff_rate.o diffusion/diff_prob.o \ + diffusion/diff_sink.o diffusion/flooding.o diffusion/omni_mcast.o \ + diffusion/hash_table.o diffusion/routing_table.o diffusion/iflist.o \ + tcp/tfrc.o tcp/tfrc-sink.o mobile/energy-model.o apps/ping.o tcp/tcp-rfc793edu.o \ + queue/rio.o queue/semantic-rio.o tcp/tcp-sack-rh.o tcp/scoreboard-rh.o \ + plm/loss-monitor-plm.o plm/cbr-traffic-PP.o \ + linkstate/hdr-ls.o \ + mpls/classifier-addr-mpls.o mpls/ldp.o mpls/mpls-module.o \ + routing/rtmodule.o classifier/classifier-hier.o \ + routing/addr-params.o \ + nix/hdr_nv.o nix/classifier-nix.o \ + nix/nixnode.o \ + routealgo/rnode.o \ + routealgo/bfs.o \ + routealgo/rbitmap.o \ + routealgo/rlookup.o \ + routealgo/routealgo.o \ + nix/nixvec.o \ + nix/nixroute.o \ + diffserv/dsred.o diffserv/dsredq.o \ + diffserv/dsEdge.o diffserv/dsCore.o \ + diffserv/dsPolicy.o diffserv/ew.o\ + queue/red-pd.o queue/pi.o queue/vq.o queue/rem.o \ + queue/gk.o \ + pushback/rate-limit.o pushback/rate-limit-strategy.o \ + pushback/ident-tree.o pushback/agg-spec.o \ + pushback/logging-data-struct.o \ + pushback/rate-estimator.o \ + pushback/pushback-queue.o pushback/pushback.o \ + common/parentnode.o trace/basetrace.o \ + common/simulator.o asim/asim.o \ + @V_STLOBJ@ + + +# don't allow comments to follow continuation lines + +# mac-csma.o mac-multihop.o\ +# sensor-nets/landmark.o mac-simple-wireless.o \ +# sensor-nets/tags.o sensor-nets/sensor-query.o \ +# sensor-nets/flood-agent.o \ + +# what was here before is now in emulate/ +OBJ_C = + +OBJ_COMPAT = $(OBJ_GETOPT) common/win32.o +#XXX compat/win32x.o compat/tkConsole.o + +OBJ_EMULATE_CC = \ + emulate/net-ip.o \ + emulate/net.o \ + emulate/tap.o \ + emulate/ether.o \ + emulate/internet.o \ + emulate/ping_responder.o \ + emulate/arp.o \ + emulate/icmp.o \ + emulate/net-pcap.o \ + emulate/nat.o \ + emulate/iptap.o \ + emulate/tcptap.o + +OBJ_EMULATE_C = \ + emulate/inet.o + +OBJ_GEN = $(GEN_DIR)version.o $(GEN_DIR)ns_tcl.o $(GEN_DIR)ptypes.o + +OBJ_PROTOLIB_CPP = \ + protolib/ns/nsProtoAgent.o protolib/common/protoSim.o \ + protolib/common/networkAddress.o protolib/common/protocolTimer.o \ + protolib/common/debug.o + +OBJ_MDP_CPP = \ + mdp/ns/nsMdpAgent.o mdp/common/mdpSimAgent.o \ + mdp/common/mdpBitMask.o mdp/common/mdpMessage.o \ + mdp/common/mdpEncoder.o mdp/common/galois.o \ + mdp/common/mdpSession.o mdp/common/mdpMsgHandler.o \ + mdp/common/mdpNode.o mdp/common/mdpObject.o \ + mdp/unix/mdpFile.o + +OBJ_NORM_CPP = \ + norm/ns/nsNormAgent.o norm/common/normSimAgent.o \ + norm/common/normMessage.o norm/common/normSession.o \ + norm/common/normNode.o norm/common/normObject.o \ + norm/common/normSegment.o norm/common/normBitmask.o \ + norm/common/normEncoder.o norm/common/galois.o \ + norm/common/normFile.o + +OBJ_CPP = $(OBJ_PROTOLIB_CPP) $(OBJ_MDP_CPP) $(OBJ_NORM_CPP) + +SRC = $(OBJ_C:.o=.c) $(OBJ_CC:.o=.cc) $(OBJ_CPP:.o=.cpp) \ + $(OBJ_EMULATE_C:.o=.c) $(OBJ_EMULATE_CC:.o=.cc) \ + common/tclAppInit.cc common/tkAppInit.cc + +OBJ = $(OBJ_C) $(OBJ_CC) $(OBJ_GEN) $(OBJ_COMPAT) $(OBJ_CPP) + +CLEANFILES = ns nse nsx ns.dyn $(OBJ) $(OBJ_EMULATE_CC) \ + $(OBJ_EMULATE_C) common/tclAppInit.o \ + $(GEN_DIR)* $(NS).core core core.$(NS) core.$(NSX) core.$(NSE) \ + common/ptypes2tcl common/ptypes2tcl.o + +SUBDIRS=\ + indep-utils/cmu-scen-gen/setdest \ + indep-utils/webtrace-conv/dec \ + indep-utils/webtrace-conv/epa \ + indep-utils/webtrace-conv/nlanr \ + indep-utils/webtrace-conv/ucb + +BUILD_NSE = @build_nse@ + +all: $(NS) $(BUILD_NSE) all-recursive + + +all-recursive: + for i in $(SUBDIRS); do ( cd $$i; $(MAKE) all; ) done + +$(NS): $(OBJ) common/tclAppInit.o Makefile + $(LINK) $(LDFLAGS) $(LDOUT)$@ \ + common/tclAppInit.o $(OBJ) $(LIB) + +Makefile: Makefile.in + @echo "Makefile.in is newer than Makefile." + @echo "You need to re-run configure." + false + +$(NSE): $(OBJ) common/tclAppInit.o $(OBJ_EMULATE_CC) $(OBJ_EMULATE_C) + $(LINK) $(LDFLAGS) $(LDOUT)$@ \ + common/tclAppInit.o $(OBJ) \ + $(OBJ_EMULATE_CC) $(OBJ_EMULATE_C) $(LIB) + +ns.dyn: $(OBJ) common/tclAppInit.o + $(LINK) $(LDFLAGS) -o $@ \ + common/tclAppInit.o $(OBJ) $(LIB) + +PURIFY = purify -cache-dir=/tmp +ns-pure: $(OBJ) common/tclAppInit.o + $(PURIFY) $(LINK) $(LDFLAGS) -o $@ \ + common/tclAppInit.o $(OBJ) $(LIB) + +NS_TCL_LIB = \ + tcl/lib/ns-compat.tcl \ + tcl/lib/ns-default.tcl \ + tcl/lib/ns-errmodel.tcl \ + tcl/lib/ns-lib.tcl \ + tcl/lib/ns-link.tcl \ + tcl/lib/ns-mobilenode.tcl \ + tcl/lib/ns-sat.tcl \ + tcl/lib/ns-cmutrace.tcl \ + tcl/lib/ns-node.tcl \ + tcl/lib/ns-rtmodule.tcl \ + tcl/lib/ns-hiernode.tcl \ + tcl/lib/ns-packet.tcl \ + tcl/lib/ns-queue.tcl \ + tcl/lib/ns-source.tcl \ + tcl/lib/ns-nam.tcl \ + tcl/lib/ns-trace.tcl \ + tcl/lib/ns-agent.tcl \ + tcl/lib/ns-random.tcl \ + tcl/lib/ns-namsupp.tcl \ + tcl/lib/ns-address.tcl \ + tcl/lib/ns-intserv.tcl \ + tcl/lib/ns-autoconf.tcl \ + tcl/rtp/session-rtp.tcl \ + tcl/lib/ns-mip.tcl \ + tcl/rtglib/dynamics.tcl \ + tcl/rtglib/route-proto.tcl \ + tcl/rtglib/algo-route-proto.tcl \ + tcl/rtglib/ns-rtProtoLS.tcl \ + tcl/interface/ns-iface.tcl \ + tcl/mcast/BST.tcl \ + tcl/mcast/ns-mcast.tcl \ + tcl/mcast/McastProto.tcl \ + tcl/mcast/DM.tcl \ + tcl/mcast/srm.tcl \ + tcl/mcast/srm-adaptive.tcl \ + tcl/mcast/srm-ssm.tcl \ + tcl/mcast/timer.tcl \ + tcl/mcast/McastMonitor.tcl \ + tcl/mcast/mftp_snd.tcl \ + tcl/mcast/mftp_rcv.tcl \ + tcl/mcast/mftp_rcv_stat.tcl \ + tcl/mobility/dsdv.tcl \ + tcl/mobility/dsr.tcl \ + tcl/ctr-mcast/CtrMcast.tcl \ + tcl/ctr-mcast/CtrMcastComp.tcl \ + tcl/ctr-mcast/CtrRPComp.tcl \ + tcl/rlm/rlm.tcl \ + tcl/rlm/rlm-ns.tcl \ + tcl/session/session.tcl \ + tcl/lib/ns-route.tcl \ + tcl/emulate/ns-emulate.tcl \ + tcl/lan/vlan.tcl \ + tcl/lan/abslan.tcl \ + tcl/lan/ns-ll.tcl \ + tcl/lan/ns-mac.tcl \ + tcl/webcache/http-agent.tcl \ + tcl/webcache/http-server.tcl \ + tcl/webcache/http-cache.tcl \ + tcl/webcache/http-mcache.tcl \ + tcl/webcache/webtraf.tcl \ + tcl/webcache/empweb.tcl \ + tcl/webcache/empftp.tcl \ + tcl/plm/plm.tcl \ + tcl/plm/plm-ns.tcl \ + tcl/plm/plm-topo.tcl \ + tcl/mpls/ns-mpls-classifier.tcl \ + tcl/mpls/ns-mpls-ldpagent.tcl \ + tcl/mpls/ns-mpls-node.tcl \ + tcl/mpls/ns-mpls-simulator.tcl \ + tcl/lib/ns-pushback.tcl \ + tcl/lib/ns-srcrt.tcl \ + @V_NS_TCL_LIB_STL@ + +$(GEN_DIR)ns_tcl.cc: $(NS_TCL_LIB) + $(TCLSH) bin/tcl-expand.tcl tcl/lib/ns-lib.tcl @V_NS_TCL_LIB_STL@ | $(TCL2C) et_ns_lib > $@ + +$(GEN_DIR)version.c: VERSION + $(RM) $@ + $(TCLSH) bin/string2c.tcl version_string < VERSION > $@ + +$(GEN_DIR)ptypes.cc: common/ptypes2tcl common/packet.h + ./common/ptypes2tcl > $@ + +common/ptypes2tcl: common/ptypes2tcl.o + $(LINK) $(LDFLAGS) $(LDOUT)$@ common/ptypes2tcl.o + +common/ptypes2tcl.o: common/ptypes2tcl.cc common/packet.h + +install: force install-ns install-man install-recursive + +install-ns: force + $(INSTALL) -m 555 -o bin -g bin ns $(DESTDIR)$(BINDEST) + +install-man: force + $(INSTALL) -m 444 -o bin -g bin ns.1 $(DESTDIR)$(MANDEST)/man1 + +install-recursive: force + for i in $(SUBDIRS); do ( cd $$i; $(MAKE) install; ) done + +clean: + $(RM) $(CLEANFILES) + +AUTOCONF_GEN = tcl/lib/ns-autoconf.tcl +distclean: + $(RM) $(CLEANFILES) Makefile config.cache config.log config.status \ + gnuc.h os-proto.h $(AUTOCONF_GEN); \ + $(MV) .configure .configure- ;\ + echo "Moved .configure to .configure-" + +tags: force + ctags -wtd *.cc *.h webcache/*.cc webcache/*.h dsdv/*.cc dsdv/*.h \ + dsr/*.cc dsr/*.h webcache/*.cc webcache/*.h lib/*.cc lib/*.h \ + ../Tcl/*.cc ../Tcl/*.h + +TAGS: force + etags *.cc *.h webcache/*.cc webcache/*.h dsdv/*.cc dsdv/*.h \ + dsr/*.cc dsr/*.h webcache/*.cc webcache/*.h lib/*.cc lib/*.h \ + ../Tcl/*.cc ../Tcl/*.h + +tcl/lib/TAGS: force + ( \ + cd tcl/lib; \ + $(TCLSH) ../../bin/tcl-expand.tcl ns-lib.tcl | grep '^### tcl-expand.tcl: begin' | awk '{print $$5}' >.tcl_files; \ + etags --lang=none -r '/^[ \t]*proc[ \t]+\([^ \t]+\)/\1/' `cat .tcl_files`; \ + etags --append --lang=none -r '/^\([A-Z][^ \t]+\)[ \t]+\(instproc\|proc\)[ \t]+\([^ \t]+\)[ \t]+/\1::\3/' `cat .tcl_files`; \ + ) + +depend: $(SRC) + $(MKDEP) $(CFLAGS) $(INCLUDES) $(SRC) + +srctar: + @cwd=`pwd` ; dir=`basename $$cwd` ; \ + name=ns-`cat VERSION | tr A-Z a-z` ; \ + tar=ns-src-`cat VERSION`.tar.gz ; \ + list="" ; \ + for i in `cat FILES` ; do list="$$list $$name/$$i" ; done; \ + echo \ + "(rm -f $$tar; cd .. ; ln -s $$dir $$name)" ; \ + (rm -f $$tar; cd .. ; ln -s $$dir $$name) ; \ + echo \ + "(cd .. ; tar cfh $$tar [lots of files])" ; \ + (cd .. ; tar cfh - $$list) | gzip -c > $$tar ; \ + echo \ + "rm ../$$name; chmod 444 $$tar" ; \ + rm ../$$name; chmod 444 $$tar + +force: + +test: force + ./validate + +# Create makefile.vc for Win32 development by replacing: +# "# !include ..." -> "!include ..." +makefile.vc: Makefile.in + $(PERL) bin/gen-vcmake.pl < Makefile.in > makefile.vc +# $(PERL) -pe 's/^# (\!include)/\!include/o' < Makefile.in > makefile.vc diff --git a/ns/nsNormAgent.cpp b/ns/nsNormAgent.cpp index 5d4afa2..99b238b 100644 --- a/ns/nsNormAgent.cpp +++ b/ns/nsNormAgent.cpp @@ -39,17 +39,18 @@ static class NsNormAgentClass : public TclClass { public: NsNormAgentClass() : TclClass("Agent/NORM") {} - TclObject *create(int argc, const char*const* argv) + TclObject* create(int argc, const char*const* argv) {return (new NsNormAgent());} } class_norm_agent; NsNormAgent::NsNormAgent() + : NormSimAgent(ProtoSimAgent::TimerInstaller, + static_cast(this), + ProtoSimAgent::SocketInstaller, + static_cast(this)) { - NormSimAgent::Init(ProtoSimAgent::TimerInstaller, - static_cast(this), - ProtoSimAgent::SocketInstaller, - static_cast(this)); + } NsNormAgent::~NsNormAgent() @@ -62,6 +63,42 @@ int NsNormAgent::command(int argc, const char*const* argv) int i = 1; while (i < argc) { + // Intercept ns-specific commands + if (!strcmp(argv[i], "attach-mgen")) + { + // (TBD) this could be done as a generic NormSimAgent command + // Attach Agent/MGEN to this NormSimAgent + if (++i >= argc) + { + DMSG(0, "NsNormAgent::command() ProcessCommand(attach-mgen) error: insufficent arguments\n"); + return TCL_ERROR; + } + Tcl& tcl = Tcl::instance(); + NsMgenAgent* mgenAgent = dynamic_cast (tcl.lookup(argv[i])); + if (mgenAgent) + { + AttachMgen(mgenAgent->GetMgenInstance()); + i++; + continue; + } + else + { + DMSG(0, "NsNormAgent::command() ProcessCommand(attach-mgen) error: invalid mgen agent\n"); + return TCL_ERROR; + } + } + else if (!strcmp(argv[i], "active")) + { + // query agent's current state of activity + Tcl& tcl = Tcl::instance(); + if (IsActive()) + sprintf(tcl.result(), "on"); + else + sprintf(tcl.result(), "off"); + i++; + continue; + } + // Other commands are interpreted by the NormSimAgent base class NormSimAgent::CmdType cmdType = CommandType(argv[i]); switch (cmdType) { @@ -92,6 +129,13 @@ int NsNormAgent::command(int argc, const char*const* argv) return TCL_OK; } // end NsNormAgent::command() +bool NsNormAgent::SendMgenMessage(const NetworkAddress* dstAddr, + const char* txBuffer, + unsigned int len) +{ + return SendMessage(len, txBuffer); +} // end NsNormAgent::SendMgenMessage() + diff --git a/ns/nsNormAgent.h b/ns/nsNormAgent.h index 10a7154..c2c8215 100644 --- a/ns/nsNormAgent.h +++ b/ns/nsNormAgent.h @@ -36,21 +36,28 @@ #include "nsProtoAgent.h" // from ProtoLib #include "normSimAgent.h" -// The "NsNormAgent" is based on the ns agent class. +#include "nsMgenAgent.h" + +// The "NsProtoAgent" is based on the ns Agent class. // This lets us have a send/recv attachment to the NS // simulation environment // IMPORTANT NOTE! NsProtoAgent must be listed _first_ here // (because we can't dynamic_cast install_data void* pointers) -class NsNormAgent : public NormSimAgent, public NsProtoAgent +class NsNormAgent : public NsProtoAgent, public NormSimAgent, public MgenSink { public: NsNormAgent(); ~NsNormAgent(); - // NsProtoAgent base class overrides - int command(int argc, const char*const* argv); - unsigned long GetAgentId() {return (unsigned long)addr();} + // NsProtoAgent base class override + int command(int argc, const char*const* argv); + // NormSimAgent override + unsigned long GetAgentId() {return (unsigned long)addr();} + // MgenSink override + bool SendMgenMessage(const NetworkAddress* dstAddr, + const char* txBuffer, + unsigned int len); }; // end class NsNormAgent diff --git a/ns/simplenorm.tcl b/ns/simplenorm.tcl index 5d6d9b4..7482bc1 100644 --- a/ns/simplenorm.tcl +++ b/ns/simplenorm.tcl @@ -10,46 +10,47 @@ set backoffFactor "4.0" set sendRate "64kb" set duration "120.0" set nsTracing 0 +set unicastNacks "off" +set cc "off" #Parse optionList for parameters -set state flag +set state "flag" foreach option $optionList { + if {"flag" != $state} { + set reset true + } else { + set reset false + } switch -- $state { "flag" { switch -glob -- $option { - "gsize" {set state "gsize"} - "rate" {set state "rate"} - "backoff" {set state "backoff"} - "duration" {set state "duration"} + "unicast" {set unicastNacks "on"} + "cc" {set cc "on"} "trace" {set nsTracing 1} - default {error "simplenorm: Bad option argument $option"} + default {set state $option} } } "gsize" { set groupSize $option - set state "flag" } "backoff" { set backoffFactor $option - set state "flag" } "rate" { set sendRate $option - set state "flag" } "duration" { set duration $option - set state "flag" } default { - error "simplenorm: Bad option parse state!" + error "simplenorm: bad option: $state" } } + if {$reset == "true"} {set state "flag"} } - # 1) An ns-2 simulator instance is created an configured # for multicast operation with a dense mode (DM) multicast # routing protocol: @@ -87,6 +88,7 @@ for {set i 1} {$i <= $numNodes} {incr i} { puts "simplenorm: Creating spoke links ..." set linkRate [expr $groupSize * [bw_parse $sendRate]] +puts "simplenorm: linkRate = [expr $linkRate / 1000.0] kbps" for {set i 1} {$i <= $numNodes} {incr i} { $ns_ duplex-link $n(0) $n($i) $linkRate 100ms DropTail $ns_ queue-limit $n(0) $n($i) 100 @@ -108,26 +110,28 @@ puts "simplenorm: Configuring NORM agents ..." # 6) Configure global NORM agent commands (using norm(1)) $norm(1) debug 2 -$norm(1) log normLog.txt +#$norm(1) log normLog.txt # 7) Configure NORM server agent at node 1 $norm(1) address $group/5000 $norm(1) rate [bw_parse $sendRate] $norm(1) backoff $backoffFactor $norm(1) parity 0 -$norm(1) repeat -1 +$norm(1) repeat 50 $norm(1) interval 0.0 $norm(1) txloss 10.0 $norm(1) gsize $groupSize -#$norm(1) trace +$norm(1) cc off +#$norm(1) trace on # 8) Configure NORM client agents at other nodes for {set i 2} {$i <= $numNodes} {incr i} { $norm($i) address $group/5000 $norm($i) backoff $backoffFactor - $norm($i) rxbuffer 100000 - $norm($i) txloss 10.0 + $norm($i) rxbuffer 1000000 + #$norm($i) txloss 10.0 $norm($i) gsize $groupSize + $norm($i) unicastNacks $unicastNacks #$norm($i) trace } diff --git a/ns/suppress.tcl b/ns/suppress.tcl index 9a573c0..f9db98c 100755 --- a/ns/suppress.tcl +++ b/ns/suppress.tcl @@ -8,7 +8,7 @@ set rate 32kb set duration 120.0 set backoff 4.0 -set fileName "results.gp" +set fileName "supress.gp" exec rm -f $fileName @@ -20,8 +20,12 @@ puts $outFile "set xlabel 'Number of Receivers'" puts $outFile "set ylabel 'NACK Transmission Fraction'" puts $outFile "plot \\" puts $outFile "'-' using 1:2 \\" -puts $outFile "'Receivers:%lf alpha:%lf' t 'Theoretical Multicast Nacking', \\" +puts $outFile "'Receivers:%lf alpha:%lf' t 'Theoretical Unicast Nacking', \\" puts $outFile "'-' using 1:2 \\" +puts $outFile "'Receivers:%lf alpha:%lf' t 'Theoretical Multicast Nacking', \\" +puts $outFile "'-' using 1:4 \\" +puts $outFile "'Receivers:%lf Sent:%lf Suppressed:%lf alpha:%lf' t 'Measured Unicast Nacking', \\" +puts $outFile "'-' using 1:4 \\" puts $outFile "'Receivers:%lf Sent:%lf Suppressed:%lf alpha:%lf' t 'Measured Multicast Nacking'\n" close $outFile @@ -29,7 +33,19 @@ puts "Computing theoretical results ..." # T = number of GRTT for NACK backoff timers ... set T $backoff -# Theoretical Multicast NACK results +# Theoretical Unicast NACK suppression results +set outFile [open $fileName "a"] +puts $outFile "#Theoretical Unicast NACK Transmission Fraction" +foreach groupSize $sizeList { + set lambda [expr log($groupSize) + 1.0] + set N [expr exp((1.2/($T)) * ($lambda))] + set alpha [expr $N / $groupSize] + puts $outFile "Receivers:$groupSize alpha:$alpha" +} +puts $outFile "e\n" +close $outFile + +# Theoretical Multicast NACK suppression results set outFile [open $fileName "a"] puts $outFile "#Theoretical Multicast NACK Transmission Fraction" foreach groupSize $sizeList { @@ -41,6 +57,19 @@ foreach groupSize $sizeList { puts $outFile "e\n" close $outFile +# Measured Unicast NACK results +set outFile [open $fileName "a"] +puts $outFile "#Measured Unicast NACK Transmission Fraction" +close $outFile +foreach groupSize $sizeList { + puts "Starting simulation run for groupSize:$groupSize ..." + puts " ns simplenorm.tcl unicast gsize $groupSize backoff $backoff rate $rate duration $duration" + catch {eval exec ns simplenorm.tcl unicast gsize $groupSize backoff $backoff rate $rate duration $duration} + puts " cat normLog.txt | nc >> $fileName" + catch {eval exec cat normLog.txt | nc >> $fileName} +} +puts $outFile "e\n" + # Measured Multicast NACK results set outFile [open $fileName "a"] puts $outFile "#Measured Multicast NACK Transmission Fraction" diff --git a/unix/Makefile.common b/unix/Makefile.common index 898eaa3..a859567 100644 --- a/unix/Makefile.common +++ b/unix/Makefile.common @@ -13,7 +13,7 @@ NS = ../ns INCLUDES = $(TCL_INCL_PATH) $(SYSTEM_INCLUDES) -I$(UNIX) -I$(COMMON) -I$(PROTOLIB)/common -CFLAGS = -g -DPROTO_DEBUG -DUNIX -Wall -O -fPIC $(SYSTEM_HAVES) $(INCLUDES) +CFLAGS = -g -DPROTO_DEBUG -DUNIX -O -fPIC $(SYSTEM_HAVES) $(INCLUDES) LDFLAGS = $(SYSTEM_LDFLAGS) @@ -49,7 +49,7 @@ TARGETS = mdp tkMdp # MDP depends upon the NRL Protean Group's development library LIBPROTO = $(PROTOLIB)/unix/libProto.a $(PROTOLIB)/unix/libProto.a: - make -C $(PROTOLIB)/unix -f Makefile.common libProto.a + cd $(PROTOLIB)/unix; $(MAKE) -f Makefile.$(SYSTEM) libProto.a NORM_SRC = $(COMMON)/normMessage.cpp $(COMMON)/normSession.cpp \ $(COMMON)/normNode.cpp $(COMMON)/normObject.cpp \ @@ -63,8 +63,8 @@ LIB_SRC = $(NORM_SRC) LIB_OBJ = $(LIB_SRC:.cpp=.o) libnorm.a: $(LIB_OBJ) - ar rcv $@ $(LIB_OBJ) - ranlib $@ + $(AR) rcv $@ $(LIB_OBJ) + $(RANLIB) $@ .cpp-sim.o: $(CC) -c $(CFLAGS) -DSIMULATE -o $*-sim.o $*.cpp @@ -77,15 +77,15 @@ libnormSim.a: $(SIM_OBJ) ranlib $@ # (mdp) command-line file broadcaster/receiver -APP_SRC = $(COMMON)/normApp.cpp +APP_SRC = $(COMMON)/normApp.cpp $(COMMON)/normPostProcess.cpp APP_OBJ = $(APP_SRC:.cpp=.o) norm: $(APP_OBJ) libnorm.a $(LIBPROTO) - $(CC) $(CFLAGS) -o $@ $(APP_OBJ) $(LDFLAGS) $(LIBS) libnorm.a $(LIBPROTO) + $(CC) $(CFLAGS) -o $@ $(APP_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS) clean: rm -f $(COMMON)/*.o $(UNIX)/*.o $(UNIX)/libnorm.a $(UNIX)/norm $(NS)/*.o; - make -C $(PROTOLIB)/unix -f Makefile.common clean + cd $(PROTOLIB)/unix; $(MAKE) -f Makefile.$(SYSTEM) clean # DO NOT DELETE THIS LINE -- mkdep uses it. # DO NOT PUT ANYTHING AFTER THIS LINE, IT WILL GO AWAY. diff --git a/unix/Makefile.freebsd b/unix/Makefile.freebsd index f48a36d..ea2eb6f 100644 --- a/unix/Makefile.freebsd +++ b/unix/Makefile.freebsd @@ -65,5 +65,7 @@ SYSTEM_LIBS = export SYSTEM_HAVES = -DHAVE_GETLOGIN -DHAVE_FLOCK -DHAVE_DIRFD $(DNETSEC) export CC = gcc +export RANLIB = ranlib +export AR = ar include Makefile.common diff --git a/unix/Makefile.hpux b/unix/Makefile.hpux index 5f53aa3..46f616c 100644 --- a/unix/Makefile.hpux +++ b/unix/Makefile.hpux @@ -65,5 +65,7 @@ SYTSTEM_LIBS = export SYSTEM_HAVES = -DHAVE_CUSERID -DHAVE_LOCKF export CC = gcc +export RANLIB = touch +export AR = ar include Makefile.common diff --git a/unix/Makefile.linux b/unix/Makefile.linux index e4bee57..d2dced7 100644 --- a/unix/Makefile.linux +++ b/unix/Makefile.linux @@ -10,19 +10,19 @@ # 1) Where to find the Tcl standard library scripts # (e.g. init.tcl, ...) -TCL_SCRIPT_PATH = /usr/local/lib/tcl8.0 +TCL_SCRIPT_PATH = /usr/lib/tcl # 2) Where to find the Tk standard library scripts # (e.g. button.tcl, entry.tcl, ...) -TK_SCRIPT_PATH = /usr/local/lib/tk8.0 +TK_SCRIPT_PATH = /usr/lib/tk # 3) Where to find Tcl/Tk header files # (e.g. tcl.h, tk.h, ...) TCL_INCL_PATH = -I/usr/local/include # 4) Point to specific libtcl.a and libtk.a to use -TCL_LIB = /usr/local/lib/libtcl8.0.a -TK_LIB = /usr/local/lib/libtk8.0.a +TCL_LIB = -ltcl #/usr/local/lib/libtcl8.0.a +TK_LIB = -ltk #/usr/local/lib/libtk8.0.a # 5) System specific additional libraries, include paths, etc @@ -66,5 +66,7 @@ export SYSTEM_HAVES = -DLINUX -DHAVE_IPV6 -DHAVE_GETLOGIN -DHAVE_LOCKF \ -DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT $(NETSEC) export CC = gcc +export RANLIB = ranlib +export AR = ar include Makefile.common diff --git a/unix/Makefile.macosx b/unix/Makefile.macosx index 9a222f9..7158ad5 100644 --- a/unix/Makefile.macosx +++ b/unix/Makefile.macosx @@ -1,38 +1,15 @@ # -# MacOS X Protean Makefile definitions +# Protean MacOS X (Darwin) Makefile definitions # -# TO BUILD THE TK GUI VERSION, EDIT THE FOLLOWING NUMBERED -# ITEMS AS NEEDED FOR YOUR SYSTEM -# (This has only been tested with TCL/TK 8.0 but it probably -# will work with Tcl7.x/Tk4.x with a little tweaking to -# the list of TCL_SCRIPTS (library scripts) given below) - -# 1) Where to find the Tcl standard library scripts -# (e.g. init.tcl, ...) -TCL_SCRIPT_PATH = /usr/local/lib/tcl8.0 - -# 2) Where to find the Tk standard library scripts -# (e.g. button.tcl, entry.tcl, ...) -TK_SCRIPT_PATH = /usr/local/lib/tk8.0 - -# 3) Where to find Tcl/Tk header files -# (e.g. tcl.h, tk.h, ...) -TCL_INCL_PATH = -I/usr/local/include/tcl8.0 -I/usr/local/include/tk8.0 - -# 4) Point to specific libtcl.a and libtk.a to use -TCL_LIB = /usr/local/lib/libtcl80.a -TK_LIB = /usr/local/lib/libtk80.a - - -# 5) System specific additional libraries, include paths, etc +# 1) System specific additional libraries, include paths, etc # (Where to find X11 libraries, etc) # SYSTEM_INCLUDES = -I/usr/X11R6/include SYSTEM_LDFLAGS = -L/usr/X11R6/lib SYSTEM_LIBS = -# 6) System specific capabilities +# 2) System specific capabilities # Must choose appropriate for the following: # # A) -DHAVE_CUSERID (preferred) or -DHAVE_GETLOGIN for cuserid() or getlogin() @@ -48,22 +25,20 @@ SYSTEM_LIBS = # D) Optionally specify -DHAVE_ASSERT if your system has a built-in ASSERT() # routine. # -# E) The MDP code's use of offset pointers requires special treatment -# for some different compilers. Set -DUSE_INHERITANCE for some -# to keep some compilers (gcc 2.7.2) happy. -# -# F) Some systems (SOLARIS/SUNOS) have a few gotchas which require +# E) Some systems (SOLARIS/SUNOS) have a few gotchas which require # some #ifdefs to avoid compiler warnings ... so you might need # to specify -DSOLARIS or -DSUNOS depending on your OS. # -# G) Uncomment this if you have the NRL IPv6+IPsec software +# F) Uncomment this if you have the NRL IPv6+IPsec software #DNETSEC = -DNETSEC -I/usr/inet6/include # # (We export these for other Makefiles as needed) # -export SYSTEM_HAVES = -DHAVE_ASSERT -DHAVE_GETLOGIN -DHAVE_FLOCK -DHAVE_DIRFD $(DNETSEC) +export SYSTEM_HAVES = -DHAVE_ASSERT -DHAVE_GETLOGIN -DHAVE_FLOCK -DHAVE_DIRFD $(DNETSEC) -DSOCKLEN_T=int -export CC = cc +export CC = g++ +export RANLIB = ranlib +export AR = ar include Makefile.common diff --git a/unix/Makefile.mklinux b/unix/Makefile.mklinux index 8e071c6..9c739e9 100644 --- a/unix/Makefile.mklinux +++ b/unix/Makefile.mklinux @@ -66,6 +66,8 @@ SYSTEM_LIBS = -ldl export SYSTEM_HAVES = -DHAVE_CUSERID -DHAVE_LOCKF -DHAVE_DIRFD $(NETSEC) export CC = gcc +export RANLIB = ranlib +export AR = ar include Makefile.common diff --git a/unix/Makefile.netbsd b/unix/Makefile.netbsd index 8424d4e..a524cfe 100644 --- a/unix/Makefile.netbsd +++ b/unix/Makefile.netbsd @@ -62,8 +62,10 @@ SYSTEM_LIBS = # (We export these for other Makefiles as needed) # -export SYSTEM_HAVES = -DHAVE_GETLOGIN -DHAVE_ASSERT -DHAVE_FLOCK -DHAVE_DIRFD $(DNETSEC) +export SYSTEM_HAVES = -DHAVE_GETLOGIN -DHAVE_ASSERT -DHAVE_FLOCK -DHAVE_DIRFD $(DNETSEC) export CC = gcc +export RANLIB = ranlib +export AR = ar include Makefile.common diff --git a/unix/Makefile.sgi b/unix/Makefile.sgi index 14cec7d..6003e22 100644 --- a/unix/Makefile.sgi +++ b/unix/Makefile.sgi @@ -1,34 +1,13 @@ # -# MDPv2 IRIX Makefile +# IRIX Protean Makefile definitions # -# TO BUILD THE TK GUI VERSION OF MDP EDIT THE FOLLOWING NUMBERED -# ITEMS AS NEEDED FOR YOUR SYSTEM -# (This has only been tested with TCL/TK 8.0 but it probably -# will work with Tcl7.x/Tk4.x with a little tweaking to -# the list of TCL_SCRIPTS (library scripts) given below) - -# 1) Where to find the Tcl standard library scripts -# (e.g. init.tcl, ...) -TCL_SCRIPT_PATH = /usr/local/lib/tcl8.0 - -# 2) Where to find the Tk standard library scripts -# (e.g. button.tcl, entry.tcl, ...) -TK_SCRIPT_PATH = /usr/local/lib/tk8.0 - -# 3) Where to find Tcl/Tk header files -# (e.g. tcl.h, tk.h, ...) -TCL_INCL_PATH = -I/usr/local/include - -# 4) Point to specific libtcl.a and libtk.a to use -TCL_LIB = /usr/local/lib/libtcl8.0.a -TK_LIB = /usr/local/lib/libtk8.0.a -# 5) System specific additional libraries, include paths, etc +# 1) System specific additional libraries, include paths, etc # (Where to find X11 libraries, etc) # -SYSTEM_INCLUDES = +SYSTEM_INCLUDES = -DIRIX -n32 -woff 3333 -cfront SYSTEM_LDFLAGS = SYSTEM_LIBS = @@ -59,11 +38,17 @@ SYSTEM_LIBS = # G) Uncomment this if you have the NRL IPv6+IPsec software #DNETSEC = -DNETSEC -I/usr/inet6/include # -# (We export these for other Makefiles as needed) # -export SYSTEM_HAVES = -DUSE_INHERITANCE -DHAVE_CUSERID -DHAVE_LOCKF $(NETSEC) +SYSTEM_HAVES = -DHAVE_IPV6 -DHAVE_GETLOGIN -DHAVE_LOCKF \ + -DHAVE_DIRFD -DHAVE_ASSERT $(NETSEC) -export CC = gcc +SYSTEM_SRC = + +SYSTEM = sgi + +CC = CC +RANLIB = touch +AR = ar include Makefile.common diff --git a/unix/Makefile.solaris b/unix/Makefile.solaris index c7258e7..6ff7bcb 100644 --- a/unix/Makefile.solaris +++ b/unix/Makefile.solaris @@ -63,5 +63,7 @@ SYSTEM_LIBS = -ldl -lnsl -lsocket export SYSTEM_HAVES = -DHAVE_ASSERT -DHAVE_CUSERID -DHAVE_LOCKF $(NETSEC) -DSOLARIS export CC = gcc +export RANLIB = touch +export AR = ar include Makefile.common diff --git a/unix/Makefile.solx86 b/unix/Makefile.solx86 index fc77e3d..1ac0662 100644 --- a/unix/Makefile.solx86 +++ b/unix/Makefile.solx86 @@ -63,6 +63,8 @@ SYSTEM_LIBS = -ldl -lnsl -lsocket export SYSTEM_HAVES = -DHAVE_ASSERT -DHAVE_CUSERID -DHAVE_LOCKF $(NETSEC) -DSOLARIS export CC = gcc +export RANLIB = touch +export AR = ar include Makefile.common