From 3456db14f2280d02808d2055d9182808cca66d87 Mon Sep 17 00:00:00 2001 From: Jeff Weston Date: Wed, 11 Sep 2019 11:22:31 -0400 Subject: [PATCH] v0.2 --- README.md | 41 +- common/galois.cpp | 7 +- common/galois.h | 17 +- common/normApp.cpp | 826 +++++++++++++++++++--- common/normBitmask.cpp | 34 +- common/normBitmask.h | 2 + common/normFile.cpp | 986 +++++++++++++++++++++++++++ common/normFile.h | 196 ++++++ common/normMessage.cpp | 44 +- common/normMessage.h | 92 ++- common/normNode.cpp | 865 ++++++++++++++++------- common/normNode.h | 48 +- common/normObject.cpp | 1262 ++++++++++++++++++++++++++++------ common/normObject.h | 190 +++++- common/normSegment.cpp | 456 ++++++++++--- common/normSegment.h | 108 ++- common/normSession.cpp | 1398 ++++++++++++++++++++++++++++++++++++-- common/normSession.h | 164 ++++- common/normSimAgent.cpp | 565 +++++++++++++++ common/normSimAgent.h | 82 +++ common/normSimAgent.o | Bin 0 -> 153344 bytes common/rules.txt | 78 +++ common/version.h | 37 + ns/README.txt | 55 ++ ns/example.tcl | 96 +++ ns/nackCount.cpp | 266 ++++++++ ns/ns-2.1b7a-Makefile.in | 432 ++++++++++++ ns/nsNormAgent.cpp | 97 +++ ns/nsNormAgent.h | 57 ++ ns/simplenorm.tcl | 167 +++++ ns/sizeAve.cpp | 132 ++++ ns/suppress.tcl | 55 ++ unix/Makefile.common | 25 +- 33 files changed, 7983 insertions(+), 897 deletions(-) create mode 100644 common/normFile.cpp create mode 100644 common/normFile.h create mode 100644 common/normSimAgent.cpp create mode 100644 common/normSimAgent.h create mode 100644 common/normSimAgent.o create mode 100644 common/version.h create mode 100644 ns/README.txt create mode 100644 ns/example.tcl create mode 100644 ns/nackCount.cpp create mode 100644 ns/ns-2.1b7a-Makefile.in create mode 100644 ns/nsNormAgent.cpp create mode 100644 ns/nsNormAgent.h create mode 100644 ns/simplenorm.tcl create mode 100644 ns/sizeAve.cpp create mode 100755 ns/suppress.tcl diff --git a/README.md b/README.md index 124bf8e..f6a2ff6 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,10 @@ NORM PRELIMINARY CODE RELEASE -These directories contain source code for a very preliminary -implementation of the Nack Oriented Reliable Multicast (NORM) -protocol. Currently, a very dumb "norm" command-line application -(which currently takes no command-line arguments) is -created. This application creates a NormSession and acts -both as a sender and receiver to the multicast group. It -creates a "NORM_STREAM" object and writes repeated, -continuous strings of "aaaaaaa ..." to the stream. As the -stream is received, the client (receiver) portion of the -"norm" app writes a notices of a successful Read() -operation from the stream. - -This currently uses a fixed transmission rate of 64 kbps -and a Reed-Solomon FEC encoding block with 20 user data -segments and calculates 8 parity segments per coding -block. Four parity segments are sent at the end of each -coding block as "auto parity". The current NORM code is -currently automatically discarding segment 9 of the -received data stream to test the FEC encoding/decoding and -stream buffer routines. - -The code does not currently generate any NACK messages, -but the routines to perform checks for losses is in place -and the routines for building NACK messages are in place, -so the addition of the timer installation routines to -schedule NACK back-off and subsequent transmission will be -added very soon. Then, routines will be added for the -server (sender) side to process received NACKs and -subsequently provide repair messages. Then, client-side -routines for NACK suppression will be added. After that, -the final details of GRTT collection, unicast feedback -suppression will be added and finally one or more -congestion control schemes will be included. - -The purpose of this release is to illustrate routines to -build and parse the messages currently defined in the NORM -Internet Draft. +The NORM code here is still preliminary but a functional "norm" +command-line application can be built which can send and receive +a set of files _or_ a stream piped to/from stdin/stdout. The command- +line options are not yet fully documented. The current code only has Makefiles for Unix platforms, but the code could be assembled as Win32 project under VC++. diff --git a/common/galois.cpp b/common/galois.cpp index c51a00c..03270c4 100644 --- a/common/galois.cpp +++ b/common/galois.cpp @@ -34,7 +34,7 @@ #include "galois.h" /* Galois field inversion table */ -const unsigned char GINV[256] = +const unsigned char Norm::GINV[256] = { 0x01, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, 0xad, 0x9d, 0xdd, 0x98, 0x3d, 0xaa, 0x5d, 0x96, 0xd8, 0x72, 0xc0, 0x58, 0xe0, 0x3e, 0x4c, 0x66, 0x90, 0xde, 0x55, 0x80, 0xa0, 0x83, 0x4b, 0x2a, @@ -55,7 +55,7 @@ const unsigned char GINV[256] = }; // end GINV[] // Galois field exponent table -const unsigned char GEXP[512] = +const unsigned char Norm::GEXP[512] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1d, 0x3a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, 0x4c, 0x98, 0x2d, 0x5a, 0xb4, 0x75, 0xea, 0xc9, 0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, @@ -92,7 +92,7 @@ const unsigned char GEXP[512] = }; // end GEXP[] // Galois field multiplication table -const unsigned char GMULT[256][256] = +const unsigned char Norm::GMULT[256][256] = { {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -4447,3 +4447,4 @@ const unsigned char GMULT[256][256] = 0x8e, 0x71, 0x6d, 0x92, 0x55, 0xaa, 0xb6, 0x49, 0x25, 0xda, 0xc6, 0x39, 0xfe, 0x01, 0x1d, 0xe2} }; // end GMULT[] + diff --git a/common/galois.h b/common/galois.h index b737a8c..076da09 100644 --- a/common/galois.h +++ b/common/galois.h @@ -30,15 +30,16 @@ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. ********************************************************************/ -#ifndef _GALOIS -#define _GALOIS +#ifndef _NORM_GALOIS +#define _NORM_GALOIS +namespace Norm +{ extern const unsigned char GINV[256]; extern const unsigned char GEXP[512]; extern const unsigned char GMULT[256][256]; - -inline unsigned char gexp(unsigned int x) {return GEXP[x];} -inline unsigned char gmult(unsigned int x, unsigned int y) {return GMULT[x][y];} -inline unsigned char ginv(unsigned int x) {return GINV[x];} - -#endif // _GALOIS +} +inline unsigned char gexp(unsigned int x) {return Norm::GEXP[x];} +inline unsigned char gmult(unsigned int x, unsigned int y) {return Norm::GMULT[x][y];} +inline unsigned char ginv(unsigned int x) {return Norm::GINV[x];} +#endif // _NORM_GALOIS diff --git a/common/normApp.cpp b/common/normApp.cpp index 440d9f0..3ccb1dd 100644 --- a/common/normApp.cpp +++ b/common/normApp.cpp @@ -6,7 +6,10 @@ #include // for stdout/stderr printouts #include // for SIGTERM/SIGINT handling +#include + +// Command-line application using Protolib EventDispatcher class NormApp : public NormController { public: @@ -16,14 +19,25 @@ class NormApp : public NormController int MainLoop() {return dispatcher.Run();} void Stop(int exitCode) {dispatcher.Stop(exitCode);} void OnShutdown(); - + + bool ProcessCommand(const char* cmd, const char* val); + bool ParseCommandLine(int argc, char* argv[]); + private: - + enum CmdType {CMD_INVALID, CMD_NOARG, CMD_ARG}; + CmdType CommandType(const char* cmd); + virtual void Notify(NormController::Event event, class NormSessionMgr* sessionMgr, class NormSession* session, class NormServerNode* server, class NormObject* object); + + void InstallTimer(ProtocolTimer* timer) + {dispatcher.InstallTimer(timer);} + bool OnIntervalTimeout(); + + static const char* const cmd_list[]; static void SignalHandler(int sigNum); @@ -31,9 +45,396 @@ class NormApp : public NormController NormSessionMgr session_mgr; NormSession* session; NormStreamObject* stream; + + // application parameters + FILE* input; // input stream + FILE* output; // output stream + char input_buffer[512]; + unsigned int input_index; + unsigned int input_length; + + // NormSession parameters + char* address; // session address + UINT16 port; // session port number + UINT8 ttl; + double tx_rate; // bits/sec + UINT16 segment_size; + UINT8 ndata; + UINT8 nparity; + UINT8 auto_parity; + unsigned long tx_buffer_size; // bytes + unsigned long rx_buffer_size; // bytes + + NormFileList tx_file_list; + double tx_object_interval; + int tx_repeat_count; + double tx_repeat_interval; + NormFileList rx_file_cache; + char* rx_cache_path; + + ProtocolTimer interval_timer; + + // protocol debug parameters + bool tracing; + double tx_loss; + double rx_loss; }; // end class NormApp +NormApp::NormApp() + : session(NULL), stream(NULL), input(NULL), output(NULL), + input_index(0), input_length(0), + address(NULL), port(0), ttl(3), tx_rate(64000.0), + segment_size(1024), ndata(32), nparity(16), auto_parity(0), + tx_buffer_size(1024*1024), rx_buffer_size(1024*1024), + tx_object_interval(0.0), tx_repeat_count(0), tx_repeat_interval(0.0), + rx_cache_path(NULL), + tracing(false), tx_loss(0.0), rx_loss(0.0) +{ + // Init tx_timer for 1.0 second interval, infinite repeats + session_mgr.Init(EventDispatcher::TimerInstaller, &dispatcher, + EventDispatcher::SocketInstaller, &dispatcher, + this); + interval_timer.Init(0.0, 0, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormApp::OnIntervalTimeout); +} + +NormApp::~NormApp() +{ + if (address) delete address; + if (rx_cache_path) delete rx_cache_path; +} + + +const char* const NormApp::cmd_list[] = +{ + "+debug", + "+log", + "-trace", + "+txloss", + "+rxloss", + "+address", + "+ttl", + "+rate", + "+input", + "+output", + "+sendfile", + "+interval", + "+repeatcount", + "+rinterval", // repeat interval + "+rxcachedir", + "+segment", + "+block", + "+parity", + "+auto", + "+txbuffer", + "+rxbuffer", + NULL +}; + +bool NormApp::ProcessCommand(const char* cmd, const char* val) +{ + CmdType type = CommandType(cmd); + ASSERT(CMD_INVALID != type); + unsigned int len = strlen(cmd); + if ((CMD_ARG == type) && !val) + { + DMSG(0, "NormApp::ProcessCommand(%s) missing argument\n", cmd); + return false; + } + + if (!strncmp("debug", cmd, len)) + { + + int debugLevel = atoi(val); + if ((debugLevel < 0) || (debugLevel > 12)) + { + DMSG(0, "NormApp::ProcessCommand(segment) invalid debug level!\n"); + return false; + } + SetDebugLevel(debugLevel); + } + else if (!strncmp("log", cmd, len)) + { + OpenDebugLog(val); + } + else if (!strncmp("trace", cmd, len)) + { + tracing = true; + if (session) session->SetTrace(true); + } + else if (!strncmp("txloss", cmd, len)) + { + double txLoss = atof(val); + if (txLoss < 0) + { + DMSG(0, "NormApp::ProcessCommand(txloss) invalid txRate!\n"); + return false; + } + tx_loss = txLoss; + if (session) session->SetTxLoss(txLoss); + } + else if (!strncmp("rxloss", cmd, len)) + { + double rxLoss = atof(val); + if (rxLoss < 0) + { + DMSG(0, "NormApp::ProcessCommand(rxloss) invalid txRate!\n"); + return false; + } + rx_loss = rxLoss; + if (session) session->SetRxLoss(rxLoss); + } + else if (!strncmp("address", cmd, len)) + { + unsigned int len = strlen(val); + if (address) delete address; + if (!(address = new char[len+1])) + { + DMSG(0, "NormApp::ProcessCommand(address) allocation error:%s\n", + strerror(errno)); + return false; + } + strcpy(address, val); + char* ptr = strchr(address, '/'); + if (!ptr) + { + delete address; + address = NULL; + DMSG(0, "NormApp::ProcessCommand(address) missing port number!\n"); + return false; + } + *ptr++ = '\0'; + int portNum = atoi(ptr); + if ((portNum < 1) || (portNum > 65535)) + { + delete address; + address = NULL; + DMSG(0, "NormApp::ProcessCommand(address) invalid port number!\n"); + return false; + } + port = portNum; + } + else if (!strncmp("ttl", cmd, len)) + { + int ttlTemp = atoi(val); + if ((ttlTemp < 1) || (ttlTemp > 255)) + { + DMSG(0, "NormApp::ProcessCommand(ttl) invalid value!\n"); + return false; + } + ttl = ttlTemp; + } + else if (!strncmp("rate", cmd, len)) + { + double txRate = atof(val); + if (txRate < 0) + { + DMSG(0, "NormApp::ProcessCommand(rate) invalid txRate!\n"); + return false; + } + tx_rate = txRate; + if (session) session->SetTxRate(txRate); + } + else if (!strncmp("input", cmd, len)) + { + if (!(input = fopen(val, "rb"))) + { + DMSG(0, "NormApp::ProcessCommand(input) fopen() error: %s\n", + strerror(errno)); + return false; + } + } + else if (!strncmp("output", cmd, len)) + { + if (!(output = fopen(val, "wb"))) + { + DMSG(0, "NormApp::ProcessCommand(output) fopen() error: %s\n", + strerror(errno)); + return false; + } + } + else if (!strncmp("sendfile", cmd, len)) + { + if (!tx_file_list.Append(val)) + { + DMSG(0, "NormApp::ProcessCommand(sendfile) Error appending \"%s\" " + "to tx file list.\n", val); + return false; + } + } + else if (!strncmp("interval", cmd, len)) + { + if (1 != sscanf(val, "%lf", &tx_object_interval)) + tx_object_interval = -1.0; + if (tx_object_interval < 0.0) + { + DMSG(0, "NormApp::ProcessCommand(interval) Invalid tx object interval: %s\n", + val); + tx_object_interval = 0.0; + return false; + } + } + else if (!strncmp("repeat", cmd, len)) + { + tx_repeat_count = atoi(val); + } + else if (!strncmp("rinterval", cmd, len)) + { + if (1 != sscanf(val, "%lf", &tx_repeat_interval)) + tx_repeat_interval = -1.0; + if (tx_repeat_interval < 0.0) + { + DMSG(0, "NormApp::ProcessCommand(rinterval) Invalid tx repeat interval: %s\n", + val); + tx_repeat_interval = 0.0; + return false; + } + } + else if (!strncmp("rxcachedir", cmd, len)) + { + unsigned int length = strlen(val); + // Make sure there is a trailing DIR_DELIMITER + if (DIR_DELIMITER != val[length-1]) + length += 2; + else + length += 1; + if (!(rx_cache_path = new char[length])) + { + DMSG(0, "NormApp::ProcessCommand(rxcachedir) alloc error: %s\n", + strerror(errno)); + return false; + } + strcpy(rx_cache_path, val); + rx_cache_path[length-2] = DIR_DELIMITER; + rx_cache_path[length-1] = '\0'; + } + else if (!strncmp("segment", cmd, len)) + { + int segmentSize = atoi(val); + if ((segmentSize < 0) || (segmentSize > 8000)) + { + DMSG(0, "NormApp::ProcessCommand(segment) invalid segment size!\n"); + return false; + } + segment_size = segmentSize; + } + else if (!strncmp("block", cmd, len)) + { + int blockSize = atoi(val); + if ((blockSize < 1) || (blockSize > 255)) + { + DMSG(0, "NormApp::ProcessCommand(block) invalid block size!\n"); + return false; + } + ndata = blockSize; + } + else if (!strncmp("parity", cmd, len)) + { + int numParity = atoi(val); + if ((numParity < 0) || (numParity > 254)) + { + DMSG(0, "NormApp::ProcessCommand(parity) invalid value!\n"); + return false; + } + nparity = numParity; + } + else if (!strncmp("auto", cmd, len)) + { + int autoParity = atoi(val); + if ((autoParity < 0) || (autoParity > 254)) + { + DMSG(0, "NormApp::ProcessCommand(auto) invalid value!\n"); + return false; + } + auto_parity = autoParity; + if (session) session->ServerSetAutoParity(autoParity); + } + else if (!strncmp("txbuffer", cmd, len)) + { + if (1 != sscanf(val, "%lu", &tx_buffer_size)) + { + DMSG(0, "NormApp::ProcessCommand(txbuffer) invalid value!\n"); + return false; + } + } + else if (!strncmp("rxbuffer", cmd, len)) + { + if (1 != sscanf(val, "%lu", &rx_buffer_size)) + { + DMSG(0, "NormApp::ProcessCommand(rxbuffer) invalid value!\n"); + return false; + } + } + return true; +} // end NormApp::ProcessCommand() + + +NormApp::CmdType NormApp::CommandType(const char* cmd) +{ + if (!cmd) return CMD_INVALID; + unsigned int len = strlen(cmd); + bool matched = false; + CmdType type = CMD_INVALID; + const char* const* nextCmd = cmd_list; + while (*nextCmd) + { + if (!strncmp(cmd, *nextCmd+1, len)) + { + if (matched) + { + // ambiguous command (command should match only once) + return CMD_INVALID; + } + else + { + matched = true; + if ('+' == *nextCmd[0]) + type = CMD_ARG; + else + type = CMD_NOARG; + } + } + nextCmd++; + } + return type; +} // end NormApp::CommandType() + + +bool NormApp::ParseCommandLine(int argc, char* argv[]) +{ + int i = 1; + while ( i < argc) + { + CmdType cmdType = CommandType(argv[i]); + switch (cmdType) + { + case CMD_INVALID: + DMSG(0, "NormApp::ParseCommandLine() Invalid command:%s\n", + argv[i]); + return false; + case CMD_NOARG: + if (!ProcessCommand(argv[i], NULL)) + { + DMSG(0, "NormApp::ParseCommandLine() ProcessCommand(%s) error\n", + argv[i]); + return false; + } + i++; + break; + case CMD_ARG: + if (!ProcessCommand(argv[i], argv[i+1])) + { + DMSG(0, "NormApp::ParseCommandLine() ProcessCommand(%s, %s) error\n", + argv[i], argv[i+1]); + return false; + } + i += 2; + break; + } + } + return true; +} // end NormApp::ParseCommandLine() void NormApp::Notify(NormController::Event event, class NormSessionMgr* sessionMgr, @@ -44,85 +445,286 @@ void NormApp::Notify(NormController::Event event, switch (event) { case TX_QUEUE_EMPTY: - //TRACE("NormApp::Notify(TX_QUEUE_EMPTY) ...\n"); - if (object == stream) - { - char text[256]; - memset(text, 'a', 256); - stream->Write(text, 256); - } - break; + // Write to stream as needed + //DMSG(0, "NormApp::Notify(TX_QUEUE_EMPTY) ...\n"); + if (object && (object == stream)) + { + bool flush = false; + if (input) + { + if (input_index >= input_length) + { + size_t result = fread(input_buffer, sizeof(char), 512, input); + if (result) + { + input_index = 0; + input_length = result; + } + else + { + input_length = input_index = 0; + if (feof(input)) + { + DMSG(0, "norm: input end-of-file.\n"); + fclose(input); + input = NULL; + flush = true; + } + else if (ferror(input)) + { + DMSG(0, "norm: input error:%s\n", strerror(errno)); + clearerr(input); + break; + } + } + } + input_index += stream->Write(input_buffer+input_index, + input_length-input_index, + flush); + } // end if (input) + } + else + { + // Can queue a new object for transmission + if (interval_timer.Interval() > 0.0) + { + InstallTimer(&interval_timer); + } + else + { + OnIntervalTimeout(); + } + } + break; case RX_OBJECT_NEW: + { //TRACE("NormApp::Notify(RX_OBJECT_NEW) ...\n"); + // It's up to the app to "accept" the object + switch (object->GetType()) { - switch (object->GetType()) + case NormObject::STREAM: { - case NormObject::STREAM: + const NormObjectSize& size = object->Size(); + if (!((NormStreamObject*)object)->Accept(size.LSB())) { - const NormObjectSize& size = object->Size(); - if (!((NormStreamObject*)object)->Accept(size.LSB())) + DMSG(0, "NormApp::Notify(RX_OBJECT_NEW) stream object accept error!\n"); + } + } + break; + case NormObject::FILE: + { + if (rx_cache_path) + { + // (TBD) re-arrange so if we've already recv'd INFO, + // we can use that for file name right away ??? + // (TBD) Manage recv file name collisions, etc ... + char fileName[PATH_MAX]; + strcpy(fileName, rx_cache_path); + strcat(fileName, "normTempXXXXXX"); +#ifdef WIN32 + if (!_mktemp(fileName)) +#else + int fd = mkstemp(fileName); + if (fd >= 0) { - TRACE("stream object accept error!\n"); + close(fd); + } + else +#endif // if/else WIN32 + { + DMSG(0, "NormApp::Notify(RX_OBJECT_NEW) Warning: mkstemp() error: %s\n", + strerror(errno)); + } + if (!((NormFileObject*)object)->Accept(fileName)) + { + DMSG(0, "NormApp::Notify(RX_OBJECT_NEW) file object accept error!\n"); } } - break; - case NormObject::FILE: - case NormObject::DATA: - TRACE("NormApp::Notify() FILE/DATA objects not supported...\n"); - break; - } - } + else + { + DMSG(0, "NormApp::Notify(RX_OBJECT_NEW) no rx cache for file\n"); + } + } + break; + case NormObject::DATA: + DMSG(0, "NormApp::Notify() FILE/DATA objects not _yet_ supported...\n"); + break; + } + break; + } + + case RX_OBJECT_INFO: + //TRACE("NormApp::Notify(RX_OBJECT_INFO) ...\n"); + switch(object->GetType()) + { + case NormObject::FILE: + { + // Rename rx file using newly received info + char fileName[PATH_MAX]; + strncpy(fileName, rx_cache_path, PATH_MAX); + UINT16 pathLen = strlen(rx_cache_path); + pathLen = MIN(pathLen, PATH_MAX); + UINT16 len = object->InfoLength(); + len = MIN(len, (PATH_MAX - pathLen)); + strncat(fileName, object->GetInfo(), len); + // Convert '/' in file info to directory delimiters + for (UINT16 i = pathLen; i < (pathLen+len); i++) + { + if ('/' == fileName[i]) + fileName[i] = DIR_DELIMITER; + } + pathLen += len; + if (pathLen < PATH_MAX) fileName[pathLen] = '\0'; + + // Deal with concurrent rx name collisions + // (TBD) and implement overwrite policy + // and cache files in cache mode + + if (!((NormFileObject*)object)->Rename(fileName)) + { + DMSG(0, "NormApp::Notify() Error renaming rx file: %s\n", + fileName); + } + break; + } + case NormObject::DATA: + case NormObject::STREAM: + break; + } // end switch(object->GetType()) break; case RX_OBJECT_UPDATE: //TRACE("NormApp::Notify(RX_OBJECT_UPDATE) ...\n"); switch (object->GetType()) { + case NormObject::FILE: + // (TBD) update progress + break; + case NormObject::STREAM: { - // Read the stream - char buffer[2048]; + // Read the stream when it's updated + ASSERT(output); + char buffer[256]; unsigned int nBytes; - while ((nBytes = ((NormStreamObject*)object)->Read(buffer, 2048))) + while ((nBytes = ((NormStreamObject*)object)->Read(buffer, 256))) { - for (unsigned int i =0; i < nBytes; i++) + unsigned int put = 0; + while (put < nBytes) { - if ('a' != buffer[i]) + size_t result = fwrite(buffer, sizeof(char), nBytes, output); + fflush(output); + if (result) { - TRACE("NormApp::Notify() bad data received!\n"); - break; + put += result; } - } - buffer[32] = '\0'; - DMSG(0, "NormApp::Notify() stream read %u bytes: \"%s\"\n", nBytes, buffer); + else + { + if (ferror(output)) + { + if (EINTR == errno) + { + clearerr(output); + continue; + } + else + { + DMSG(0, "norm: output error:%s\n", strerror(errno)); + clearerr(output); + break; + } + } + } + } // end while(put < nBytes) } - + break; } - break; - case NormObject::FILE: case NormObject::DATA: - TRACE("NormApp::Notify() FILE/DATA objects not supported...\n"); + DMSG(0, "NormApp::Notify() FILE/DATA objects not _yet_ supported...\n"); + break; + } // end switch (object->GetType()) + break; + case RX_OBJECT_COMPLETE: + { + switch(object->GetType()) + { + case NormObject::FILE: + DMSG(0, "norm: Completed rx file: %s\n", ((NormFileObject*)object)->Path()); + break; + + case NormObject::STREAM: + ASSERT(0); + break; + case NormObject::DATA: + ASSERT(0); break; } break; + } + } // end switch(event) +} // end NormApp::Notify() + + +bool NormApp::OnIntervalTimeout() +{ + char fileName[PATH_MAX]; + if (tx_file_list.GetNextFile(fileName)) + { + char pathName[PATH_MAX]; + tx_file_list.GetCurrentBasePath(pathName); + unsigned int len = strlen(pathName); + len = MIN(len, PATH_MAX); + unsigned int maxLen = PATH_MAX - len; + char* ptr = fileName + len; + len = strlen(ptr); + len = MIN(len, maxLen); + // (TBD) Make sure len <= segment_size) + char fileNameInfo[PATH_MAX]; + strncpy(fileNameInfo, ptr, len); + // Normalize directory delimiters in file name info + for (unsigned int i = 0; i < len; i++) + { + if (DIR_DELIMITER == fileNameInfo[i]) + fileNameInfo[i] = '/'; + } + char temp[PATH_MAX]; + strncpy(temp, fileNameInfo, len); + temp[len] = '\0'; + if (!session->QueueTxFile(fileName, fileNameInfo, len)) + { + DMSG(0, "NormApp::OnIntervalTimeout() Error opening tx file: %s\n", + fileName); + // Try the next file in the list + return OnIntervalTimeout(); + } + DMSG(0, "norm: File \"%s\" queued for transmission.\n", fileName); + interval_timer.SetInterval(tx_object_interval); } -} - -NormApp::NormApp() - : session(NULL), stream(NULL) -{ - // Init tx_timer for 1.0 second interval, infinite repeats - session_mgr.Init(EventDispatcher::TimerInstaller, &dispatcher, - EventDispatcher::SocketInstaller, &dispatcher, - this); -} - -NormApp::~NormApp() -{ - -} + else if (tx_repeat_count) + { + // (TBD) When repeating, remove previous instance from tx queue??? + if (tx_repeat_count > 0) tx_repeat_count--; + tx_file_list.ResetIterator(); + if (tx_repeat_interval > tx_object_interval) + { + if (interval_timer.IsActive()) interval_timer.Deactivate(); + interval_timer.SetInterval(tx_repeat_interval = tx_object_interval); + InstallTimer(&interval_timer); + return false; + } + else + { + return OnIntervalTimeout(); + } + } + else + { + DMSG(0, "norm: End of tx file list reached.\n"); + } + return true; +} // end NormApp::OnIntervalTimeout() bool NormApp::OnStartup() { @@ -137,34 +739,69 @@ bool NormApp::OnStartup() signal(SIGTERM, SignalHandler); signal(SIGINT, SignalHandler); + // Validate our application settings + if (!address) + { + DMSG(0, "NormApp::OnStartup() Error! no session address given."); + return false; + } + if (!input && !output && tx_file_list.IsEmpty() && !rx_cache_path) + { + DMSG(0, "NormApp::OnStartup() Error! no \"input\", \"output\", " + "\"sendfile\", or \"rxcache\" given."); + return false; + } - session = session_mgr.NewSession("224.225.1.5", 5005); + // Create a new session on multicast group/port + session = session_mgr.NewSession(address, port); if (session) { - if (!session->StartServer(1024*1024, 256, 20, 8)) - { - DMSG(0, "NormApp::OnStartup() start server error!\n"); - session_mgr.Destroy(); - return false; - } - session->ServerSetAutoParity(4); - //session->SetLoopback(true); + // Common session parameters + session->SetTxRate(tx_rate); + session->SetTrace(tracing); + session->SetTxLoss(tx_loss); + session->SetRxLoss(rx_loss); - // Open a stream object to write to - stream = session->QueueTxStream(10*1024); - if (!stream) + if (input || !tx_file_list.IsEmpty()) { - DMSG(0, "NormApp::OnStartup() queue tx stream error!\n"); - session_mgr.Destroy(); - return false; + // StartServer(bufferMax, segmentSize, fec_ndata, fec_nparity) + if (!session->StartServer(tx_buffer_size, segment_size, ndata, nparity)) + { + DMSG(0, "NormApp::OnStartup() start server error!\n"); + session_mgr.Destroy(); + return false; + } + session->ServerSetAutoParity(auto_parity); + + if (input) + { + // Open a stream object to write to (QueueTxStream(stream bufferSize)) + stream = session->QueueTxStream(tx_buffer_size); + if (!stream) + { + DMSG(0, "NormApp::OnStartup() queue tx stream error!\n"); + session_mgr.Destroy(); + return false; + } + } + } + + if (output || rx_cache_path) + { + TRACE("starting client ...\n"); + // StartClient(bufferMax (per-sender)) + if (!session->StartClient(rx_buffer_size)) + { + DMSG(0, "NormApp::OnStartup() start client error!\n"); + session_mgr.Destroy(); + return false; + } } - char text[256]; - memset(text, 'a', 256); - stream->Write(text, 256); return true; } else { + DMSG(0, "NormApp::OnStartup() new session error!\n"); return false; } } // end NormApp::OnStartup() @@ -172,6 +809,10 @@ bool NormApp::OnStartup() void NormApp::OnShutdown() { session_mgr.Destroy(); + if (input) fclose(input); + input = NULL; + if (output) fclose(output); + output = NULL; } // end NormApp::OnShutdown() @@ -180,14 +821,18 @@ NormApp theApp; // Use "main()" for UNIX and WIN32 console apps, // "WinMain()" for non-console WIN32 -// (VC++ uses the "_CONSOLE_ macro to indicate build type) +// (VC++ uses the "_CONSOLE_" macro to indicate build type) -#if defined(WIN32) && !defined(_CONSOLE) +#if defined(WIN32) && !defined(_CONSOLE_) int PASCAL WinMain(HINSTANCE instance, HINSTANCE prevInst, LPSTR cmdline, int cmdshow) +{ + // (TBD) transform WinMain "cmdLine" to (argc, argv[]) values + int argc = 0; + char* argv[] = NULL; #else int main(int argc, char* argv[]) -#endif { +#endif #ifdef WIN32 // Hack to determine if Win32 console application was launched // independently or from a pre-existing console window @@ -207,16 +852,29 @@ int main(int argc, char* argv[]) OpenDebugWindow(); pauseForUser = true; } - - #endif // WIN32 - + int exitCode = 0; - if (theApp.OnStartup()) + if (theApp.ParseCommandLine(argc, argv)) { - exitCode = theApp.MainLoop(); - theApp.OnShutdown(); - fprintf(stderr, "norm: Done.\n"); + if (theApp.OnStartup()) + { + exitCode = theApp.MainLoop(); + theApp.OnShutdown(); + fprintf(stderr, "norm: Done.\n"); + } + else + { + fprintf(stderr, "norm: Error initializing application!\n"); + exitCode = -1; + } + } + else + { + fprintf(stderr, "norm: Error parsing command line!\n"); + exitCode = -1; + } + #ifdef WIN32 // If Win32 console is going to disappear, pause for user before exiting if (pauseForUser) @@ -224,13 +882,7 @@ int main(int argc, char* argv[]) printf ("Program Finished - Hit to exit"); getchar(); } -#endif // WIN32 - } - else - { - fprintf(stderr, "norm: Error initializing application!\n"); - return -1; - } +#endif // WIN32 return exitCode; // exitCode contains "signum" causing exit } // end main(); @@ -240,7 +892,7 @@ void NormApp::SignalHandler(int sigNum) { case SIGTERM: case SIGINT: - theApp.Stop(sigNum); // causes theApp's main loop to exit + theApp.Stop(sigNum); // causes theApp's main loop to exit break; default: diff --git a/common/normBitmask.cpp b/common/normBitmask.cpp index 9b6d361..52b39bb 100644 --- a/common/normBitmask.cpp +++ b/common/normBitmask.cpp @@ -1,5 +1,7 @@ #include "normBitmask.h" +#include "debug.h" +#include "sysdefs.h" // Hamming weights for given one-byte bit masks static unsigned char WEIGHT[256] = @@ -377,7 +379,9 @@ bool NormBitmask::XCopy(const NormBitmask& b) { if (b.num_bits > num_bits) return false; unsigned int len = b.mask_len; - for (unsigned int i = 0; i < len; i++) + unsigned int begin = b.first_set >> 3; + if (begin) memset(mask, 0, begin); + for (unsigned int i = begin; i < len; i++) mask[i] = b.mask[i] & ~mask[i]; if (len < mask_len) memset(&mask[len], 0, mask_len - len); if (b.first_set < first_set) @@ -601,6 +605,7 @@ bool NormSlidingMask::Unset(unsigned long index) mask[(pos >> 3)] &= ~(0x80 >> (pos & 0x07)); if (start == end) { + ASSERT(pos == start); start = end = num_bits; return true; } @@ -1133,7 +1138,7 @@ bool NormSlidingMask::Add(const NormSlidingMask& b) } // end NormSlidingMask::Add() // A sort of logical subtraction "this = (this & ~b)" -// This leaves us with bits uniquely set in this instance +// This leaves us with bits uniquely set in "this" bool NormSlidingMask::Subtract(const NormSlidingMask& b) { if (b.IsSet()) @@ -1150,11 +1155,15 @@ bool NormSlidingMask::Subtract(const NormSlidingMask& b) } } } + else + { + Copy(b); + } return true; } // end NormSlidingMask::Subtract() // A sort of logical subtraction "this = (~this & b)" -// This leaves us with bits uniquely set in this instance +// This leaves us with bits uniquely set in "b" bool NormSlidingMask::XCopy(const NormSlidingMask& b) { if (b.IsSet()) @@ -1174,6 +1183,10 @@ bool NormSlidingMask::XCopy(const NormSlidingMask& b) Set(index); index++; } + } + else + { + Copy(b); } } return true; @@ -1232,3 +1245,18 @@ void NormSlidingMask::Display(FILE* stream) if (0x3f == (i & 0x3f)) fprintf(stream, "\n"); } } // end NormSlidingMask::Display() + +void NormSlidingMask::Debug(long theCount) +{ + unsigned long index = offset; + theCount = MIN(theCount, num_bits); + DMSG(0, "NormSlidingMask::Debug() offset:%lu\n ", index); + long i; + for (i = 0; i < theCount; i++) + { + if (Test(index++)) DMSG(0, "1"); else DMSG(0, "0"); + if (0x07 == (i & 0x07)) DMSG(0, " "); + if (0x3f == (i & 0x3f)) DMSG(0, "\n "); + } + if (0x3f != (i & 0x3f)) DMSG(0, "\n"); +} // end NormSlidingMask::Display() diff --git a/common/normBitmask.h b/common/normBitmask.h index 97f1031..1426717 100644 --- a/common/normBitmask.h +++ b/common/normBitmask.h @@ -128,6 +128,7 @@ class NormSlidingMask } void Reset(unsigned long index = 0) { + ASSERT(num_bits); memset(mask, 0xff, mask_len); mask[mask_len-1] = 0x00ff << ((8 - (num_bits & 0x07)) & 0x07); start = 0; @@ -171,6 +172,7 @@ class NormSlidingMask bool Xor(const NormSlidingMask & b); // this = this ^ b void Display(FILE* stream); + void Debug(long theCount); private: unsigned char* mask; diff --git a/common/normFile.cpp b/common/normFile.cpp new file mode 100644 index 0000000..085e326 --- /dev/null +++ b/common/normFile.cpp @@ -0,0 +1,986 @@ + +#include "normFile.h" + +#include // for strerror() +#include // for errno + +#ifdef WIN32 +#include +#include +#else +#include +// Most don't have the dirfd() function +#ifndef HAVE_DIRFD +static inline int dirfd(DIR *dir) {return (dir->dd_fd);} +#endif // HAVE_DIRFD +#endif // if/else WIN32 + +NormFile::NormFile() + : fd(-1) +{ +} + +NormFile::~NormFile() +{ + if (IsOpen()) Close(); +} // end NormFile::~NormFile() + +// This should be called with a full path only! +bool NormFile::Open(const char* thePath, int theFlags) +{ + ASSERT(!IsOpen()); + if (theFlags & O_CREAT) + { + // Create sub-directories as needed. + char tempPath[PATH_MAX]; + strncpy(tempPath, thePath, PATH_MAX); + char* ptr = strrchr(tempPath, DIR_DELIMITER); + if (ptr) *ptr = '\0'; + ptr = NULL; + while (!NormFile::Exists(tempPath)) + { + char* ptr2 = ptr; + ptr = strrchr(tempPath, DIR_DELIMITER); + if (ptr2) *ptr2 = DIR_DELIMITER; + if (ptr) + { + *ptr = '\0'; + } + else + { + ptr = tempPath; + break; + } + } + if (ptr && ('\0' == *ptr)) *ptr++ = DIR_DELIMITER; + while (ptr) + { + ptr = strchr(ptr, DIR_DELIMITER); + if (ptr) *ptr = '\0'; + if (mkdir(tempPath, 0755)) + { + DMSG(0, "NormFile::Rename() mkdir(%s) error: %s\n", + tempPath, strerror(errno)); + return false; + } + if (ptr) *ptr++ = DIR_DELIMITER; + } + } +#ifdef WIN32 + // Make sure we're in binary mode (important for WIN32) + theFlags |= _O_BINARY; + // Allow sharing of read-only files but not of files being written + if (theFlags & _O_RDONLY) + fd = _sopen(thePath, theFlags, _SH_DENYNO); + else + fd = _open(thePath, theFlags, 0640); + if(fd >= 0) + { + offset = 0; + flags = theFlags; + return true; // no error + } + else + { + DMSG(0, "Error opening file \"%s\": %s\n", path, + strerror(errno)); + flags = 0; + return false; + } +#else + if((fd = open(thePath, theFlags, 0640)) >= 0) + { + offset = 0; + return true; // no error + } + else + { + DMSG(0, "norm: Error opening file \"%s\": %s\n", + thePath, strerror(errno)); + return false; + } +#endif // if/else WIN32 +} // end NormFile::Open() + +void NormFile::Close() +{ + if (IsOpen()) + { +#ifdef WIN32 + _close(fd); +#else + close(fd); +#endif // if/else WIN32 + fd = -1; + } +} // end NormFile::Close() + + +// Routines to try to get an exclusive lock on a file +bool NormFile::Lock() +{ +#ifndef WIN32 // WIN32 files are automatically locked + fchmod(fd, 0640 | S_ISGID); +#ifdef HAVE_FLOCK + if (flock(fd, LOCK_EX | LOCK_NB)) +#else +//#ifdef HAVE_LOCKF + if (lockf(fd, F_LOCK, 0)) // assume lockf if not flock +//#endif // HAVE_LOCKF +#endif // if/else HAVE_FLOCK + return false; + else +#endif // !WIN32 + return true; +} // end NormFile::Lock() + +void NormFile::Unlock() +{ +#ifndef WIN32 +#ifdef HAVE_FLOCK + flock(fd, LOCK_UN); +#else +#ifdef HAVE_LOCKF + lockf(fd, F_ULOCK, 0); +#endif // HAVE_LOCKF +#endif // HAVE_FLOCK + fchmod(fd, 0640); +#endif // !WIN32 +} // end NormFile::UnLock() + +bool NormFile::Rename(const char* oldName, const char* newName) +{ + if (!strcmp(oldName, newName)) return true; // no change required + // Make sure the new file name isn't an existing "busy" file + // (This also builds sub-directories as needed) + if (NormFile::IsLocked(newName)) return false; +#ifdef WIN32 + // In Win32, the new file can't already exist + if (NormFile::Exists(newName) _unlink(newName); + // In Win32, the old file can't be open + int oldFlags = 0; + if (IsOpen()) + { + oldFlags = flags; + oldFlags &= ~(O_CREAT | O_TRUNC); // unset these + Close(); + } +#else + // Create sub-directories as needed. + char tempPath[PATH_MAX]; + strncpy(tempPath, newName, PATH_MAX); + char* ptr = strrchr(tempPath, DIR_DELIMITER); + if (ptr) *ptr = '\0'; + ptr = NULL; + while (!NormFile::Exists(tempPath)) + { + char* ptr2 = ptr; + ptr = strrchr(tempPath, DIR_DELIMITER); + if (ptr2) *ptr2 = DIR_DELIMITER; + if (ptr) + { + *ptr = '\0'; + } + else + { + ptr = tempPath; + break; + } + } + if (ptr && ('\0' == *ptr)) *ptr++ = DIR_DELIMITER; + while (ptr) + { + ptr = strchr(ptr, DIR_DELIMITER); + if (ptr) *ptr = '\0'; + if (mkdir(tempPath, 0755)) + { + DMSG(0, "NormFile::Rename() mkdir(%s) error: %s\n", + tempPath, strerror(errno)); + return false; + } + if (ptr) *ptr++ = DIR_DELIMITER; + } +#endif // if/else WIN32 + if (rename(oldName, newName)) + { + DMSG(0, "NormFile::Rename() rename() error: %s\n", strerror(errno)); +#ifdef WIN32 + //if (oldFlags) Open(oldName, oldFlags); +#endif // WIN32 + return false; + + } + else + { +#ifdef WIN32 + // (TBD) Is the file offset OK doing this??? + //if (oldFlags) Open(newName, oldFlags); +#endif // WIN32 + return true; + } +} // end NormFile::Rename() + +int NormFile::Read(char* buffer, int len) +{ + ASSERT(IsOpen()); +#ifdef WIN32 + int result = _read(fd, buffer, len); +#else + int result = read(fd, buffer, len); +#endif // if/else WIN32 + if (result > 0) offset += result; + return result; +} // end NormFile::Read() + +int NormFile::Write(const char* buffer, int len) +{ + ASSERT(IsOpen()); +#ifdef WIN32 + int result = _write(fd, buffer, len); +#else + int result = write(fd, buffer, len); +#endif // if/else WIN32 + if (result > 0) offset += result; + return result; +} // end NormFile::Write() + +bool NormFile::Seek(off_t theOffset) +{ + ASSERT(IsOpen()); +#ifdef WIN32 + off_t result = _lseek(fd, theOffset, SEEK_SET); +#else + off_t result = lseek(fd, theOffset, SEEK_SET); +#endif // if/else WIN32 + if (result == (off_t)-1) + { + DMSG(0, "NormFile::Seek() lseek() error: %s", strerror(errno)); + return false; + } + else + { + offset = result; + return true; // no error + } +} // end NormFile::Seek() + +off_t NormFile::GetSize() const +{ + ASSERT(IsOpen()); +#ifdef WIN32 + struct _stat info; + int result = _fstat(fd, &info); +#else + struct stat info; + int result = fstat(fd, &info); +#endif // if/else WIN32 + if (result) + { + DMSG(0, "Error getting file size: %s\n", strerror(errno)); + return 0; + } + else + { + return info.st_size; + } +} // end NormFile::GetSize() + + +/*********************************************** + * The NormDirectoryIterator classes is used to + * walk directory trees for file transmission + */ + +NormDirectoryIterator::NormDirectoryIterator() + : current(NULL) +{ + +} + +NormDirectoryIterator::~NormDirectoryIterator() +{ + Close(); +} + +bool NormDirectoryIterator::Open(const char *thePath) +{ + if (current) Close(); +#ifdef WIN32 + if (thePath && _access(thePath, 0)) +#else + if (thePath && access(thePath, X_OK)) +#endif // if/else WIN32 + { + DMSG(0, "NormDirectoryIterator: can't access directory: %s\n", thePath); + return false; + } + current = new NormDirectory(thePath); + if (current && current->Open()) + { + path_len = strlen(current->Path()); + path_len = MIN(PATH_MAX, path_len); + return true; + } + else + { + DMSG(0, "NormDirectoryIterator: can't open directory: %s\n", thePath); + if (current) delete current; + current = NULL; + return false; + } +} // end NormDirectoryIterator::Init() + +void NormDirectoryIterator::Close() +{ + NormDirectory* d; + while ((d = current)) + { + current = d->parent; + d->Close(); + delete d; + } +} // end NormDirectoryIterator::Close() + +bool NormDirectoryIterator::GetPath(char* pathBuffer) +{ + if (current) + { + NormDirectory* d = current; + while (d->parent) d = d->parent; + strncpy(pathBuffer, d->Path(), PATH_MAX); + return true; + } + else + { + pathBuffer[0] = '\0'; + return false; + } +} + +#ifdef WIN32 +bool NormDirectoryIterator::GetNextFile(char* fileName) +{ + if (!current) return false; + bool success = true; + while(success) + { + WIN32_findData findData; + if (current->hSearch == (HANDLE)-1) + { + // Construct search string + current->GetFullName(fileName); + strcat(fileName, "\\*"); + if ((HANDLE)-1 == + (current->hSearch = FindFirstFile(fileName, &findData)) ) + success = false; + else + success = true; + } + else + { + success = (0 != FindNextFile(current->hSearch, &findData)); + } + + // Do we have a candidate file? + if (success) + { + char* ptr = strrchr(findData.cFileName, DIR_DELIMITER); + if (ptr) + ptr += 1; + else + ptr = findData.cFileName; + + // Skip "." and ".." directories + if (ptr[0] == '.') + { + if ((1 == strlen(ptr)) || + ((ptr[1] == '.') && (2 == strlen(ptr)))) + { + continue; + } + } + current->GetFullName(fileName); + strcat(fileName, ptr); + NormFile::Type type = NormFile::GetType(fileName); + if (NormFile::NORMAL == type) + { + int nameLen = strlen(fileName); + nameLen = MIN(PATH_MAX, nameLen); + nameLen -= path_len; + memmove(fileName, fileName+path_len, nameLen); + if (nameLen < PATH_MAX) fileName[nameLen] = '\0'; + return true; + } + else if (NormFile::DIRECTORY == type) + { + + NormDirectory *dir = new NormDirectory(ptr, current); + if (dir && dir->Open()) + { + // Push sub-directory onto stack and search it + current = dir; + return GetNextFile(fileName); + } + else + { + // Couldn't open try next one + if (dir) delete dir; + } + } + else + { + // NormFile::INVALID file, try next one + } + } // end if(success) + } // end while(success) + + // if parent, popup a level and continue search + if (current->parent) + { + current->Close(); + NormDirectory *dir = current; + current = current->parent; + delete dir; + return GetNextFile(fileName); + } + else + { + current->Close(); + delete current; + current = NULL; + return false; + } +} // end NormDirectoryIterator::GetNextFile() (WIN32) +#else +bool NormDirectoryIterator::GetNextFile(char* fileName) +{ + if (!current) return false; + struct dirent *dp; + while((dp = readdir(current->dptr))) + { + // Make sure it's not "." or ".." + if (dp->d_name[0] == '.') + { + if ((1 == strlen(dp->d_name)) || + ((dp->d_name[1] == '.' ) && (2 == strlen(dp->d_name)))) + { + continue; // skip "." and ".." directory names + } + } + current->GetFullName(fileName); + strcat(fileName, dp->d_name); + NormFile::Type type = NormFile::GetType(fileName); + if (NormFile::NORMAL == type) + { + int nameLen = strlen(fileName); + nameLen = MIN(PATH_MAX, nameLen); + nameLen -= path_len; + memmove(fileName, fileName+path_len, nameLen); + if (nameLen < PATH_MAX) fileName[nameLen] = '\0'; + return true; + } + else if (NormFile::DIRECTORY == type) + { + NormDirectory *dir = new NormDirectory(dp->d_name, current); + if (dir && dir->Open()) + { + // Push sub-directory onto stack and search it + current = dir; + return GetNextFile(fileName); + } + else + { + // Couldn't open this one, try next one + if (dir) delete dir; + } + } + else + { + // NormFile::INVALID, try next one + } + } // end while(readdir()) + + // Pop up a level and recursively continue or finish if done + if (current->parent) + { + char path[PATH_MAX]; + current->parent->GetFullName(path); + current->Close(); + NormDirectory *dir = current; + current = current->parent; + delete dir; + return GetNextFile(fileName); + } + else + { + current->Close(); + delete current; + current = NULL; + return false; // no more files remain + } +} // end NormDirectoryIterator::GetNextFile() (UNIX) +#endif // if/else WIN32 + +NormDirectoryIterator::NormDirectory::NormDirectory(const char* thePath, + NormDirectory* theParent) + : parent(theParent), +#ifdef WIN32 + hSearch((HANDLE)-1) +#else + dptr(NULL) +#endif // if/else WIN32 +{ + strncpy(path, thePath, PATH_MAX); + int len = MIN(PATH_MAX, strlen(path)); + if ((len < PATH_MAX) && (DIR_DELIMITER != path[len-1])) + { + path[len++] = DIR_DELIMITER; + if (len < PATH_MAX) path[len] = '\0'; + } +} + +NormDirectoryIterator::NormDirectory::~NormDirectory() +{ + Close(); +} + +bool NormDirectoryIterator::NormDirectory::Open() +{ + Close(); // in case it's already open + char fullName[PATH_MAX]; + GetFullName(fullName); + // Get rid of trailing DIR_DELIMITER + int len = MIN(PATH_MAX, strlen(fullName)); + if (DIR_DELIMITER == fullName[len-1]) fullName[len-1] = '\0'; +#ifdef WIN32 + DWORD attr = GetFileAttributes(fullName); + if (0xFFFFFFFF == attr) + return false; + else if (attr & FILE_ATTRIBUTE_DIRECTORY) + return true; + else + return false; +#else + if((dptr = opendir(fullName))) + return true; + else + return false; +#endif // if/else WIN32 + +} // end NormDirectoryIterator::NormDirectory::Open() + +void NormDirectoryIterator::NormDirectory::Close() +{ +#ifdef WIN32 + if (hSearch != (HANDLE)-1) + { + FindClose(hSearch); + hSearch = (HANDLE)-1; + } +#else + closedir(dptr); + dptr = NULL; +#endif // if/else WIN32 +} // end NormDirectoryIterator::NormDirectory::Close() + + +void NormDirectoryIterator::NormDirectory::GetFullName(char* ptr) +{ + ptr[0] = '\0'; + RecursiveCatName(ptr); +} // end NormDirectoryIterator::NormDirectory::GetFullName() + +void NormDirectoryIterator::NormDirectory::RecursiveCatName(char* ptr) +{ + if (parent) parent->RecursiveCatName(ptr); + int len = MIN(PATH_MAX, strlen(ptr)); + strncat(ptr, path, PATH_MAX-len); +} // end NormDirectoryIterator::NormDirectory::RecursiveCatName() + +// Below are some static routines for getting file/directory information + +// Is the named item a valid directory or file (or neither)?? +NormFile::Type NormFile::GetType(const char* path) +{ +#ifdef WIN32 + DWORD attr = GetFileAttributes(path); + if (0xFFFFFFFF == attr) + return INVALID; // error + else if (attr & FILE_ATTRIBUTE_DIRECTORY) + return DIRECTORY; + else + return NORMAL; +#else + struct stat file_info; + if (stat(path, &file_info)) + return INVALID; // stat() error + else if ((S_ISDIR(file_info.st_mode))) + return DIRECTORY; + else + return NORMAL; +#endif // if/else WIN32 +} // end NormFile::GetType() + +off_t NormFile::GetSize(const char* path) +{ +#ifdef WIN32 + struct _stat info; + int result = _stat(path, &info); +#else + struct stat info; + int result = stat(path, &info); +#endif // if/else WIN32 + if (result) + { + //DMSG(0, "Error getting file size: %s\n", strerror(errno)); + return 0; + } + else + { + return info.st_size; + } +} // end NormFile::GetSize() + +time_t NormFile::GetUpdateTime(const char* path) +{ +#ifdef WIN32 + struct _stat info; + int result = _stat(path, &info); +#else + struct stat info; + int result = stat(path, &info); +#endif // if/else WIN32 + if (result) + { + return (time_t)0; // stat() error + } + else + { +#ifdef WIN32 + // Hack because Win2K and Win98 seem to work differently + time_t updateTime = MAX(info.st_ctime, info.st_atime); + updateTime = MAX(updateTime, info.st_mtime); + return updateTime; +#else + return info.st_ctime; +#endif // if/else WIN32 + } +} // end NormFile::GetUpdateTime() + +bool NormFile::IsLocked(const char* path) +{ + // If file doesn't exist, it's not locked + if (!Exists(path)) return false; + NormFile testFile; +#ifdef WIN32 + if(!testFile.Open(path, _O_WRONLY | _O_CREAT)) +#else + if(!testFile.Open(path, O_WRONLY | O_CREAT)) +#endif // if/else WIN32 + { + return true; + } + else if (testFile.Lock()) + { + // We were able to lock the file successfully + testFile.Unlock(); + testFile.Close(); + return false; + } + else + { + testFile.Close(); + return true; + } +} // end NormFile::IsLocked() + +bool NormFile::Unlink(const char* path) +{ + // Don't unlink a file that is open (locked) + if (NormFile::IsLocked(path)) + { + return false; + } +#ifdef WIN32 + else if (_unlink(path)) +#else + else if (unlink(path)) +#endif // if/else WIN32 + { + //DMSG(0, "NormFile::Unlink() unlink error: %s\n", strerror(errno)); + return false; + } + else + { + return true; + } +} // end NormFile::Unlink() + +NormFileList::NormFileList() + : this_time(0), big_time(0), last_time(0), + updates_only(false), head(NULL), tail(NULL), next(NULL) +{ +} + +NormFileList::~NormFileList() +{ + Destroy(); +} + +void NormFileList::Destroy() +{ + while ((next = head)) + { + head = next->next; + delete next; + } + tail = NULL; +} // end NormFileList::Destroy() + +bool NormFileList::Append(const char* path) +{ + FileItem* theItem = NULL; + switch(NormFile::GetType(path)) + { + case NormFile::NORMAL: + theItem = new FileItem(path); + break; + case NormFile::DIRECTORY: + theItem = new DirectoryItem(path); + break; + default: + // Allow non-existent files for update_only mode + // (TBD) allow non-existent directories? + if (updates_only) + { + theItem = new FileItem(path); + } + else + { + DMSG(0, "NormFileList::Append() Bad file/directory name: %s\n", + path); + return false; + } + break; + } + if (theItem) + { + theItem->next = NULL; + if ((theItem->prev = tail)) + tail->next = theItem; + else + head = theItem; + tail = theItem; + return true; + } + else + { + DMSG(0, "NormFileList::Append() Error creating file/directory item: %s\n", + strerror(errno)); + return false; + } +} // end NormFileList::Append() + +bool NormFileList::Remove(const char* path) +{ + FileItem* nextItem = head; + unsigned int pathLen = strlen(path); + pathLen = MIN(pathLen, PATH_MAX); + while (nextItem) + { + unsigned nameLen = strlen(nextItem->Path()); + nameLen = MIN(nameLen, PATH_MAX); + nameLen = MAX(nameLen, pathLen); + if (!strncmp(path, nextItem->Path(), nameLen)) + { + if (nextItem == next) next = nextItem->next; + if (nextItem->prev) + nextItem->prev = next = nextItem->next; + else + head = nextItem->next; + if (nextItem->next) + nextItem->next->prev = nextItem->prev; + else + tail = nextItem->prev; + return true; + } + } + return false; +} // end NormFileList::Remove() + +bool NormFileList::GetNextFile(char* pathBuffer) +{ + if (!next) + { + next = head; + reset = true; + } + if (next) + { + if (next->GetNextFile(pathBuffer, reset, updates_only, + last_time, this_time, big_time)) + { + reset = false; + return true; + } + else + { + if (next->next) + { + next = next->next; + reset = true; + return GetNextFile(pathBuffer); + } + else + { + reset = false; + return false; // end of list + } + } + } + else + { + return false; // empty list + } +} // end NormFileList::GetNextFile() + +void NormFileList::GetCurrentBasePath(char* pathBuffer) +{ + if (next) + { + if (NormFile::DIRECTORY == next->GetType()) + { + strncpy(pathBuffer, next->Path(), PATH_MAX); + unsigned int len = strlen(pathBuffer); + len = MIN(len, PATH_MAX); + if (DIR_DELIMITER != pathBuffer[len-1]) + { + if (len < PATH_MAX) pathBuffer[len++] = DIR_DELIMITER; + if (len < PATH_MAX) pathBuffer[len] = '\0'; + } + } + else // NormFile::NORMAL + { + const char* ptr = strrchr(next->Path(), DIR_DELIMITER); + if (ptr++) + { + unsigned int len = ptr - next->Path(); + strncpy(pathBuffer, next->Path(), len); + pathBuffer[len] = '\0'; + } + else + { + pathBuffer[0] = '\0'; + } + } + } + else + { + pathBuffer[0] = '\0'; + } +} // end NormFileList::GetBasePath() + + +NormFileList::FileItem::FileItem(const char* thePath) + : prev(NULL), next(NULL) +{ + unsigned int len = strlen(thePath); + len = MIN(len, PATH_MAX); + strncpy(path, thePath, PATH_MAX); + size = NormFile::GetSize(thePath); +} + +NormFileList::FileItem::~FileItem() +{ +} + +bool NormFileList::FileItem::GetNextFile(char* thePath, + bool reset, + bool updatesOnly, + time_t lastTime, + time_t thisTime, + time_t& bigTime) +{ + if (reset) + { + if (updatesOnly) + { + time_t updateTime = NormFile::GetUpdateTime(thePath); + if (updateTime > bigTime) bigTime = updateTime; + if ((updateTime <= lastTime) || (updateTime > thisTime)) + return false; + } + strncpy(thePath, path, PATH_MAX); + return true; + } + else + { + return false; + } +} // end NormFileList::FileItem::GetNextFile() + +NormFileList::DirectoryItem::DirectoryItem(const char* thePath) + : NormFileList::FileItem(thePath) +{ +} + +NormFileList::DirectoryItem::~DirectoryItem() +{ + diterator.Close(); +} + +bool NormFileList::DirectoryItem::GetNextFile(char* thePath, + bool reset, + bool updatesOnly, + time_t lastTime, + time_t thisTime, + time_t& bigTime) +{ + if (reset) + { + /* For now we are going to poll all files in a directory individually + since directory update times aren't always changed when files are + are replaced within the directory tree ... uncomment this code + if you only want to check directory nodes that have had their + change time updated + if (updates_only) + { + // Check to see if directory has been touched + time_t update_time = MdpFileGetUpdateTime(path); + if (updateTime > bigTime) *bigTime = updateTime; + if ((updateTime <= lastTime) || (updateTime > thisTime)) + return false; + } */ + if (!diterator.Open(path)) + { + DMSG(0, "NormFileList::DirectoryItem::GetNextFile() Directory iterator init error\n"); + return false; + } + } + strncpy(thePath, path, PATH_MAX); + unsigned int len = strlen(thePath); + len = MIN(len, PATH_MAX); + if ((DIR_DELIMITER != thePath[len-1]) && (len < PATH_MAX)) + { + thePath[len++] = DIR_DELIMITER; + if (len < PATH_MAX) thePath[len] = '\0'; + } + char tempPath[PATH_MAX]; + while (diterator.GetNextFile(tempPath)) + { + unsigned int maxLen = PATH_MAX - len; + strncat(thePath, tempPath, maxLen); + if (updatesOnly) + { + time_t updateTime = NormFile::GetUpdateTime(thePath); + if (updateTime > bigTime) bigTime = updateTime; + if ((updateTime <= lastTime) || (updateTime > thisTime)) + { + thePath[len] = '\0'; + continue; + } + } + return true; + } + return false; +} // end NormFileList::DirectoryItem::GetNextFile() diff --git a/common/normFile.h b/common/normFile.h new file mode 100644 index 0000000..63cf272 --- /dev/null +++ b/common/normFile.h @@ -0,0 +1,196 @@ +#ifndef _NORM_FILE +#define _NORM_FILE + +// This module defines some simple classes for manipulating files. +// Unix and Win32 platforms are supported. Routines for iterating +// over directories are also provided. And a file/directory list +// class is provided to manage a list of files. + +#ifdef WIN32 +#include +#else +#include +#include +#endif // if/else WIN32 + +#include +#include +#include + +// From PROTOLIB +#include "sysdefs.h" // for bool definition, DIR_DELIMITER, PATH_MAX, etc +#include "debug.h" // for DEBUG stuff + +class NormFile +{ + // Methods + public: + enum Type {INVALID, NORMAL, DIRECTORY}; + NormFile(); + ~NormFile(); + bool Open(const char* path, int theFlags); + bool Lock(); + void Unlock(); + bool Rename(const char* oldName, const char* newName); + bool Unlink(const char *path); + void Close(); + bool IsOpen() const {return (fd >= 0);} + int Read(char* buffer, int len); + int Write(const char* buffer, int len); + bool Seek(off_t theOffset); + off_t GetOffset() const {return (offset);} + off_t GetSize() const; + + // static helper methods + static NormFile::Type GetType(const char *path); + static off_t GetSize(const char* path); + static time_t GetUpdateTime(const char* path); + static bool IsLocked(const char *path); + + static bool Exists(const char* path) + { +#ifdef WIN32 + return (0xFFFFFFFF != GetFileAttributes(path)); +#else + return (0 == access(path, F_OK)); +#endif // if/else WIN32 + } + + static bool IsWritable(const char* path) + { +#ifdef WIN32 + DWORD attr = GetFileAttributes(path); + return ((0xFFFFFFFF == attr) ? + false : (0 == (attr & FILE_ATTRIBUTE_READONLY))); +#else + return (0 == access(path, W_OK)); + } +#endif // if/else WIN32 + + + // Members + private: + int fd; + int flags; + off_t offset; +}; + +/****************************************** +* The NormDirectory and NormDirectoryIterator classes +* are used to walk directory trees for file transmission +*/ + +class NormDirectoryIterator +{ + public: + NormDirectoryIterator(); + ~NormDirectoryIterator(); + bool Open(const char*thePath); + void Close(); + bool GetPath(char* pathBuffer); + // "buffer" should be PATH_MAX long! + bool GetNextFile(char* buffer); + + private: + class NormDirectory + { + friend class NormDirectoryIterator; + private: + char path[PATH_MAX]; + NormDirectory* parent; + #ifdef WIN32 + HANDLE hSearch; + #else + DIR* dptr; + #endif // if/else WIN32 + NormDirectory(const char *thePath, NormDirectory* theParent = NULL); + ~NormDirectory(); + void GetFullName(char* namePtr); + bool Open(); + void Close(); + + const char* Path() const {return path;} + void RecursiveCatName(char* ptr); + }; // end class NormDirectoryIterator::NormDirectory + + NormDirectory* current; + int path_len; +}; // end class NormDirectoryIterator + + +class NormFileList +{ + public: + NormFileList(); + ~NormFileList(); + void Destroy(); + bool IsEmpty() {return (NULL == head);} + void ResetIterator() + { + last_time = this_time; + this_time = big_time; + next = NULL; + reset = true; + } + void InitUpdateTime(bool updatesOnly, time_t initTime = 0) + { + updates_only = updatesOnly; + last_time = this_time = big_time = initTime; + } + + bool Append(const char* path); + bool Remove(const char* path); + bool GetNextFile(char* pathBuffer); + void GetCurrentBasePath(char* pathBuffer); + + private: + class FileItem + { + friend class NormFileList; + public: + FileItem(const char* thePath); + virtual ~FileItem(); + NormFile::Type GetType() {return NormFile::GetType(path);} + off_t Size() const {return size;} + virtual bool GetNextFile(char* thePath, + bool reset, + bool updatesOnly, + time_t lastTime, + time_t thisTime, + time_t& bigTime); + + protected: + const char* Path() {return path;} + + char path[PATH_MAX]; + off_t size; + FileItem* prev; + FileItem* next; + }; + class DirectoryItem : public FileItem + { + friend class NormFileList; + public: + DirectoryItem(const char* thePath); + ~DirectoryItem(); + virtual bool GetNextFile(char* thePath, + bool reset, + bool updatesOnly, + time_t lastTime, + time_t thisTime, + time_t& bigTime); + private: + NormDirectoryIterator diterator; + }; + + time_t this_time; + time_t big_time; + time_t last_time; + bool updates_only; + FileItem* head; + FileItem* tail; + FileItem* next; + bool reset; +}; // end class NormFileList + +#endif // _NORM_FILE diff --git a/common/normMessage.cpp b/common/normMessage.cpp index 86aedc7..dec56d1 100644 --- a/common/normMessage.cpp +++ b/common/normMessage.cpp @@ -1,7 +1,5 @@ #include "normMessage.h" -#include "debug.h" - NormRepairRequest::NormRepairRequest() : form(INVALID), flags(0), length(0), buffer(NULL), buffer_len(0) { @@ -11,7 +9,13 @@ bool NormRepairRequest::AppendRepairItem(const NormObjectId& objectId, const NormBlockId& blockId, UINT16 symbolId) { - if (buffer_len >= (CONTENT_OFFSET+length+RepairItemLength())) + if (RANGES == form) + DMSG(4, "NormRepairRequest::AppendRepairRange(obj>%hu blk>%lu seg>%hu) ...\n", + (UINT16)objectId, (UINT32)blockId, (UINT32)symbolId); + else + DMSG(4, "NormRepairRequest::AppendRepairItem(obj>%hu blk>%lu seg>%hu) ...\n", + (UINT16)objectId, (UINT32)blockId, (UINT32)symbolId); + if (buffer_len >= (length+CONTENT_OFFSET+RepairItemLength())) { UINT16 temp16 = htons((UINT16)objectId); memcpy(buffer+CONTENT_OFFSET+length, &temp16, 2); @@ -35,7 +39,10 @@ bool NormRepairRequest::AppendRepairRange(const NormObjectId& startObjectId, const NormBlockId& endBlockId, UINT16 endSymbolId) { - if (buffer_len >= (CONTENT_OFFSET+length+RepairRangeLength())) + DMSG(4, "NormRepairRequest::AppendRepairRange(%hu:%lu:%hu->%hu:%lu:%hu) ...\n", + (UINT16)startObjectId, (UINT32)startBlockId, (UINT32)startSymbolId, + (UINT16)endObjectId, (UINT32)endBlockId, (UINT32)endSymbolId); + if (buffer_len >= (length+CONTENT_OFFSET+RepairRangeLength())) { // range start UINT16 temp16; @@ -257,6 +264,28 @@ NormMessage* NormMessageQueue::RemoveTail() } } // end NormMessageQueue::RemoveTail() + +/**************************************************************** + * RTT quantization routines: + * These routines are valid for 1.0e-06 <= RTT <= 1000.0 seconds + * They allow us to pack our RTT estimates into a 1 byte fields + */ + +// valid for rtt = 1.0e-06 to 1.0e+03 +unsigned char NormQuantizeRtt(double rtt) +{ + if (rtt > NORM_RTT_MAX) + rtt = NORM_RTT_MAX; + else if (rtt < NORM_RTT_MIN) + rtt = NORM_RTT_MIN; + if (rtt < 3.3e-05) + return ((unsigned char)ceil((rtt*NORM_RTT_MIN)) - 1); + else + return ((unsigned char)(ceil(255.0 - (13.0*log(NORM_RTT_MAX/rtt))))); +} // end NormQuantizeRtt() + + + bool NormObjectSize::operator<(const NormObjectSize& size) const { UINT16 diff = msb - size.msb; @@ -314,16 +343,15 @@ NormObjectSize NormObjectSize::operator*(const NormObjectSize& b) const return result; } // end NormObjectSize::operator*(NormObjectSize size) +// Note: This always rounds up if there is _any_ remainder NormObjectSize NormObjectSize::operator/(const NormObjectSize& b) const { // Zero dividend is special case if ((0 == lsb) && (0 == msb)) return NormObjectSize(0, 0); // Zero divisor is special case if ((0 == b.lsb) && (0 == b.msb)) return NormObjectSize(0xffff, 0xffffffff); - // Dividend equals divisor is special case - if (*this == b) return NormObjectSize(0,1); - // Divisor > dividend is special case - if ((b.lsb > lsb) && (b.msb > msb)) return NormObjectSize(0,1); + // Divisor >= dividend is special case + if ((b.lsb >= lsb) && (b.msb >= msb)) return NormObjectSize(0,1); // divisor UINT32 divisor[2]; divisor[0] = (UINT32)b.msb; diff --git a/common/normMessage.h b/common/normMessage.h index bc8d556..a53a80b 100644 --- a/common/normMessage.h +++ b/common/normMessage.h @@ -2,12 +2,13 @@ #define _NORM_MESSAGE // PROTOLIB includes -#include // for NetworkAddress class -#include // for UINT typedefs -#include +#include "networkAddress.h" // for NetworkAddress class +#include "sysdefs.h" // for UINT typedefs +#include "debug.h" // standard includes #include // for memcpy(), etc +#include // (TBD) Alot of the "memcpy()" calls could be eliminated by // taking advantage of the alignment of NORM messsages @@ -15,6 +16,32 @@ const UINT16 NORM_MSG_SIZE_MAX = 8192; const int NORM_ROBUST_FACTOR = 20; // default robust factor +// These are the GRTT estimation bounds set for the current +// NORM protocol. (Note that our Grtt quantization routines +// are good for the range of 1.0e-06 <= 1000.0) +const double NORM_GRTT_MIN = 0.001; // 1 msec +const double NORM_GRTT_MAX = 15.0; // 15 sec +const double NORM_RTT_MIN = 1.0e-06; +const double NORM_RTT_MAX = 1000.0; +inline double NormUnquantizeRtt(unsigned char qrtt) +{ + return ((qrtt < 31) ? + (((double)(qrtt+1))/(double)NORM_RTT_MIN) : + (NORM_RTT_MAX/exp(((double)(255-qrtt))/(double)13.0))); +} +unsigned char NormQuantizeRtt(double rtt); + +inline double NormUnquantizeGroupSize(unsigned char gsize) +{ + return ((double)(gsize >> 4) * pow(10.0, (double)(gsize & 0x0f))); +} +inline unsigned char NormQuantizeGroupSize(double gsize) +{ + unsigned char exponent = (unsigned char)log10(gsize); + return ((((unsigned char)(gsize / pow(10.0, (double)exponent) + 0.5)) << 4) + | exponent); +} + // This class is used to describe object "size" and/or "offset" class NormObjectSize { @@ -139,9 +166,8 @@ class NormMsg UINT32 temp32 = htonl(sender); memcpy(buffer+SENDER_OFFSET, &temp32, 4); } - - void SetDestination(const NetworkAddress& dest) - {addr = dest;} + void SetDestination(const NetworkAddress& dst) {addr = dst;} + void SetLength(UINT16 len) {length = len;} // Message processing routines UINT8 GetVersion() const {return buffer[VERSION_OFFSET];} @@ -158,13 +184,14 @@ class NormMsg memcpy(&temp32, buffer+SENDER_OFFSET, 4); return (ntohl(temp32)); } - + const NetworkAddress& GetDestination() {return addr;} + const NetworkAddress& GetSource() {return addr;} + UINT16 GetLength() {return length;} + // For message buffer transmission/reception and misc. - NetworkAddress* Src() {return &addr;} - NetworkAddress* Dest() {return &addr;} - char* GetBuffer() {return buffer;} - UINT16 GetLength() const {return length;} - void SetLength(UINT16 len) {length = len;} + char* AccessBuffer() {return buffer;} + UINT16 AccessLength() const {return length;} + NetworkAddress& AccessAddress() {return addr;} protected: @@ -230,9 +257,9 @@ class NormObjectMsg : public NormMsg } UINT16 GetFecBlockLen() const { - UINT16 nparity; - memcpy(&nparity, buffer+FEC_NDATA_OFFSET, 2); - return (ntohs(nparity)); + UINT16 ndata; + memcpy(&ndata, buffer+FEC_NDATA_OFFSET, 2); + return (ntohs(ndata)); } NormObjectId GetObjectId() const { @@ -242,7 +269,7 @@ class NormObjectMsg : public NormMsg } // Message building routines - + void ResetFlags() {buffer[FLAGS_OFFSET] = 0;} void SetFlag(NormObjectMsg::Flag flag) {buffer[FLAGS_OFFSET] |= flag;} void SetGrtt(UINT8 grtt) {buffer[GRTT_OFFSET] = grtt;} @@ -370,10 +397,10 @@ class NormDataMsg : public NormObjectMsg } bool IsData() const {return (GetFecSymbolId() < GetFecBlockLen());} const char* GetData() {return (buffer + DATA_OFFSET);} - UINT16 GetDataLen() const {return (length - DATA_OFFSET);} + UINT16 GetDataLength() const {return (length - DATA_OFFSET);} const char* GetPayload() {return (buffer+LENGTH_OFFSET);} - UINT16 GetPayloadLen() const {return (length - LENGTH_OFFSET);} + UINT16 GetPayloadLength() const {return (length - LENGTH_OFFSET);} bool IsParity() const {return (GetFecSymbolId() >= GetFecBlockLen());} // (Note: "payload_len" and "offset" field spaces are in the FEC payload) @@ -447,9 +474,9 @@ class NormCmdMsg : public NormMsg {buffer[FLAVOR_OFFSET] = flavor;} // Message processing - UINT8 GetGrtt() {return buffer[GRTT_OFFSET];} - UINT8 GetGroupSize() {return buffer[GSIZE_OFFSET];} - NormCmdMsg::Flavor GetFlavor() {return (Flavor)buffer[FLAVOR_OFFSET];} + UINT8 GetGrtt() const {return buffer[GRTT_OFFSET];} + UINT8 GetGroupSize() const {return buffer[GSIZE_OFFSET];} + NormCmdMsg::Flavor GetFlavor() const {return (Flavor)buffer[FLAVOR_OFFSET];} protected: enum @@ -466,6 +493,11 @@ class NormCmdFlushMsg : public NormCmdMsg enum Flag {NORM_FLUSH_FLAG_EOT = 0x01}; // Message building + void Reset() + { + buffer[FLAGS_OFFSET] = 0; + length = SYMBOL_ID_OFFSET + 2; + } void SetFlag(NormCmdFlushMsg::Flag flag) {buffer[FLAGS_OFFSET] |= flag;} void UnsetFlag(NormCmdFlushMsg::Flag flag) @@ -542,12 +574,13 @@ class NormCmdSquelchMsg : public NormCmdMsg } void ResetInvalidObjectList() {length = OBJECT_LIST_OFFSET;} - UINT16 AppendInvalidObject(NormObjectId objectId) + bool AppendInvalidObject(NormObjectId objectId, UINT16 segmentSize) { + if ((length-OBJECT_LIST_OFFSET+2) > segmentSize) return false; UINT16 temp16 = htons((UINT16)objectId); memcpy(buffer+length, &temp16, 2); length += 2; - return length; + return true; } // Message processing @@ -716,8 +749,10 @@ class NormCmdApplicationMsg : public NormCmdMsg class NormRepairRequest { - friend class NormRepairRequest::Iterator; public: + class Iterator; + friend class NormRepairRequest::Iterator; + enum Form { INVALID, @@ -748,8 +783,9 @@ class NormRepairRequest // Repair request building void SetForm(NormRepairRequest::Form theForm) {form = theForm;} + void ResetFlags() {flags = 0;} void SetFlag(NormRepairRequest::Flag theFlag) {flags |= theFlag;} - void SetFlags(int theFlags) {flags |= theFlags;} + //void SetFlags(int theFlags) {flags |= theFlags;} void UnsetFlag(NormRepairRequest::Flag theFlag) {flags &= ~theFlag;} @@ -917,7 +953,7 @@ class NormReportMsg : public NormMsg // do some unions so we can easily use these // via casting or dereferencing the union members -class NormCommandMessage +class NormCommandMsg { public: union @@ -930,7 +966,7 @@ class NormCommandMessage NormCmdCCMsg cc; NormCmdApplicationMsg app; }; -}; // end class NormCommandMessage +}; // end class NormCommandMsg class NormMessage { @@ -942,7 +978,7 @@ class NormMessage NormObjectMsg object; NormInfoMsg info; NormDataMsg data; - NormCommandMessage cmd; + NormCommandMsg cmd; NormNackMsg nack; NormAckMsg ack; NormReportMsg report; diff --git a/common/normNode.cpp b/common/normNode.cpp index ac638f7..5ab1ad2 100644 --- a/common/normNode.cpp +++ b/common/normNode.cpp @@ -21,10 +21,18 @@ const NormNodeId& NormNode::LocalNodeId() {return session->LocalNodeId();} NormServerNode::NormServerNode(class NormSession* theSession, NormNodeId nodeId) : NormNode(theSession, nodeId), synchronized(false), - is_open(false), segment_size(0), ndata(0), nparity(0), erasure_loc(NULL) + is_open(false), segment_size(0), ndata(0), nparity(0), erasure_loc(NULL), + recv_total(0), recv_goodput(0), resync_count(0), + nack_count(0), suppress_count(0), completion_count(0), failure_count(0) { - repair_timer.Init(0.0, -1, (ProtocolTimerOwner*)this, + repair_timer.Init(0.0, 1, (ProtocolTimerOwner*)this, (ProtocolTimeoutFunc)&NormServerNode::OnRepairTimeout); + grtt_send_time.tv_sec = 0; + grtt_send_time.tv_usec = 0; + grtt_quantized = NormQuantizeRtt(NormSession::DEFAULT_GRTT_ESTIMATE); + grtt_estimate = NormUnquantizeRtt(grtt_quantized); + gsize_quantized = NormQuantizeGroupSize(NormSession::DEFAULT_GSIZE_ESTIMATE); + gsize_estimate = NormUnquantizeGroupSize(gsize_quantized); } NormServerNode::~NormServerNode() @@ -109,14 +117,10 @@ void NormServerNode::Close() delete []erasure_loc; erasure_loc = NULL; } - NormObjectTable::Iterator iterator(rx_table); NormObject* obj; - while ((obj = iterator.GetNextObject())) - { - // (TBD) Notify app of object closing - obj->Close(); - delete obj; - } + while ((obj = rx_table.Find(rx_table.RangeLo()))) DeleteObject(obj); + segment_pool.Destroy(); + block_pool.Destroy(); rx_repair_mask.Destroy(); rx_pending_mask.Destroy(); rx_table.Destroy(); @@ -124,188 +128,223 @@ void NormServerNode::Close() is_open = false; } // end NormServerNode::Close() -void NormServerNode::HandleObjectMessage(NormMessage& msg) + +void NormServerNode::HandleCommand(NormCommandMsg& cmd) { - if (IsOpen()) + UINT8 grttQuantized = cmd.generic.GetGrtt(); + if (grttQuantized != grtt_quantized) { - // (TBD - also verify encoder name ...) - if ((msg.object.GetSegmentSize() != segment_size) && - (ndata != msg.object.GetFecBlockLen()) && - (nparity != msg.object.GetFecNumParity())) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node:%lu remote server:%lu parameter change.\n", - LocalNodeId(), Id()); - Close(); - if (!Open(msg.object.GetSegmentSize(), msg.object.GetFecBlockLen(), - msg.object.GetFecNumParity())) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node:%lu remote server:%lu open error\n", - LocalNodeId(), Id()); - // (TBD) notify app of error ?? - return; - } - } + grtt_quantized = grttQuantized; + grtt_estimate = NormUnquantizeRtt(grttQuantized); + DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new grtt: %lf sec\n", + LocalNodeId(), Id(), grtt_estimate); } - else + UINT8 gsizeQuantized = cmd.generic.GetGroupSize(); + if (gsizeQuantized != gsize_quantized) { - if (!Open(msg.object.GetSegmentSize(), - msg.object.GetFecBlockLen(), - msg.object.GetFecNumParity())) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node:%lu remote server:%lu open error\n", - LocalNodeId(), Id()); - // (TBD) notify app of error ?? - return; - } + gsize_quantized = gsizeQuantized; + gsize_estimate = NormUnquantizeGroupSize(gsizeQuantized); + DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new group size: %lf\n", + LocalNodeId(), Id(), gsize_estimate); } - NormObjectId objectId = msg.object.GetObjectId(); - ObjectStatus objectStatus = GetObjectStatus(objectId); - if (synchronized) + NormCmdMsg::Flavor flavor = cmd.generic.GetFlavor(); + switch (flavor) { - if (OBJ_INVALID == objectStatus) + case NormCmdMsg::NORM_CMD_SQUELCH: + if (IsOpen()) + { + // 1) Sync to squelch + NormObjectId objectId = cmd.squelch.GetObjectId(); + Sync(objectId); + // 2) Prune stream object if applicable + NormObject* obj = rx_table.Find(objectId); + if (obj && (NormObject::STREAM == obj->GetType())) + { + NormBlockId blockId = cmd.squelch.GetFecBlockId(); + ((NormStreamObject*)obj)->Prune(blockId); + } + // 3) (TBD) Discard any invalidated objects + } + break; + + case NormCmdMsg::NORM_CMD_ACK_REQ: + GetSystemTime(&grtt_recv_time); + cmd.ack_req.GetRttSendTime(grtt_send_time); + switch(cmd.ack_req.GetAckFlavor()) + { + case NormCmdAckReqMsg::RTT: + break; + default: + break; + } + break; + + case NormCmdMsg::NORM_CMD_FLUSH: + if (synchronized) UpdateSyncStatus(cmd.flush.GetObjectId()); + RepairCheck(NormObject::THRU_SEGMENT, + cmd.flush.GetObjectId(), + cmd.flush.GetFecBlockId(), + cmd.flush.GetFecSymbolId()); + break; + + default: + DMSG(0, "NormServerNode::HandleCommand() recv'd unimplemented command!\n"); + break; + } // end switch(flavor) + +} // end NormServerNode::HandleCommand() + +void NormServerNode::HandleNackMessage(NormNackMsg& nack) +{ + // Clients only care about recvd NACKS for suppression + if (repair_timer.IsActive() && repair_timer.RepeatCount()) + { + // Parse NACK and incorporate into repair state masks + NormRepairRequest req; + UINT16 requestOffset = 0; + UINT16 requestLength = 0; + bool freshObject = true; + NormObjectId prevObjectId; + NormObject* object = NULL; + bool freshBlock = true; + NormBlockId prevBlockId; + NormBlock* block = NULL; + while ((requestLength = nack.UnpackRepairRequest(req, requestOffset))) { - // (TBD) We may want to control re-sync policy options - // or at least revert to fresh sync if sync is totally lost. - DMSG(0, "NormServerNode::HandleObjectMessage() re-syncing ...\n"); - Sync(objectId); - objectStatus = OBJ_NEW; - } - } - else - { - // Does this object message meet our sync policy? - if (SyncTest(msg)) + enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; + NormRepairRequest::Form requestForm = req.GetForm(); + requestOffset += requestLength; + NormRequestLevel requestLevel; + if (req.FlagIsSet(NormRepairRequest::SEGMENT)) + requestLevel = SEGMENT; + else if (req.FlagIsSet(NormRepairRequest::BLOCK)) + requestLevel = BLOCK; + else if (req.FlagIsSet(NormRepairRequest::OBJECT)) + requestLevel = OBJECT; + else + { + requestLevel = INFO; + ASSERT(req.FlagIsSet(NormRepairRequest::INFO)); + } + bool repairInfo = req.FlagIsSet(NormRepairRequest::INFO); + + NormRepairRequest::Iterator iterator(req); + NormObjectId nextObjectId, lastObjectId; + NormBlockId nextBlockId, lastBlockId; + NormSegmentId nextSegmentId, lastSegmentId; + while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, &nextSegmentId)) + { + if (NormRepairRequest::RANGES == requestForm) + { + if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, &lastSegmentId)) + { + DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu recvd incomplete RANGE request!\n", + LocalNodeId()); + continue; // (TBD) break/return instead??? + } + // (TBD) test for valid range form/level + } + else + { + lastObjectId = nextObjectId; + lastBlockId = nextBlockId; + lastSegmentId = nextSegmentId; + } + switch(requestLevel) + { + case INFO: + { + while (nextObjectId <= lastObjectId) + { + NormObject* obj = rx_table.Find(nextObjectId); + if (obj) obj->SetRepairInfo(); + nextObjectId++; + } + break; + } + case OBJECT: + rx_repair_mask.SetBits(nextObjectId, lastObjectId - nextObjectId + 1); + break; + case BLOCK: + { + if (nextObjectId != prevObjectId) freshObject = true; + if (freshObject) + { + object = rx_table.Find(nextObjectId); + prevObjectId = nextObjectId; + } + if (object) + { + if (repairInfo) object->SetRepairInfo(); + object->SetRepairs(nextBlockId, lastBlockId); + } + break; + } + case SEGMENT: + { + if (nextObjectId != prevObjectId) freshObject = true; + if (freshObject) + { + object = rx_table.Find(nextObjectId); + prevObjectId = nextObjectId; + } + if (object) + { + if (repairInfo) object->SetRepairInfo(); + if (nextBlockId != prevBlockId) freshBlock = true; + if (freshBlock) + { + block = object->FindBlock(nextBlockId); + prevBlockId = nextBlockId; + } + if (block) block->SetRepairs(nextSegmentId,lastSegmentId); + } + break; + } + } // end switch(requestLevel) + } // end while (iterator.NextRepairItem()) + } // end while (nack.UnpackRepairRequest()) + } // end if (repair_timer.IsActive() && repair_timer.RepeatCount()) +} // end NormServerNode::HandleNackMessage() + + +void NormServerNode::CalculateGrttResponse(struct timeval& grttResponse) +{ + grttResponse.tv_sec = grttResponse.tv_usec = 0; + if (grtt_send_time.tv_sec || grtt_send_time.tv_usec) + { + // 1st - Get current time + ::GetSystemTime(&grttResponse); + // 2nd - Calculate hold_time (current_time - recv_time) + if (grttResponse.tv_usec < grtt_recv_time.tv_usec) { - Sync(objectId); + grttResponse.tv_sec = grttResponse.tv_sec - grtt_recv_time.tv_sec - 1; + grttResponse.tv_usec = 1000000 - (grtt_recv_time.tv_usec - + grttResponse.tv_usec); } else { - DMSG(0, "NormServerNode::HandleObjectMessage() waiting to sync ...\n"); - return; + grttResponse.tv_sec = grttResponse.tv_sec - grtt_recv_time.tv_sec; + grttResponse.tv_usec = grttResponse.tv_usec - grtt_recv_time.tv_usec; } - } - NormObject* obj = NULL; - switch (objectStatus) - { - case OBJ_NEW: - SetPending(objectId); - break; - - case OBJ_PENDING: - obj = rx_table.Find(objectId); - break; - - case OBJ_COMPLETE: - return; + // 3rd - Calculate adjusted grtt_send_time (hold_time + send_time) + grttResponse.tv_sec += grtt_send_time.tv_sec; + grttResponse.tv_usec += grtt_send_time.tv_usec; + if (grttResponse.tv_usec > 1000000) + { + grttResponse.tv_usec -= 1000000; + grttResponse.tv_sec += 1; + } } - if (!obj) - { - if (msg.object.FlagIsSet(NormObjectMsg::FLAG_STREAM)) - { - if (!(obj = new NormStreamObject(session, this, objectId))) - { - DMSG(0, "NormServerNode::HandleObjectMessage() new NORM_OBJECT_STREAM error\n"); - return; - } - - } - else if (msg.object.FlagIsSet(NormObjectMsg::FLAG_FILE)) - { - DMSG(0, "NormServerNode::HandleObjectMessage() NORM_OBJECT_FILE not yet supported!\n"); - return; - } - else - { - DMSG(0, "NormServerNode::HandleObjectMessage() NORM_OBJECT_DATA not yet supported!\n"); - return; - } - // Open receive object and notify app for accept. - NormObjectSize objectSize = msg.object.GetObjectSize(); - if (!obj->Open(objectSize, msg.object.FlagIsSet(NormObjectMsg::FLAG_INFO))) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node:%lu server:%lu " - "obj:%hu was not opened.\n", LocalNodeId(), Id(), (UINT16)objectId); - delete obj; - rx_pending_mask.Unset(objectId); - return; - } - - session->Notify(NormController::RX_OBJECT_NEW, this, obj); - - if (!obj->Accepted()) - { - delete obj; - rx_pending_mask.Unset(objectId); - return; - } - - rx_table.Insert(obj); - DMSG(0, "NormServerNode::HandleObjectMessage() node:%lu server:%lu new obj:%hu\n", - LocalNodeId(), Id(), (UINT16)objectId); - } - obj->HandleObjectMessage(msg); -} // end NormServerNode::HandleObjectMessage() - -void NormServerNode::SetPending(NormObjectId objectId) -{ - ASSERT(synchronized); - ASSERT(OBJ_NEW == GetObjectStatus(objectId)); - if (objectId < next_id) - { - rx_pending_mask.Set(objectId); - } - else - { - rx_pending_mask.SetBits(next_id, next_id - objectId + 1); - next_id = objectId + 1; - } -} // end NormServerNode::SetPending() - - -void NormServerNode::Sync(NormObjectId objectId) -{ - if (synchronized) - { - // Dump pending objects < objectId - if (rx_pending_mask.IsSet() && (objectId > sync_id)) - { - NormObjectTable::Iterator iterator(rx_table); - NormObject* obj = iterator.GetNextObject(); - while (obj && (obj->Id() < objectId)) - { - DeleteObject(obj); - obj = iterator.GetNextObject(); - } - rx_pending_mask.UnsetBits(sync_id, (objectId - sync_id)); - } - sync_id = objectId; - if (objectId > next_id) next_id = objectId; - } - else - { - ASSERT(!rx_pending_mask.IsSet()); - sync_id = next_id = objectId; - synchronized = true; - } -} // end NormServerNode::Sync() - - -bool NormServerNode::SyncTest(const NormMessage& msg) const -{ - // (TBD) Additional sync policies - - // Sync if non-repair and (INFO or block zero message) - bool result = !msg.object.FlagIsSet(NormObjectMsg::FLAG_REPAIR) && - (NORM_MSG_INFO == msg.generic.GetType()) ? - true : (0 == msg.data.GetFecBlockId()); - return result; - -} // end NormServerNode::SyncTest() +} // end NormServerNode::CalculateGrttResponse() void NormServerNode::DeleteObject(NormObject* obj) { + // (TBD) Notify app of object's closing/demise? + obj->Close(); rx_table.Remove(obj); + rx_pending_mask.Unset(obj->Id()); delete obj; } // end NormServerNode::DeleteObject() @@ -314,7 +353,7 @@ NormBlock* NormServerNode::GetFreeBlock(NormObjectId objectId, NormBlockId block NormBlock* b = block_pool.Get(); if (!b) { - // reverse iteration to find newest object with resources + // reverse iteration to find newer object with resources NormObjectTable::Iterator iterator(rx_table); NormObject* obj; while ((obj = iterator.GetPrevObject())) @@ -342,25 +381,299 @@ NormBlock* NormServerNode::GetFreeBlock(NormObjectId objectId, NormBlockId block char* NormServerNode::GetFreeSegment(NormObjectId objectId, NormBlockId blockId) { - char* ptr = segment_pool.Get(); - while (!ptr) + while (segment_pool.IsEmpty()) { NormBlock* b = GetFreeBlock(objectId, blockId); if (b) - ptr = segment_pool.Get(); + block_pool.Put(b); else break; } - return ptr; + return segment_pool.Get(); } // end NormServerNode::GetFreeSegment() -NormServerNode::ObjectStatus NormServerNode::GetObjectStatus(NormObjectId objectId) const +void NormServerNode::HandleObjectMessage(NormMessage& msg) +{ + UINT8 grttQuantized = msg.object.GetGrtt(); + if (grttQuantized != grtt_quantized) + { + grtt_quantized = grttQuantized; + grtt_estimate = NormUnquantizeRtt(grttQuantized); + DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new grtt: %lf sec\n", + LocalNodeId(), Id(), grtt_estimate); + } + UINT8 gsizeQuantized = msg.object.GetGroupSize(); + if (gsizeQuantized != gsize_quantized) + { + gsize_quantized = gsizeQuantized; + gsize_estimate = NormUnquantizeGroupSize(gsizeQuantized); + DMSG(4, "NormServerNode::HandleCommand() node>%lu server>%lu new group size: %lf\n", + LocalNodeId(), Id(), gsize_estimate); + } + if (IsOpen()) + { + // (TBD - also verify encoder name ...) + if ((msg.object.GetSegmentSize() != segment_size) && + (ndata != msg.object.GetFecBlockLen()) && + (nparity != msg.object.GetFecNumParity())) + { + DMSG(2, "NormServerNode::HandleObjectMessage() node>%lu server>%lu parameter change - resyncing.\n", + LocalNodeId(), Id()); + Close(); + if (!Open(msg.object.GetSegmentSize(), + msg.object.GetFecBlockLen(), + msg.object.GetFecNumParity())) + { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu open error\n", + LocalNodeId(), Id()); + // (TBD) notify app of error ?? + return; + } + resync_count++; + } + } + else + { + if (!Open(msg.object.GetSegmentSize(), + msg.object.GetFecBlockLen(), + msg.object.GetFecNumParity())) + { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu open error\n", + LocalNodeId(), Id()); + // (TBD) notify app of error ?? + return; + } + } + NormMsgType msgType = msg.generic.GetType(); + NormObjectId objectId = msg.object.GetObjectId(); + NormBlockId blockId; + NormSegmentId segmentId; + if (NORM_MSG_INFO == msgType) + { + blockId = 0; + segmentId = 0; + } + else + { + blockId = msg.data.GetFecBlockId(); + segmentId = msg.data.GetFecSymbolId(); + } + + ObjectStatus status; + if (synchronized) + { + status = UpdateSyncStatus(objectId); + } + else + { + // Does this object message meet our sync policy? + if (SyncTest(msg)) + { + Sync(objectId); + SetPending(objectId); + status = OBJ_NEW; + } + else + { + DMSG(0, "NormServerNode::HandleObjectMessage() waiting to sync ...\n"); + return; + } + } + + NormObject* obj = NULL; + switch (status) + { + case OBJ_PENDING: + if ((obj = rx_table.Find(objectId))) break; + case OBJ_NEW: + { + if (msg.object.FlagIsSet(NormObjectMsg::FLAG_STREAM)) + { + if (!(obj = new NormStreamObject(session, this, objectId))) + { + DMSG(0, "NormServerNode::HandleObjectMessage() new NORM_OBJECT_STREAM error: %s\n", + strerror(errno)); + } + } + else if (msg.object.FlagIsSet(NormObjectMsg::FLAG_FILE)) + { +#ifdef SIMULATE + if (!(obj = new NormSimObject(session, this, objectId))) +#else + if (!(obj = new NormFileObject(session, this, objectId))) +#endif + { + DMSG(0, "NormServerNode::HandleObjectMessage() new NORM_OBJECT_FILE error: %s\n", + strerror(errno)); + } + } + else + { + obj = NULL; + DMSG(0, "NormServerNode::HandleObjectMessage() NORM_OBJECT_DATA not yet supported!\n"); + } + + if (obj) + { + // Open receive object and notify app for accept. + NormObjectSize objectSize = msg.object.GetObjectSize(); + if (obj->Open(objectSize, msg.object.FlagIsSet(NormObjectMsg::FLAG_INFO))) + { + session->Notify(NormController::RX_OBJECT_NEW, this, obj); + if (obj->Accepted()) + { + rx_table.Insert(obj); + DMSG(8, "NormServerNode::HandleObjectMessage() node>%lu server>%lu new obj>%hu\n", + LocalNodeId(), Id(), (UINT16)objectId); + } + else + { + DeleteObject(obj); + obj = NULL; + } + } + else + { + DeleteObject(obj); + obj = NULL; + } + } + break; + } + case OBJ_COMPLETE: + obj = NULL; + break; + default: + ASSERT(0); + break; + } // end switch(status) + + if (obj) + { + obj->HandleObjectMessage(msg, msgType, blockId, segmentId); + if (!obj->IsPending()) + { + + if (NormObject::FILE == obj->GetType()) +#ifdef SIMULATE + ((NormSimObject*)obj)->Close(); +#else + ((NormFileObject*)obj)->Close(); +#endif // !SIMULATE + session->Notify(NormController::RX_OBJECT_COMPLETE, this, obj); + DeleteObject(obj); + completion_count++; + } + } + RepairCheck(NormObject::TO_BLOCK, objectId, blockId, segmentId); +} // end NormServerNode::HandleObjectMessage() + +bool NormServerNode::SyncTest(const NormMessage& msg) const +{ + // (TBD) Additional sync policies + + // Sync if non-repair and (INFO or block zero message) + bool result = !msg.object.FlagIsSet(NormObjectMsg::FLAG_REPAIR) && + (NORM_MSG_INFO == msg.generic.GetType()) ? + true : (0 == msg.data.GetFecBlockId()); + return result; + +} // end NormServerNode::SyncTest() + +void NormServerNode::Sync(NormObjectId objectId) +{ + if (synchronized) + { + if (rx_pending_mask.IsSet()) + { + NormObjectId firstSet = NormObjectId(rx_pending_mask.FirstSet()); + if (objectId > NormObjectId(rx_pending_mask.LastSet())) + { + NormObject* obj; + while ((obj = rx_table.Find(rx_table.RangeLo()))) + { + DeleteObject(obj); + failure_count++; + } + rx_pending_mask.Clear(); + } + else if (objectId > firstSet) + { + NormObject* obj; + while ((obj = rx_table.Find(rx_table.RangeLo())) && + (obj->Id() < objectId)) + { + DeleteObject(obj); + failure_count++; + } + unsigned long numBits = (UINT16)(objectId - firstSet) + 1; + rx_pending_mask.UnsetBits(firstSet, numBits); + } + } + if (next_id < objectId) next_id = objectId; + sync_id = objectId; + ASSERT(OBJ_INVALID != GetObjectStatus(objectId)); + } + else + { + ASSERT(!rx_pending_mask.IsSet()); + sync_id = next_id = objectId; + synchronized = true; + } +} // end NormServerNode::Sync() + +NormServerNode::ObjectStatus NormServerNode::UpdateSyncStatus(const NormObjectId& objectId) +{ + ASSERT(synchronized); + ObjectStatus status = GetObjectStatus(objectId); + switch (status) + { + case OBJ_INVALID: + // (TBD) We may want to control re-sync policy options + // or revert to fresh sync if sync is totally lost, + // otherwise SQUELCH process will get things in order + DMSG(2, "NormServerNode::HandleObjectMessage() node>%lu re-syncing to server>%lu...\n", + LocalNodeId(), Id()); + Sync(objectId); + resync_count++; + status = OBJ_NEW; + case OBJ_NEW: + SetPending(objectId); + break; + default: + break; + } + return status; +} // end NormServerNode::UpdateSyncStatus() + +void NormServerNode::SetPending(NormObjectId objectId) +{ + ASSERT(synchronized); + ASSERT(OBJ_NEW == GetObjectStatus(objectId)); + if (objectId < next_id) + { + rx_pending_mask.Set(objectId); + } + else + { + rx_pending_mask.SetBits(next_id, objectId - next_id + 1); + next_id = objectId + 1; + // This prevents the "sync_id" from getting stale + sync_id = rx_pending_mask.FirstSet(); + } +} // end NormServerNode::SetPending() + + +NormServerNode::ObjectStatus NormServerNode::GetObjectStatus(const NormObjectId& objectId) const { if (synchronized) { if (objectId < sync_id) { - return OBJ_INVALID; + if ((sync_id - objectId) > 256) + return OBJ_INVALID; + else + return OBJ_COMPLETE; } else { @@ -402,7 +715,7 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, NormObjectId objectId, NormBlockId blockId, NormSegmentId segmentId) -{ +{ ASSERT(synchronized); if (!repair_timer.IsActive()) { @@ -418,8 +731,11 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, NormObject* obj = rx_table.Find(nextId); if (obj) { - NormObject::CheckLevel level = - (nextId == lastId) ? checkLevel : NormObject::THRU_OBJECT; + NormObject::CheckLevel level; + if (nextId < lastId) + level = NormObject::THRU_OBJECT; + else + level = checkLevel; startTimer |= obj->ClientRepairCheck(level, blockId, segmentId, false); } @@ -433,7 +749,13 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, current_object_id = objectId; if (startTimer) { - DMSG(0, "NormServerNode::RepairCheck() starting NACK back-off ...\n"); + double backoffTime = + ExponentialRand(grtt_estimate*session->BackoffFactor(), + gsize_estimate); + repair_timer.SetInterval(backoffTime); + DMSG(4, "NormServerNode::RepairCheck() node>%lu begin NACK back-off: %lf sec)...\n", + LocalNodeId(), backoffTime); + session->InstallTimer(&repair_timer); } } else @@ -446,8 +768,13 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, { // Repair timer in back-off phase // Trim server current transmit position reference - if (objectId < current_object_id) - current_object_id = objectId; + NormObject* obj = rx_table.Find(objectId); + if (obj) obj->ClientRepairCheck(checkLevel, blockId, segmentId, true); + if (objectId < current_object_id) current_object_id = objectId; + } + else + { + // Holding-off on repair cycle initiation } } // end NormServerNode::RepairCheck() @@ -455,18 +782,22 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, // and queue for transmission to this server node bool NormServerNode::OnRepairTimeout() { - DMSG(0, "NormServerNode::OnRepairTimeout() ...\n"); + switch(repair_timer.RepeatCount()) { case 0: // hold-off time complete + DMSG(4, "NormServerNode::OnRepairTimeout() node>%lu end NACK hold-off ...\n", + LocalNodeId()); break; case 1: // back-off timeout complete { + DMSG(4, "NormServerNode::OnRepairTimeout() node>%lu end NACK back-off ...\n", + LocalNodeId()); // 1) Were we suppressed? if (rx_pending_mask.IsSet()) { - bool repair_pending = false; + bool repairPending = false; NormObjectId nextId = rx_pending_mask.FirstSet(); NormObjectId lastId = rx_pending_mask.LastSet(); if (current_object_id < lastId) lastId = current_object_id; @@ -477,18 +808,25 @@ bool NormServerNode::OnRepairTimeout() NormObject* obj = rx_table.Find(nextId); if (!obj || obj->IsRepairPending(nextId != current_object_id)) { - repair_pending = true; + repairPending = true; break; } } nextId++; nextId = rx_pending_mask.NextSet(nextId); } // end while (nextId <= current_block_id) - if (repair_pending) + if (repairPending) { - // Build NACK - NormMessage msg; - NormNackMsg& nack = msg.nack; + // We weren't completely suppressed, so build NACK + NormMessage* msg = session->GetMessageFromPool(); + if (!msg) + { + DMSG(0, "NormServerNode::OnRepairTimeout() node>%lu Warning! " + "message pool empty ...\n", LocalNodeId()); + repair_timer.Deactivate(); + return false; + } + NormNackMsg& nack = msg->nack; nack.ResetNackContent(); NormRepairRequest req; NormObjectId prevId; @@ -498,97 +836,110 @@ bool NormServerNode::OnRepairTimeout() lastId = rx_pending_mask.LastSet(); if (current_object_id < lastId) lastId = current_object_id; lastId++; // force loop to fully flush nack building. - while (nextId <= lastId) + while ((nextId <= lastId) || (reqCount > 0)) { NormObject* obj = NULL; + bool objPending = false; if (nextId == lastId) nextId++; // force break of possible ending consecutive series else obj = rx_table.Find(nextId); - if (obj) + if (obj) objPending = obj->IsPending(nextId != current_object_id); + + if (!objPending && reqCount && (reqCount == (nextId - prevId))) { - if (obj->IsPending(nextId != current_object_id)) + reqCount++; // consecutive series of missing objs continues + } + else + { + NormRepairRequest::Form nextForm; + switch (reqCount) + { + case 0: + nextForm = NormRepairRequest::INVALID; + break; + case 1: + case 2: + nextForm = NormRepairRequest::ITEMS; + break; + default: + nextForm = NormRepairRequest::RANGES; + break; + } + if (prevForm != nextForm) { if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + if (NormRepairRequest::INVALID != nextForm) { - nack.PackRepairRequest(req); - prevForm = NormRepairRequest::INVALID; + nack.AttachRepairRequest(req, segment_size); // (TBD) error check + req.SetForm(nextForm); + req.ResetFlags(); + req.SetFlag(NormRepairRequest::OBJECT); } - obj->AppendRepairRequest(nack); // (TBD) error check - reqCount = 0; + prevForm = nextForm; } - } - else - { - if (reqCount && (reqCount == (nextId - prevId))) + if (NormRepairRequest::INVALID != nextForm) + DMSG(6, "NormServerNode::AppendRepairRequest() OBJECT request\n"); + switch (nextForm) { - // Consecutive series of missing objects continues - reqCount++; + case NormRepairRequest::ITEMS: + req.AppendRepairItem(prevId, 0, 0); + if (2 == reqCount) + req.AppendRepairItem(prevId+1, 0, 0); + break; + case NormRepairRequest::RANGES: + req.AppendRepairItem(prevId, 0, 0); + req.AppendRepairItem(prevId+reqCount-1, 0, 0); + break; + default: + break; } + prevId = nextId; + if (obj || (nextId >= lastId)) + reqCount = 0; else - { - NormRepairRequest::Form nextForm; - switch (reqCount) - { - case 0: - nextForm = NormRepairRequest::INVALID; - break; - case 1: - case 2: - nextForm = NormRepairRequest::ITEMS; - break; - default: - nextForm = NormRepairRequest::RANGES; - break; - } - if (prevForm != nextForm) - { - if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - if (NormRepairRequest::INVALID != nextForm) - { - nack.AttachRepairRequest(req, segment_size); // (TBD) error check - req.SetForm(nextForm); - req.SetFlag(NormRepairRequest::OBJECT); - } - prevForm = nextForm; - } - switch (nextForm) - { - case NormRepairRequest::ITEMS: - req.AppendRepairItem(prevId, 0, 0); - if (2 == reqCount) - req.AppendRepairItem(prevId+1, 0, 0); - break; - case NormRepairRequest::RANGES: - req.AppendRepairItem(prevId, 0, 0); - req.AppendRepairItem(prevId+reqCount-1, 0, 0); - default: - break; - } - prevId = nextId; reqCount = 1; - } + } // end if/else (!objPending && reqCount && (reqCount == (nextId - prevId))) + if (objPending) + { + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + prevForm = NormRepairRequest::INVALID; + reqCount = 0; + bool flush = (nextId != current_object_id); + obj->AppendRepairRequest(nack, flush); // (TBD) error check } nextId++; - if (nextId <= lastId) + if (nextId <= lastId) nextId = rx_pending_mask.NextSet(nextId); - } // end while (nextId <= lastId) - + } // end while(nextId <= lastId) + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check // (TBD) Queue NACK for transmission - DMSG(0, "NormServerNode::OnRepairTimeout() NACK TRANSMITTED ...\n"); + msg->generic.SetType(NORM_MSG_NACK); + msg->nack.SetServerId(Id()); + msg->generic.SetDestination(session->Address()); + session->QueueMessage(msg); + nack_count++; } else { - DMSG(0, "NormServerNode::OnRepairTimeout() NACK SUPPRESSED ...\n"); - // (TBD) repair_timer.SetInterval(HOLD_OFF_INTERVAL) - } + suppress_count++; + DMSG(6, "NormServerNode::OnRepairTimeout() node>%lu NACK SUPPRESSED ...\n", + LocalNodeId()); + } // end if/else(repairPending) + repair_timer.SetInterval(grtt_estimate*(session->BackoffFactor() + 2.0)); + DMSG(4, "NormServerNode::OnRepairTimeout() node>%lu begin NACK hold-off: %lf sec ...\n", + LocalNodeId(), repair_timer.Interval()); + } else { - DMSG(0, "NormServerNode::OnRepairTimeout() nothing pending ...\n"); + DMSG(4, "NormServerNode::OnRepairTimeout() node>%lu nothing pending ...\n", + LocalNodeId()); // (TBD) cancel hold-off timeout ??? - } // end if/else (repair_pending) + } // end if/else (repair_mask.IsSet()) } break; diff --git a/common/normNode.h b/common/normNode.h index 29f393a..63e1176 100644 --- a/common/normNode.h +++ b/common/normNode.h @@ -46,16 +46,19 @@ class NormServerNode : public NormNode NormServerNode(class NormSession* theSession, NormNodeId nodeId); ~NormServerNode(); + void HandleCommand(NormCommandMsg& cmd); void HandleObjectMessage(NormMessage& msg); - + void HandleNackMessage(NormNackMsg& nack); + bool Open(UINT16 segmentSize, UINT16 numData, UINT16 numParity); void Close(); bool IsOpen() const {return is_open;} bool SyncTest(const NormMessage& msg) const; void Sync(NormObjectId objectId); - ObjectStatus GetObjectStatus(NormObjectId objectId) const; + ObjectStatus UpdateSyncStatus(const NormObjectId& objectId); void SetPending(NormObjectId objectId); + ObjectStatus GetObjectStatus(const NormObjectId& objectId) const; void DeleteObject(NormObject* obj); @@ -77,7 +80,7 @@ class NormServerNode : public NormNode void SetErasureLoc(UINT16 index, UINT16 value) { - ASSERT(index < (nparity)); + ASSERT(index < nparity); erasure_loc[index] = value; } UINT16 GetErasureLoc(UINT16 index) @@ -87,7 +90,28 @@ class NormServerNode : public NormNode return decoder.Decode(segmentList, numData, erasureCount, erasure_loc); } - + void CalculateGrttResponse(struct timeval& grttResponse); + + unsigned long CurrentBufferUsage() + {return (segment_size * segment_pool.CurrentUsage());} + unsigned long PeakBufferUsage() + {return (segment_size * segment_pool.PeakUsage());} + unsigned long BufferOverunCount() + {return segment_pool.OverunCount() + block_pool.OverrunCount();} + + unsigned long RecvTotal() {return recv_total;} + unsigned long RecvGoodput() {return recv_goodput;} + void IncrementRecvTotal(unsigned long count) {recv_total += count;} + void IncrementRecvGoodput(unsigned long count) {recv_goodput += count;} + void ResetRecvStats() {recv_total = recv_goodput = 0;} + void IncrementResyncCount() {resync_count++;} + unsigned long ResyncCount() {return resync_count;} + unsigned long NackCount() {return nack_count;} + unsigned long SuppressCount() {return suppress_count;} + unsigned long CompletionCount() {return completion_count;} + unsigned long PendingCount() {return rx_table.Count();} + unsigned long FailureCount() {return failure_count;} + private: void RepairCheck(NormObject::CheckLevel checkLevel, NormObjectId objectId, @@ -116,6 +140,22 @@ class NormServerNode : public NormNode ProtocolTimer repair_timer; NormObjectId current_object_id; // index for repair + + double grtt_estimate; + UINT8 grtt_quantized; + struct timeval grtt_send_time; + struct timeval grtt_recv_time; + double gsize_estimate; + UINT8 gsize_quantized; + + // For statistics tracking + unsigned long recv_total; // total recvd accumulator + unsigned long recv_goodput; // goodput recvd accumulator + unsigned long resync_count; + unsigned long nack_count; + unsigned long suppress_count; + unsigned long completion_count; + unsigned long failure_count; // due to re-syncs }; // end class NodeServerNode diff --git a/common/normObject.cpp b/common/normObject.cpp index 74a1106..5bd33fc 100644 --- a/common/normObject.cpp +++ b/common/normObject.cpp @@ -24,6 +24,7 @@ NormObject::~NormObject() if (info) delete info; } +// This is mainly used for debug messages NormNodeId NormObject::LocalNodeId() const { return session->LocalNodeId(); @@ -39,14 +40,22 @@ bool NormObject::Open(const NormObjectSize& objectSize, // DATA or FILE objects, buffer size for STREAM objects // In either case, we need our sliding bit masks to be of // appropriate size. - UINT16 segmentSize, numData, numParity; if (server) { segmentSize = server->SegmentSize(); numData = server->BlockSize(); numParity = server->NumParity(); - if (infoLen > 0) pending_info = true; + if (infoLen > 0) + { + pending_info = true; + info_len = 0; + if (!(info = new char[segmentSize])) + { + DMSG(0, "NormObject::Open() info allocation error\n"); + return false; + } + } } else { @@ -56,7 +65,7 @@ bool NormObject::Open(const NormObjectSize& objectSize, if (infoPtr) { if (info) delete []info; - if (infoLen > segment_size) + if (infoLen > segmentSize) { DMSG(0, "NormObject::Open() info too big error\n"); info_len = 0; @@ -107,20 +116,26 @@ bool NormObject::Open(const NormObjectSize& objectSize, if (STREAM == type) { last_block_id = 0; // not applicable for STREAM - last_segment_id = numData; // assumed for STREAM + last_block_size = numData; // assumed for STREAM + stream_next_id = numBlocks.LSB(); } else { last_block_id = numBlocks.LSB() - 1; - NormObjectSize size = NormObjectSize(numBlocks.LSB()) * blockSize; - size = objectSize - size; - size = size / NormObjectSize(segmentSize); - ASSERT(!size.MSB()); - ASSERT(size.LSB() < numData); - last_segment_id = size.LSB(); + NormObjectSize lastBlockBytes = NormObjectSize(last_block_id) * blockSize; + lastBlockBytes = objectSize - lastBlockBytes; + NormObjectSize lastBlockSize = lastBlockBytes / NormObjectSize(segmentSize); + ASSERT(!lastBlockSize.MSB()); + ASSERT(lastBlockSize.LSB() < numData); + last_block_size = lastBlockSize.LSB(); + NormObjectSize lastSegmentSize = + NormObjectSize(0,last_block_size-1) * NormObjectSize(segmentSize); + lastSegmentSize = lastBlockBytes - lastSegmentSize; + ASSERT(!lastSegmentSize.MSB()); + ASSERT(lastSegmentSize.LSB() <= segmentSize); + last_segment_size = lastSegmentSize.LSB(); } - object_size = objectSize; segment_size = segmentSize; ndata = numData; @@ -137,19 +152,224 @@ void NormObject::Close() { block_buffer.Remove(block); if (server) - { server->PutFreeBlock(block); - } else - { session->ServerPutFreeBlock(block); - } } repair_mask.Destroy(); pending_mask.Destroy(); block_buffer.Destroy(); } // end NormObject::Close(); +bool NormObject::HandleInfoRequest() +{ + // (TBD) immediately make info pending? + bool increasedRepair = false; + if (info) + { + if (!repair_info) + { + repair_info = true; + increasedRepair = true; + } + } + return increasedRepair; +} // end NormObject::HandleInfoRequest() + +bool NormObject::HandleBlockRequest(NormBlockId nextId, NormBlockId lastId) +{ + DMSG(6, "NormObject::HandleBlockRequest() node>%lu blk>%lu -> blk>%lu\n", + LocalNodeId(), (UINT32)nextId, (UINT32)lastId); + bool increasedRepair = false; + while (nextId <= lastId) + { + if (!repair_mask.Test(nextId)) + { + // (TBD) these tests can probably go away if everything else is done right + if (!pending_mask.CanSet(nextId)) + DMSG(0, "NormObject::HandleBlockRequest() pending_mask.CanSet(%lu) error\n", + (UINT32)nextId); + if (!repair_mask.Set(nextId)) + DMSG(0, "NormObject::HandleBlockRequest() repair_mask.Set(%lu) error\n", + (UINT32)nextId); + increasedRepair = true; + } + nextId++; + } + return increasedRepair; +} // end NormObject::HandleBlockRequest(); + +bool NormObject::TxReset(NormBlockId firstBlock) +{ + bool increasedRepair = false; + if (!pending_info && HaveInfo()) + { + increasedRepair = true; + pending_info = true; + } + repair_info = false; + repair_mask.Reset((UINT32)firstBlock); + repair_mask.Xor(pending_mask); + if (repair_mask.IsSet()) + { + increasedRepair = true; + pending_mask.Reset((UINT32)firstBlock); + } + repair_mask.Clear(); + NormBlockBuffer::Iterator iterator(block_buffer); + NormBlock* block; + while ((block = iterator.GetNextBlock())) + { + NormBlockId blockId = block->Id(); + if (blockId >= firstBlock) + { + increasedRepair |= block->TxReset(BlockSize(blockId), + nparity, + session->ServerAutoParity(), + segment_size); + } + } + return increasedRepair; +} // end NormObject::TxReset() + +bool NormObject::TxResetBlocks(NormBlockId nextId, NormBlockId lastId) +{ + bool increasedRepair = false; + UINT16 autoParity = session->ServerAutoParity(); + while (nextId <= lastId) + { + if (!pending_mask.Test(nextId)) + { + pending_mask.Set(nextId); + increasedRepair = true; + } + NormBlock* block = block_buffer.Find(nextId); + if (block) + increasedRepair |= block->TxReset(BlockSize(block->Id()), nparity, autoParity, segment_size); + nextId++; + } + return increasedRepair; +} // end NormObject::TxResetBlocks() + + + +bool NormObject::ActivateRepairs() +{ + bool repairsActivated = false; + // Activate repair of info if applicable (TBD - how to flag info message as repair???) + if (repair_info) + { + pending_info = true; + repair_info = false; + repairsActivated = true; + } + // Activate any complete block repairs + if (repair_mask.IsSet()) + { + repairsActivated = true; + NormBlockId nextId = repair_mask.FirstSet(); + NormBlockId lastId = repair_mask.LastSet(); + DMSG(6, "NormObject::ActivateRepairs() node>%lu obj>%hu activated blk>%lu->%lu repairs\n", + LocalNodeId(), (UINT16)id, + (UINT32)nextId, (UINT32)lastId); + UINT16 autoParity = session->ServerAutoParity(); + while (nextId <= lastId) + { + NormBlock* block = block_buffer.Find(nextId); + if (block) block->TxReset(BlockSize(nextId), nparity, autoParity, segment_size); + // (TBD) This check can be eventually eliminated if everything else is done right + if (!pending_mask.Set(nextId)) + DMSG(0, "NormObject::ActivateRepairs() pending_mask.Set(%lu) error!\n", + (UINT32)nextId); + nextId++; + nextId = repair_mask.NextSet(nextId); + } + repair_mask.Clear(); + } + // Activate partial block (segment) repairs + NormBlockBuffer::Iterator iterator(block_buffer); + NormBlock* block; + while ((block = iterator.GetNextBlock())) + { + if (block->ActivateRepairs(nparity)) + { + repairsActivated = true; + DMSG(6, "NormObject::ActivateRepairs() node>%lu obj>%hu activated blk>%lu segment repairs ...\n", + LocalNodeId(), (UINT16)id, (UINT32)block->Id()); + // (TBD) This check can be eventually eliminated if everything else is done right + if (!pending_mask.Set(block->Id())) + DMSG(0, "NormObject::ActivateRepairs() pending_mask.Set(%lu) error!\n", (UINT32)block->Id()); + } + } + return repairsActivated; +} // end NormObject::ActivateRepairs() + +// Called by server only +bool NormObject::IsRepairPending() const +{ + ASSERT(!server); + if (repair_info) return true; + if (repair_mask.IsSet()) return true; + NormBlockBuffer::Iterator iterator(block_buffer); + NormBlock* block; + while ((block = iterator.GetNextBlock())) + { + if (block->IsRepairPending()) return true; + } + return false; +} // end NormObject::IsRepairPending() + + +bool NormObject::IsPending(bool flush) const +{ + if (pending_info) return true; + if (flush) + { + return pending_mask.IsSet(); + } + else + { + if (pending_mask.IsSet()) + { + NormBlockId firstId = pending_mask.FirstSet(); + if (firstId < current_block_id) + { + return true; + } + else if (firstId > current_block_id) + { + return false; + } + else + { + if (next_segment_id > 0) + { + NormBlock* block = block_buffer.Find(current_block_id); + if (block) + { + if (block->FirstPending() < next_segment_id) + return true; + else + return false; + } + else + { + return true; + } + } + else + { + return false; + } + } + } + else + { + return false; + } + } +} // end NormObject::IsPending() + bool NormObject::ClientRepairCheck(CheckLevel level, NormBlockId blockId, @@ -187,7 +407,7 @@ bool NormObject::ClientRepairCheck(CheckLevel level, if (blockId < current_block_id) { current_block_id = blockId; - next_segment_id = ndata; + next_segment_id = BlockSize(blockId); } break; default: @@ -198,7 +418,7 @@ bool NormObject::ClientRepairCheck(CheckLevel level, bool needRepair = false; if (pending_info) { - repair_info = true; + repair_info = false; needRepair = true; } if ((level > THRU_INFO) && pending_mask.IsSet()) @@ -244,7 +464,7 @@ bool NormObject::ClientRepairCheck(CheckLevel level, break; case TO_BLOCK: current_block_id = blockId; - next_segment_id = segmentId + 1; + next_segment_id = 0; break; case THRU_SEGMENT: current_block_id = blockId; @@ -252,55 +472,204 @@ bool NormObject::ClientRepairCheck(CheckLevel level, break; case THRU_BLOCK: current_block_id = blockId; - next_segment_id = ndata; + next_segment_id = BlockSize(blockId); break; case THRU_OBJECT: current_block_id = last_block_id; - next_segment_id = ndata; + next_segment_id = BlockSize(blockId); default: break; } return needRepair; } // end NormObject::ClientRepairCheck() +// Note this clears "repair_mask" state (called on client repair_timer timeout) bool NormObject::IsRepairPending(bool flush) { - if (!flush) - { - // Mask repair mask for region of interest - } + ASSERT(server); + if (pending_info && !repair_info) return true; // Calculate repair_mask = pending_mask - repair_mask repair_mask.XCopy(pending_mask); if (repair_mask.IsSet()) { - NormBlockId lastId = repair_mask.LastSet(); NormBlockId nextId = repair_mask.FirstSet(); + NormBlockId lastId = repair_mask.LastSet(); + if (!flush && (current_block_id < lastId)) lastId = current_block_id; while (nextId <= lastId) { NormBlock* block = block_buffer.Find(nextId); if (block) { - if (block->IsRepairPending(ndata)) return true; + bool isPending; + UINT16 numData = BlockSize(nextId); + if (flush || (nextId < lastId)) + { + isPending = block->IsRepairPending(numData, nparity); + } + else + { + if (next_segment_id < numData) + isPending = block->IsRepairPending(next_segment_id, 0); + else + isPending = block->IsRepairPending(numData, nparity); + } + if (isPending) return true; } else { - // We need the whole thing - return true; + return true; // We need the whole thing } nextId++; nextId = repair_mask.NextSet(nextId); } } - else - { - return false; - } return false; } // end NormObject::IsRepairPending() -void NormObject::HandleObjectMessage(NormMessage& msg) +bool NormObject::AppendRepairRequest(NormNackMsg& nack, + bool flush) +{ + // If !flush, we request only up to current_block_id::next_segment_id. + NormRepairRequest req; + NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; + NormBlockId prevId; + UINT16 reqCount = 0; + if (pending_mask.IsSet()) + { + NormBlockId nextId = pending_mask.FirstSet(); + NormBlockId lastId = pending_mask.LastSet(); + if (!flush && (current_block_id < lastId)) lastId = current_block_id; + lastId++; + //DMSG(6, "NormObject::AppendRepairRequest() node>%lu obj>%hu, blk>%lu->%lu (current:%lu)\n", + // LocalNodeId(), (UINT16)id, + // (UINT32)nextId, (UINT32)lastId, (UINT32)current_block_id); + while ((nextId <= lastId) || (reqCount > 0)) + { + NormBlock* block = NULL; + bool blockPending = false; + if (nextId == lastId) + nextId++; // force break of possible ending consec. series + else + block = block_buffer.Find(nextId); + if (block) + { + if (nextId == current_block_id) + { + if (block->FirstPending() < next_segment_id) + blockPending = true; + } + else + { + blockPending = true; + } + } // end if (block) + + if (!blockPending && reqCount && (reqCount == (nextId - prevId))) + { + reqCount++; // consecutive series of missing blocks continues + } + else + { + NormRepairRequest::Form nextForm; + switch(reqCount) + { + case 0: + nextForm = NormRepairRequest::INVALID; + break; + case 1: + case 2: + nextForm = NormRepairRequest::ITEMS; + break; + default: + nextForm = NormRepairRequest::RANGES; + break; + } // end switch(reqCount) + if (prevForm != nextForm) + { + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + if (NormRepairRequest::INVALID != nextForm) + { + nack.AttachRepairRequest(req, segment_size); + req.SetForm(nextForm); + req.ResetFlags(); + req.SetFlag(NormRepairRequest::BLOCK); + if (pending_info) req.SetFlag(NormRepairRequest::INFO); + } + prevForm = nextForm; + } + if (NormRepairRequest::INVALID != nextForm) + DMSG(6, "NormObject::AppendRepairRequest() BLOCK request\n"); + switch (nextForm) + { + case NormRepairRequest::ITEMS: + req.AppendRepairItem(id, prevId, 0); // (TBD) error check + if (2 == reqCount) + req.AppendRepairItem(id, prevId+1, 0); // (TBD) error check + break; + case NormRepairRequest::RANGES: + req.AppendRepairItem(id, prevId, 0); // (TBD) error check + req.AppendRepairItem(id, prevId+reqCount-1, 0); // (TBD) error check + break; + default: + break; + } // end switch(nextForm) + prevId = nextId; + if (block || (nextId >= lastId)) + reqCount = 0; + else + reqCount = 1; + } // end if/else (!blockPending && reqCount && (reqCount == (nextId - prevId))) + if (blockPending) + { + UINT16 numData = BlockSize(nextId); + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + reqCount = 0; + prevForm = NormRepairRequest::INVALID; + if (flush || (nextId != current_block_id)) + { + block->AppendRepairRequest(nack, numData, nparity, id, + pending_info, segment_size); // (TBD) error check + } + else + { + if (next_segment_id < numData) + block->AppendRepairRequest(nack, next_segment_id, 0, id, + pending_info, segment_size); // (TBD) error check + else + block->AppendRepairRequest(nack, numData, nparity, id, + pending_info, segment_size); // (TBD) error check + } + } + nextId++; + if (nextId <= lastId) + nextId = pending_mask.NextSet(nextId); + } // end while (nextId <= lastId) + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + } + else + { + // INFO_ONLY repair request + ASSERT(pending_info); + nack.AttachRepairRequest(req, segment_size); + req.SetForm(NormRepairRequest::ITEMS); + req.ResetFlags(); + req.SetFlag(NormRepairRequest::INFO); + req.AppendRepairItem(id, 0, 0); // (TBD) error check + nack.PackRepairRequest(req); // (TBD) error check + } // end if/else pending_mask.IsSet() + return true; +} // end NormObject::AppendRepairRequest() + + +void NormObject::HandleObjectMessage(NormMessage& msg, + NormMsgType msgType, + NormBlockId blockId, + NormSegmentId segmentId) { - if (NORM_MSG_INFO == msg.generic.GetType()) + if (NORM_MSG_INFO == msgType) { if (pending_info) { @@ -308,35 +677,56 @@ void NormObject::HandleObjectMessage(NormMessage& msg) if (info_len > segment_size) { info_len = segment_size; - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(0, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "Warning! info too long.\n", LocalNodeId(), server->Id(), (UINT16)id); } memcpy(info, msg.info.GetInfo(), info_len); pending_info = false; + session->Notify(NormController::RX_OBJECT_INFO, server, this); } else { // (TBD) Verify info hasn't changed? - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(6, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "received duplicate info ...\n", LocalNodeId(), server->Id(), (UINT16)id); } } else // NORM_MSG_DATA { - NormBlockId blockId = msg.data.GetFecBlockId(); // For stream objects, a little extra mgmt is required if (STREAM == type) { if (!StreamUpdateStatus(blockId)) { - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " - "broken stream ...\n", LocalNodeId(), server->Id(), - (UINT16)id); - // (TBD) deal with broken stream - return; + //DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj>%hu blk>%lu " + // "broken stream ...\n", LocalNodeId(), server->Id(), (UINT16)id, (UINT32)blockId); + + // ??? Ignore this new packet and try to fix stream ??? + return; + server->IncrementResyncCount(); + } + while (!StreamUpdateStatus(blockId)) + { + // Server is too far ahead of me ... + if (pending_mask.IsSet()) + { + NormBlockId firstId = pending_mask.FirstSet(); + NormBlock* block = block_buffer.Find(firstId); + if (block) + { + block_buffer.Remove(block); + server->PutFreeBlock(block); + } + pending_mask.Unset(firstId); + } + else + { + stream_next_id = blockId - NormBlockId(pending_mask.Size()); + } } } + UINT16 numData = BlockSize(blockId); if (pending_mask.Test(blockId)) { NormBlock* block = block_buffer.Find(blockId); @@ -344,35 +734,29 @@ void NormObject::HandleObjectMessage(NormMessage& msg) { if (!(block = server->GetFreeBlock(id, blockId))) { - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(2, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "Warning! no free blocks ...\n", LocalNodeId(), server->Id(), (UINT16)id); return; } - block->RxInit(blockId, ndata); + block->RxInit(blockId, numData, nparity); block_buffer.Insert(block); } - NormSegmentId segmentId = msg.data.GetFecSymbolId(); - if (9 == segmentId) - { - return; - } if (block->IsPending(segmentId)) { // 1) Store data in block buffer in case its needed for decoding char* segment = server->GetFreeSegment(id, blockId); if (!segment) { - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(2, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "Warning! no free segments ...\n", LocalNodeId(), server->Id(), (UINT16)id); return; } - - UINT16 segmentLen = msg.data.GetDataLen(); + UINT16 segmentLen = msg.data.GetDataLength(); if (segmentLen > segment_size) { - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(0, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "Error! segment too large ...\n", LocalNodeId(), server->Id(), (UINT16)id); server->PutFreeSegment(segment); @@ -382,27 +766,35 @@ void NormObject::HandleObjectMessage(NormMessage& msg) block->AttachSegment(segmentId, segment); block->UnsetPending(segmentId); // 2) Write segment to object (if it's data) - if (segmentId < ndata) - WriteSegment(blockId, segmentId, segment); - // 3) Decode block if ready and return to pool - if (block->ErasureCount() <= nparity) + if (segmentId < numData) { - // Decode (if pending_mask.FirstSet() < ndata) + block->DecrementErasureCount(); + if (WriteSegment(blockId, segmentId, segment)) + server->IncrementRecvGoodput(segmentLen); + } + else + { + block->IncrementParityCount(); + } + // 3) Decode block if ready and return to pool + if (block->ErasureCount() <= block->ParityCount()) + { + // Decode (if pending_mask.FirstSet() < numData) // and write any decoded data segments to object - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " - "decoding block ...\n", LocalNodeId(), server->Id(), - (UINT16)id); + DMSG(8, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu blk>%lu " + "completed block ...\n", LocalNodeId(), server->Id(), + (UINT16)id, (UINT32)block->Id()); UINT16 nextErasure = block->FirstPending(); UINT16 erasureCount = 0; - UINT16 blockLen = ndata + nparity; + UINT16 blockLen = numData + nparity; while (nextErasure < blockLen) { server->SetErasureLoc(erasureCount++, nextErasure); - if (nextErasure < ndata) + if (nextErasure < numData) { if (!(segment = server->GetFreeSegment(id, blockId))) { - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(2, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "Warning! no free segments ...\n", LocalNodeId(), server->Id(), (UINT16)id); // (TBD) Dump the block ...??? @@ -415,14 +807,22 @@ void NormObject::HandleObjectMessage(NormMessage& msg) } if (erasureCount) { - server->Decode(block->SegmentList(), ndata, erasureCount); + server->Decode(block->SegmentList(), numData, erasureCount); for (UINT16 i = 0; i < erasureCount; i++) { NormSegmentId sid = server->GetErasureLoc(i); - if (sid < ndata) - WriteSegment(blockId, sid, block->Segment(sid)); + if (sid < numData) + { + if (WriteSegment(blockId, sid, block->Segment(sid))) + { + // For statistics only (TBD) #ifdef NORM_DEBUG + server->IncrementRecvGoodput(NormDataMsg::ReadLength(block->Segment(sid))); + } + } else + { break; + } } } // OK, we're done with this block @@ -431,23 +831,23 @@ void NormObject::HandleObjectMessage(NormMessage& msg) server->PutFreeBlock(block); } - // (TBD) Notify application if new data available + // Notify application of new data available session->Notify(NormController::RX_OBJECT_UPDATE, server, this); } else { - DMSG(0, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(6, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "received duplicate segment ...\n", LocalNodeId(), server->Id(), (UINT16)id); } } else { - DMSG(6, "NormObject::HandleObjectMessage() node:%lu server:%lu obj:%hu " + DMSG(6, "NormObject::HandleObjectMessage() node>%lu server>%lu obj>%hu " "received duplicate block message ...\n", LocalNodeId(), server->Id(), (UINT16)id); - } - } + } // end if/else pending_mask.Test(blockId) + } // end if/else (NORM_MSG_INFO) } // end NormObject::HandleObjectMessage() bool NormObject::StreamUpdateStatus(NormBlockId blockId) @@ -473,13 +873,13 @@ bool NormObject::StreamUpdateStatus(NormBlockId blockId) { if (pending_mask.CanSet(blockId)) { - pending_mask.SetBits(stream_next_id, - blockId - stream_next_id + 1); + pending_mask.SetBits(stream_next_id, blockId - stream_next_id + 1); stream_next_id = blockId + 1; // Handle potential sync block id wrap NormBlockId delta = stream_next_id - stream_sync_id; if (delta > NormBlockId(2*pending_mask.Size())) stream_sync_id = NormBlockId(pending_mask.FirstSet()); + return true; } else { @@ -503,6 +903,7 @@ bool NormObject::StreamUpdateStatus(NormBlockId blockId) delta = stream_next_id - stream_sync_id; if (delta > NormBlockId(2*pending_mask.Size())) stream_sync_id = blockId; + return true; } } @@ -516,12 +917,14 @@ bool NormObject::StreamUpdateStatus(NormBlockId blockId) pending_mask.SetBits(blockId, pending_mask.Size()); stream_sync = true; stream_sync_id = blockId; - stream_next_id = blockId + pending_mask.Size(); + stream_next_id = blockId + pending_mask.Size(); + return true; } - return true; } // end NormObject::StreamUpdateStatus() -NormBlock* NormObject::StealOldestBlock(bool excludeBlock, NormBlockId excludeId) +// Steals non-pending block (oldest first) for server resource management +// (optionally excludes block indicated by blockId) +NormBlock* NormObject::StealNonPendingBlock(bool excludeBlock, NormBlockId excludeId) { if (block_buffer.IsEmpty()) { @@ -529,19 +932,30 @@ NormBlock* NormObject::StealOldestBlock(bool excludeBlock, NormBlockId excludeId } else { - NormBlock* block = block_buffer.Find(block_buffer.RangeLo()); - if (excludeBlock && (excludeId == block->Id())) + NormBlockBuffer::Iterator iterator(block_buffer); + NormBlock* block; + while ((block = iterator.GetNextBlock())) { - return NULL; + NormBlockId bid = block->Id(); + if (block->IsTransmitPending() || + pending_mask.Test(bid) || + repair_mask.Test(bid) || + (excludeBlock && (excludeId == bid))) + { + continue; + } + else + { + block_buffer.Remove(block); + return block; + } } - else - { - block_buffer.Remove(block); - return block; - } } -} // end NormObject::StealOldestBlock() + return NULL; +} // end NormObject::StealNonPendingBlock() +// For client resource management, steals newer block resources when +// needing resources for ordinally older blocks. NormBlock* NormObject::StealNewestBlock(bool excludeBlock, NormBlockId excludeId) { if (block_buffer.IsEmpty()) @@ -565,9 +979,7 @@ NormBlock* NormObject::StealNewestBlock(bool excludeBlock, NormBlockId excludeId bool NormObject::NextServerMsg(NormMessage* msg) { - if (!pending_mask.IsSet()) return false; - - + msg->object.ResetFlags(); switch(type) { case STREAM: @@ -579,26 +991,37 @@ bool NormObject::NextServerMsg(NormMessage* msg) default: break; } + if (info) msg->object.SetFlag(NormObjectMsg::FLAG_INFO); msg->object.SetSegmentSize(segment_size); msg->object.SetObjectSize(object_size); msg->object.SetFecNumParity(nparity); msg->object.SetFecBlockLen(ndata); msg->object.SetObjectId(id); - - + + if (pending_info) + { + msg->generic.SetType(NORM_MSG_INFO); + msg->info.SetInfo(info, info_len); + pending_info = false; + return true; + } + if (!pending_mask.IsSet()) return false; + NormBlockId blockId = pending_mask.FirstSet(); + UINT16 numData = BlockSize(blockId); NormBlock* block = block_buffer.Find(blockId); if (!block) { if (!(block = session->ServerGetFreeBlock(id, blockId))) { - DMSG(0, "NormObject::NextServerMsg() Warning! server resource " - "constrained (no free blocks).\n"); + DMSG(2, "NormObject::NextServerMsg() node>%lu Warning! server resource " + "constrained (no free blocks).\n", LocalNodeId()); return false; } // Load block with zero initialized parity segments - UINT16 totalBlockLen = ndata + nparity; - for (UINT16 i = ndata; i < totalBlockLen; i++) + + UINT16 totalBlockLen = numData + nparity; + for (UINT16 i = numData; i < totalBlockLen; i++) { char* s = session->ServerGetFreeSegment(id, blockId); if (s) @@ -608,14 +1031,14 @@ bool NormObject::NextServerMsg(NormMessage* msg) } else { - DMSG(0, "NormObject::NextServerMsg() Warning! server resource " - "constrained (no free segments).\n"); + DMSG(2, "NormObject::NextServerMsg() node>%lu Warning! server resource " + "constrained (no free segments).\n", LocalNodeId()); session->ServerPutFreeBlock(block); return false; } } - block->TxInit(blockId, ndata, session->ServerAutoParity()); + block->TxInit(blockId, numData, session->ServerAutoParity()); if (!block_buffer.Insert(block)) { ASSERT(STREAM == type); @@ -632,34 +1055,41 @@ bool NormObject::NextServerMsg(NormMessage* msg) NormSegmentId segmentId = block->FirstPending(); // Try to read segment - if (segmentId < ndata) + if (segmentId < numData) { - // Try to read data segment - NormObjectSize offset; - char* dataPtr = msg->data.AccessData(); - UINT16 length = ReadSegment(blockId, segmentId, - &offset, dataPtr, segment_size); - if (!length) + // Try to read data segment + char* buffer = msg->data.AccessPayload(); + if (!ReadSegment(blockId, segmentId, buffer)) { - // (TBD) deal with read error (may be ok for streams) + // (TBD) deal with read error + //(for streams, it currently means the stream is non-pending) + TRACE("NormObject::NextServerMsg() ReadSegment() error\n"); return false; } - msg->data.SetDataOffset(offset); - msg->data.SetDataLength(length); + UINT16 length = NormDataMsg::ReadLength(buffer); + msg->data.SetDataLength(length); // Perform incremental FEC encoding as needed - if (!block->ParityReady() && nparity) + if ((block->ParityReadiness() <= segmentId) && + nparity) // (TBD) && incrementalParity == true { // (TBD) for non-stream objects, catch alternate "last block/segment len" if (length < segment_size) - memset(dataPtr+length, 0, segment_size - length); - session->ServerEncode(msg->data.AccessPayload(), block->SegmentList(ndata)); - if (segmentId == (ndata-1)) block->SetFlag(NormBlock::PARITY_READY); + { + memset(msg->data.AccessData()+length, 0, segment_size-length); + } + session->ServerEncode(msg->data.AccessPayload(), block->SegmentList(numData)); + block->IncreaseParityReadiness(); } } else { - ASSERT(block->ParityReady()); + if (!block->ParityReady(numData)) + { + ASSERT(0 == block->ParityReadiness()); + CalculateBlockParity(block); + } + ASSERT(block->ParityReady(numData)); char* segment = block->Segment(segmentId); ASSERT(segment); msg->data.SetPayload(segment, segment_size+NormDataMsg::PayloadHeaderLen()); @@ -669,21 +1099,268 @@ bool NormObject::NextServerMsg(NormMessage* msg) if (block->InRepair()) msg->object.SetFlag(NormObjectMsg::FLAG_REPAIR); msg->data.SetFecBlockId(blockId); msg->data.SetFecSymbolId(segmentId); - if (!block->IsPending()) pending_mask.Unset(blockId); + if (!block->IsPending()) + { + block->ResetParityCount(nparity); + pending_mask.Unset(blockId); + } - // This lets us continue stream objects indefinitely + // This lets NORM_STREAM objects continue indefinitely if (IsStream() && !pending_mask.IsSet()) { - pending_mask.Set(block_buffer.RangeHi()+1); - } - + NormBlockId nextBlockId = stream_next_id; + // Make sure we won't prevent about any pending repairs + if (repair_mask.CanSet(nextBlockId)) + { + if (block_buffer.CanInsert(nextBlockId)) + { + if (pending_mask.Set(nextBlockId)) + stream_next_id++; + else + DMSG(0, "NormObject::NextServerMsg() error setting stream pending mask (1)\n"); + } + else + { + block = block_buffer.Find(block_buffer.RangeLo()); + ASSERT(block); + if (!block->IsTransmitPending()) + { + if (pending_mask.Set(nextBlockId)) + stream_next_id++; + else + DMSG(0, "NormObject::NextServerMsg() error setting stream pending mask (2)\n"); + } + else + { + DMSG(4, "NormObject::NextServerMsg() node>%lu Pending segment repairs (blk>%lu) " + "delaying stream advance ...\n", LocalNodeId(), (UINT32)block->Id()); + } + } + } + else + { + DMSG(0, "NormObject::NextServerMsg() Pending block repair delaying stream advance ...\n"); + } + } return true; } // end NormObject::NextServerMsg() +bool NormObject::CalculateBlockParity(NormBlock* block) +{ + char buffer[NORM_MSG_SIZE_MAX]; + UINT16 numData = BlockSize(block->Id()); + for (UINT16 i = 0; i < numData; i++) + { + if (ReadSegment(block->Id(), i, buffer)) + { + UINT16 length = NormDataMsg::ReadLength(buffer); + if (length < segment_size) + { + memset(buffer+NormDataMsg::PayloadHeaderLen()+length, 0, segment_size-length); + } + session->ServerEncode(buffer, block->SegmentList(numData)); + } + else + { + return false; + } + } + block->SetParityReadiness(numData); + return true; +} // end NormObject::CalculateBlockParity() + +NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) +{ + NormBlock* block = session->ServerGetFreeBlock(id, blockId); + if (block) + { + UINT16 numData = BlockSize(blockId); + // Init block parameters + block->TxRecover(blockId, numData, nparity); + // Fill block with zero initialized parity segments + UINT16 totalBlockLen = numData + nparity; + for (UINT16 i = numData; i < totalBlockLen; i++) + { + char* s = session->ServerGetFreeSegment(id, blockId); + if (s) + { + memset(s, 0, segment_size + NormDataMsg::PayloadHeaderLen()); + block->AttachSegment(i, s); + } + else + { + DMSG(2, "NormObject::ServerRecoverBlock() node>%lu Warning! server resource " + "constrained (no free segments).\n", LocalNodeId()); + session->ServerPutFreeBlock(block); + return (NormBlock*)NULL; + } + } + // Attempt to re-generate parity + if (CalculateBlockParity(block)) + { + return block; + } + else + { + session->ServerPutFreeBlock(block); + return (NormBlock*)NULL; + } + } + else + { + DMSG(2, "NormObject::ServerRecoverBlock() node>%lu Warning! server resource " + "constrained (no free blocks).\n", LocalNodeId()); + return (NormBlock*)NULL; + } +} // end NormObject::ServerRecoverBlock() + ///////////////////////////////////////////////////////////////// // -// NormObjectTable Implementation +// NormFileObject Implementation // +NormFileObject::NormFileObject(class NormSession* theSession, + class NormServerNode* theServer, + const NormObjectId& objectId) + : NormObject(FILE, theSession, theServer, objectId), block_size(0,0) +{ + path[0] = '\0'; +} + +NormFileObject::~NormFileObject() +{ + Close(); +} + +// Open file +bool NormFileObject::Open(const char* thePath, + const char* infoPtr, + UINT16 infoLen) +{ + if (server) + { + // We're receiving this file + if (NormFile::IsLocked(thePath)) + { + DMSG(0, "NormFileObject::Open() Error trying to open locked file for recv!\n"); + return false; + } + else + { + if (file.Open(thePath, O_WRONLY | O_CREAT | O_TRUNC)) + { + file.Lock(); + } + else + { + DMSG(0, "NormFileObject::Open() recv file.Open() error!\n"); + return false; + } + } + block_size = NormObjectSize(server->BlockSize()) * + NormObjectSize(server->SegmentSize()); + } + else + { + // We're sending this file + if (file.Open(thePath, O_RDONLY)) + { + unsigned long size = file.GetSize(); + if (size) + { + if (!NormObject::Open(size, infoPtr, infoLen)) + { + DMSG(0, "NormFileObject::Open() send object open error\n"); + Close(); + return false; + } + } + else + { + DMSG(0, "NormFileObject::Open() send file.GetSize() error!\n"); + file.Close(); + return false; + } + block_size = NormObjectSize(session->ServerBlockSize()) * + NormObjectSize(session->ServerSegmentSize()); + } + else + { + DMSG(0, "NormFileObject::Open() send file.Open() error!\n"); + return false; + } + } + strncpy(path, thePath, PATH_MAX); + unsigned int len = strlen(thePath); + len = MIN(len, PATH_MAX); + if (len < PATH_MAX) path[len] = '\0'; + return true; +} // end NormFileObject::Open() + +bool NormFileObject::Accept(const char* thePath) +{ + if (Open(thePath)) + { + NormObject::Accept(); + return true; + } + else + { + return false; + } +} // end NormFileObject::Accept() + +void NormFileObject::Close() +{ + NormObject::Close(); + file.Close(); +} // end NormFileObject::Close() + +bool NormFileObject::WriteSegment(NormBlockId blockId, + NormSegmentId segmentId, + const char* buffer) +{ + UINT16 len; + if ((blockId == last_block_id) && + (segmentId == (last_block_size-1))) + len = last_segment_size; + else + len = segment_size; + NormObjectSize segmentOffset = NormDataMsg::ReadOffset(buffer); + off_t offset = segmentOffset.LSB() + (segmentOffset.MSB() * 0x100000000LL); + if (offset != file.GetOffset()) + { + if (!file.Seek(offset)) return false; + } + UINT16 nbytes = file.Write(buffer+NormDataMsg::PayloadHeaderLen(), len); + return (nbytes == len); +} // end NormFileObject::WriteSegment() + + +bool NormFileObject::ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer) +{ + // Determine segment length + UINT16 len; + if ((blockId == last_block_id) && + (segmentId == (last_block_size - 1))) + len = last_segment_size; + else + len = segment_size; + // Determine segment offset from blockId::segmentId + NormObjectSize segmentOffset = NormObjectSize(blockId) * block_size; + segmentOffset = segmentOffset + (NormObjectSize(segmentId) * NormObjectSize(segment_size)); + NormDataMsg::WriteLength(buffer, len); + NormDataMsg::WriteOffset(buffer, segmentOffset); + off_t offset = segmentOffset.LSB() + (segmentOffset.MSB() * 0x100000000LL); + if (offset != file.GetOffset()) + { + if (!file.Seek(offset)) return false; + } + UINT16 nbytes = file.Read(buffer+NormDataMsg::PayloadHeaderLen(), len); + return (len == nbytes); +} // end NormFileObject::ReadSegment() + ///////////////////////////////////////////////////////////////// // @@ -693,7 +1370,7 @@ bool NormObject::NextServerMsg(NormMessage* msg) NormStreamObject::NormStreamObject(class NormSession* theSession, class NormServerNode* theServer, const NormObjectId& objectId) - : NormObject(STREAM, theSession, theServer, objectId) + : NormObject(STREAM, theSession, theServer, objectId), flush_pending(false) { } @@ -732,7 +1409,6 @@ bool NormStreamObject::Open(unsigned long bufferSize, NormObjectSize numBlocks = NormObjectSize(bufferSize) / blockSize; ASSERT(0 == numBlocks.MSB()); - // Buffering requires at least 2 segments & 2 blocks numSegments = MAX(2, numSegments); numBlocks = MAX(2, numBlocks.LSB()); @@ -798,40 +1474,82 @@ void NormStreamObject::Close() b->EmptyToPool(segment_pool); block_pool.Put(b); } - segment_pool.Destroy(); stream_buffer.Destroy(); - block_pool.Destroy(); + segment_pool.Destroy(); + block_pool.Destroy(); } // end NormStreamObject::Close() -UINT16 NormStreamObject::ReadSegment(NormBlockId blockId, NormSegmentId segmentId, - NormObjectSize* offset, char* buffer, UINT16 maxlen) +bool NormStreamObject::LockBlocks(NormBlockId nextId, NormBlockId lastId) +{ + while (nextId <= lastId) + { + NormBlock* block = stream_buffer.Find(nextId); + if (block) + { + block->SetPending(0, ndata); + } + else + { + return false; + } + nextId++; + } + return true; +} // end NormStreamObject::LockBlocks() + + +bool NormStreamObject::LockSegments(NormBlockId blockId, NormSegmentId firstId, NormSegmentId lastId) +{ + NormBlock* block = stream_buffer.Find(blockId); + if (block) + { + ASSERT(firstId <= lastId); + block->SetPending(firstId, (lastId - firstId + 1)); + return true; + } + else + { + return false; + } +} // end NormStreamObject::LockSegments() + +bool NormStreamObject::ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer) { // (TBD) compare blockId with stream_buffer.RangeLo() and stream_buffer.RangeHi() NormBlock* block = stream_buffer.Find(blockId); if (!block) { //DMSG(0, "NormStreamObject::ReadSegment() stream starved (1)\n"); - return 0; + return false; } - if (!block->IsPending(segmentId)) + if ((blockId == write_index.block) && + (segmentId >= write_index.segment)) { //DMSG(0, "NormStreamObject::ReadSegment() stream starved (2)\n"); - return 0; + return false; } + block->UnsetPending(segmentId); char* segment = block->Segment(segmentId); - block->UnsetPending(segmentId); - ASSERT(segment); + ASSERT(segment); UINT16 length = NormDataMsg::ReadLength(segment); - *offset = NormDataMsg::ReadOffset(segment); - ASSERT(length <= maxlen); - memcpy(buffer, segment+NormDataMsg::PayloadHeaderLen(), length); - return length; + ASSERT(length <= segment_size); + memcpy(buffer, segment, length+NormDataMsg::PayloadHeaderLen()); + return true; } // end NormStreamObject::Read() bool NormStreamObject::WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* segment) { + if ((blockId < read_index.block) || + ((blockId == read_index.block) && + (segmentId < read_index.segment))) + { + //DMSG(0, "NormStreamObject::WriteSegment() block/segment < read_index!?\n"); + return false; + } NormBlock* block = stream_buffer.Find(blockId); if (!block) { @@ -841,9 +1559,9 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, { block = stream_buffer.Find(stream_buffer.RangeLo()); stream_buffer.Remove(block); + if (block->IsPending()) broken = true; block->EmptyToPool(segment_pool); block_pool.Put(block); - broken = true; } block = block_pool.Get(); block->SetId(blockId); @@ -852,22 +1570,58 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, ASSERT(success); if (broken) { + DMSG(2, "NormStreamObject::WriteSegment() node>%lu obj>%hu blk>%lu seg>%hu broken stream ...\n", + LocalNodeId(), (UINT16)id, (UINT32)blockId, (UINT16)segmentId); NormBlock* first = stream_buffer.Find(stream_buffer.RangeLo()); read_index.block = first->Id(); read_index.segment = 0; } + + } + // Make sure this segment hasn't already been written + if(!block->Segment(segmentId)) + { + char* s = segment_pool.Get(); + ASSERT(s); // for now, this should always succeed + UINT16 length = NormDataMsg::ReadLength(segment); + memcpy(s, segment, length + NormDataMsg::PayloadHeaderLen()); + block->AttachSegment(segmentId, s); + block->SetPending(segmentId); } - ASSERT(!block->Segment(segmentId)); - char* s = segment_pool.Get(); - ASSERT(s); - UINT16 length = NormDataMsg::ReadLength(segment); - memcpy(s, segment, length + NormDataMsg::PayloadHeaderLen()); - block->AttachSegment(segmentId, s); - block->SetPending(segmentId); return true; } // end NormStreamObject::WriteSegment() - +void NormStreamObject::Prune(NormBlockId blockId) +{ + bool resync = false; + NormBlock* block; + while ((block = block_buffer.Find(block_buffer.RangeLo()))) + { + if (block->Id() < blockId) + { + resync = true; + pending_mask.Unset(block->Id()); + block_buffer.Remove(block); + server->PutFreeBlock(block); + } + else + { + break; + } + } + if (pending_mask.IsSet()) + { + NormBlockId firstId = pending_mask.FirstSet(); + if (firstId < blockId) + { + resync = true; + UINT32 count = blockId - firstId; + pending_mask.UnsetBits(firstId, count); + } + } + if (resync) server->IncrementResyncCount(); + StreamUpdateStatus(blockId); +} // end NormStreamObject::Prune() // Sequential (in order) read/write routines (TBD) Add a "Seek()" method unsigned long NormStreamObject::Read(char* buffer, unsigned long len) @@ -889,8 +1643,12 @@ unsigned long NormStreamObject::Read(char* buffer, unsigned long len) } NormObjectSize segmentOffset = NormDataMsg::ReadOffset(segment); - ASSERT((segmentOffset < read_offset) || - (segmentOffset == read_offset)); + if (segmentOffset > read_offset) + { + DMSG(0, "NormStreamObject::Read() node>%lu broken stream!\n", LocalNodeId()); + read_offset = segmentOffset; + return nBytes; + } NormObjectSize delta = read_offset - segmentOffset; ASSERT(!delta.MSB()); UINT16 index = delta.LSB(); @@ -911,6 +1669,7 @@ unsigned long NormStreamObject::Read(char* buffer, unsigned long len) if (read_index.segment >= ndata) { stream_buffer.Remove(block); + block->EmptyToPool(segment_pool); block_pool.Put(block); read_index.block++; read_index.segment = 0; @@ -932,6 +1691,7 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus if (!(block = block_pool.Get())) { block = stream_buffer.Find(stream_buffer.RangeLo()); + ASSERT(block); if (block->IsPending()) { DMSG(0, "NormStreamObject::Write() stream buffer full (1)\n"); @@ -960,18 +1720,18 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus DMSG(0, "NormStreamObject::Write() stream buffer full (2)\n"); break; } + stream_buffer.Remove(b); b->EmptyToPool(segment_pool); + block_pool.Put(b); segment = segment_pool.Get(); ASSERT(segment); } NormDataMsg::WriteOffset(segment, write_offset); NormDataMsg::WriteLength(segment, 0); - block->AttachSegment(write_index.segment, segment); + block->AttachSegment(write_index.segment, segment); } - UINT16 index = NormDataMsg::ReadLength(segment); - ASSERT(write_offset == (NormDataMsg::ReadOffset(segment) + - NormObjectSize(index))); + ASSERT(write_offset == (NormDataMsg::ReadOffset(segment)+NormObjectSize(index))); UINT16 space = segment_size - index; UINT16 count = MIN(space, len); memcpy(segment+index+NormDataMsg::PayloadHeaderLen(), buffer+nBytes, count); @@ -990,13 +1750,59 @@ unsigned long NormStreamObject::Write(char* buffer, unsigned long len, bool flus } } } while (len > 0); + if (flush) + flush_pending = true; + else + flush_pending = false; if (nBytes) session->TouchServer(); return nBytes; } // end NormStreamObject::Write() +#ifdef SIMULATE +///////////////////////////////////////////////////////////////// +// +// NormSimObject Implementation +// + +NormSimObject::NormSimObject(class NormSession* theSession, + class NormServerNode* theServer, + const NormObjectId& objectId) + : NormObject(FILE, theSession, theServer, objectId) +{ + +} + +NormSimObject::~NormSimObject() +{ + +} + +bool NormSimObject::ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer) +{ + // Determine segment length + UINT16 len; + if ((blockId == last_block_id) && + (segmentId == (last_block_size - 1))) + len = last_segment_size; + else + len = segment_size; + // the "len" is needed to build the correct size message + NormDataMsg::WriteLength(buffer, len); + return true; +} // end NormSimObject::ReadSegment() + +#endif // SIMULATE + +///////////////////////////////////////////////////////////////// +// +// NormObjectTable Implementation +// NormObjectTable::NormObjectTable() - : table((NormObject**)NULL), range_max(0), range(0) + : table((NormObject**)NULL), range_max(0), range(0), + count(0), size(0,0) { } @@ -1045,7 +1851,6 @@ NormObject* NormObjectTable::Find(const NormObjectId& objectId) const { if ((objectId < range_lo) || (objectId > range_hi)) return (NormObject*)NULL; NormObject* theObject = table[((UINT16)objectId) & hash_mask]; - //printf("NormObjectTable::Find() table[%lu] = %p\n", ((UINT16)objectId) & hash_mask, theObject); while (theObject && (objectId != theObject->Id())) theObject = theObject->next; return theObject; } @@ -1055,50 +1860,75 @@ NormObject* NormObjectTable::Find(const NormObjectId& objectId) const } } // end NormObjectTable::Find() -bool NormObjectTable::Insert(NormObject* theObject) +bool NormObjectTable::CanInsert(NormObjectId objectId) const { - if (range < range_max) + if (0 != range) { - const NormObjectId& objectId = theObject->Id(); - if (!range) - { - range_lo = range_hi = objectId; - range = 1; - } if (objectId < range_lo) { - UINT16 newRange = range_lo - objectId + range; - if (newRange > range_max) return false; - range_lo = objectId; - range = newRange; + if ((range_lo - objectId + range) > range_max) + return false; + else + return true; } else if (objectId > range_hi) - { - UINT16 newRange = objectId - range_hi + range; - if (newRange > range_max) return false; - range_hi = objectId; - range = newRange; - } - UINT16 index = ((UINT16)objectId) & hash_mask; - NormObject* prev = NULL; - NormObject* entry = table[index]; - while (entry && (entry->Id() < objectId)) { - prev = entry; - entry = entry->next; - } - if (prev) - prev->next = theObject; + if ((objectId - range_hi + range) > range_max) + return false; + else + return true; + } else - table[index] = theObject; - ASSERT((entry ? (objectId != entry->Id()) : true)); - theObject->next = entry; - return true; + { + return true; + } } else { - return false; + return true; + } +} // end NormObjectTable::CanInsert() + + +bool NormObjectTable::Insert(NormObject* theObject) +{ + const NormObjectId& objectId = theObject->Id(); + if (!range) + { + range_lo = range_hi = objectId; + range = 1; } + if (objectId < range_lo) + { + UINT16 newRange = range_lo - objectId + range; + if (newRange > range_max) return false; + range_lo = objectId; + range = newRange; + } + else if (objectId > range_hi) + { + UINT16 newRange = objectId - range_hi + range; + if (newRange > range_max) return false; + range_hi = objectId; + range = newRange; + } + UINT16 index = ((UINT16)objectId) & hash_mask; + NormObject* prev = NULL; + NormObject* entry = table[index]; + while (entry && (entry->Id() < objectId)) + { + prev = entry; + entry = entry->next; + } + if (prev) + prev->next = theObject; + else + table[index] = theObject; + ASSERT((entry ? (objectId != entry->Id()) : true)); + theObject->next = entry; + count++; + size = size + theObject->Size(); + return true; } // end NormObjectTable::Insert() bool NormObjectTable::Remove(const NormObject* theObject) @@ -1120,7 +1950,7 @@ bool NormObjectTable::Remove(const NormObject* theObject) if (prev) prev->next = entry->next; else - table[index] = (NormObject*)NULL; + table[index] = entry->next; if (range > 1) { if (objectId == range_lo) @@ -1129,7 +1959,7 @@ bool NormObjectTable::Remove(const NormObject* theObject) UINT16 i = index; UINT16 endex; if (range <= hash_mask) - endex = (index + range) & hash_mask; + endex = (index + range - 1) & hash_mask; else endex = index; entry = NULL; @@ -1144,8 +1974,8 @@ bool NormObjectTable::Remove(const NormObject* theObject) NormObjectId id = (UINT16)index + offset; while(entry && (entry->Id() != id)) { - if ((entry->Id() > objectId) && (entry->Id() < nextId)) - nextId = entry->Id(); + if ((entry->Id() > objectId) && + (entry->Id() < nextId)) nextId = entry->Id(); entry = entry->next; } @@ -1153,19 +1983,10 @@ bool NormObjectTable::Remove(const NormObject* theObject) } } while (i != endex); if (entry) - { range_lo = entry->Id(); - range = range_hi - range_lo + 1; - } - else if (nextId != range_hi) - { - range_lo = nextId; - range = range_hi - range_lo + 1; - } else - { - range = 0; - } + range_lo = nextId; + range = range_hi - range_lo + 1; } else if (objectId == range_hi) { @@ -1173,7 +1994,7 @@ bool NormObjectTable::Remove(const NormObject* theObject) UINT16 i = index; UINT16 endex; if (range <= hash_mask) - endex = (index - range) & hash_mask; + endex = (index - range + 1) & hash_mask; else endex = index; entry = NULL; @@ -1190,33 +2011,26 @@ bool NormObjectTable::Remove(const NormObject* theObject) //printf("Looking for id:%lu at index:%lu\n", (UINT16)id, i); while(entry && (entry->Id() != id)) { - if ((entry->Id() < objectId) && (entry->Id() > prevId)) - prevId = entry->Id(); + if ((entry->Id() < objectId) && + (entry->Id() > prevId)) prevId = entry->Id(); entry = entry->next; } if (entry) break; } } while (i != endex); if (entry) - { range_hi = entry->Id(); - - } - else if (prevId != range_lo) - { - range_hi = prevId; - range = range_hi - range_lo + 1; - } else - { - range = 0; - } + range_hi = prevId; + range = range_hi - range_lo + 1; } } else { range = 0; } + count--; + size = size - theObject->Size(); return true; } else @@ -1249,13 +2063,13 @@ NormObject* NormObjectTable::Iterator::GetNextObject() { if (table.range && (index < table.range_hi) && - !(index < table.range_lo)) + (index >= table.range_lo)) { // Find next entry _after_ current "index" UINT16 i = index; - UINT16 endex = table.range_hi - index; - if (endex <= table.hash_mask) - endex = (index + endex) & table.hash_mask; + UINT16 endex; + if ((UINT16)(table.range_hi - index) <= table.hash_mask) + endex = table.range_hi & table.hash_mask; else endex = index; UINT16 offset = 0; @@ -1266,7 +2080,7 @@ NormObject* NormObjectTable::Iterator::GetNextObject() offset++; NormObjectId id = (UINT16)index + offset; NormObject* entry = table.table[i]; - while ((NULL != entry )& (entry->Id() != id)) + while ((NULL != entry) && (entry->Id() != id)) { if ((entry->Id() > index) && (entry->Id() < nextId)) nextId = entry->Id(); @@ -1307,14 +2121,14 @@ NormObject* NormObjectTable::Iterator::GetPrevObject() else { if (table.range && - (index < table.range_hi) && - !(index < table.range_lo)) + (index <= table.range_hi) && + (index > table.range_lo)) { - // Find next entry _after_ current "index" + // Find prev entry _before_ current "index" UINT16 i = index; - UINT16 endex = index - table.range_lo; - if (endex <= table.hash_mask) - endex = ((UINT16)index - endex) & table.hash_mask; + UINT16 endex; + if ((UINT16)(index - table.range_lo) <= table.hash_mask) + endex = table.range_lo & table.hash_mask; else endex = index; UINT16 offset = 0; @@ -1325,7 +2139,7 @@ NormObject* NormObjectTable::Iterator::GetPrevObject() offset--; NormObjectId id = (UINT16)index + offset; NormObject* entry = table.table[i]; - while ((NULL != entry )& (entry->Id() != id)) + while ((NULL != entry ) && (entry->Id() != id)) { if ((entry->Id() > index) && (entry->Id() < nextId)) nextId = entry->Id(); diff --git a/common/normObject.h b/common/normObject.h index dcd272e..4fb9f9f 100644 --- a/common/normObject.h +++ b/common/normObject.h @@ -2,6 +2,8 @@ #define _NORM_OBJECT #include "normSegment.h" // NORM segmentation classes +#include "normEncoder.h" +#include "normFile.h" #include @@ -30,9 +32,12 @@ class NormObject virtual ~NormObject(); NormObject::Type GetType() const {return type;} const NormObjectId& Id() const {return id;} - const NormObjectSize& Size() {return object_size;} - - bool IsStream() {return (STREAM == type);} + const NormObjectSize& Size() const {return object_size;} + bool HaveInfo() const {return (info_len > 0);} + const char* GetInfo() const {return info;} + UINT16 InfoLength() const {return info_len;} + bool IsStream() const {return (STREAM == type);} + NormNodeId LocalNodeId() const; // Opens (inits) object for tx operation @@ -47,30 +52,66 @@ class NormObject virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* buffer) = 0; - - virtual UINT16 ReadSegment(NormBlockId blockId, NormSegmentId segmentId, - NormObjectSize* offset, char* buffer, UINT16 maxlen) = 0; + virtual bool ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer) = 0; // These are only valid after object is open - NormBlockId LastBlockId(); - NormSegmentId LastSegmentId(); - NormSegmentId LastSegmentId(NormBlockId blockId); + NormBlockId LastBlockId() const {return last_block_id;} + NormSegmentId LastBlockSize() const {return last_block_size;} + NormSegmentId BlockSize(NormBlockId blockId) + {return ((blockId == last_block_id) ? last_block_size : ndata);} - // (TBD) re-write to use current_block_id/next_segment_id when - // flush == false !!! - bool IsPending(bool flush = true) - {return (pending_info || pending_mask.IsSet());} + bool IsPending(bool flush = true) const; + bool IsRepairPending() const; + bool IsPendingInfo() {return pending_info;} + NormBlockId FirstPending() {return pending_mask.FirstSet();} // Methods available to server for transmission bool NextServerMsg(NormMessage* msg); + NormBlock* ServerRecoverBlock(NormBlockId blockId); + bool CalculateBlockParity(NormBlock* block); + + bool TxReset(NormBlockId firstBlock = NormBlockId(0)); + bool TxResetBlocks(NormBlockId nextId, NormBlockId lastId); + bool TxUpdateBlock(NormBlock* theBlock, + NormSegmentId firstSegmentId, + NormSegmentId lastSegmentId, + UINT16 numErasures) + { + NormBlockId blockId = theBlock->Id(); + bool result = theBlock->TxUpdate(firstSegmentId, lastSegmentId, + BlockSize(blockId), nparity, + segment_size, numErasures); + if (result) + { + result = pending_mask.Set(blockId); + ASSERT(result); + } + return result; + } // end NormObject::TxUpdateBlock() + bool HandleInfoRequest(); + bool HandleBlockRequest(NormBlockId nextId, NormBlockId lastId); + bool SetPending(NormBlockId blockId) + {return pending_mask.Set(blockId);} + NormBlock* FindBlock(NormBlockId blockId) + { + return block_buffer.Find(blockId); + } + bool ActivateRepairs(); + bool IsRepairSet(NormBlockId blockId) + {return repair_mask.Test(blockId);} // Used by session server for resource management scheme - NormBlock* StealOldestBlock(bool excludeBlock, NormBlockId excludeId = 0); + NormBlock* StealNonPendingBlock(bool excludeBlock, NormBlockId excludeId = 0); // Methods available to client for reception bool Accepted() {return accepted;} - void HandleObjectMessage(NormMessage& msg); + void HandleObjectMessage(NormMessage& msg, + NormMsgType msgType, + NormBlockId blockId, + NormSegmentId segmentId); bool StreamUpdateStatus(NormBlockId blockId); // Used by remote server node for resource management scheme @@ -82,7 +123,15 @@ class NormObject NormSegmentId segmentId, bool timerActive); bool IsRepairPending(bool flush); - bool AppendRepairRequest(NormNackMsg& nack) {return true;} + bool AppendRepairRequest(NormNackMsg& nack, bool flush); + void SetRepairInfo() {repair_info = true;} + bool SetRepairs(NormBlockId first, NormBlockId last) + { + if (first == last) + return repair_mask.Set(first); + else + return repair_mask.SetBits(first, last-first+1); + } protected: NormObject(NormObject::Type theType, @@ -109,7 +158,8 @@ class NormObject NormBlockId current_block_id; NormSegmentId next_segment_id; NormBlockId last_block_id; - NormSegmentId last_segment_id; + NormSegmentId last_block_size; + UINT16 last_segment_size; char* info; UINT16 info_len; @@ -125,6 +175,47 @@ class NormObject }; // end class NormObject +class NormFileObject : public NormObject +{ + public: + NormFileObject(class NormSession* theSession, + class NormServerNode* theServer, + const NormObjectId& objectId); + ~NormFileObject(); + + bool Open(const char* thePath, + const char* infoPtr = NULL, + UINT16 infoLen = 0); + bool Accept(const char* thePath); + void Close(); + const char* Path() {return path;} + bool Rename(const char* newPath) + { + if (file.Rename(path, newPath)) + { + strncpy(path, newPath, PATH_MAX); + return true; + } + else + { + return false; + } + } + + virtual bool WriteSegment(NormBlockId blockId, + NormSegmentId segmentId, + const char* buffer); + + virtual bool ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer); + + private: + char path[PATH_MAX]; + NormFile file; + NormObjectSize block_size; +}; // end class NormFileObject + class NormStreamObject : public NormObject { @@ -144,14 +235,29 @@ class NormStreamObject : public NormObject virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* buffer); - - virtual UINT16 ReadSegment(NormBlockId blockId, NormSegmentId segmentId, - NormObjectSize* offset, char* buffer, UINT16 maxlen); + virtual bool ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer); + unsigned long Read(char* buffer, unsigned long len); unsigned long Write(char* buffer, unsigned long len, bool flush = false); // For receive stream, we can rewind to earliest buffered offset void Rewind(); + + bool LockBlocks(NormBlockId nextId, NormBlockId lastId); + bool LockSegments(NormBlockId blockId, NormSegmentId firstId, + NormSegmentId lastId); + NormBlockId StreamBufferLo() {return stream_buffer.RangeLo();} + void Prune(NormBlockId blockId); + + bool IsFlushPending() {return flush_pending;} + NormBlockId FlushBlockId() + {return (write_index.segment ? write_index.block : + (NormBlockId((UINT32)write_index.block-1)));} + NormSegmentId FlushSegmentId() + {return (write_index.segment ? (write_index.segment-1) : + (ndata-1));} private: class Index @@ -167,20 +273,52 @@ class NormStreamObject : public NormObject NormObjectSize write_offset; Index read_index; NormObjectSize read_offset; + bool flush_pending; }; // end class NormStreamObject +#ifdef SIMULATE +// This class is used to simulate file objects in the +// network simulation build of NORM +class NormSimObject : public NormObject +{ + public: + NormSimObject(class NormSession* theSession, + class NormServerNode* theServer, + const NormObjectId& objectId); + ~NormSimObject(); + + bool Open(unsigned long objectSize, + const char* infoPtr = NULL, + UINT16 infoLen = 0) + { + return server ? true : NormObject::Open(objectSize, infoPtr, infoLen); + } + bool Accept() {NormObject::Accept(); return true;} + void Close() {NormObject::Close();} + + virtual bool WriteSegment(NormBlockId blockId, + NormSegmentId segmentId, + const char* buffer) {return true;} + + virtual bool ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer); +}; // end class NormSimObject +#endif // SIMULATE + class NormObjectTable { - friend class NormObjectTable::Iterator; - public: + class Iterator; + friend class NormObjectTable::Iterator; + NormObjectTable(); ~NormObjectTable(); bool Init(UINT16 rangeMax, UINT16 tableSize = 256); void Destroy(); bool IsInited() {return (NULL != table);} - + bool CanInsert(NormObjectId objectId) const; bool Insert(NormObject* theObject); bool Remove(const NormObject* theObject); NormObject* Find(const NormObjectId& objectId) const; @@ -188,7 +326,9 @@ class NormObjectTable NormObjectId RangeLo() {return range_lo;} NormObjectId RangeHi() {return range_hi;} bool IsEmpty() {return (0 == range);} - + unsigned long Count() {return count;} + const NormObjectSize& Size() {return size;} + class Iterator { public: @@ -213,6 +353,8 @@ class NormObjectTable unsigned long range; // zero if "object table" is empty NormObjectId range_lo; NormObjectId range_hi; + unsigned long count; + NormObjectSize size; }; // end class NormObjectTable #endif // _NORM_OBJECT diff --git a/common/normSegment.cpp b/common/normSegment.cpp index 04eacd4..e2a6226 100644 --- a/common/normSegment.cpp +++ b/common/normSegment.cpp @@ -67,7 +67,7 @@ char* NormSegmentPool::Get() { seg_list = *ptr; seg_count--; -#ifdef NORM_DEBUG +//#ifdef NORM_DEBUG overrun_flag = false; unsigned int usage = seg_total - seg_count; if (usage > peak_usage) peak_usage = usage; @@ -79,7 +79,7 @@ char* NormSegmentPool::Get() overruns++; overrun_flag = true; } -#endif // NORM_DEBUG +//#endif // NORM_DEBUG } return ((char*)ptr); } // end NormSegmentPool::GetSegment() @@ -122,6 +122,7 @@ bool NormBlock::Init(UINT16 blockSize) size = blockSize; erasure_count = 0; parity_count = 0; + parity_offset = 0;; return true; } // end NormBlock::Init() @@ -133,7 +134,10 @@ void NormBlock::Destroy() if (segment_table) { for (unsigned int i = 0; i < size; i++) + { + ASSERT(!segment_table[i]); if (segment_table[i]) delete []segment_table[i]; + } delete []segment_table; segment_table = (char**)NULL; } @@ -161,17 +165,319 @@ bool NormBlock::IsEmpty() return true; } // end NormBlock::EmptyToPool() -bool NormBlock::IsRepairPending(NormSegmentId end) +// Used by client side to determine if NACK should be sent +// Note: This clears the block's "repair_mask" state +bool NormBlock::IsRepairPending(UINT16 ndata, UINT16 nparity) { - ASSERT(end <= repair_mask.Size()); - repair_mask.SetBits(end, repair_mask.Size() - end); + // Clients ask for a block of parity to fulfill their + // repair needs (erasure_count), but if there isn't + // enough parity, they ask for some data segments, too + if (erasure_count > nparity) + { + if (nparity) + { + UINT16 i = nparity; + NormSegmentId nextId = pending_mask.FirstSet(); + while (i--) + { + // (TBD) for more NACK suppression, we could skip ahead + // if this bit is already set in repair_mask? + repair_mask.Set(nextId); // set bit a parity can fill + nextId++; + nextId = pending_mask.NextSet(nextId); + } + } + else if (size > ndata) + { + repair_mask.SetBits(ndata, size-ndata); + } + } + else + { + repair_mask.SetBits(0, ndata); + repair_mask.SetBits(ndata+erasure_count, nparity-erasure_count); + } repair_mask.XCopy(pending_mask); - return (repair_mask.IsSet() ? (repair_mask.FirstSet() < end) : false); + return (repair_mask.IsSet()); } // end NormBlock::IsRepairPending() - +// Called by server +bool NormBlock::TxReset(UINT16 ndata, + UINT16 nparity, + UINT16 autoParity, + UINT16 segmentSize) +{ + bool increasedRepair = false; + repair_mask.SetBits(0, ndata+autoParity); + repair_mask.UnsetBits(ndata+autoParity, nparity-autoParity); + repair_mask.Xor(pending_mask); + if (repair_mask.IsSet()) + { + increasedRepair = true; + repair_mask.Clear(); + pending_mask.SetBits(0, ndata+autoParity); + pending_mask.UnsetBits(ndata+autoParity, nparity-autoParity); + parity_offset = autoParity; // reset parity since we're resending this one + parity_count = nparity; // no parity repair this repair cycle + SetFlag(IN_REPAIR); + if (!ParityReady(ndata)) // (TBD) only when incrementalParity == true + { + // Clear _any_ existing incremental parity state + char** ptr = segment_table+ndata; + while (nparity--) + { + if (*ptr) memset(*ptr, 0, segmentSize); + ptr++; + } + erasure_count = 0; + } + } + return increasedRepair; +} // end NormBlock::TxReset() + +bool NormBlock::ActivateRepairs(UINT16 nparity) +{ + if (repair_mask.IsSet()) + { + pending_mask.Add(repair_mask); + repair_mask.Clear(); + return true; + } + else + { + return false; + } +} // end NormBlock::ActivateRepairs() + +// For NACKs arriving during server repair_timer "holdoff" time +bool NormBlock::TxUpdate(NormSegmentId nextId, NormSegmentId lastId, + UINT16 ndata, UINT16 nparity, + UINT16 segmentSize, UINT16 erasureCount) +{ + bool increasedRepair = false; + + if (nextId < ndata) + { + // Explicit data repair request + parity_offset = parity_count = nparity; + while (nextId <= lastId) + { + if (!pending_mask.Test(nextId)) + { + pending_mask.Set(nextId); + increasedRepair = true; + } + nextId++; + } + } + else + { + // parity repair request + UINT16 parityAvailable = nparity - parity_offset; + if (erasureCount <= parityAvailable) + { + // Use fresh parity for repair + if (erasureCount > parity_count) + { + pending_mask.SetBits(ndata+parity_offset+parity_count, + erasureCount - parity_count); + parity_count = erasureCount; + increasedRepair = true; + } + } + else + { + // Use any remaining fresh parity ... + if (parity_count < parityAvailable) + { + UINT16 count = parityAvailable - parity_count; + pending_mask.SetBits(ndata+parity_offset+parity_count, count); + parity_count = parityAvailable; + nextId += parityAvailable; + increasedRepair = true; + } + // and explicit repair for the rest + while (nextId <= lastId) + { + if (!pending_mask.Test(nextId)) + { + pending_mask.Set(nextId); + increasedRepair = true; + } + nextId++; + } + } + } + return increasedRepair; +} // end NormBlock::TxUpdate() + +bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, + UINT16 ndata, UINT16 nparity, UINT16 erasureCount) +{ + DMSG(4, "NormBlock::HandleSegmentRequest() blk>%lu seg>%hu:%hu erasures:%hu\n", + (UINT32)id, (UINT16)nextId, (UINT16)lastId, erasureCount); + bool increasedRepair = false; + if (nextId < ndata) + { + // Explicit data repair request + parity_count = parity_offset = nparity; + while (nextId <= lastId) + { + if (!repair_mask.Test(nextId)) + { + repair_mask.Set(nextId); + increasedRepair = true; + } + nextId++; + } + } + else + { + // parity repair request + UINT16 parityAvailable = nparity - parity_offset; + if (erasureCount <= parityAvailable) + { + // Use fresh parity for repair + if (erasureCount > parity_count) + { + repair_mask.SetBits(ndata+parity_offset+parity_count, + erasureCount - parity_count); + parity_count = erasureCount; + increasedRepair = true; + } + } + else + { + // Use any remaining fresh parity ... + if (parity_count < parityAvailable) + { + UINT16 count = parityAvailable - parity_count; + repair_mask.SetBits(ndata+parity_offset+parity_count, count); + parity_count = parityAvailable; + nextId += parityAvailable; + increasedRepair = true; + } + // and explicit repair for the rest + while (nextId <= lastId) + { + if (!repair_mask.Test(nextId)) + { + repair_mask.Set(nextId); + increasedRepair = true; + } + nextId++; + } + } + } + return increasedRepair; +} // end NormBlock::HandleSegmentRequest() + + +// Called by client +bool NormBlock::AppendRepairRequest(NormNackMsg& nack, + UINT16 ndata, + UINT16 nparity, + NormObjectId objectId, + bool pendingInfo, + UINT16 segmentSize) +{ + ASSERT(pending_mask.FirstSet() < ndata); + NormRepairRequest req; + nack.AttachRepairRequest(req, segmentSize); + req.SetFlag(NormRepairRequest::SEGMENT); + if (pendingInfo) req.SetFlag(NormRepairRequest::INFO); + NormSegmentId nextId, lastId; + + if (erasure_count > nparity) + { + // Request explicit repair + nextId = pending_mask.FirstSet(); + UINT16 i = nparity; + // Skip nparity missing data segments + while (i--) + { + nextId++; + nextId = pending_mask.NextSet(nextId); + } + lastId = ndata + nparity; + } + else + { + nextId = pending_mask.NextSet(ndata); + lastId = ndata + erasure_count; + } + NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; + UINT16 reqCount = 0; + NormSegmentId prevId = nextId; + while ((nextId <= lastId) || (reqCount > 0)) + { + // force break of possible ending consec. series + if (nextId == lastId) nextId++; + if (reqCount && (reqCount == (nextId - prevId))) + { + reqCount++; // consecutive series continues + } + else + { + NormRepairRequest::Form nextForm; + switch(reqCount) + { + case 0: + nextForm = NormRepairRequest::INVALID; + break; + case 1: + case 2: + nextForm = NormRepairRequest::ITEMS; + break; + default: + nextForm = NormRepairRequest::RANGES; + break; + } // end switch(reqCount) + if (prevForm != nextForm) + { + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + if (NormRepairRequest::INVALID != nextForm) + { + nack.AttachRepairRequest(req, segmentSize); + req.SetForm(nextForm); + req.SetFlag(NormRepairRequest::SEGMENT); + if (pendingInfo) req.SetFlag(NormRepairRequest::INFO); + } + prevForm = nextForm; + } + if (NormRepairRequest::INVALID != nextForm) + DMSG(6, "NormBlock::AppendRepairRequest() SEGMENT request\n"); + switch (nextForm) + { + case NormRepairRequest::ITEMS: + req.AppendRepairItem(objectId, id, prevId); // (TBD) error check + if (2 == reqCount) + req.AppendRepairItem(objectId, id, prevId+1); // (TBD) error check + break; + case NormRepairRequest::RANGES: + req.AppendRepairItem(objectId, id, prevId); // (TBD) error check + req.AppendRepairItem(objectId, id, prevId+reqCount-1); // (TBD) error check + break; + default: + break; + } // end switch(nextForm) + prevId = nextId; + if (nextId < lastId) + reqCount = 1; + else + reqCount = 0; + } // end if/else (reqCount && (reqCount == (nextId - prevId))) + nextId++; + if (nextId <= lastId) nextId = pending_mask.NextSet(nextId); + } // end while(nextId <= lastId) + if (NormRepairRequest::INVALID != prevForm) + nack.PackRepairRequest(req); // (TBD) error check + return true; +} // end NormBlock::AppendRepairRequest() + NormBlockPool::NormBlockPool() - : head((NormBlock*)NULL) + : head((NormBlock*)NULL), overruns(0), overrun_flag(false) { } @@ -273,10 +579,11 @@ NormBlock* NormBlockBuffer::Find(const NormBlockId& blockId) const { if (range) { - if ((blockId < range_lo) || (blockId > range_hi)) return (NormBlock*)NULL; + if ((blockId < range_lo) || (blockId > range_hi)) + return (NormBlock*)NULL; NormBlock* theBlock = table[((UINT32)blockId) & hash_mask]; - //printf("NormBlockBuffer::Find() table[%lu] = %p\n", ((UINT32)blockId) & hash_mask, theBlock); - while (theBlock && (blockId != theBlock->Id())) theBlock = theBlock->next; + while (theBlock && (blockId != theBlock->Id())) + theBlock = theBlock->next; return theBlock; } else @@ -318,56 +625,49 @@ bool NormBlockBuffer::CanInsert(NormBlockId blockId) const bool NormBlockBuffer::Insert(NormBlock* theBlock) { - if (range < range_max) + const NormBlockId& blockId = theBlock->Id(); + if (!range) { - const NormBlockId& blockId = theBlock->Id(); - if (!range) - { - range_lo = range_hi = blockId; - range = 1; - } - if (blockId < range_lo) - { - UINT32 newRange = range_lo - blockId + range; - if (newRange > range_max) return false; - range_lo = blockId; - range = newRange; - } - else if (blockId > range_hi) - { - UINT32 newRange = blockId - range_hi + range; - if (newRange > range_max) return false; - range_hi = blockId; - range = newRange; - } - UINT32 index = ((UINT32)blockId) & hash_mask; - NormBlock* prev = NULL; - NormBlock* entry = table[index]; - while (entry && (entry->Id() < blockId)) - { - prev = entry; - entry = entry->next; - } - if (prev) - prev->next = theBlock; - else - table[index] = theBlock; - ASSERT((entry ? (blockId != entry->Id()) : true)); - theBlock->next = entry; - return true; + range_lo = range_hi = blockId; + range = 1; } + if (blockId < range_lo) + { + UINT32 newRange = range_lo - blockId + range; + if (newRange > range_max) return false; + range_lo = blockId; + range = newRange; + } + else if (blockId > range_hi) + { + UINT32 newRange = blockId - range_hi + range; + if (newRange > range_max) return false; + range_hi = blockId; + range = newRange; + } + UINT32 index = ((UINT32)blockId) & hash_mask; + NormBlock* prev = NULL; + NormBlock* entry = table[index]; + while (entry && (entry->Id() < blockId)) + { + prev = entry; + entry = entry->next; + } + if (prev) + prev->next = theBlock; else - { - return false; - } + table[index] = theBlock; + ASSERT((entry ? (blockId != entry->Id()) : true)); + theBlock->next = entry; + return true; } // end NormBlockBuffer::Insert() bool NormBlockBuffer::Remove(const NormBlock* theBlock) { ASSERT(theBlock); - const NormBlockId& blockId = theBlock->Id(); if (range) { + const NormBlockId& blockId = theBlock->Id(); if ((blockId < range_lo) || (blockId > range_hi)) return false; UINT32 index = ((UINT32)blockId) & hash_mask; NormBlock* prev = NULL; @@ -377,11 +677,12 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) prev = entry; entry = entry->next; } - if (entry != theBlock) return false; + if (!entry) return false; if (prev) prev->next = entry->next; else - table[index] = (NormBlock*)NULL; + table[index] = entry->next; + if (range > 1) { if (blockId == range_lo) @@ -390,7 +691,7 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) UINT32 i = index; UINT32 endex; if (range <= hash_mask) - endex = (index + range) & hash_mask; + endex = (index + range - 1) & hash_mask; else endex = index; entry = NULL; @@ -405,8 +706,8 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) NormBlockId id = (UINT32)index + offset; while(entry && (entry->Id() != id)) { - if ((entry->Id() > blockId) && (entry->Id() < nextId)) - nextId = entry->Id(); + if ((entry->Id() > blockId) && + (entry->Id() < nextId)) nextId = entry->Id(); entry = entry->next; } @@ -414,19 +715,10 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) } } while (i != endex); if (entry) - { range_lo = entry->Id(); - range = range_hi - range_lo + 1; - } - else if (nextId != range_hi) - { - range_lo = nextId; - range = range_hi - range_lo + 1; - } else - { - range = 0; - } + range_lo = nextId; + range = range_hi - range_lo + 1; } else if (blockId == range_hi) { @@ -434,7 +726,7 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) UINT32 i = index; UINT32 endex; if (range <= hash_mask) - endex = (index - range) & hash_mask; + endex = (index - range + 1) & hash_mask; else endex = index; entry = NULL; @@ -451,27 +743,18 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) //printf("Looking for id:%lu at index:%lu\n", (UINT32)id, i); while(entry && (entry->Id() != id)) { - if ((entry->Id() < blockId) && (entry->Id() > prevId)) - prevId = entry->Id(); + if ((entry->Id() < blockId) && + (entry->Id() > prevId)) prevId = entry->Id(); entry = entry->next; } if (entry) break; } } while (i != endex); if (entry) - { range_hi = entry->Id(); - - } - else if (prevId != range_lo) - { + else range_hi = prevId; - range = range_hi - range_lo + 1; - } - else - { - range = 0; - } + range = range_hi - range_lo + 1; } } else @@ -510,13 +793,13 @@ NormBlock* NormBlockBuffer::Iterator::GetNextBlock() { if (buffer.range && (index < buffer.range_hi) && - !(index < buffer.range_lo)) + (index >= buffer.range_lo)) { // Find next entry _after_ current "index" UINT32 i = index; - UINT32 endex = buffer.range_hi - index; - if (endex <= buffer.hash_mask) - endex = (index + endex) & buffer.hash_mask; + UINT32 endex; + if ((UINT32)(buffer.range_hi - index) <= buffer.hash_mask) + endex = buffer.range_hi & buffer.hash_mask; else endex = index; UINT32 offset = 0; @@ -526,8 +809,9 @@ NormBlock* NormBlockBuffer::Iterator::GetNextBlock() ++i &= buffer.hash_mask; offset++; NormBlockId id = (UINT32)index + offset; + ASSERT(i < 256); NormBlock* entry = buffer.table[i]; - while ((NULL != entry )& (entry->Id() != id)) + while ((NULL != entry ) && (entry->Id() != id)) { if ((entry->Id() > index) && (entry->Id() < nextId)) nextId = entry->Id(); diff --git a/common/normSegment.h b/common/normSegment.h index 771e5f1..59703f4 100644 --- a/common/normSegment.h +++ b/common/normSegment.h @@ -26,14 +26,18 @@ class NormSegmentPool } bool IsEmpty() {return (NULL == seg_list);} + unsigned long CurrentUsage() {return (seg_total - seg_count);} + unsigned long PeakUsage() {return peak_usage;} + unsigned long OverunCount() {return overruns;} + private: unsigned int seg_size; unsigned int seg_count; unsigned int seg_total; char* seg_list; - unsigned int peak_usage; - unsigned int overruns; + unsigned long peak_usage; + unsigned long overruns; bool overrun_flag; }; // end class NormSegmentPool @@ -45,8 +49,7 @@ class NormBlock public: enum Flag { - PARITY_READY = 0x01, - IN_REPAIR = 0x02 + IN_REPAIR = 0x01 }; NormBlock(); @@ -57,8 +60,11 @@ class NormBlock void Destroy(); void SetFlag(NormBlock::Flag flag) {flags |= flag;} - bool ParityReady() {return (0 != (flags & PARITY_READY));} bool InRepair() {return (0 != (flags & IN_REPAIR));} + bool ParityReady(UINT16 ndata) {return (erasure_count == ndata);} + UINT16 ParityReadiness() {return erasure_count;} + void IncreaseParityReadiness() {erasure_count++;} + void SetParityReadiness(UINT16 ndata) {erasure_count = ndata;} char** SegmentList(UINT16 index = 0) {return &segment_table[index];} char* Segment(NormSegmentId sid) @@ -71,7 +77,6 @@ class NormBlock ASSERT(sid < size); ASSERT(!segment_table[sid]); segment_table[sid] = segment; - erasure_count--; } char* DetachSegment(NormSegmentId sid) { @@ -87,25 +92,62 @@ class NormBlock segment_table[sid] = segment; } + // Server routines void TxInit(NormBlockId& blockId, UINT16 ndata, UINT16 autoParity) { id = blockId; pending_mask.Clear(); pending_mask.SetBits(0, ndata+autoParity); repair_mask.Clear(); - erasure_count = size - ndata; - parity_count = 0; + erasure_count = 0; + parity_count = 0; + parity_offset = autoParity; flags = 0; } - void RxInit(NormBlockId& blockId, UINT16 ndata) + void TxRecover(NormBlockId& blockId, UINT16 ndata, UINT16 nparity) { id = blockId; - pending_mask.Reset(); + pending_mask.Clear(); repair_mask.Clear(); - erasure_count = size; + erasure_count = 0; + parity_count = nparity; // force recovered blocks to + parity_offset = nparity; // explicit repair mode ??? + flags = IN_REPAIR; + } + bool TxReset(UINT16 ndata, UINT16 nparity, UINT16 autoParity, + UINT16 segmentSize); + bool TxUpdate(NormSegmentId nextId, NormSegmentId lastId, + UINT16 ndata, UINT16 nparity, + UINT16 segmentSize, UINT16 erasureCount); + + bool HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, + UINT16 ndata, UINT16 nparity, + UINT16 erasureCount); + bool ActivateRepairs(UINT16 nparity); + void ResetParityCount(UINT16 nparity) + { + parity_offset += parity_count; + parity_offset = MIN(parity_offset, nparity); + parity_count = 0; + } + + // Client routines + void RxInit(NormBlockId& blockId, UINT16 ndata, UINT16 nparity) + { + id = blockId; + pending_mask.Clear(); + pending_mask.SetBits(0, ndata+nparity); + repair_mask.Clear(); + erasure_count = ndata; parity_count = 0; + parity_offset = 0; flags = 0; } + bool IsRepairPending(UINT16 ndata, UINT16 nparity); + void DecrementErasureCount() {erasure_count--;} + UINT16 ErasureCount() const {return erasure_count;} + void IncrementParityCount() {parity_count++;} + UINT16 ParityCount() {return parity_count;} NormSymbolId FirstPending() const {return pending_mask.FirstSet();} @@ -113,12 +155,21 @@ class NormBlock {return repair_mask.FirstSet();} bool SetPending(NormSymbolId s) {return pending_mask.Set(s);} + bool SetPending(NormSymbolId firstId, UINT16 count) + {return pending_mask.SetBits(firstId, count);} void UnsetPending(NormSymbolId s) {pending_mask.Unset(s);} void ClearPending() {pending_mask.Clear();} bool SetRepair(NormSymbolId s) {return repair_mask.Set(s);} + bool SetRepairs(NormSymbolId first, NormSymbolId last) + { + if (first == last) + return repair_mask.Set(first); + else + return (repair_mask.SetBits(first, last-first+1)); + } void UnsetRepair(NormSymbolId s) {repair_mask.Unset(s);} void ClearRepairs() @@ -126,12 +177,23 @@ class NormBlock bool IsPending(NormSymbolId s) const {return pending_mask.Test(s);} bool IsPending() const - {return pending_mask.IsSet();} - bool IsRepairPending(NormSegmentId end); + {return pending_mask.IsSet();} + bool IsRepairPending() const + {return repair_mask.IsSet();} + bool IsTransmitPending() const + {return (pending_mask.IsSet() || repair_mask.IsSet());} + NormSymbolId NextPending(NormSymbolId index) const {return pending_mask.NextSet(index);} - UINT16 ErasureCount() const {return erasure_count;} + + + bool AppendRepairRequest(NormNackMsg& nack, + UINT16 ndata, + UINT16 nparity, + NormObjectId objectId, + bool pendingInfo, + UINT16 segmentSize); //void DisplayPendingMask(FILE* f) {pending_mask.Display(f);} bool IsEmpty(); @@ -145,6 +207,7 @@ class NormBlock int flags; UINT16 erasure_count; UINT16 parity_count; + UINT16 parity_offset; NormBitmask pending_mask; NormBitmask repair_mask; @@ -164,6 +227,15 @@ class NormBlockPool { NormBlock* b = head; head = b ? b->next : NULL; + if (b) + { + overrun_flag = false; + } + else if (!overrun_flag) + { + overruns++; + overrun_flag = true; + } return b; } void Put(NormBlock* b) @@ -171,16 +243,20 @@ class NormBlockPool b->next = head; head = b; } + unsigned long OverrunCount() {return overruns;} private: NormBlock* head; + unsigned long overruns; + bool overrun_flag; }; // end class NormBlockPool class NormBlockBuffer { - friend class NormBlockBuffer::Iterator; - public: + class Iterator; + friend class NormBlockBuffer::Iterator; + NormBlockBuffer(); ~NormBlockBuffer(); bool Init(unsigned long rangeMax, unsigned long tableSize = 256); diff --git a/common/normSession.cpp b/common/normSession.cpp index 317e927..05e8e33 100644 --- a/common/normSession.cpp +++ b/common/normSession.cpp @@ -1,12 +1,32 @@ #include "normSession.h" #include +#include // for gmtime() in NormTrace() + + +const double NormSession::DEFAULT_TRANSMIT_RATE = 64000.0; // bits/sec +const double NormSession::DEFAULT_PROBE_MIN = 1.0; // sec +const double NormSession::DEFAULT_PROBE_MAX = 10.0; // sec +const double NormSession::DEFAULT_GRTT_ESTIMATE = 0.5; // sec +const double NormSession::DEFAULT_GRTT_MAX = 10.0; // sec +const unsigned int NormSession::DEFAULT_GRTT_DECREASE_DELAY = 3; +const double NormSession::DEFAULT_BACKOFF_FACTOR = 4.0; +const double NormSession::DEFAULT_GSIZE_ESTIMATE = 1000.0; NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) - : session_mgr(sessionMgr), local_node_id(localNodeId), - tx_rate(DEFAULT_TRANSMIT_RATE), - is_server(false), flush_count(NORM_ROBUST_FACTOR), + : session_mgr(sessionMgr), notify_pending(false), local_node_id(localNodeId), + tx_rate(DEFAULT_TRANSMIT_RATE/8.0), backoff_factor(DEFAULT_BACKOFF_FACTOR), + is_server(false), + tx_cache_count_min(1), tx_cache_count_max(256), + tx_cache_size_max((unsigned long)20*1024*1024), + flush_count(NORM_ROBUST_FACTOR+1), posted_tx_queue_empty(false), + probe_interval(0.0), probe_interval_min(DEFAULT_PROBE_MIN), + probe_interval_max(DEFAULT_PROBE_MAX), + grtt_max(DEFAULT_GRTT_MAX), + grtt_decrease_delay_count(DEFAULT_GRTT_DECREASE_DELAY), + grtt_response(false), grtt_current_peak(0.0), is_client(false), + trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0), next(NULL) { tx_socket.Init((UdpSocketOwner*) this, @@ -21,6 +41,29 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) tx_timer.Init(0.0, -1, (ProtocolTimerOwner *)this, (ProtocolTimeoutFunc)&NormSession::OnTxTimeout); + repair_timer.Init(0.0, 1, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormSession::OnRepairTimeout); + flush_timer.Init(0.0, 0, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormSession::OnCommandTimeout); + probe_timer.Init(0.0, -1, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormSession::OnProbeTimeout); + grtt_quantized = NormQuantizeRtt(DEFAULT_GRTT_ESTIMATE); + grtt_measured = grtt_advertised = NormUnquantizeRtt(grtt_quantized); + + gsize_measured = DEFAULT_GSIZE_ESTIMATE; + gsize_quantized = NormQuantizeGroupSize(DEFAULT_GSIZE_ESTIMATE); + gsize_advertised = NormUnquantizeGroupSize(gsize_quantized); + gsize_nack_ave = (1.2 / (2.0*DEFAULT_BACKOFF_FACTOR)) * + (log(DEFAULT_GSIZE_ESTIMATE) + 1.0); + gsize_nack_ave = exp(gsize_nack_ave); + gsize_correction_factor = 1.0; + gsize_nack_delta = 0.0; + + // This timer is for printing out occasional status reports + // (It may be used to trigger transmission of report messages + // in the future for debugging, etc + report_timer.Init(30.0, -1, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormSession::OnReportTimeout); } NormSession::~NormSession() @@ -28,14 +71,13 @@ NormSession::~NormSession() Close(); } -const double NormSession::DEFAULT_TRANSMIT_RATE = 8000.0; bool NormSession::Open() { ASSERT(address.IsValid()); if (!tx_socket.IsOpen()) - { + { if (UDP_SOCKET_ERROR_NONE != tx_socket.Open()) { DMSG(0, "NormSession::Open() tx_socket open error\n"); @@ -75,11 +117,14 @@ bool NormSession::Open() return false; } } + InstallTimer(&report_timer); + return true; } // end NormSession::Open() void NormSession::Close() { + if (report_timer.IsActive()) report_timer.Deactivate(); if (is_server) StopServer(); if (is_client) StopClient(); if (tx_timer.IsActive()) tx_timer.Deactivate(); @@ -161,12 +206,15 @@ bool NormSession::StartServer(unsigned long bufferSpace, ndata = numData; nparity = numParity; is_server = true; - flush_count = NORM_ROBUST_FACTOR; // (TBD) parameterize robust_factor + flush_count = NORM_ROBUST_FACTOR+1; // (TBD) parameterize robust_factor + probe_timer.SetInterval(0.0); + InstallTimer(&probe_timer); return true; } // end NormSession::StartServer() void NormSession::StopServer() { + if (probe_timer.IsActive()) probe_timer.Deactivate(); encoder.Destroy(); tx_table.Destroy(); block_pool.Destroy(); @@ -180,6 +228,10 @@ void NormSession::StopServer() bool NormSession::StartClient(unsigned long bufferSize) { + if (!IsOpen()) + { + if (!Open()) return false; + } is_client = true; remote_server_buffer_size = bufferSize; return true; @@ -192,12 +244,13 @@ void NormSession::StopClient() if (!is_server) Close(); } + void NormSession::Serve() { - if (tx_timer.IsActive()) - { - return; - } + // Only send new data when no other messages are queued for transmission + //if (tx_timer.IsActive()) return; + if (!message_queue.IsEmpty()) return; + NormObject* obj = NULL; // Queue next server message @@ -206,7 +259,6 @@ void NormSession::Serve() NormObjectId objectId(tx_pending_mask.FirstSet()); obj = tx_table.Find(objectId); ASSERT(obj); - flush_count = 0; } else { @@ -224,21 +276,36 @@ void NormSession::Serve() if (obj) { - NormMessage* msg = message_pool.RemoveHead(); + NormMessage* msg = GetMessageFromPool(); if (msg) { if (obj->NextServerMsg(msg)) { msg->generic.SetDestination(address); + msg->object.SetGrtt(grtt_quantized); + msg->object.SetGroupSize(gsize_quantized); QueueMessage(msg); + flush_count = 0; + if (flush_timer.IsActive()) flush_timer.Deactivate(); if (!obj->IsPending()) - tx_pending_mask.Unset(obj->Id()); + { + if (obj->IsStream()) + posted_tx_queue_empty = true; // repair-delayed stream advance + else + tx_pending_mask.Unset(obj->Id()); + } } else { - message_pool.Append(msg); + ReturnMessageToPool(msg); if (obj->IsStream()) { + if (((NormStreamObject*)obj)->IsFlushPending() && + (flush_count < NORM_ROBUST_FACTOR)) + { + // Queue flush message + ServerQueueFlush(); + } if (!posted_tx_queue_empty) { posted_tx_queue_empty = true; @@ -251,26 +318,119 @@ void NormSession::Serve() } else { - DMSG(0, "NormSession::Serve() pending obj, no message?.\n"); - + DMSG(0, "NormSession::Serve() pending obj, no message?.\n"); + ASSERT(0); } } } else { - DMSG(0, "NormSession::Serve() Warning! message_pool empty.\n"); + DMSG(0, "NormSession::Serve() node>%lu Warning! message_pool empty.\n", + LocalNodeId()); } } else if (flush_count < NORM_ROBUST_FACTOR) { // Queue flush message - flush_count++; + ServerQueueFlush(); } + else if (flush_count == NORM_ROBUST_FACTOR) + { + DMSG(6, "NormSession::Serve() node>%lu server flush complete ...\n", + LocalNodeId()); + flush_count++; + } } // end NormSession::Serve() +void NormSession::ServerQueueFlush() +{ + // (TBD) Deal with EOT or pre-queued squelch on squelch case + if (flush_timer.IsActive()) return; + NormMessage* msg = GetMessageFromPool(); + if (msg) + { + msg->generic.SetType(NORM_MSG_CMD); + msg->generic.SetDestination(address); + msg->cmd.generic.SetGrtt(grtt_quantized); + msg->cmd.generic.SetGroupSize(gsize_quantized); + msg->cmd.generic.SetFlavor(NormCmdMsg::NORM_CMD_FLUSH); + NormCmdFlushMsg& flush = (NormCmdFlushMsg&)msg->cmd.flush; + flush.Reset(); // reset flags and length + NormObject* obj = tx_table.Find(tx_table.RangeHi()); + NormObjectId objectId; + NormBlockId blockId; + NormSegmentId segmentId; + if (obj) + { + if (obj->IsStream()) + { + NormStreamObject* stream = (NormStreamObject*)obj; + objectId = stream->Id(); + blockId = stream->FlushBlockId(); + segmentId = stream->FlushSegmentId(); + } + else + { + objectId = obj->Id(); + blockId = obj->LastBlockId(); + segmentId = obj->LastBlockSize() - 1; + } + } + else + { + ReturnMessageToPool(msg); + if (ServerQueueSquelch(next_tx_object_id)) + { + flush_count++; + flush_timer.SetInterval(2*grtt_advertised); + InstallTimer(&flush_timer); + } + DMSG(8, "NormSession::ServerQueueFlush() node>%lu squelch queued (flush_count:%u)...\n", + LocalNodeId(), flush_count); + return; + } + flush.SetObjectId(objectId); + flush.SetFecBlockId(blockId); + flush.SetFecSymbolId(segmentId); + QueueMessage(msg); + flush_count++; + flush_timer.SetInterval(2*grtt_advertised); + InstallTimer(&flush_timer); + DMSG(8, "NormSession::ServerQueueFlush() node>%lu, flush queued (flush_count:%u)...\n", + LocalNodeId(), flush_count); + + } + else + { + DMSG(0, " NormSession::ServerQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n", + LocalNodeId()); + } +} // end NormSession::ServerQueueFlush() + +bool NormSession::OnCommandTimeout() +{ + flush_timer.Deactivate(); + Serve(); + return false; +} // NormSession::OnCommandTimeout() + void NormSession::QueueMessage(NormMessage* msg) { - if (message_queue.IsEmpty()) + +/* A little test jig + static struct timeval lastTime = {0,0}; + struct timeval currentTime; + GetSystemTime(¤tTime); + if (0 != lastTime.tv_sec) + { + double delta = currentTime.tv_sec - lastTime.tv_sec; + delta += (((double)currentTime.tv_usec)*1.0e-06 - + ((double)lastTime.tv_usec)*1.0e-06); + TRACE("NormSession::QueueMessage() deltaT:%lf\n", delta); + } + lastTime = currentTime; +*/ + if (!tx_timer.IsActive()) { tx_timer.SetInterval(0.0); InstallTimer(&tx_timer); @@ -278,28 +438,65 @@ void NormSession::QueueMessage(NormMessage* msg) message_queue.Append(msg); } // end NormSesssion::QueueMessage(NormMessage& msg) + + +NormFileObject* NormSession::QueueTxFile(const char* path, + const char* infoPtr, + UINT16 infoLen) +{ + if (!IsServer()) + { + DMSG(0, "NormSession::QueueTxFile() Error: server is closed\n"); + return NULL; + } + NormFileObject* file = new NormFileObject(this, NULL, next_tx_object_id); + if (!file) + { + DMSG(0, "NormSession::QueueTxFile() new file object error: %s\n", + strerror(errno)); + return NULL; + } + if (!file->Open(path, infoPtr, infoLen)) + { + DMSG(0, "NormSession::QueueTxFile() file open error\n"); + delete file; + return NULL; + } + if (QueueTxObject(file, false)) + { + return file; + } + else + { + file->Close(); + delete file; + return NULL; + } +} // end NormSession::QueueTxFile() + NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, const char* infoPtr, UINT16 infoLen) { if (!IsServer()) { - DMSG(0, "NormSession::QueueTxStream() non-server session!\n"); + DMSG(0, "NormSession::QueueTxStream() Error: server is closed\n"); return NULL; } - NormStreamObject* stream = new NormStreamObject(this, NULL, current_tx_object_id); + NormStreamObject* stream = new NormStreamObject(this, NULL, next_tx_object_id); if (!stream) { - DMSG(0, "NormSession::QueueTxStream() new stream error!\n"); + DMSG(0, "NormSession::QueueTxStream() new stream object error: %s\n", + strerror(errno)); return NULL; } if (!stream->Open(bufferSize, infoPtr, infoLen)) { - DMSG(0, "NormSession::QueueTxStream() stream open error!\n"); + DMSG(0, "NormSession::QueueTxStream() stream open error\n"); delete stream; return NULL; } - if (QueueTxObject(stream, false)) + if (QueueTxObject(stream, true)) { // (???: stream has nothing pending until user writes to it???) //stream->Reset(); @@ -313,56 +510,149 @@ NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, } } // end NormSession::QueueTxStream() +#ifdef SIMULATE +NormSimObject* NormSession::QueueTxSim(unsigned long objectSize) +{ + if (!IsServer()) + { + DMSG(0, "NormSession::QueueTxSim() Error: server is closed\n"); + return NULL; + } + NormSimObject* simObject = new NormSimObject(this, NULL, next_tx_object_id); + if (!simObject) + { + DMSG(0, "NormSession::QueueTxSim() new sim object error: %s\n", + strerror(errno)); + return NULL; + } + + if (!simObject->Open(objectSize)) + { + DMSG(0, "NormSession::QueueTxSim() open error\n"); + delete simObject; + return NULL; + } + if (QueueTxObject(simObject, false)) + { + return simObject; + } + else + { + delete simObject; + return NULL; + } +} // end NormSession::QueueTxSim() +#endif // SIMULATE bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) { if (!IsServer()) { - DMSG(0, "NormSession::QueueTxObject() non-server session!\n"); + DMSG(0, "NormSession::QueueTxObject() non-server session error!\n"); return false; } + + // Manage tx_table min/max count and max size bounds + if (tx_table.Count() >= tx_cache_count_min) + { + unsigned long count = tx_table.Count(); + while ((count >= tx_cache_count_min) && + ((count >= tx_cache_count_max) || + ((tx_table.Size() + obj->Size()) > tx_cache_size_max))) + { + // Remove oldest non-pending + NormObject* oldest = tx_table.Find(tx_table.RangeLo()); + ASSERT(!oldest->IsPending()); + if (oldest->IsRepairPending()) + { + DMSG(0, "NormSession::QueueTxObject() all held objects repair pending\n"); + //posted_tx_queue_empty = false; + return false; + } + else + { + tx_table.Remove(oldest); + oldest->Close(); + delete oldest; + } + count = tx_table.Count(); + } + } // Attempt to queue the object if (!tx_table.Insert(obj)) { - // (TBD) steal an old non-pending object DMSG(0, "NormSession::QueueTxObject() tx_table insert error\n"); + ASSERT(0); return false; } tx_pending_mask.Set(obj->Id()); ASSERT(tx_pending_mask.Test(obj->Id())); - current_tx_object_id++; - if (touchServer) TouchServer(); + next_tx_object_id++; + posted_tx_queue_empty = false; + Serve(); return true; } // end NormSession::QueueTxObject() +void NormSession::DeleteTxObject(NormObject* obj) +{ + NormObjectId objectId = obj->Id(); + ASSERT(obj == tx_table.Find(objectId)); + tx_table.Remove(obj); + obj->Close(); + tx_pending_mask.Unset(objectId); + tx_repair_mask.Unset(objectId); + delete obj; +} // end NormSession::DeleteTxObject() + NormBlock* NormSession::ServerGetFreeBlock(NormObjectId objectId, NormBlockId blockId) { // First, try to get one from our block pool NormBlock* b = block_pool.Get(); + // Second, try to steal oldest non-pending block if (!b) { + NormObjectTable::Iterator iterator(tx_table); NormObject* obj; while ((obj = iterator.GetNextObject())) + { + if (obj->Id() == objectId) + b = obj->StealNonPendingBlock(true, blockId); + else + b = obj->StealNonPendingBlock(false); + if (b) + { + b->EmptyToPool(segment_pool); + break; + } + } + } + // Finally, try to steal newer pending block + if (!b) + { + // reverse iteration to find newest object with resources + NormObjectTable::Iterator iterator(tx_table); + NormObject* obj; + while ((obj = iterator.GetPrevObject())) { - if (obj->Id() > objectId) + if (obj->Id() < objectId) { break; } else { - if (obj->Id() < objectId) - b = obj->StealOldestBlock(false); - else - b = obj->StealOldestBlock(true, blockId); + if (obj->Id() > objectId) + b = obj->StealNewestBlock(false); + else + b = obj->StealNewestBlock(true, blockId); if (b) { b->EmptyToPool(segment_pool); break; } } - } + } } return b; } // end NormSession::ServerGetFreeBlock() @@ -370,8 +660,14 @@ NormBlock* NormSession::ServerGetFreeBlock(NormObjectId objectId, char* NormSession::ServerGetFreeSegment(NormObjectId objectId, NormBlockId blockId) { - while (segment_pool.IsEmpty() && - ServerGetFreeBlock(objectId, blockId)); + while (segment_pool.IsEmpty()) + { + NormBlock* b = ServerGetFreeBlock(objectId, blockId); + if (b) + block_pool.Put(b); + else + return NULL; + } return segment_pool.Get(); } // end NormSession::ServerGetFreeSegment() @@ -379,14 +675,15 @@ bool NormSession::TxSocketRecvHandler(UdpSocket* /*theSocket*/) { NormMessage msg; unsigned int buflen = NORM_MSG_SIZE_MAX; - if (UDP_SOCKET_ERROR_NONE == tx_socket.RecvFrom(msg.generic.GetBuffer(), &buflen, msg.generic.Src())) + if (UDP_SOCKET_ERROR_NONE == tx_socket.RecvFrom(msg.generic.AccessBuffer(), &buflen, + &msg.generic.AccessAddress())) { msg.generic.SetLength(buflen); HandleReceiveMessage(msg, true); } else { - DMSG(0, "NormSession::TxSocketRecvHandler() recv from error"); + DMSG(0, "NormSession::TxSocketRecvHandler() recvfrom error"); } return true; } // end NormSession::TxSocketRecvHandler() @@ -395,31 +692,161 @@ bool NormSession::RxSocketRecvHandler(UdpSocket* /*theSocket*/) { NormMessage msg; unsigned int buflen = NORM_MSG_SIZE_MAX; - if (UDP_SOCKET_ERROR_NONE == rx_socket.RecvFrom(msg.generic.GetBuffer(), &buflen, msg.generic.Src())) + if (UDP_SOCKET_ERROR_NONE == rx_socket.RecvFrom(msg.generic.AccessBuffer(), &buflen, + &msg.generic.AccessAddress())) { msg.generic.SetLength(buflen); HandleReceiveMessage(msg, false); } else { - DMSG(0, "NormSession::RxSocketRecvHandler() recv from error"); + DMSG(0, "NormSession::RxSocketRecvHandler() recvfrom error"); } return true; } // end NormSession::RxSocketRecvHandler() -void NormSession::HandleReceiveMessage(NormMessage& msg, bool wasUnicast) +#ifdef SIMULATE +bool NormSession::SimSocketRecvHandler(char* buffer, unsigned short buflen, + const NetworkAddress& src, bool unicast) { + NormMessage msg; + memcpy(msg.generic.AccessBuffer(), buffer, buflen); + msg.generic.SetLength(buflen); + msg.generic.AccessAddress() = src; + HandleReceiveMessage(msg, unicast); + return true; +} // end NormSession::SimSocketRecvHandler() +#endif // SIMULATE + +static double THE_TIME = 0.0; + +void NormTrace(NormNodeId localId, NormMessage& msg, bool sent) +{ + //if (DebugLevel() < 8) return; // (TBD) provide per-session trace on/off switch + static const char* MSG_NAME[] = + { + "INVALID", + "INFO", + "DATA", + "CMD", + "NACK", + "ACK", + "REPORT" + }; + static const char* CMD_NAME[] = + { + "CMD_INVALID", + "CMD_FLUSH", + "CMD_SQUELCH", + "CMD_ACK_REQ", + "CMD_REPAIR_ADV", + "CMD_CC", + "CMD_APP" + }; + static const char* REQ_NAME[] = + { + "INVALID", + "WATERMARK", + "RTT", + "APP" + }; + + NormMsgType msgType = msg.generic.GetType(); + UINT16 length = msg.generic.AccessLength(); + const char* status = sent ? "dst" : "src"; + + struct timeval currentTime; + GetSystemTime(¤tTime); + struct tm* ct = gmtime((time_t*)¤tTime.tv_sec); + DMSG(0, "trace>%02d:%02d:%02d.%06lu node>%lu %s>%s ", + ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, + (UINT32)localId, status, msg.generic.AccessAddress().HostAddressString()); + + THE_TIME = currentTime.tv_sec + 1.0e-06 * (double)currentTime.tv_usec; + switch (msgType) + { + case NORM_MSG_INFO: + DMSG(0, "INFO obj>%hu ", (UINT16)msg.object.GetObjectId()); + break; + case NORM_MSG_DATA: + if (msg.data.GetFecSymbolId() < msg.object.GetFecBlockLen()) + DMSG(0, "DATA "); + else + DMSG(0, "PRTY "); + DMSG(0, "obj>%hu blk>%lu seg>%hu ", + (UINT16)msg.object.GetObjectId(), + (UINT32)msg.data.GetFecBlockId(), + (UINT16)msg.data.GetFecSymbolId()); + break; + + case NORM_MSG_CMD: + { + NormCmdMsg::Flavor flavor = msg.cmd.generic.GetFlavor(); + DMSG(0, "%s", CMD_NAME[flavor]); + switch (flavor) + { + case NormCmdMsg::NORM_CMD_ACK_REQ: + { + int index = msg.cmd.ack_req.GetAckFlavor(); + index = MIN(index, 3); + DMSG(0, "(%s)", REQ_NAME[index]); + break; + } + case NormCmdMsg::NORM_CMD_SQUELCH: + DMSG(0, " obj>%hu blk>%lu seg>%hu", + (UINT16)msg.cmd.squelch.GetObjectId(), + (UINT32)msg.cmd.squelch.GetFecBlockId(), + (UINT16)msg.cmd.squelch.GetFecSymbolId()); + break; + case NormCmdMsg::NORM_CMD_FLUSH: + DMSG(0, " obj>%hu blk>%lu seg>%hu", + (UINT16)msg.cmd.flush.GetObjectId(), + (UINT32)msg.cmd.flush.GetFecBlockId(), + (UINT16)msg.cmd.flush.GetFecSymbolId()); + break; + default: + break; + } + DMSG(0, " "); + break; + } + + default: + DMSG(0, "%s ", MSG_NAME[msgType]); + break; + } + DMSG(0, "len>%hu\n", length); +} // end NormTrace(); + + +void NormSession::HandleReceiveMessage(NormMessage& msg, bool wasUnicast) +{ + + ASSERT(this == session_mgr.top_session); + // Drop some rx messages for testing + if (UniformRand(100.0) < rx_loss_rate) return; + + if (trace) NormTrace(LocalNodeId(), msg, false); switch (msg.generic.GetType()) { case NORM_MSG_INFO: - DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_INFO)\n"); + //DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_INFO)\n"); + if (IsClient()) ClientHandleObjectMessage(msg); break; case NORM_MSG_DATA: - //DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_DATA)\n"); - ClientHandleObjectMessage(msg); + //DMSG(0, "NormSession::HandleReceiveMessage(NORM_MSG_DATA) ...\n"); + if (IsClient()) ClientHandleObjectMessage(msg); break; case NORM_MSG_CMD: + if (IsClient()) ClientHandleCommand(msg); + break; case NORM_MSG_NACK: + DMSG(4, "NormSession::HandleReceiveMessage(NORM_MSG_NACK) node>%lu ...\n", + LocalNodeId()); + if (IsServer() && (msg.nack.GetServerId() == LocalNodeId())) + ServerHandleNackMessage(msg.nack); + if (IsClient()) ClientHandleNackMessage(msg.nack); + break; case NORM_MSG_ACK: case NORM_MSG_REPORT: case NORM_MSG_INVALID: @@ -428,6 +855,8 @@ void NormSession::HandleReceiveMessage(NormMessage& msg, bool wasUnicast) } } // end NormSession::HandleReceiveMessage() + + void NormSession::ClientHandleObjectMessage(NormMessage& msg) { NormNodeId serverId = msg.generic.GetSender(); @@ -437,50 +866,860 @@ void NormSession::ClientHandleObjectMessage(NormMessage& msg) if ((theServer = new NormServerNode(this, serverId))) { server_tree.AttachNode(theServer); - DMSG(0, "NormSession::ClientHandleObjectMessage() new remote server:%lu ...\n", - serverId); + DMSG(4, "NormSession::ClientHandleObjectMessage() node>%lu new remote server:%lu ...\n", + LocalNodeId(), serverId); } else { - DMSG(0, "NormSession::ClientHandleObjectMessage() new server node error\n"); + DMSG(0, "NormSession::ClientHandleObjectMessage() new server node error: %s\n", + strerror(errno)); // (TBD) notify application of error return; } } - theServer->SetAddress(*msg.generic.Src()); + // for statistics only (TBD) #ifdef NORM_DEBUG + theServer->IncrementRecvTotal(msg.generic.GetLength()); + theServer->SetAddress(msg.generic.GetSource()); theServer->HandleObjectMessage(msg); } // end NormSession::ClientHandleObjectMessage() +void NormSession::ClientHandleCommand(NormMessage& msg) +{ + NormNodeId serverId = msg.generic.GetSender(); + NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(serverId); + if (!theServer) + { + //DMSG(0, "NormSession::ClientHandleCommand() node>%lu recvd command from unknown server ...\n", + // LocalNodeId()); + if ((theServer = new NormServerNode(this, serverId))) + { + server_tree.AttachNode(theServer); + DMSG(4, "NormSession::ClientHandleCommand() node>%lu new remote server:%lu ...\n", + LocalNodeId(), serverId); + } + else + { + DMSG(0, "NormSession::ClientHandleCommand() new server node error: %s\n", + strerror(errno)); + // (TBD) notify application of error + return; + } + } + // for statistics only (TBD) #ifdef NORM_DEBUG + theServer->IncrementRecvTotal(msg.generic.GetLength()); + theServer->SetAddress(msg.generic.GetSource()); + theServer->HandleCommand(msg.cmd); +} // end NormSession::ClientHandleCommand() + + +void NormSession::ServerUpdateGrttEstimate(const struct timeval& grttResponse) +{ + if (grttResponse.tv_sec || grttResponse.tv_usec) + { + struct timeval currentTime; + GetSystemTime(¤tTime); + double clientRtt; + // Calculate rtt estimate for this client and process the response + if (currentTime.tv_usec < grttResponse.tv_usec) + { + clientRtt = + (double)(currentTime.tv_sec - grttResponse.tv_sec - 1); + clientRtt += + ((double)(1000000 - (grttResponse.tv_usec - currentTime.tv_usec))) / 1.0e06; + } + else + { + clientRtt = + (double)(currentTime.tv_sec - grttResponse.tv_sec); + clientRtt += + ((double)(currentTime.tv_usec - grttResponse.tv_usec)) / 1.0e06; + } + // Lower limit on RTT (because of coarse timer resolution on some systems, + // this can sometimes actually end up a negative value!) + if (clientRtt < 1.0e-06) clientRtt = 1.0e-06; + grtt_response = true; + if (clientRtt > grtt_current_peak) + { + // Immediately incorporate bigger RTT's + grtt_current_peak = clientRtt; + if (clientRtt > grtt_measured) + { + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + grtt_measured = 0.25 * grtt_measured + 0.75 * clientRtt; + if (grtt_measured > grtt_max) grtt_measured = grtt_max; + double pktInterval = ((double)(44+segment_size))/tx_rate; + grtt_advertised = MAX(pktInterval, grtt_measured); + UINT8 grttQuantizedOld = grtt_quantized; + grtt_quantized = NormQuantizeRtt(grtt_advertised); + // Recalculate grtt_advertise since quantization rounds upward + grtt_advertised = NormUnquantizeRtt(grtt_quantized); + if (grttQuantizedOld != grtt_quantized) + DMSG(4, "NormSession::ServerUpdateGrttEstimate() node>%lu new grtt: %lf sec.\n", + LocalNodeId(), grtt_advertised); + } + } + } +} // end NormSession::ServerUpdateGrttEstimate() + + +void NormSession::ServerHandleNackMessage(NormNackMsg& nack) +{ + // Update GRTT estimate + struct timeval grttResponse; + nack.GetGrttResponse(grttResponse); + ServerUpdateGrttEstimate(grttResponse); + gsize_nack_count++; + + // Parse and process NACK + UINT16 requestOffset = 0; + UINT16 requestLength = 0; + NormRepairRequest req; + NormObject* object = NULL; + bool freshObject = true; + NormObjectId prevObjectId; + NormBlock* block = NULL; + bool freshBlock = true; + NormBlockId prevBlockId; + + bool startTimer = false; + UINT16 numErasures = 0; + + bool squelchQueued = false; + + NormObjectId txObjectIndex; + NormBlockId txBlockIndex; + if (tx_pending_mask.IsSet()) + { + txObjectIndex = tx_pending_mask.FirstSet(); + NormObject* obj = tx_table.Find(txObjectIndex); + ASSERT(obj); + if (obj->IsPending()) + { + if (obj->IsPendingInfo()) + txBlockIndex = 0; + else + txBlockIndex = obj->FirstPending() + 1; + } + else + { + txObjectIndex = txObjectIndex + 1; + txBlockIndex = 0; + } + } + else + { + txObjectIndex = next_tx_object_id; + txBlockIndex = 0; + } + + bool holdoff = (repair_timer.IsActive() && !repair_timer.RepeatCount()); + + enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; + while ((requestLength = nack.UnpackRepairRequest(req, requestOffset))) + { + NormRepairRequest::Form requestForm = req.GetForm(); + requestOffset += requestLength; + NormRequestLevel requestLevel; + if (req.FlagIsSet(NormRepairRequest::SEGMENT)) + requestLevel = SEGMENT; + else if (req.FlagIsSet(NormRepairRequest::BLOCK)) + requestLevel = BLOCK; + else if (req.FlagIsSet(NormRepairRequest::OBJECT)) + requestLevel = OBJECT; + else + { + requestLevel = INFO; + ASSERT(req.FlagIsSet(NormRepairRequest::INFO)); + } + + NormRepairRequest::Iterator iterator(req); + NormObjectId nextObjectId, lastObjectId; + NormBlockId nextBlockId, lastBlockId; + NormSegmentId nextSegmentId, lastSegmentId; + while (iterator.NextRepairItem(&nextObjectId, &nextBlockId, &nextSegmentId)) + { + if (NormRepairRequest::RANGES == requestForm) + { + if (!iterator.NextRepairItem(&lastObjectId, &lastBlockId, &lastSegmentId)) + { + DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu recvd incomplete RANGE request!\n", + LocalNodeId()); + continue; // (TBD) break/return instead??? + } + // (TBD) test for valid range form/level + } + else + { + lastObjectId = nextObjectId; + lastBlockId = nextBlockId; + lastSegmentId = nextSegmentId; + } + + bool inRange = true; + while (inRange) + { + if (nextObjectId != prevObjectId) freshObject = true; + if (freshObject) + { + freshBlock = true; + if (!(object = tx_table.Find(nextObjectId))) + { + DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu recvd repair request " + "for unknown object ...\n", LocalNodeId()); + if (!squelchQueued) + { + ServerQueueSquelch(nextObjectId); + squelchQueued = true; + } + if ((OBJECT == requestLevel) || + (INFO == requestLevel)) + { + nextObjectId++; + if (nextObjectId > lastObjectId) inRange = false; + } + else + { + inRange = false; + } + continue; + } + prevObjectId = nextObjectId; + freshObject = false; + // Deal with INFO request if applicable + if (req.FlagIsSet(NormRepairRequest::INFO)) + { + if (holdoff) + { + if (nextObjectId > txObjectIndex) + object->HandleInfoRequest(); + } + else + { + object->HandleInfoRequest(); + startTimer = true; + } + } + } + ASSERT(object); + + switch (requestLevel) + { + case OBJECT: + if (holdoff) + { + if (nextObjectId > txObjectIndex) + { + if (object->IsStream()) + object->TxReset(((NormStreamObject*)object)->StreamBufferLo()); + else + object->TxReset(); + if (!tx_pending_mask.Set(nextObjectId)) + DMSG(0, "NormSession::ServerHandleNackMessage() tx_pending_mask.Set(%hu) error (1)\n", + (UINT16)nextObjectId); + } + } + else + { + tx_repair_mask.Set(nextObjectId); + startTimer = true; + } + nextObjectId++; + if (nextObjectId > lastObjectId) inRange = false; + break; + + case BLOCK: + // (TBD) if entire object is TxReset(), continue + if (object->IsStream()) + { + bool attemptLock = true; + NormBlockId firstLockId = nextBlockId; + if (holdoff) + { + // Only lock blocks for which we're going to accept the repair request + if (nextObjectId == txObjectIndex) + { + if (lastBlockId < txBlockIndex) + attemptLock = false; + else if (nextBlockId < txBlockIndex) + firstLockId = txBlockIndex; + } + else if (nextObjectId < txObjectIndex) + { + attemptLock = false; // NACK arrived too late + } + } + + // Make sure the stream' pending_mask can be set as needed + // (TBD) + + // Lock stream_buffer pending for block data retransmissions + if (attemptLock) + { + if (!((NormStreamObject*)object)->LockBlocks(firstLockId, lastBlockId)) + { + DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu LockBlocks() failure\n", + LocalNodeId()); + inRange = false; + if (!squelchQueued) + { + ServerQueueSquelch(nextObjectId); + squelchQueued = true; + } + continue; + } + } + else + { + inRange = false; + continue; + } + } + if (holdoff) + { + if (nextObjectId == txObjectIndex) + { + if (nextBlockId >= txBlockIndex) + object->TxResetBlocks(nextBlockId, lastBlockId); + else if (lastBlockId >= txBlockIndex) + object->TxResetBlocks(txBlockIndex, lastBlockId); + } + else if (nextObjectId > txObjectIndex) + { + if (object->TxResetBlocks(nextBlockId, lastBlockId)) + { + if (!tx_pending_mask.Set(nextObjectId)) + DMSG(0, "NormSession::ServerHandleNackMessage() tx_pending_mask.Set(%hu) error (2)\n", + (UINT16)nextObjectId); + } + } + } + else + { + object->HandleBlockRequest(nextBlockId, lastBlockId); + startTimer = true; + } + inRange = false; + break; + case SEGMENT: + if (nextBlockId != prevBlockId) freshBlock = true; + if (freshBlock) + { + // Is this entire block already repair pending? + if (object->IsRepairSet(nextBlockId)) + { + inRange = false; + continue; + } + if (!(block = object->FindBlock(nextBlockId))) + { + // Try to recover block including parity calculation + if (!(block = object->ServerRecoverBlock(nextBlockId))) + { + if (NormObject::STREAM == object->GetType()) + { + DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu " + "recvd repair request for old stream block(%lu) ...\n", + LocalNodeId(), (UINT32)nextBlockId); + inRange = false; + if (!squelchQueued) + { + ServerQueueSquelch(nextObjectId); + squelchQueued = true; + } + continue; + } + else + { + // Resource constrained, move on. + DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu " + "Warning - server is resource contrained ...\n"); + inRange = false; + continue; + } + } + } + freshBlock = false; + numErasures = 0; + prevBlockId = nextBlockId; + } + // If stream && explicit data repair, lock the data for retransmission + + if (object->IsStream() && (nextSegmentId < ndata)) + { + bool attemptLock = true; + NormSegmentId firstLockId = nextSegmentId; + NormSegmentId lastLockId = ndata - 1; + lastLockId = MIN(lastLockId, lastSegmentId); + if (holdoff) + { + if (nextObjectId == txObjectIndex) + { + if (nextBlockId < txBlockIndex) + { + if (1 == (txBlockIndex - nextBlockId)) + { + // We're currently sending this block + NormSegmentId firstPending = block->FirstPending(); + if (lastLockId <= firstPending) + attemptLock = false; + else if (nextSegmentId < firstPending) + firstLockId = firstPending; + } + else + { + attemptLock = false; // NACK arrived way too late + } + } + } + else if (nextObjectId < txObjectIndex) + { + attemptLock = false; // NACK arrived too late + } + } + if (attemptLock) + { + if (!((NormStreamObject*)object)->LockSegments(nextBlockId, firstLockId, lastLockId)) + { + DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu " + "LockSegments() failure\n", LocalNodeId()); + inRange = false; + if (!squelchQueued) + { + ServerQueueSquelch(nextObjectId); + squelchQueued = true; + } + continue; + } + } + else + { + inRange = false; + continue; + } + } + + // With a series of SEGMENT repair requests for a block, "numErasures" will + // eventually total the number of missing segments in the block. + numErasures += (lastSegmentId - nextSegmentId + 1); + if (holdoff) + { + if (nextObjectId > txObjectIndex) + { + if (object->TxUpdateBlock(block, nextSegmentId, + lastSegmentId, numErasures)) + { + if (!tx_pending_mask.Set(nextObjectId)) + DMSG(0, "NormSession::ServerHandleNackMessage() tx_pending_mask.Set(%hu) error (3)\n", + (UINT16)nextObjectId); + } + } + else if (nextObjectId == txObjectIndex) + { + if (nextBlockId >= txBlockIndex) + { + object->TxUpdateBlock(block, nextSegmentId, + lastSegmentId, numErasures); + } + else if (1 == (txBlockIndex - nextBlockId)) + { + NormSegmentId firstPending = block->FirstPending(); + if (nextSegmentId > firstPending) + object->TxUpdateBlock(block, nextSegmentId, + lastSegmentId, numErasures); + else if (lastSegmentId > firstPending) + object->TxUpdateBlock(block, firstPending, + lastSegmentId, numErasures); + else if (numErasures > block->ParityCount()) + object->TxUpdateBlock(block, firstPending, + firstPending, numErasures); + } + } + } + else + { + block->HandleSegmentRequest(nextSegmentId, lastSegmentId, ndata, nparity, numErasures); + startTimer = true; + } + inRange = false; + break; + + case INFO: + nextObjectId++; + if (nextObjectId > lastObjectId) inRange = false; + break; + } // end switch(requestLevel) + } // end while(inRange) + } // end while(NextRepairItem()) + } // end while(UnpackRepairRequest()) + if (startTimer && !repair_timer.IsActive()) + { + repair_timer.SetInterval(grtt_advertised * (backoff_factor + 1.0)); + DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu starting server " + "NACK aggregation timer (%lf sec)...\n", LocalNodeId(), repair_timer.Interval()); + gsize_nack_count = 1; + InstallTimer(&repair_timer); + } +} // end NormSession::ServerHandleNackMessage() + +void NormSession::ClientHandleNackMessage(NormNackMsg& nack) +{ + NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(nack.GetServerId()); + if (theServer) + { + theServer->HandleNackMessage(nack); + } + else + { + DMSG(4, "NormSession::ClientHandleNackMessage() node>%lu heard NACK for unkown server\n", + (UINT32)nack.GetServerId()); + } +} // end NormSession::ClientHandleNackMessage() + + +bool NormSession::ServerQueueSquelch(NormObjectId objectId) +{ + // (TBD) if a squelch is already queued, update it if (objectId < squelch->objectId) + NormMessage* msg = GetMessageFromPool(); + if (msg) + { + msg->generic.SetType(NORM_MSG_CMD); + msg->generic.SetDestination(address); + msg->cmd.generic.SetFlavor(NormCmdMsg::NORM_CMD_SQUELCH); + NormCmdSquelchMsg& squelch = (NormCmdSquelchMsg&)msg->cmd.squelch; + NormObject* obj = tx_table.Find(objectId); + NormObjectTable::Iterator iterator(tx_table); + NormObjectId nextId; + if (obj) + { + ASSERT(NormObject::STREAM == obj->GetType()); + squelch.SetObjectId(objectId); + squelch.SetFecBlockId(((NormStreamObject*)obj)->StreamBufferLo()); + squelch.SetFecSymbolId(0); + squelch.ResetInvalidObjectList(); + while ((obj = iterator.GetNextObject())) + if (objectId == obj->Id()) break; + nextId = objectId + 1; + } + else + { + obj = iterator.GetNextObject(); + if (obj) + { + squelch.SetObjectId(obj->Id()); + if (obj->IsStream()) + squelch.SetFecBlockId(((NormStreamObject*)obj)->StreamBufferLo()); + else + squelch.SetFecBlockId(0); + squelch.SetFecSymbolId(0); + nextId = obj->Id() + 1; + } + else + { + // Squelch to point to future object + squelch.SetObjectId(next_tx_object_id); + squelch.SetFecBlockId(0); + squelch.SetFecSymbolId(0); + nextId = next_tx_object_id; + } + } + bool buildingList = true; + while (buildingList && (obj = iterator.GetNextObject())) + { + while (nextId != obj->Id()) + { + if (!squelch.AppendInvalidObject(nextId, segment_size)) + { + buildingList = false; + break; + } + nextId++; + } + } + QueueMessage(msg); + DMSG(2, "NormSession::ServerQueueSquelch() node>%lu server queued squelch ...\n", + LocalNodeId()); + return true; + } + else + { + DMSG(0, " NormSession::ServerQueueSquelch() node>%lu message_pool exhausted! (couldn't squelch)\n", + LocalNodeId()); + return false; + } +} // end NormSession::ServerQueueSquelch() + + +void NormSession::ServerUpdateGroupSize() // (TBD) not doing anything yet +{ + // Possible approach ... use smoothed history of gsize_nack_count information + // to adjust a group size estimator (if needed to keep the nack count average + // with an acceptable volume) there may be some issue here since either + // underestimating _or_ overestimating the group size may reduce + // NACK suppression performance ... but it's somewhat safer to overestimate + // and a check could be provided to help guess if we're over or under estimating? + gsize_nack_ave = 0.9*gsize_nack_ave + 0.1*(double)gsize_nack_count; + +} // end NormSession::ServerUpdateGroupSize() + +bool NormSession::OnRepairTimeout() +{ + if (repair_timer.RepeatCount()) + { + // NACK aggregation period has ended. (incorporate accumulated repair requests) + DMSG(4, "NormSession::OnRepairTimeout() node>%lu server NACK aggregation time ended.\n", + LocalNodeId()); + + // Update our group size estimate (we don't include nacks during repair hold-off interval) + ServerUpdateGroupSize(); + + NormObjectTable::Iterator iterator(tx_table); + NormObject* obj; + while ((obj = iterator.GetNextObject())) + { + NormObjectId objectId = obj->Id(); + if (tx_repair_mask.Test(objectId)) + { + DMSG(6, "NormSession::OnRepairTimeout() node>%lu tx reset obj>%hu ...\n", + LocalNodeId(), (UINT16)objectId); + if (obj->IsStream()) + obj->TxReset(((NormStreamObject*)obj)->StreamBufferLo()); + else + obj->TxReset(); + tx_repair_mask.Unset(objectId); + if (!tx_pending_mask.Set(objectId)) + DMSG(0, "NormSession::OnRepairTimeout() rx_pending_mask.Set(%hu) error (1)\n", + (UINT16)objectId); + } + else + { + //DMSG(6, "NormSession::OnRepairTimeout() node>%lu activating obj>%hu repairs ...\n", + // LocalNodeId(), (UINT16)objectId); + if (obj->ActivateRepairs()) + { + DMSG(6, "NormSession::OnRepairTimeout() node>%lu activated obj>%hu repairs ...\n", + LocalNodeId(), (UINT16)objectId); + if (!tx_pending_mask.Set(objectId)) + DMSG(0, "NormSession::OnRepairTimeout() rx_pending_mask.Set(%hu) error (2)\n", + (UINT16)objectId); + } + } + } // end while (iterator.GetNextObject()) + TouchServer(); + // Holdoff initiation of new repair cycle for one GRTT + repair_timer.SetInterval(grtt_advertised); // repair holdoff interval = 1*GRTT + } + else + { + // REPAIR holdoff interval has now ended. + DMSG(4, "NormSession::OnRepairTimeout() node>%lu server holdoff time ended.\n", + LocalNodeId()); + } + return true; +} // end NormSession::OnRepairTimeout() + + + bool NormSession::OnTxTimeout() { NormMessage* msg = message_queue.RemoveHead(); if (msg) { + // Fill in any last minute timestamps + switch (msg->generic.GetType()) + { + case NORM_MSG_CMD: + switch (msg->cmd.generic.GetFlavor()) + { + case NormCmdMsg::NORM_CMD_ACK_REQ: + if (NormCmdAckReqMsg::RTT == msg->cmd.ack_req.GetAckFlavor()) + { + struct timeval currentTime; + GetSystemTime(¤tTime); + msg->cmd.ack_req.SetRttSendTime(currentTime); + } + break; + default: + break; + } + break; + + case NORM_MSG_NACK: + { + NormServerNode* theServer = + (NormServerNode*)server_tree.FindNodeById(msg->nack.GetServerId()); + ASSERT(theServer); + struct timeval grttResponse; + theServer->CalculateGrttResponse(grttResponse); + msg->nack.SetGrttResponse(grttResponse); + // (TBD) fill in cc_sequence + // (TBD) fill in loss estimate + break; + } + + case NORM_MSG_ACK: + { + NormServerNode* theServer = + (NormServerNode*)server_tree.FindNodeById(msg->nack.GetServerId()); + ASSERT(theServer); + struct timeval grttResponse; + theServer->CalculateGrttResponse(grttResponse); + //msg->ack.SetGrttResponse(grttResponse); + // (TBD) fill in cc_sequence + // (TBD) fill in loss estimate + break; + } + + default: + break; + } // Fill in common message fields msg->generic.SetVersion(1); msg->generic.SetSequence(tx_sequence++); msg->generic.SetSender(local_node_id); - NetworkAddress* dest = msg->generic.Dest(); - // Use session address by default - if (!dest) dest = &address; UINT16 len = msg->generic.GetLength(); - if (UDP_SOCKET_ERROR_NONE != tx_socket.SendTo(dest, msg->generic.GetBuffer(), len)) + // Drop some tx messages for testing purposes + bool drop = (UniformRand(100.0) < tx_loss_rate); + if (!drop) { - DMSG(0, "NormSession::OnTxTimeout() sendto() error\n"); + if (UDP_SOCKET_ERROR_NONE != + tx_socket.SendTo(&msg->generic.GetDestination(), + msg->generic.AccessBuffer(), len)) + { + DMSG(0, "NormSession::OnTxTimeout() sendto() error\n"); + } + else + { + // Separate send/recv tracing + if (trace) NormTrace(LocalNodeId(), *msg, true); + } + } + else + { + //TRACE("TX MESSAGE DROPPED! (tx_loss_rate:%lf\n", tx_loss_rate); } tx_timer.SetInterval(((double)len) / tx_rate); - message_pool.Append(msg); + ReturnMessageToPool(msg); + return true; // reinstall tx_timer } else { - tx_timer.Deactivate(); if (IsServer()) Serve(); - return false; + if (message_queue.IsEmpty()) + { + tx_timer.Deactivate(); + return false; + } + else + { + // We have a new message as a result of serving, so send it immediately + OnTxTimeout(); + return true; + } } - return true; } // end NormSession::OnTxTimeout() +bool NormSession::OnProbeTimeout() +{ + NormMessage* msg = GetMessageFromPool(); + if (msg) + { + msg->generic.SetType(NORM_MSG_CMD); + msg->generic.SetDestination(address); + msg->cmd.generic.SetGrtt(grtt_quantized); + msg->cmd.generic.SetGroupSize(gsize_quantized); + msg->cmd.generic.SetFlavor(NormCmdMsg::NORM_CMD_ACK_REQ); + NormCmdAckReqMsg& ackReq = msg->cmd.ack_req; + ackReq.SetAckFlavor(NormCmdAckReqMsg::RTT); + // SetRttSendTime() when message is being sent + ackReq.ResetAckingNodeList(); + QueueMessage(msg); + } + else + { + DMSG(0, "NormSession::OnProbeTimeout() node>%lu message_pool empty! can't probe\n", + LocalNodeId()); + } + + if (grtt_response) + { + // Update grtt estimate + if (grtt_current_peak < grtt_measured) + { + if (grtt_decrease_delay_count-- == 0) + { + grtt_measured = 0.5 * grtt_measured + + 0.5 * grtt_current_peak; + grtt_current_peak = 0.0; + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + } + } + else + { + // Increase already incorporated + grtt_current_peak = 0.0; + grtt_decrease_delay_count = DEFAULT_GRTT_DECREASE_DELAY; + } + if (grtt_measured < NORM_GRTT_MIN) + grtt_measured = NORM_GRTT_MIN; + else if (grtt_measured > grtt_max) + grtt_measured = grtt_max; + double pktInterval = (double)(44+segment_size)/tx_rate; + grtt_advertised = MAX(pktInterval, grtt_measured); + double grttQuantizedOld = grtt_quantized; + grtt_quantized = NormQuantizeRtt(grtt_advertised); + // Recalculate grtt_advertise since quantization rounds upward + grtt_advertised = NormUnquantizeRtt(grtt_quantized); + grtt_response = false; + if (grttQuantizedOld != grtt_quantized) + DMSG(2, "NormSession::OnProbeTimeout() node>%lu new grtt: %lf\n", + LocalNodeId(), grtt_advertised); + } + + // Manage probe_timer interval + if (probe_interval < probe_interval_min) + probe_interval = probe_interval_min; + else + probe_interval *= 2.0; + if (probe_interval > probe_interval_max) + probe_interval = probe_interval_max; + probe_timer.SetInterval(probe_interval); + return true; +} // end NormSession::OnProbeTimeout() + +bool NormSession::OnReportTimeout() +{ + // Client reporting (just print out for now) + struct timeval currentTime; + GetSystemTime(¤tTime); + struct tm* ct = gmtime((time_t*)¤tTime.tv_sec); + DMSG(2, "Report time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", + ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, LocalNodeId()); + if (IsClient()) + { + NormNodeTreeIterator iterator(server_tree); + NormServerNode* next; + while ((next = (NormServerNode*)iterator.GetNextNode())) + { + DMSG(2, "Remote server:%lu\n", next->Id()); + double rxRate = 8.0e-03*((double)next->RecvTotal()) / report_timer.Interval(); // kbps + double rxGoodput = 8.0e-03*((double)next->RecvGoodput()) / report_timer.Interval(); // kbps + next->ResetRecvStats(); + DMSG(2, " rx_rate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput); + DMSG(2, " objects completed>%lu pending>%lu failed:%lu\n", + next->CompletionCount(), next->PendingCount(), next->FailureCount()); + DMSG(2, " buffer usage> current:%lu peak:%lu (overuns:%lu)\n", next->CurrentBufferUsage(), + next->PeakBufferUsage(), + next->BufferOverunCount()); + DMSG(2, " resyncs>%lu nacks>%lu suppressed>%lu\n", next->ResyncCount(), + next->NackCount(), next->SuppressCount()); + + } + } // end if (IsClient()) + DMSG(2, "***************************************************************************\n"); + return true; +} // end NormSession::OnReportTimeout() + NormSessionMgr::NormSessionMgr() : socket_installer(NULL), socket_install_data(NULL), controller(NULL), top_session(NULL) @@ -518,30 +1757,43 @@ NormSession* NormSessionMgr::NewSession(const char* sessionAddress, // (TBD) test IPv6 "EndIdentifier" ??? localNodeId = localAddr.EndIdentifier(); } - - - NetworkAddress a; - if (!a.LookupHostAddress(sessionAddress)) + NetworkAddress theAddress; + if (!theAddress.LookupHostAddress(sessionAddress)) { DMSG(0, "NormSessionMgr::NewSession() session address lookup error!\n"); return ((NormSession*)NULL); } - a.SetPort(sessionPort); - - NormSession* s = new NormSession(*this, localNodeId); - - if (!s) + theAddress.SetPort(sessionPort); + NormSession* theSession = new NormSession(*this, localNodeId); + if (!theSession) { DMSG(0, "NormSessionMgr::NewSession() new session error: %s\n", strerror(errno)); return ((NormSession*)NULL); - } - - s->SetAddress(a); - + } + theSession->SetAddress(theAddress); // Add new session to our session list - s->next = top_session; - top_session = s; - return s; + theSession->next = top_session; + top_session = theSession; + return theSession; } // end NormSessionMgr::NewSession(); +void NormSessionMgr::DeleteSession(class NormSession* theSession) +{ + NormSession* prev = NULL; + NormSession* next = top_session; + while (next && (next != theSession)) + { + prev = next; + next = next->next; + } + if (next) + { + if (prev) + prev->next = theSession->next; + else + top_session = theSession->next; + delete theSession; + } +} // end NormSessionMgr::DeleteSession() + diff --git a/common/normSession.h b/common/normSession.h index d0e15e1..fdffd3c 100644 --- a/common/normSession.h +++ b/common/normSession.h @@ -17,7 +17,9 @@ class NormController { TX_QUEUE_EMPTY, RX_OBJECT_NEW, - RX_OBJECT_UPDATE + RX_OBJECT_INFO, + RX_OBJECT_UPDATE, + RX_OBJECT_COMPLETE, }; virtual void Notify(NormController::Event event, @@ -51,6 +53,8 @@ class NormSessionMgr UINT16 sessionPort, NormNodeId localNodeId = NORM_NODE_ANY); + void DeleteSession(class NormSession* theSession); + UdpSocketInstallFunc* SocketInstaller() {return socket_installer;} const void* SocketInstallData() {return socket_install_data;} @@ -63,17 +67,16 @@ class NormSessionMgr if (controller) controller->Notify(event, this, session, server, object); } - - private: - void InstallTimer(ProtocolTimer* timer) - {timer_mgr.InstallTimer(timer);} + + void InstallTimer(ProtocolTimer* timer) {timer_mgr.InstallTimer(timer);} + private: ProtocolTimerMgr timer_mgr; UdpSocketInstallFunc* socket_installer; const void* socket_install_data; NormController* controller; - class NormSession* top_session; + class NormSession* top_session; // top of NormSession list }; // end class NormSessionMgr @@ -85,33 +88,62 @@ class NormSession public: enum {DEFAULT_MESSAGE_POOL_DEPTH = 16}; static const double DEFAULT_TRANSMIT_RATE; // in bytes per second - - NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId); - ~NormSession(); - + static const double DEFAULT_PROBE_MIN; + static const double DEFAULT_PROBE_MAX; + static const double DEFAULT_GRTT_ESTIMATE; + static const double DEFAULT_GRTT_MAX; + static const unsigned int DEFAULT_GRTT_DECREASE_DELAY; + static const double DEFAULT_BACKOFF_FACTOR; // times GRTT = backoff max + static const double DEFAULT_GSIZE_ESTIMATE; + // General methods const NormNodeId& LocalNodeId() {return local_node_id;} bool Open(); void Close(); bool IsOpen() {return (rx_socket.IsOpen() || tx_socket.IsOpen());} - + const NetworkAddress& Address() {return address;} void SetAddress(const NetworkAddress& addr) {address = addr;} + + + // Session parameters + double TxRate() {return (tx_rate * 8.0);} + void SetTxRate(double txRate) {tx_rate = txRate / 8.0;} + double BackoffFactor() {return backoff_factor;} + void SetBackoffFactor(double value) {backoff_factor = value;} + void SetUnicastNacks(bool state) {unicast_nacks = state;} + bool UnicastNacks() {return unicast_nacks;} void SetLoopback(bool state) - {rx_socket.SetLoopback(state); - tx_socket.SetLoopback(state);} + { + rx_socket.SetLoopback(state); + tx_socket.SetLoopback(state); + } + void Notify(NormController::Event event, class NormServerNode* server, class NormObject* object) { + notify_pending = true; session_mgr.Notify(event, this, server, object); + notify_pending = false; } + NormMessage* GetMessageFromPool() {return message_pool.RemoveHead();} + void ReturnMessageToPool(NormMessage* msg) {message_pool.Append(msg);} + void QueueMessage(NormMessage* msg); + // Server methods bool StartServer(unsigned long bufferSpace, UINT16 segmentSize, UINT16 numData, UINT16 numParity); void StopServer(); + NormStreamObject* QueueTxStream(UINT32 bufferSize, + const char* infoPtr = NULL, + UINT16 infoLen = 0); + NormFileObject* QueueTxFile(const char* path, + const char* infoPtr = NULL, + UINT16 infoLen = 0); + bool IsServer() {return is_server;} UINT16 ServerSegmentSize() {return segment_size;} UINT16 ServerBlockSize() {return ndata;} @@ -120,23 +152,30 @@ class NormSession void ServerSetAutoParity(UINT16 autoParity) {ASSERT(autoParity <= nparity); auto_parity = autoParity;} + double ServerGroupSize() {return gsize_measured;} + void ServerSetGroupSize(double gsize) + { + gsize_measured = gsize; + gsize_quantized = NormQuantizeGroupSize(gsize); + gsize_advertised = NormUnquantizeGroupSize(gsize_quantized); + } + void ServerEncode(const char* segment, char** parityVectorList) {encoder.Encode(segment, parityVectorList);} - NormStreamObject* QueueTxStream(UINT32 bufferSize, - const char* infoPtr = NULL, - UINT16 infoLen = 0); + NormBlock* ServerGetFreeBlock(NormObjectId objectId, NormBlockId blockId); - char* ServerGetFreeSegment(NormObjectId objectId, NormBlockId blockId); void ServerPutFreeBlock(NormBlock* block) { block->EmptyToPool(segment_pool); block_pool.Put(block); } + char* ServerGetFreeSegment(NormObjectId objectId, NormBlockId blockId); + void ServerPutFreeSegment(char* segment) {segment_pool.Put(segment);} void TouchServer() { posted_tx_queue_empty = false; - Serve(); + if (!notify_pending) Serve(); } // Client methods @@ -145,39 +184,69 @@ class NormSession bool IsClient() {return is_client;} unsigned long RemoteServerBufferSize() {return remote_server_buffer_size;} - - private: - void QueueMessage(NormMessage* msg); - void Serve(); - bool QueueTxObject(NormObject* obj, bool touchServer = true); - void InstallTimer(ProtocolTimer* timer) {session_mgr.InstallTimer(timer);} - + + // Debug settings + void SetTrace(bool state) {trace = state;} + void SetTxLoss(double percent) {tx_loss_rate = percent;} + void SetRxLoss(double percent) {rx_loss_rate = percent;} + +#ifdef SIMULATE + // Simulation specific methods + NormSimObject* QueueTxSim(unsigned long objectSize); + bool SimSocketRecvHandler(char* buffer, unsigned short buflen, + const NetworkAddress& src, bool unicast); +#endif // SIMULATE + + private: + // Only NormSessionMgr can create/delete sessions + NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId); + ~NormSession(); + + void Serve(); + bool QueueTxObject(NormObject* obj, bool touchServer = true); + void DeleteTxObject(NormObject* obj); + bool OnTxTimeout(); - + bool OnRepairTimeout(); + bool OnCommandTimeout(); + bool OnProbeTimeout(); + bool OnReportTimeout(); + bool TxSocketRecvHandler(UdpSocket* theSocket); bool RxSocketRecvHandler(UdpSocket* theSocket); + void HandleReceiveMessage(NormMessage& msg, bool wasUnicast); + void ServerHandleNackMessage(NormNackMsg& nack); + void ServerUpdateGrttEstimate(const struct timeval& grttResponse); + bool ServerQueueSquelch(NormObjectId objectId); + void ServerQueueFlush(); + void ServerUpdateGroupSize(); + void ClientHandleObjectMessage(NormMessage& msg); + void ClientHandleCommand(NormMessage& msg); + void ClientHandleNackMessage(NormNackMsg& nack); NormSessionMgr& session_mgr; + bool notify_pending; ProtocolTimer tx_timer; UdpSocket tx_socket; UdpSocket rx_socket; NormMessageQueue message_queue; NormMessageQueue message_pool; + ProtocolTimer report_timer; + UINT16 tx_sequence; // General session parameters NormNodeId local_node_id; NetworkAddress address; // session destination address - UINT8 ttl; // session multicast ttl - - UINT16 tx_sequence; + UINT8 ttl; // session multicast ttl double tx_rate; // bytes per second + double backoff_factor; - // Server parameters + // Server parameters and state bool is_server; UINT16 segment_size; UINT16 ndata; @@ -187,20 +256,51 @@ class NormSession NormObjectTable tx_table; NormSlidingMask tx_pending_mask; NormSlidingMask tx_repair_mask; + ProtocolTimer repair_timer; NormBlockPool block_pool; NormSegmentPool segment_pool; NormEncoder encoder; - NormObjectId current_tx_object_id; + NormObjectId next_tx_object_id; + unsigned int tx_cache_count_min; + unsigned int tx_cache_count_max; + NormObjectSize tx_cache_size_max; + ProtocolTimer flush_timer; int flush_count; bool posted_tx_queue_empty; + ProtocolTimer probe_timer; // GRTT/congestion control probes + double probe_interval; + double probe_interval_min; + double probe_interval_max; + + double grtt_max; + unsigned int grtt_decrease_delay_count; + bool grtt_response; + double grtt_current_peak; + double grtt_measured; + double grtt_advertised; + UINT8 grtt_quantized; + double gsize_measured; + double gsize_advertised; + UINT8 gsize_quantized; + unsigned int gsize_nack_count; + double gsize_nack_ave; + double gsize_correction_factor; + double gsize_nack_delta; + // Client parameters bool is_client; NormNodeTree server_tree; unsigned long remote_server_buffer_size; + bool unicast_nacks; + + // Protocol test/debug parameters + bool trace; + double tx_loss_rate; // for correlated loss + double rx_loss_rate; // for uncorrelated loss - // Misc + // Linkers NormSession* next; }; // end class NormSession diff --git a/common/normSimAgent.cpp b/common/normSimAgent.cpp new file mode 100644 index 0000000..7641197 --- /dev/null +++ b/common/normSimAgent.cpp @@ -0,0 +1,565 @@ +#include "normSimAgent.h" + +#include + +NormSimAgent::NormSimAgent() + : session(NULL), stream(NULL), + address(NULL), port(0), ttl(3), + tx_rate(NormSession::DEFAULT_TRANSMIT_RATE), + backoff_factor(NormSession::DEFAULT_BACKOFF_FACTOR), + segment_size(1024), ndata(32), nparity(16), auto_parity(0), + group_size(NormSession::DEFAULT_GSIZE_ESTIMATE), + tx_buffer_size(1024*1024), rx_buffer_size(1024*1024), + tx_object_size(0), tx_object_interval(0.0), + tx_repeat_count(0), tx_repeat_interval(0.0), + tracing(false), tx_loss(0.0), rx_loss(0.0) +{ + // Bind NormSessionMgr to this agent and simulation environment + session_mgr.Init(ProtoSimAgent::TimerInstaller, this, + ProtoSimAgent::SocketInstaller, this, + static_cast(this)); + + interval_timer.Init(0.0, 0, (ProtocolTimerOwner*)this, + (ProtocolTimeoutFunc)&NormSimAgent::OnIntervalTimeout); +} + +NormSimAgent::~NormSimAgent() +{ + if (address) delete address; +} + + +const char* const NormSimAgent::cmd_list[] = +{ + "+debug", + "+log", + "-trace", + "+txloss", + "+rxloss", + "+address", + "+ttl", + "+rate", + "+backoff", + "+input", + "+output", + "+interval", + "+repeat", + "+rinterval", // repeat interval + "+segment", + "+block", + "+parity", + "+auto", + "+gsize", + "+txbuffer", + "+rxbuffer", + "+start", + "-stop", + "+sendFile", + NULL +}; + +bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) +{ + CmdType type = CommandType(cmd); + ASSERT(CMD_INVALID != type); + unsigned int len = strlen(cmd); + if ((CMD_ARG == type) && !val) + { + DMSG(0, "NormSimAgent::ProcessCommand(%s) missing argument\n", cmd); + return false; + } + + if (!strncmp("debug", cmd, len)) + { + + int debugLevel = atoi(val); + if ((debugLevel < 0) || (debugLevel > 12)) + { + DMSG(0, "NormSimAgent::ProcessCommand(segment) invalid debug level!\n"); + return false; + } + SetDebugLevel(debugLevel); + } + else if (!strncmp("log", cmd, len)) + { + OpenDebugLog(val); + } + else if (!strncmp("trace", cmd, len)) + { + tracing = true; + if (session) session->SetTrace(true); + } + else if (!strncmp("txloss", cmd, len)) + { + double txLoss = atof(val); + if (txLoss < 0) + { + DMSG(0, "NormSimAgent::ProcessCommand(txloss) invalid txRate!\n"); + return false; + } + tx_loss = txLoss; + if (session) session->SetTxLoss(txLoss); + } + else if (!strncmp("rxloss", cmd, len)) + { + double rxLoss = atof(val); + if (rxLoss < 0) + { + DMSG(0, "NormSimAgent::ProcessCommand(rxloss) invalid txRate!\n"); + return false; + } + rx_loss = rxLoss; + if (session) session->SetRxLoss(rxLoss); + } + else if (!strncmp("address", cmd, len)) + { + unsigned int len = strlen(val); + if (address) delete address; + if (!(address = new char[len+1])) + { + DMSG(0, "NormSimAgent::ProcessCommand(address) allocation error:%s\n", + strerror(errno)); + return false; + } + strcpy(address, val); + char* ptr = strchr(address, '/'); + if (!ptr) + { + delete address; + address = NULL; + DMSG(0, "NormSimAgent::ProcessCommand(address) missing port number!\n"); + return false; + } + *ptr++ = '\0'; + int portNum = atoi(ptr); + if ((portNum < 1) || (portNum > 65535)) + { + delete address; + address = NULL; + DMSG(0, "NormSimAgent::ProcessCommand(address) invalid port number!\n"); + return false; + } + port = portNum; + } + else if (!strncmp("ttl", cmd, len)) + { + int ttlTemp = atoi(val); + if ((ttlTemp < 1) || (ttlTemp > 255)) + { + DMSG(0, "NormSimAgent::ProcessCommand(ttl) invalid value!\n"); + return false; + } + ttl = ttlTemp; + } + else if (!strncmp("rate", cmd, len)) + { + double txRate = atof(val); + if (txRate < 0) + { + DMSG(0, "NormSimAgent::ProcessCommand(rate) invalid txRate!\n"); + return false; + } + tx_rate = txRate; + if (session) session->SetTxRate(txRate); + } + else if (!strncmp("backoff", cmd, len)) + { + double backoffFactor = atof(val); + if (backoffFactor < 0) + { + DMSG(0, "NormSimAgent::ProcessCommand(backoff) invalid txRate!\n"); + return false; + } + backoff_factor = backoffFactor; + if (session) session->SetTxRate(backoffFactor); + } + else if (!strncmp("interval", cmd, len)) + { + if (1 != sscanf(val, "%lf", &tx_object_interval)) + tx_object_interval = -1.0; + if (tx_object_interval < 0.0) + { + DMSG(0, "NormSimAgent::ProcessCommand(interval) Invalid tx object interval: %s\n", + val); + tx_object_interval = 0.0; + return false; + } + } + else if (!strncmp("repeat", cmd, len)) + { + tx_repeat_count = atoi(val); + } + else if (!strncmp("segment", cmd, len)) + { + int segmentSize = atoi(val); + if ((segmentSize < 0) || (segmentSize > 8000)) + { + DMSG(0, "NormSimAgent::ProcessCommand(segment) invalid segment size!\n"); + return false; + } + segment_size = segmentSize; + } + else if (!strncmp("block", cmd, len)) + { + int blockSize = atoi(val); + if ((blockSize < 1) || (blockSize > 255)) + { + DMSG(0, "NormSimAgent::ProcessCommand(block) invalid block size!\n"); + return false; + } + ndata = blockSize; + } + else if (!strncmp("parity", cmd, len)) + { + int numParity = atoi(val); + if ((numParity < 0) || (numParity > 254)) + { + DMSG(0, "NormSimAgent::ProcessCommand(parity) invalid value!\n"); + return false; + } + nparity = numParity; + } + else if (!strncmp("auto", cmd, len)) + { + int autoParity = atoi(val); + if ((autoParity < 0) || (autoParity > 254)) + { + DMSG(0, "NormSimAgent::ProcessCommand(auto) invalid value!\n"); + return false; + } + auto_parity = autoParity; + if (session) session->ServerSetAutoParity(autoParity); + } + else if (!strncmp("gsize", cmd, len)) + { + if (1 != sscanf(val, "%lf", &group_size)) + { + DMSG(0, "NormSimAgent::ProcessCommand(gize) invalid value!\n"); + return false; + } + if (session) session->ServerSetGroupSize(group_size); + } + else if (!strncmp("txbuffer", cmd, len)) + { + if (1 != sscanf(val, "%lu", &tx_buffer_size)) + { + DMSG(0, "NormSimAgent::ProcessCommand(txbuffer) invalid value!\n"); + return false; + } + } + else if (!strncmp("rxbuffer", cmd, len)) + { + if (1 != sscanf(val, "%lu", &rx_buffer_size)) + { + DMSG(0, "NormSimAgent::ProcessCommand(rxbuffer) invalid value!\n"); + return false; + } + } + else if (!strncmp("start", cmd, len)) + { + if (!strcmp("server", val)) + { + return StartServer(); + } + else if (!strcmp("client", val)) + { + return StartClient(); + } + else + { + DMSG(0, "NormSimAgent::ProcessCommand(start) invalid value!\n"); + return false; + } + } + else if (!strncmp("stop", cmd, len)) + { + Stop(); + } + else if (!strncmp("sendFile", cmd, len)) + { + if (1 != sscanf(val, "%lu", &tx_object_size)) + { + DMSG(0, "NormSimAgent::ProcessCommand(sendFile) invalid size!\n"); + return false; + } + if (session) + { + return (NULL != session->QueueTxSim(tx_object_size)); + } + else + { + DMSG(0, "NormSimAgent::ProcessCommand(sendFile) no session started!\n"); + return false; + } + } + return true; +} // end NormSimAgent::ProcessCommand() + + +NormSimAgent::CmdType NormSimAgent::CommandType(const char* cmd) +{ + if (!cmd) return CMD_INVALID; + unsigned int len = strlen(cmd); + bool matched = false; + CmdType type = CMD_INVALID; + const char* const* nextCmd = cmd_list; + while (*nextCmd) + { + if (!strncmp(cmd, *nextCmd+1, len)) + { + if (matched) + { + // ambiguous command (command should match only once) + return CMD_INVALID; + } + else + { + matched = true; + if ('+' == *nextCmd[0]) + type = CMD_ARG; + else + type = CMD_NOARG; + } + } + nextCmd++; + } + return type; +} // end NormSimAgent::CommandType() + + +void NormSimAgent::Notify(NormController::Event event, + class NormSessionMgr* sessionMgr, + class NormSession* session, + class NormServerNode* server, + class NormObject* object) +{ + switch (event) + { + case TX_QUEUE_EMPTY: + // Can queue a new object or write to stream for transmission + if (interval_timer.Interval() > 0.0) + { + InstallTimer(interval_timer); + } + else + { + OnIntervalTimeout(); + } + break; + + case RX_OBJECT_NEW: + { + //TRACE("NormSimAgent::Notify(RX_OBJECT_NEW) ...\n"); + // It's up to the app to "accept" the object + switch (object->GetType()) + { + case NormObject::STREAM: + { + const NormObjectSize& size = object->Size(); + if (!((NormStreamObject*)object)->Accept(size.LSB())) + { + DMSG(0, "NormSimAgent::Notify(RX_OBJECT_NEW) stream object accept error!\n"); + } + } + break; + case NormObject::FILE: + { + if (!((NormSimObject*)object)->Accept()) + { + DMSG(0, "NormSimAgent::Notify(RX_OBJECT_NEW) sim object accept error!\n"); + } + } + break; + case NormObject::DATA: + DMSG(0, "NormSimAgent::Notify() FILE/DATA objects not _yet_ supported...\n"); + break; + } + break; + } + + case RX_OBJECT_INFO: + //TRACE("NormSimAgent::Notify(RX_OBJECT_INFO) ...\n"); + switch(object->GetType()) + { + case NormObject::FILE: + case NormObject::DATA: + case NormObject::STREAM: + break; + } // end switch(object->GetType()) + break; + + case RX_OBJECT_UPDATE: + //TRACE("NormSimAgent::Notify(RX_OBJECT_UPDATE) ...\n"); + switch (object->GetType()) + { + case NormObject::FILE: + // (TBD) update progress + break; + + case NormObject::STREAM: + { + // Read the stream when it's updated + char buffer[256]; + unsigned int nBytes; + while ((nBytes = ((NormStreamObject*)object)->Read(buffer, 256))) + { + + } + break; + } + + case NormObject::DATA: + DMSG(0, "NormSimAgent::Notify() FILE/DATA objects not _yet_ supported...\n"); + break; + } // end switch (object->GetType()) + break; + case RX_OBJECT_COMPLETE: + { + switch(object->GetType()) + { + case NormObject::FILE: + //DMSG(0, "norm: Completed rx file: %s\n", ((NormFileObject*)object)->Path()); + break; + + case NormObject::STREAM: + ASSERT(0); + break; + case NormObject::DATA: + ASSERT(0); + break; + } + break; + } + } // end switch(event) +} // end NormSimAgent::Notify() + + +bool NormSimAgent::OnIntervalTimeout() +{ + if (tx_repeat_count) + { + if (stream) + { + + } + else + { + // Queue a NORM_OBJECT_SIM as long as there are repeats + if (tx_repeat_count > 0) tx_repeat_count--; + if (!session->QueueTxSim(tx_object_size)) + { + DMSG(0, "NormSimAgent::OnIntervalTimeout() Error queueing tx object.\n"); + } + } + interval_timer.SetInterval(tx_object_interval); + return true; + } + else + { + // Done + interval_timer.Deactivate(); + return false; + } + +} // end NormSimAgent::OnIntervalTimeout() + + +bool NormSimAgent::StartServer() +{ + if (session) + { + DMSG(0, "NormSimAgent::StartServer() Error! server or client already started!\n"); + return false; + } + // Validate our session settings + if (!address) + { + DMSG(0, "NormSimAgent::StartServer() Error! no session address given."); + return false; + } + + // Create a new session on multicast group/port + session = session_mgr.NewSession(address, port, GetAgentId()); + if (session) + { + // Common session parameters + session->SetTxRate(tx_rate); + session->SetBackoffFactor(backoff_factor); + session->SetTrace(tracing); + session->SetTxLoss(tx_loss); + session->SetRxLoss(rx_loss); + + // StartServer(bufferMax, segmentSize, fec_ndata, fec_nparity) + if (!session->StartServer(tx_buffer_size, segment_size, ndata, nparity)) + { + DMSG(0, "NormSimAgent::OnStartup() start server error!\n"); + session_mgr.Destroy(); + return false; + } + session->ServerSetAutoParity(auto_parity); + session->ServerSetGroupSize(group_size); + return true; + } + else + { + DMSG(0, "NormSimAgent::StartServer() new session error!\n"); + return false; + } +} // end NormSimAgent::StartServer() + + +bool NormSimAgent::StartClient() +{ + + if (session) + { + DMSG(0, "NormSimAgent::StartClient() Error! server or client already started!\n"); + return false; + } + // Validate our session settings + if (!address) + { + DMSG(0, "NormSimAgent::StartClient() Error! no session address given."); + return false; + } + + // Create a new session on multicast group/port + session = session_mgr.NewSession(address, port, GetAgentId()); + if (session) + { + // Common session parameters + session->SetTxRate(tx_rate); + session->SetBackoffFactor(backoff_factor); + session->SetTrace(tracing); + session->SetTxLoss(tx_loss); + session->SetRxLoss(rx_loss); + + // StartClient(bufferSize) + if (!session->StartClient(rx_buffer_size)) + { + DMSG(0, "NormSimAgent::StartClient() start client error!\n"); + session_mgr.Destroy(); + return false; + } + return true; + } + else + { + DMSG(0, "NormSimAgent::StartClient() new session error!\n"); + return false; + } +} // end NormSimAgent::StartServer() + +void NormSimAgent::Stop() +{ + if (session) + { + if (session->IsServer()) session->StopServer(); + if (session->IsClient()) session->StopClient(); + session_mgr.DeleteSession(session); + session = NULL; + } +} // end NormSimAgent::StopServer() + + + diff --git a/common/normSimAgent.h b/common/normSimAgent.h new file mode 100644 index 0000000..1265767 --- /dev/null +++ b/common/normSimAgent.h @@ -0,0 +1,82 @@ + +// normSimAgent.h - Generic (base class) NORM simulation agent + +#include "protoLib.h" +#include "normSession.h" + +// Base class for Norm simulation agents (e.g. ns-2, OPNET, etc) +class NormSimAgent : public NormController +{ + public: + NormSimAgent(); + virtual ~NormSimAgent(); + void Init(ProtocolTimerInstallFunc* timerInstaller, + const void* timerInstallData, + UdpSocketInstallFunc* socketInstaller, + void* socketInstallData) + { + session_mgr.Init(timerInstaller, timerInstallData, + socketInstaller, socketInstallData, + this); + } + + bool ProcessCommand(const char* cmd, const char* val); + + // Note: don't allow client _and_ server operation at same time + bool StartServer(); + bool StartClient(); + void Stop(); + + + protected: + enum CmdType {CMD_INVALID, CMD_NOARG, CMD_ARG}; + CmdType CommandType(const char* cmd); + virtual unsigned long GetAgentId() = 0; + + private: + virtual void Notify(NormController::Event event, + class NormSessionMgr* sessionMgr, + class NormSession* session, + class NormServerNode* server, + class NormObject* object); + + void InstallTimer(ProtocolTimer& theTimer) + {session_mgr.InstallTimer(&theTimer);} + + bool OnIntervalTimeout(); + + static const char* const cmd_list[]; + + NormSessionMgr session_mgr; + NormSession* session; + NormStreamObject* stream; + + // session parameters + char* address; // session address + UINT16 port; // session port number + UINT8 ttl; + double tx_rate; // bits/sec + double backoff_factor; + UINT16 segment_size; + UINT8 ndata; + UINT8 nparity; + UINT8 auto_parity; + double group_size; + unsigned long tx_buffer_size; // bytes + unsigned long rx_buffer_size; // bytes + + // for simulated transmission (streams or files) + unsigned long tx_object_size; + double tx_object_interval; + int tx_repeat_count; + double tx_repeat_interval; + + ProtocolTimer interval_timer; + + // protocol debug parameters + bool tracing; + double tx_loss; + double rx_loss; + +}; // end class NormSimAgent + diff --git a/common/normSimAgent.o b/common/normSimAgent.o new file mode 100644 index 0000000000000000000000000000000000000000..757bf3f4e6b5c6fa3b079bdeb41034b8f6deff65 GIT binary patch literal 153344 zcmcG%37lP3kw1Q42#SgZ_YD;#LclEjmfq5WNxC}_CD3F6T(IeMzfKx@qu+ZW!L3P% zrj76eQQRHeMsyr?#@%r=ED~{a+;9{D#SM+1;)V{Q`G3Dvb?Th^x_eYUpZ|mOt6R6K z&N+4J)TvYF-1}bDd)nGUp^$}7Av-LKS6SB4wLorF+m;+zCzG%$fFN*zZ{6{ z!0u^&;WK<-_oV{dwENZgJ`C4y=JmpxcE1&0$K160ZTLFsrrmERc4qfA{Cm}V@%O;) zYw;KN0DZ`zj|#-QA9v`}4t>s{{SMt|&}AR{5}=vNVrl4basMN;?9idl^6r7%4;ayb z-9dW?b}uwN9@u?^K{xIGlaQa%$FG_GO$SaHQfS*u|LE=;k2a&g?&8cXmij0eT^fvyXr@R(Inh>l#1NV)+z3s7o|mj;I2Y_(agC=Tvq+li(h)~LH6?% z3g2)=u!Y5k8b=;#93fShxqja(jx6qf;4>gR^xF^r@f&E_?)w*9_QFGrC+%B*)SjCg zkKO?CZEW1Z$36_^vpwtENDTF_4ZHEEJ^LFg&(F?(`1#qP#!&&e`%g!1XN9+|BJX$5 zcJQW+Irufjoh|aN&&zuPC=+1^-=nl=3GGdJ+Gp+Ce8j#rJNE3qX%{=Ku;|hTsBfD7 z1ja+*;0WL=&J@0{xzbHBTX3nzDGO%#fMt#{*is# zsF;JdJ_L1z_RypI)*o@`-$<_{7$YlS_lG_fQ8yj~dpeH&ACb|^yy%77R#D`uA#&zE zSON`i)%i9aX0E?%%^`H$Nq;?*W!q1H5H#k{CtdgfX%$M(`p?-mO*tov9Q5Vz%oTKB z2Rm6;)zgn~9a=iu#$#C^+x|!z$9_t@=Wukx`XlyU`D%*$1+Hf9Q~SFsFK@$xvA>MH zIRl}g@ApFJ%-+}G@7?ELp=0yzTse#Oup$q}&Q`IkrzK^{M6j0=f!W!SY~AMX&aA&3 zW9N3b*(}32+PC?;`_|t+v-jt?oVnsB_yKs#jIP-8i`o3)( zQOVdpwf;ohJTUFEz_iW+_kID^cgHTGlTnMox@b=WrNM6!EqRwmctAa@8|nCgMZxq+Y*> z>G%qm{_u=UlLxQkwd!Y4URINDox|>q0q?;&{=nMzUd`F|;1sTc-90`pZ;rbC6)ak} z^C;>B{J8=O+q;WX_QB&s)&}vRN9Sd&o!!nT>l(_s1C6}`dfRvAjy>Ppd&%1&aM7i= z069Ffi#>htOMq4P&&nrZrxA~r%dVhK3tz-AhY`*6Q=l%k~@V?$(uy=17 zUi8WR!+k$kxOn$H54q#$#oMxMF+7ZL-u{X^Z$BK~Kck5CE@FKmV($&t{dnQSemC=7 z$l879qP_pLmkqpa;o@^x6&JVv?uEh57Jc%DzV98m_?$cbiS~N%Q&0ou3$=ygb2@1( zhdwqQey!-E1-H_1vgp!R07E}?3@r2D5H8g3RA$TAaQKJjk~#=(1VbLPHJKYw0o#0Vaa9Q-_} z*??O2U-U7d~mnYs?N^J)1EInV>sEjtyyPvwILLDcRR`g<-SvQ z?76xAEY#xAN6B+&a!2T=;x{qBzw!FLU%&gwpcO^K|CwbJ_uqZk6(22xh;HV-5N{p1 zZ~c)7q!xl;8)w-gW3=@y$j{al@53Cr{@w$<)K&Hc{;7A(+yDo5?`7SEK15rg0+@uJ zICJl%`x_rC?w{^@&Z0N$U-fykHJjip_vl5hxfxuC7VnwF*B!fV!QbFQ@4Kghb_>F(1pU(3>6k61`=u@(o>eCqLvf9G4q(5e^!_7T6&%9vAqDwCY(;bH&h5FpSZ!iAd z@s^+9@4nqv@b6V`z~7lG@8w_gf0n&g7Bu^=qABdXgrV_1SqjZuB9ZEqj8yl@LTKMr z3?yf+WZXD&$$tDr%@%!9R&Ray=Jdk)y_w&KK1N@3$TfKkdi?3spR~05Bcf?f6G@t36ke`_v)+v?7sS)f4{GOCw(<6 z0fPmrg4o*P{xJB|9(7%}5tE(mI;AP#+u^G59NqtM?^p$5)F{hRvx?#IC&s(!ACcc~vPPPyjQvC3sreeLw8e-Fu7 zeUXnSm^#AD=}%t^MQ6=x5V)BgadD0HxTZ5~45B#sz>u?+;_{Xvz6od9M-D%9OCfs< zpdq==vQh${0rYL zz;8_8e@WmkCNQJPAg6#}D8MTd_}m2EmB1fL;BP1JLl;Il%K`7QdNNcF@K6Ht_PdSd zl?ndM3H;Fn{!#*`yg!@=pVHi)&^!+FN~ljtvoyh10Z(TKtah=U7YqIv!CxuxGX=g@ z;AaW^p8_8%@V5khw!n7_e4M}qIQHp`7M5jC5||bRyA!x9@F@aU1m+wwowW+QL*O=n zuN1gL;13AgDe(0IcM1F>flm_nPXeDT@T1^VrnBb>yjb8>0iFe*_i@w7I=fe4S_cb z{5pX*3H*M6Hw%1&z-I}3o51G?{5yePEbt>APWc?6S@vv!s{(fkyiMTE0*?wjA@GF) z?-h7T;P(jJ5cu-~Zx{GBfp-dgpTHLj{MbjZ+*b(v9D%0=?h$ym!2JSWD)7q$evQCy z5cstMe;9DE+pi_?L7`FolknX_on_Y_$#S16SD(i+dpc8_{ENUR3jRM95wA9Oo51SH zj(jxn+Skv13}NllH3Dm2eCn~pzewbK`*DO-x4#qkbyka5A0k5Cj(mJ<%P|R@(yRnN zl$+xF68ez@PHA=vZx2!O3FV&hgjnuC0+T4v?3@RG)jaqQC-|EIPiIe*GVc-?3wZkk z`48asbatFzABRw5m)&0icsjcvkb8aZO=x&C$a&b4Nv}R+vB2%Z`&@yaFYxB4kVeN; z$~z|bPN8XoiA-ne6E+F_62ZS!U>ysiN0VlU;8QvKfDiS*CV^9$PtSw@%{=%=JdI^7 zld_&8u(oS8;850C2|SU&X}L2A{(8XESqElP{RAC;C!x6~fm1opetIl7#Xk@D>Ffn2 zG4{*l#}JR3_`D0#Ll}2AJuAw&ErCfF>hrqF5a|HpjkZ+o@9aM#q-d-0r=@m z{qW}n*82Za;M1g}qZgBAv%qHvtYiGO0>4!7HwZi+@Sg?VB5?WHSZ(-a0rQkm(*oUrQSsxYn_X2-W;B#7tKV0nReVCr7vvUOgFM$UH{;j|x z0zVNhbvhdrm;ypsCkeh!WNri;>Xy=6I1fIh*_Y7Vn7|AXg3QhYJ|lrsIa?F_s}uOW z37pc;2k-v}>F1O4gGA2V3Hv0V=Cv_^WdKc{B-s#l4hF(hOy&wzQBtGex<-L zUVN?+SpD<00v|87ttB<-{VD-Z^)9LIap?QM9YXn{@@F@a!3#?=9#R98; zzEI$^h2}K^4+{Jafi*t)cY((Q|7C$|0{=+hy1@4fe6hesVW49Wd_ds+0^cm~_W%by+?&9>8T4=j+()2E@hN=lJTxcG zgMSh5)7f1Vk<|qLg}|2!{40U^C)D8s^T_#lg8zO3r@Vhm@Q>`8yDiV22cOD$X+m>J z0)H0pbT&>>`Gh)uL-41Gue)2|VSx`mm-O2OJ^@2wI=e{Vet~xg{6>Ly3j8&JFBbUG zCzJkV0qmk4~ez`F##O5oiBe^Fp&!?Nsm z0`C#{d8^2KmB3YjZxi^u)x`f$;ORAl?-lq%0&`WKWxo*kaRPs@n>5D@yr75h5`mWp ze4@Z-3j7>_Ck0+A@KpjY6ZlgCuMqf$0+$4SbT4IA1YRw0hrn9}?iBbkfmaIrZvsD8 z;5!69S>Oe0DQA_y&lGsIz^epih9S!a1@0F3YJqzM{3wE`aoH-{KQ;1dOY zzQCIVeu2QR5O}@7?-BTPfxjZ~83O;Wz-J2lWP~j6Q39_Oc$2{60&f=htpcAd@Ye)> zp}-5*k#|7gmm@5m&Mp@CZKo6dAA!3uB~NER6Zi_aHEUyGP)+3H-E&}-cO7IJXrU%Q9>FnVG^WMAp$36l6%O@l6;!j1GWPv7yPnm~iM}q$s zz#%u1@*euk-13)wmh=cx@Okro!Y7D-;B~0OSFa-;hKbKz0>d!zIV7;g0}s1lZu!rc z2j7<9PfK9R4do8agMSC`)7ci1W*@wfFcgo^^()<@_rZY+0679VAC%livbiPaO>YpJ0ISIZufm1o>Ciq$cr!@QK!M|@F z{Pn#G?*CidAh)B1YRj{x4>r!+#~Rm!0QCQT;TNr|BJwI zTKL>7@EHRCOkg-Cd=CFFmIWt-&oc!c5cnj4w+Q?~frkWc2)s?;*9)xk`$q(x5d2pJ z)^*%LfnO>3!dF<Z&#Ug)H;1dP@PD1~C!9Pv#smw*+An)r1 z|A4^n5%{EU5&vC*|M?ceKNI+-ZxcRI#?dwZ6YKfe1SX-kC4s*Un(3@fcvG5x{tofC z5|w=uMTW8-`#r+H5&R**LH|#gcp+}TPd*8SARtOa|F*nq0TA1 z^{1qHnb2G;@XH0hTwt~BcM1F|!GA*FX@S2a@FfECPmq5*@S&c+PvD2&74a$keDJdp z`kn+HO5l|DmGj`Qng{=Z1pl82oXY%mg8yv-r!*=2aF}&ydx}499(>O{_$~9`r-7f& z{z%c;e+&Ewv8A8=eC~47Iy~ecc~2AGp9-wL^r~MF-zWGpe?|C<0)IhZ40L>cCol%F zeL`O!`@2}~^Ah;H1SU(M-d?-~LK`Mc}^*yh`AWhZTU|E%3by32Po^F^(ZlXCD?k z|AhJ?Avzmd@2oxq1bbMCqwHxGW*Jot0w!H>^_-vxXa``>&v;g_N$k>3Vg1>GXlR-;QuG^!`pLv=Zsh%|3GZA$`9Bl*W`Uo#hWM`u{1$<~ zE$~eO*9CqA4qIT&E^wE?w+q~R3h_S?c5Ibsqc~3I2S*)0z6= zly?tOsMGk5pAP^IW%2sm{DaR8^Wc9p51u@M_o>rSW=jI6G^Zx`EeV{`ybSp1O#M%a zPvM=H#IjQS>l6G36F8;0ejfbo3H}d&r?Y31O+KNGya@0~NGtEs+c}!E=fO_^A7ZV4 z68Oo&w_q>v=L!6UeS|j&{PD{OUo0^HgmOQ2#oXn73HWI|C&$md0zXmUQ?Dfc2pM;~ zUQc+vzzg0;c=j6e8*h&Cf1bb;7V5CzEs-Y0KXo2_>pb|)z=s&%Jp#W^>U`zfh}S&8 zBi=#yD}sOEorJMCuurJZMc2$-pDX9V|JywHuL2+ZUgx_AKTAqF`hA2qhhkk1t3ONp zF|+U&0l&+zuS;N}LY?26;6I@Au9a)6GpJ7_fE;E|vJI(Se!Yi^f9p92+ zn#|sB^q3mu!@3c^(eN*F{8$#@@eI?9(NAO(4o_xNfPZ4Vm@16sNg3h7=L*>>=pjtH zi*ff_;2#D2Vmm{%1n@P0skc^xsVjbZj7H(36Wu=5MjmO7&U9oGewxfwYo*+$O1T>i z)?UK%BM>%-e?!0@ga7z>8{oxuR_6fVYXGx8w;0U&{Lo+>@ht0PvdX6i=lve$@apUd z4zF>q)|0LF3`$S7CR^?_J$4F%GJByf{q!KGeJa3TpH_>Ao34xcHqc-fO}w3Q0s&Aw9U$jeJuhS!}Xw($|F9n{q;J>Z_TDbzr^tGcKq>$tARh+@c-`k^Rs^j{>6sh z@A%^j9|!(I!{6-q^D|T>yWH>^1*;zR1OHybe?6hU5qN&+mu_-+Ecowj>P_+s0=B>eJh+U2|=dxOKT%--)XR`w2GlJSWRetIwq z-Q@7@?5hs%$!-DsCu=(&!BqNXh3^3Ws4u8({LJxtvfF@vj^Tgq_+Pw~l{J zb{Ft3HT)kOzpTKOR^9M_a{Ozudw|zE%w)fF__f(W@2`E?qa1!+_IQUc&yIHZ3faZI z(&V3hc#yfQum&{mH~dQ-e^s^t_!|vB;P_<)uIj#L_#ww%m7U{dy)o-|`08w{!*9+; z9e!)}a=?EyIq!5i#}{4?{3Bubm!gUvcKkasj=K{L|2fAWU-&5Ss|^1|$G;={4Dja~ z{!5NOzVJEVYli=doug#tf z{4&E=9DjUa3Gm&9?{NIJ*;3%Oy&uf@?1IAolC?Sfui0}Q{&4m@hyOKO?eK@Q^?=Va z`9m&$JUa*Yiw%E;<3E~R>b(DMyYbZd$Fl!${KvCzI{c~Z*A9O=JLK?ZvM0E1KbIW? z_)S*UnumtAEGx7*%}Alg;bnzW93Ck=AMn41`aAFb>>S{~Z1{^EzpOA0{GEp1=lC14 zSGmkDX0LYmrtETuzm&bf;V)|_kSG!)$Gf_w;29b$1f{< z75KG=zuoa)&Atx2_UYHM?>PMR><12iGy93dw`9KuJZN&BV1aT^wyZ!~-D&va9six| zsotlzX3ui?d)bK&-!9K~XdS+vtpfgPle5L;EGwJ^{D%x*bNmmptuE)s*|@_$$+kOu zSN3v;f0n(<;k&X+9sXJNTEI7({I|ROw`8=*9~=H!#~;kz?!0$rpLF<_*|!}2Rd$=h z_hxqzzCruzewXu(>{q})#qft&0NRr+$^Hm@#qbYt{5!Mzfj`~wM>>89Hq)gWF-DHf zCLCUpUFh(!8Q){U5By1Xk;BJkuK>KwWPZ+N{xF=k0=jgo27t9mlDO0ld8MI3n@s$x4M|6%_muhgTF%aJWmTq&$_xUF!u!zUNcb@-&hu)`-8E^xS~Fz)c$!cK=zDR52nbF_CF z^|_a?+yT5QTfs-jvIPi)v4>K)2lz(Q4L%Yb)yZQv7=w=oe>ah;FY0 zd<|gg_MHY(xBqG|*OJ#8OnrXKU}h5^F!*S|{PdVih5HJR60G7ib4<9ez%0}wzo@=q zv2|n*_7+*cWrlBe{EHHuzeM!>e4{zlX-<)K=#b%G;P|ETl*X$Jf2QM4k&_N@G5mRs z-%>aab@=2g{4+D+`Jw;05Z_-jnhTwGYk_!vda{uM^~?{}DTUYJ`yQje!s##1t`G{+ zEGfJR@Zng{knb%{b5Y?fPP0?sXU(E{W=5JW!@u2mX9{lz{+wC(XJ*7-X!uV!{bhwu zcv){q;5Qe(>G-!7ZgKb>g&#Zok-}XL|GU7KL;fn*!vf&nW-v)UXz*cx{{t}2&lLCo z|Ca@Sm95a!SA_c93EEHx+Q={Q@4KL1oPAR07hvn?8o=fPp-;tw4DhEC{AUWfOG*6x zLd?lrSD;*edN7GS7vGB@lYMdWBZ(!xZci%`ZphPbD~#qWr#X;}@vjs{l@;_~ExgR( zuNUYMPXq5N)RTJPhvo0WzZU|(*jBKw0DKML*sid$WgWg7V$Wvl}F#Jax|BkH7dEc3x40zF()HXit zG)uA`r#UwJn8Qo5PdI#R_GyQY&ybF^`W%}f{LM}>|L|0FJh#c;4&D79{wE3#3zj>)XhFr{qZYi#;YADj9X@Ko7QmNUdw=fjMgG=lCViKKHuHMN?@#i9 zU&!8>;NR=;b=fBYztiNev1DjZ_LK$J1OFd}?{)ljasuevhCkEsPg(E{mwEJp=--~U z;17;}`hveWe9VH(`(^Qhg$_S^f$nfphsQ5?Jn+9oS*y?&X@4CJ{2|~MXZno5(KoZd zPE7hm@hq+sm=VHLu0H&?(HJHBc?FO@&{|=aK4}pS-O5bJAR5bez?a8t!qd(s9j~5%b z4ER;3Cu7LB8oU7T#|(yfX8&z4`}*ewQ*R3}Ev>S1JGjjD^N)S*#P?EzSw7)MgC4QH zVAldhY%jXqXmHen(hLKCDX2pq^<<|AJZ<=>pQWOow*kKjG~u&90QkNW|DIyktPkN2 zfY#_JI|`uwrmxY>aoCnYRuOmilpXa%8kRv^*Ob zudl2xX<1r4;iM_~>RMJRR@y5a<+e&kmxJwH+2FQ;DIqN>Y3YQnmabw6cSa}1w;mG& zmGy4ehdEc#wi8ZTRcotsvDt0l zE~aHW+|}YWNCB&j2%J@=_C#{~ta=qwISzMawQUnqb=OG;Wfa?_b{)`Vxzyg)(Z#<| zxRTB)w=?1CM7dpUt*zx&kgK}A1B)#r>=4<2yOtG89omyFGSb2@oVG}zrB;aCGBGi# zb^~kR{bi_YHat2pVCH0I0FDZHvZ0B_meHzmSa;)A2XBcE)BDz=xL+)hfW|sn-QlV! zqUUNIQ`Lddm140=Tb7nfT_a-yTkX1BA|f&-)k4>ZnWI~z@E|)aAF3A`s2z<;Dln_2 zP`FS+*(GUafB%-o$Y_0Jynp*Z|L90fl~IDePefHq(4~H5{rwkF#r;MQh7`use8aE5 ze|vq)X#Zp#Ogkr4l_h9d1Vg36qKo;*{#Bb-b~nR&Q*X z#j$0ijv~H4oq07n%=%hx!4TMg^7K@6O5eRecT#Bv(b*G}$33e_;zeEu<8zDt60o$si7d8fl zq{L2Oh;Ju8!NSUcZ6g+9JB$_v3@k=1YnR|*w8TsGfG)NfJxmn%jzI2s`T*RJdV{`P zGI~f5L4hIO(1wayz|d9+4BA$yCo2?hXmhn$14Bb<>`)||i!ZeAM0^Y5P<49{#|Ba* z(!QXfBAZ9G0=sp@v22`SN5&^i0!JILQqi%AA?;T56B}m4<1mE_0y*1d_=(}+fYnw> zfujkUZL&H~dOb?ofuj4GTM~-YE)@LK=*U=*txc2)673o*X)C4eQFzkG;hLk%Xl1Pf zPJvz!)ko_FrU-~Dk?;R=Hicg+cF|%^fcwNOI5AnB8mLc9owQQKq4~#BTFVJb`Gsa5 zmt3N~?7DCl&GftC<+@h(_Yc(i`^(GwTQ+p_Z&zvc@-BR5DNmYSqDs@u6C{m5lUS z^F18^!dAOz%V@B;ikgey2S+Cc9k1pR@!r`L8}*@HVJKB>)DwR}b!Vtk(#ny^!TxBd zt;wKAeG*e`)nPAX-xCK3Gc7~T zsFyl*t(=pa)abxK-L*0dLQ_Hi@Ib9jgO|pYq;YUzwK`&i8XHxnGeYPF+~ml(P^yd4 zh%VRGQfMoJJ!vcDd0X2ND2dv=Ow@QbQrp~2%66ARyGxv&Pb7lk=aoVNYdbk+w|%Hu z8|3;4bhKl9!E8k?%ydW1>nn87$y8&oz7jSODmPF^`ScBq;ky{%BQ$utj&=ReR4nmwJrJ>Vi;mq2bv@0E7ggu#f*uHi>#m^^3%7>9Sk9U4fIzxTdlrBzdoQ3SrOfk@Q zzLcP#@ZHg*m_!Wq!QgIw0bU0vMoW&02~hxr=ro4eFPPa_qh8$+?2Lh8t%SK{tRL&M z>eTk?kPTED!(+%2HIN-jJus;!E*h^+<;{=i3r5C+fZ#O7au!Ut(G+O1mca&4AL$W| zHF+l>g8y|y=rcjMHmpa=(j#T-k+Swk8NDhBbr}pcSxES9jye-DLKjBVoC3TK=n*YB z_J}AwwN(b8oYUn-qP4l~m4T_&yHo313Bwdpqydag#ab8xwPGASYtT$B1@*0bU1aM@x>{nHV3mbvLBYGjc`Z25Y0+!e0brx#()H zd_yiv;30gm0;v*MmYQTJ{TzvlS%SFGVocUXQMHcx_I|7cxK=_BXw(7}R{(;8dzm9< zX$vt6W;O9D6c*;Kc2*aqZ0HV<^pauOQ*9w8X|tF_+4Bq}gCaQ@k<90!oa&gasZ~d- zgY~~RVUyEGPL|x6TbSM&*@_@QYy+~CpbIzXo!~KSfeT6~ zEG%%bb}h@|6@!<#5aixcKM!ItXnfO!;0xw?9KY=n>NzpFgm3V>ZmTw4T4lVn+IVS2 zFF=g=f>fLoLJtBYUA#+y#U~PYTc(CNd2>uspk!`?PKq6(EMc8<9MDi|ldwNnX-QWO z^H}M)SSkF?Tt*63&g;igh8U^H3P#!yO4-oupqr_b4h@Ah0|4>eCZUOibZV$VH@Bgx z+3EPqudksG)RW^+SD8coV*|Ad7-Jzi6|2JHZg>czRJ=$BZ9*+-7^`>6#fa))1!tL+ zEJJ3Q6^L0~GD+~z!8jg~fnUNb;EDZGLoU7azBRLI6SHbFvufkuM)#_-cL59%fVMBX zW)}myN<;`VMp9b0ggM7$h6Tk$buo&j3@D7B(%jt zo5Q=ff2dX`OD;%A$6~7)Q>)q%MdnP2Wtm+NbW@T#tV9(!l8?o=7?QF#(c`9?5sBQ2 zG=W1gSPW>sq*WGPq{6ZaLG)O4s{f*GBZJ#uG_4%e)peWJt?%8?zoB>aX)9Zn+SOU> zHf-7qL@^-xHk`3`Woceg|d*Rvo(8$E!nm!&(En+(EZS3i<${ z$U93^Xm+jpPeH6ZK0LCuxjDJgrCtxFK^MUwM3)E&bi#wn%C%B9gF@jCrih$0sdW&E zKy!=a2ebw@F<>Q`1osy4n|+pL252Beg^ZO;q|!Rz1+xVxnHo~R$y9QPgVOO2N|}pA z=29}kn2zH&=VE-xTpTaOrP`(YEjrCJOYnK^_}8(aJHqBbJ7D=>l6g7G;)IMV`^wGB zQ9`pca>yJ#ryQ?R6rc`uj%W`8)T0y7s_3vzu3^6?vczy-CS1GrvTV+1mpP-|n%u5U zmQkEaFtx%9NaHcM^V3EmMcB|C$Ii#^HK6LXof(G8@Z^N1zsn5PHCX`^<6dk?21~J= zlr{!7)a2~IOGmXtWT2{yMrataBf&424OHWoYBqZ5^)k(Lie@@ZGo927H_md>Ic1Af z`Qob2rJ!|*1X9UrF_xY1=9uy-vAi?bV279pkcCpEv|{R(DL_(ROw_4TX3qfNRokNs zsVFT-v=XkNa8XORh7$e0SB`qAAZvon7x^;H6uJGvWR8Rd$?rU;DJkcmw!+(P5AH5(AvbU127GX258D0aiq84Q2Q3o2kZeN#Fmkl@C1qI? z@=M3FtnO?-K2VN~BLHx6sE_0@YILTo*P6(yNC-ML%#lQe*b zwa7rLAqV}96u_XMsRU&~E07aPYDU`-RvmwkflAKFdSRnFwKFLb15OxGTP+jyos%)I ziXxgrKFvrHK&9v8%jty4M`1&Kr8uSl$u?UpLj&~z#iS#mB2u?mRu|1Cj5=$~N5(lSe^{CaWlcroI9BZxc zd$nx{VfwcXj1S?+kBIPbf!;#EX1q8pG*lhxpV)Ff5)msy4`9!Je1J;|mMvBYZHASq z#Vt%g%3KiCK0Gxs)~{?>5yA@v-Pb2_Xy3cw;9?0N^<4`$Ox}RgFNPh9w6hUM_OVV=7nx^^TM`+nisnjW|6zjnUwUq_K{mG_vU3RipaUxrBB`X zK%cBUV!_z6_YunssO)r2H0oO#!%`xXpxmeDd}Zp=yXdDtU;$Zw|M2L*R@qzxaj=$= z@$D0X19hBm(lFS~DOioH;Km0=nb5PIamx|qPof-zZLmwXo?O5~(?P^z-a;yc!4Lm4 zSEn$CtHZH^@gbdpW_zc!r=23Yfof+XXLXONQ5Ax5WF>?-M@pDikck|Ggd3U^<*5wxuoMEDNfr{zkk{kjgZD&dwKSV= z#chX28B|j^$`^M2++10?)c$y#(~s0%D!`Ku+DaaE5c5TQ#3|;)k)2iJsRyT@r4s_NwejWPd6Dur4m6R&?}h=k}7r4Gqg7OKajKcq&{2%P(FG}8yyfjKv3g= zP)BJ9FEPZ}*eDX+1VwPiiE>^4K}_>E*QG8^L?n_#1bQrA*HlEFRog7K=7;}S3-mDx z;qgCMp&#?Z9nR`>p>l`k_UvRJZ=nxyi$?=bku!#N29UQHb;;St2N60COfkO#XExp# zOwPwvLQ({#kujI!N1{sM)TP99xHC2|bpd=e6Wns}lKbQuT;Uo$i>yBJHYwC(z5qso z2+0Kn|063iw%(?Mbe05zPLcqm>qTZ53B?vxQ7ZxulUzv`#G5b!FzVb0fJwI5^dw;~ zBipVdp&v0X4OIuX*SR;_7^UX`0XnnF07&mlT!0*9yu_)Y`UrB4#RONKnwpqGsxnfI zPtX;uEF*jAX)~%*Qfakfr6V_AvctPb#{*2vqsrbEv}SHR+>v=#<1U+G!<{VyoIrCS z8E##KQzq5^^j?@<{bo{Fm|^{1Qd|hpa1Xtm6B(j3yOl7701bC)1KaZzfh!G--h#s@ zjmezMa>-R9oQY648(~z_z$_>|ib+pnGn^wH85bl1ON-6=arUh`9zwrLMN77Wc@}n5 zI<&-zQ3T4v!`#iVJWH$Wk$3eEHl|<`qm40$=crxUTETW_TNloY58%wbbHUuFmgMB!DIg*fIem=Z>&Z7;K%bQ7_vE9sgs69D1-z)U`Hl&lnjsSVC4iX z>@q#cJz>W0Wc1LG#mP5ER;yMo=nqt!R)q@s@U<~Y#PQKuHBzDZLWRZ~mFMc93SedC zYsx)KMnsbNcw%9!IV%(IhCBa2dK&Vu&-Ne~u|NHwdGm}Ri!?WOzMPY^8k5Axj^eyZ z!8C;h0_2AF#iIC}!Q2H#!6jpr({_!~>J-GlH_x#NAU0ys1CJfTahm zh{HE)r~`nd0?Rp_(|1B(FyECR7C9O4I(!$L+y7M=|NQu1180G=rbPXQ2X%|gK9=lzMc zcA>fPyg%{UHUySg!%kQetBu2h23a`nPpq~MQDYWPQ4_0;Loz8#1Zd+Bab-0>SWRMU z9yAn>q7$zz#3`LD9z_S9jsPvk3MdPQxro&^V*QifIN~m5v;F0|Uk;duTC(p&I21TL zD;g|DcSgk!Xcoc%wLFN3PoJ%6L<^Fvojd~52v5+kaqJNZI2nkVQL;ZI(gaE$ejz3M zLnboVN(fnrXOBpn3$%r+M6*vM$OYQ+^F*^txI>^8AEmQPKrRC$C*PuFXkW%iEWu)B z*sk1&Wmu#PMNd_?;z2O%HHV?4twkZZEX~-y8h^4`Ky4V31`WEXC0O-l%_>G4ULqeq`)`*LjcqL&^yu@M$ zj2LYy3FAvDL`k#xNQ#jbNIiz4xYf0tHD2=ms9E&&@^hPp% zAem5aVWN{$CZRHpz7f-lxH{0&vnf5Qs`Q{S>J8Pcy*nng0C6Z}fYy}()l&u~GoDTI zpa#do3#vPfm5!UN!ICgd8f%U^Ct;mc6jP@#Ynm8Rm|xX$m3dvL?IxZ9+B9)N)w+=B zR~CySWm;n%Y&A|N4P{OZ4Du;llAFaa4=N4ib6LhlD&sgyo}VYZSqgKQLeCte(u2mR z%|bUJ@Hby=k?v#i6%$v3 z)n@Qou0-3)np+V2sSRp2SI6|aE#1<}XiA%;;(6P~Yj_gboTvERv4J|C)&hz&SlntN z1p9l?2u8DONj+_PupYY7-GwGy@zF7>6gpCLg~DA|m3;M}s3y!TEujrsJ#?=L4sJmN z&n-e4*z&*CXJnyu2eL+RB`-KX`hJAD%p zR)l)a*t`Y@xr%bpyKd8|y&L2_opUq82L-olConG6`6bMLh|Q!F_Ke%0KEm&KIDIDf znvR`Nsw=!bC>#X^g?D--aAwY7SF(!N+PHq(gMLL$GoQ4?H4ECDW>-&6uEWs)naj1& zrr&nx*B0@thD?l+s>6I-2eg||nFmPdph8xl1C6%WK<7y&4iYkGB6S7bV?#YS9-exO zjwWvbl7!zpU&Q5D*pbv9m zH?o=4sYHF9*5FQSu*nbfYeU>5y6$XhcJFosW=Y{}GL+Ye)+MbHs_t#o!3z>nG~Dzo z?Gc)06?R(VJM&#Dv>U7SM$&G)jHF56j?_qsEzn2WVQY~|jJ3hpfGTEIAVIo3%K3uzGxFN<`zJ zGzOJPLpD!h)>0`sEy4O%nt~vtY1kGSjjBko0a=L%tp*jeFN@k!!9fLFho`O5 zzL|$97y#qalnMPMvx?3QMMkSQ%9Eb?@Rbp4D|iv9G(%U3m1xU0XAgvOywb5w`m=j`LrjL4q7uhOkHyv_@UPANQQt=m_z_GM)hj?7F&dE8CbbVZM#RpcD}T2m-ng9UQrW+;fZ`MAi)8qW+)RfoHqPY-X|i3bG2 zPLD=K*xT4z#{ps{U`(By%KNyDB(24H01b&oqU+aN4BZbj_{~w}_>U4&?Tk=Mp=A zVJZm&vboVNoPf7xF5wzPqU&Y{VnsR4p=H2A1kht0(f~*K;AQqhsnCI@bfUZh+><}E zlv5{9732^>iG1jW<;Z!Crb>7@4zRnum?nvH7NN3X9Hu&#lmq4yXn>Hl62N37n!$$0&4sFm~YIDtexkjy2 z!YjID?|R<6RgnuaEw}|%YGZE8`6^9@vGfr9gs!IsCMU7ZvcgQPdJw&{dV4q$XTDa^ zo3Wp0ZpikMwPqU`C#V&0y_ak>mRiClC^wkFls{j?TE%r6#%$;WwA|fxEA^dd2d=jn z<=v8yS@*QXKE`6qv?Su^kt6-o7CW_h3Fm=PaOPad%{hTKMn_4UcjkjJX(=UzE*Xl; z1g!xqMY|O7mbrHvO<-Id{5}_qN`flPH!E`4Xy(afpxQX-UA<+h*Bb?;o|B_Bw4=LlSZlnNw$dgnzLP*^8Vr54 zad}OF%r!u%X$z_xbgo7%xJtm8Nei;ok^06BLAC`O*JG=KYY&8W8bXjAj${JA)FsGH zfypBojEAvIs*!dn3EaP11r&vz>J55PtF3Z z+nf81i+NKO>Sg@~t$4plkcs6HwFZr+m$A=Gsp>(w^baC^J`s^~@N0j?Nu{G{Qt8n} zN^n)o0@O;yl5PV9o0MM41(`a!Wce4T0v%t>PmK&F9-kS7#N!t`JMGvDItjYYA3dcB zFtcC{WO6}m9W~4A%xWImY?rRH&@j~*^AABJdPEIOM9#r4I%6s#a^!6M)PdTz-tqd> zPS+ZGgbp6rNw*26AgmZUc8bw2r-ULw7;TsWLsFWOH}NddO7mY^8iN;9U1*8P2ovg4 z^=uMWce}bI+cb8nWO#+2FbeSWUB*n5js5N@iyGP3it%5jcz;76?tY zUSj$}JPC`G__4%9hV);83X^Pt3LZ^S9;<1Tsq>91ongB?nm89;x}OB<-AYVyG*^jX zI?SxZNJxRECl==t%+PZ#JPTAI9AgT32V=Db2DYUot?OoJcfVjZ4L%Ypo@<-e6qOlIS4Th-KS7 z=7AE!D|fNPB8Cu$0ep7Xcw@{NWR+`98hDPx5w3_8uT%0=m9K{#+9QvMp@Ptd~az6GxC#TeyS*?^( zM0QDB#xCDj9hky%%hu=IPsJ8`o$f1ja_T<*-nu<*cOSv-h8c{wwznX zK@_8sW(H|#Oq{WLxt5;Us5Yv7IKLOS=Sw`PA{$X>ZtmUO+rMe`I=$66$hJQdJUVReAN#$W$Myb|H!p1WRo|l(y=oW7up2nt9l?RsHcMq z8Ww{yrvY^3Nb#y0CgDtu?{JqyP7|Xj3HhdkI;SM$j%5Q}T!-w3l)?shiD|7g`Fb|q zI=6OWO8O|JhskR(;%ey9WE}QPA*`NS!;}2$al$v{fbNt-yiF5FcYa%{6n4r>%nK!% zY?Uw6r9`;A#T2?m*qKyg-S}i9Nw$~54tj~HpIJiBXw;j?p$e%n$P=j>aXvRKI%t9! znOQ=i>N6&3iz!FY1k*3GI8K?117;&NVS;5A$Jqnenoi_1xk75JK$2=ht|sbJO4x|X zol7BWM{!m!rKUc3V%he2v@d35|4YnZh*X@0kP7rpjAC11yf!LN-kBd_azjStrVXo4 z-+0>UO}+iAH*Y$_MrK5IZ?Lf$QK#`OEF8ZP)x7~oKHR_olpKc$BbWIRJfg}pbeiH2 ze`oJJmq7*-F()S{tflp;A@r5{VVI^Z-ACNMlmxw`B$T32FCI?}MkO?!0x984U5KuD z;yaea1c-@=t3PZqBzB~t(0N9OfYBC%Tj!ud-x?iiZgIy&ZZj$iU8^)Oo)CmUR$?(q zhpsg`G|@s6(UguxB|^_yfe>b)%9fl07<&)c)0LHlA_X=1sjeTm-kF7o%w1S>b$z;LZZo z`d+-)$ZqlNMUp?8sE-Vf=ni8z@>60)0PnAWt;OjMLnLhmjdpb7&? z;?CwhKP3T3ekPnkw$eiTN28h8R1or{fG}&@>ZwlqQU3(b~^In$B_~~i$(Lb}1 z-qSf!>uaU~LLokKq>y=KNqZ4uC49Nw4(Al4$1Q~AnjgDvRXa4OjthIh2$D2Xt50mM z565B98M+BAH|6u`^ky1UbnYZ+$@2q9P5iBN)i@Hx$x_DFKF#BehaFT&Wo8t#C(tDC zi8dNy+HNRs7S8)2CDd*xGyM=saAznb%qNMxP-a>ovgLX~j|8gv%gi5OIB?FeRf0?x zNGS2Nm!>Q%@f6o}nsmZCup2*Kf+WW{UpFT!KARBo_F85}B2)(5Em~s9K}!-lOSLQ% z7TC3-6_%;1+=A{8qQW>-QO&HXhNOR*7SMeIQ+RsM)WEfVt=!&*XOzd#9GnP?%@>7) z8&L_W72@Hka}27Ua3^x8a-cRQg@VQ`J6h*C&9EXXBZU%Dq9B*{JH0Npg}%0W`@qPk z^@k1TzaD$enjCai|4vyj+NTgQ}Ee|ZoSzEtp zJ`x206{WBYv=cLpJh&8iB7r&3oWql+nW!3hf(c#1>W2;~meSV2IpwZH$HeAKGeeFI zf}jYgjjo7ib4zI!e+YTnGpYy1Cu^1H7-*>@Q zOfcI=hN`{Dl~6=bC+$VoBBl=`-%!1yo{&-z<{C)bv!(ljgc|Bn zYBZV;Y4RX~rRV}Rr%|aL&*CdFnrw%h@nIm7 z1l!TG1%7R6gP)gT(F}3*k!Yn@H$Df2Js)ipwT*D8Jj1{egWqGW#!9qDrP2ht z>ulA5@z5}PoB_O|HtL%V<3rTeu%0dC!V#|3Q&R&woiC`52aTdaI$DzUfv-n?+s&DR z7MZ(Am^2vhDxQXC;sT332LabCN(}Yu&?q6Y;5eAG1TAO|$62CFktR8LFdQD8iIwWq zS8#a=YH-dJn%|mhey47^3sYkfKd^x_mCh0x&*u}8ax5Yl#G!$z6ElmW){p29r)%|B zFen#E8tX#oayur2=?Ja1F2`Y9H{(do6NsTMJfI?`$>S-`rLQ*#g@hi@@dXFQ)XXrC z?9$y8i~8yV&QftBrl)xSVjaq zi5jrUk>f!~jzPlo1U%TkfrZYs^fot9P_JPrncyt$MR<)6E#yFDcjkF=9 zPLLNuWydEFm?WC6RJ`FiuHMlqgZsb;@zGNmArxg)>cg2tk!CaV5wz5*uXkYg8=LL2 zpjDO@0g$M*mFrpUdSf`vPLHxTIg#^i6nP5hVN?%rK~_*clNX^D*u@Vlvz;Q(D`>q% z?ZJcKT$gw7Tt1NxE(ja55OIVT+oH3YyBvk&`>Ym4J3!6B%F_n3oa|Y)M9i);6bU) zY{Axw;h_m>9%>uvf|zgOqTU^R-~sy|>NASA$mAj=(J}&XZ=-s#)E^a5MdYocLsF7( z8Wa_kM=eZssj`v%@Y?YCVvVU~D9HmC%A%_Yju!aUQaL&#dGtyjSx_D{(fTwzU?Ghz zM-L?|@Y_gP7>nzP8^-3K(A?NwR%|`qZ{K1!!5eVjx#Cq)TnlS;?lL z5eL^+{RShAjn^mCa+YlS zwkV9zd?=zl`Xzg;O`fu-4&`~>lENM>hednzPioz(M@I&1urcb_2Ju1VP+CXySqX>x zPJ5P_RZO(d5#5$DnK`I381ZXCx?Rh$gsB^=->41Q1u!n#GgM~A{>GfzLud4ARwB2E z^PH{`zIKuAli}DI-JNl2Z(DJ$6y308UFjq{VFJNyrLMB&y>X&3B}c`f53UBRqG6Pv z2aw!$m#4+`;)%U{JzYIv|CaTHb86BKW*5EzLWZP={zZ-bQB6$F$nm%Hf&#BeW0beQZ)8Ar69f7q?C=LIQ4HebaiO_Z#67NXQ*Y# zUQ$jU_5?9hs*3<{VoeL}N3UqC@~FMk(ANM6kJ#7bLARAl(N(IKSzW754B9bSmf$uL zf_c$>DLPDLo69miu*YN=?WZTO5#g{BDWyJ{`0{d6)n0k?wP&L7aAx9Lw4)1X1KDnFZ)_FIC88pI5l1UeXZXnv*>Afo_i?yBU z%A}Iy)H%~iS(2VklIM684T6Q-6vqU3#SWP;Jpq!brd%8DM}Lvr50z#*qGCIFK}0?u zhTG8#P(pr@iHVO{^r%Tta0#fI=jM#HH;&{=k{qYNbj66d4Vm3I(s@7BHJk>9y@Zt( zVvdTvzBANbZ7=CAzR1&}ptU1-Cxh(J(k!@rW0gzXWu+|+YtRq%?x+qn}2Fo{}8GU2qKWTbMk`@NjiXk7SsKz`KTG_z@{J4pY7PLJTpK&m34JrY*NRtsBSA zH5O$aS7DhP)TZk(Yf)!(2C-0{1dUzQn$LGz33|ANmAAJ>5lSYX)_mIAxRTWQNwaOdyCJ2Lu`d@$8st+}D7mWDSUwecHP9 z_;q&v5(2gF%Rh*vQ9vOWd9C^EH*0XlaPwgIRiGW(<@}2M#MXQs9K$FXoF!lV$?4pf zS8Yx5;Aqwv*hTc2Mb7W4BE!NDqp3B?h=aqA&q=em{bL+rRMO8>Z;}uPlg5T6U+IOf*FaEH&F($eURb zuJz33+Y&c6I!IZ7e!IY*LYuyw@hlEGp& z)Do}Nj;ot>wT%PA=0f#ej8mRJF)|*9I~TdyJ6ybmXs<2FQ{%0W7kFoKU2Ej%R$Dx@ zS&n+jejOZ{dLCiJ49+ca*ZTT_3#uHFxoL}QSrdp+p~D%y)t2O_$(L_VTauvWUA!nq zpRdh{=1NyZb19pMhM8H&vc5XCHLP;bziml2`gAsIQ%s@`j8!4wh zcjoKiD(2o(FpIV%KMgz0jR7uNtsv1d3z7o{Nq!nUSUR9%Y}{~BYCNt;<%!D{Iz_Yl zTn+7I%%TBE+myHQm42X=2PB6?fTv!^H z>x71ah%)qMpBbS1S_{H7+>BnM5ATJwEgY+cD~tk- zev-Q8GekRN&%^u}26kJLqm~Da;xfFoJCubpJo!@3J)5S)1RU6Bs~q%NzGSO z<@r9FmUu9#tt~p24b?%MmRyanJFLjcc!)GqBN@<1RyxPy4wcWPTJ24F@IY-zUK(#= zly@1Y_QG?5#>Gt}S9y|>4s0mMV=g(qsD*LG<7`P@THpH!)1C~A$+90iD{~7;tc<}$ zTauVoE=DD)X0CgzoJ1E%PFfj|LFKQV!i&fuskS8XtOOed#?7mg`MNF>nHpJ=ao$kH z0UZAH1%|2EY?4>Tq?^l3$jYQW$scbV92ifNDOk~l^0+24OTjA0{S!In*eob>;iMJi z`o+cU?{?UfFiEA;MwvEqB2)b zR;cw2GkUPo*fL)bHLq@vIs|3(E|6|!nJfp8eDxBdNw+6?U$V^+)*Xl6+%ahU`u@eOjc!ZH_psvEA*!uV)UGPU5r zIBrbUu(5g?e`>)xhH6hTwL+$E;t{!S6I{GSxzVA3VOF3jeVHU^L4(mOp3xg#<{Hqd zoXm;sNrG0{lCYCJOWTtKEemgoXr-rkW*tc$HFB6Txshosi8Jj; z(yRIcNQVB^p-_^xHp!-Lo(#9JZ{iY@?nx4?LTkm_aYlG9E?aG`=}1zo5E~;DE@-u~ zT(3HHT+o18jKz~J*jaIwF!y7!=twfLDNm@CD}^go4NJ)svn6Nq$EbAZ;DH<5ir(Su zDP(0&Z;Y*;;!jA@d2)}_s%;wK;Oa=SvptP5yx)_r6%iOJW`Xq-d&lVsivU7*b6)f#Y61nkyc=mu} zC&}c-hZ9uU%<&_>b0>oP_B4Xr*8-A6E#?fL$XqWtklv9bX3coRZC{0!brmX(t|Q6H z@@PC?oiA47OUyENJ~e8k*(AAHnfzV4cS^dpgmiBb7nb%`crC%rJiLNg>@vy2GP9AG znoL!lNe&iWJa|DauJ^TPS>M*o^(s7LXOe=2Q)^}ux7eA_w~Fj)E=`5~x-gP*t5byC z(#Vi?D0Ngq)}hn#s__J4ovww4(W4z8@8<5usTFL zlYA=Os*R1ZZ@iXu<5euPGfAV87}~KFuy40wb*~{caJ;-1yg&Ln3VXu>1CzDUFP@^*3eC}$QGKB8%bW3UC^+?mghb~mQ@uEV7Gik|?5 zQppE%LR^Q3Yh(O@(cDxPT&0$!yRX3n(|l><@wlksr#P{(n)dGyF?YP)r}{pjv7evuJmU}dYP3~e4DYt zrMAx6OnY_avaTcb*o9Ix*P3oKLTfbUBo8Y;Y;qdjN}_FnkFLmAOqVQp5Qm`Zm}E51XB1F6ySaV$F?N8t-hlT{sA zQ1v9Ci#c}VPV6s@<|lbv&~KUuvznHINf6zYByn>z&66HTliaP4Mw{V*WfH`FA|Z&j5eTC95d!Ob zYqdL{`2HdYO_H(7#NdI%kHneAGP)p% zQBaaDO|3%1fpb&>x2L6YJ7DA%^{zQYX{6Dp7HxAN={cv_ExC>q3)4?O_fX8Rz(OQPw{&ENu;_G`H zZ?Z{2=RBOvcP%ks)mJ)5nC3gcm2_Q?RbPy4PMg@;ocYH2D6gKZ?scsJw&-1?kTpzJ zSrFnY{EDvd)k~v>t_oqXvb5KI{ai`b&#>MY>j)2zvKa%MvAS7ZLR%q8=2sZbY)sUv zL#f+9?V#Nx91^~ouIOso)RUZ=6>@Az`6vPl z3Qv-}sE9S?sz=GQjQ~5YyJDCTw>|ok-^#?&y@Mfl1qF6C!w{=5f2O zCi}Xg3H3BMLF-REc!7O*;6c?Dh@v~vp~qdoFu_!Ohq|IW)T*OWlIT??bhalQ5uSMa z(*c2(QTxcltrn%b&>p2C_27*~WZT5o$~Cy~wQ`kAZOa;0+FubFQS1N=dF5Or|x_DHhw#Ar|jpW+}-QD-+vgOhFZ`T+$*oRsVojj19gZKw>$P7=u4Jz|`pQ+a*|G||;Ssd1`C4O@M8x;5RM z#>3&!)u7JE!%ht9k7Kf3$mfQN>*loJRB1dK$U>78G6bMc_`{=gDmu7%#)wpLWz|Gw zD;oF)(j^&WzR3&i!`s(Y@3wlzLI=CwBuyMmSbc_^hHzB4;|w}fI-H2%&we)1NcEB& zahlVzg1y7J9uH$5d#7%~TwQJ$xJbycvSF!7B3RiZ5#p0dN$%I~ z>fUd9bgEcN62OV_t>$ib`h!|F%_Ax2RX3hAi;B51^g1&GvijmX95L3T_X#y(O2FT3 zDmLSr?X9}m4r4EA=O%e%W0$8C?UijXH{X13<*8yRxP%~sCC6>3lr_9(k-XkZ`ZHTn zEHdOc#t?m!@y1u7YK>h@NSTLXWke;ZW@Am>iz>XnJ&i-5ffvCIi;XKQ(;U zOE>oTN>#16tIOXIQ$m~t*5sIQELjg($C7;mTF{}#WE3NrOn;5IE z7#JEDt4)lr7_XJ^Z$;Cuc+W)w?xvq4Up9zeFdvbh8vi>P#s(%Qv7K%&FT+-0gRjt# zmzTl!V{FulB^=UVTDX7ZiWbc2uyt`m_?7U%h*B2=+;+i%SH=lfeXuc&Y|@_M#3DKa zfF3?>iIkRm*O*tv-Zkdep}QP`E(1G!5BdOrxD=B&7cXgG8G*QK0p!37z(;~?&gT)mIpU$|1YdJ{Xffi4 z9Id)D=}Fyr!YtDF@x_anjPd(0QaSACR{K#y+Bh4i8dE7Z_9p4kwi68y@3zN3G+-;fu)(K6u};8S#lykKl#WIXVQA%Yh=!)x#^=sz1Nw~34T z{pUa?g-|D1HB^iVc{37DCdoasVx7#k$U*2J9Qsb)e253wTJ48eG{nX<-!-x2{F;!{ zbulbNb^xN#ysmUapoZEX0HERHR1IN7u+ba3`B!=p-_SQG)s6v(cgMHBUM~=mL@iNB z%rlMf89{M#B?JvC)qb8wPllv8u8S>@-)~Q0$iR4BS5UYXs~bCz*>(bdS?YORLwxyz z)QEwqo1j<+4!QD?aJ{PrEuCe?Q`Id!yPsNiU+x#+jE=_3cBYm-(w8TN$}OL-w5nJX z%FKfmi;Y;M+Exc~!KkUfJ{^}VMfIygq)&7&h3n*CXc?AWD$gnxMn57kh)bJ6c1CJY zk(iv;MzBF~%A^j-#15-m25F+(KvETJ(BL>mLVH~P0z zj#MJ3RZIBWOkKJ`oG-i>jB0ro(?N2?OskdL!I-yJY4?~qV2z^wF7~$(gwD}Qsi&qM z6KSL#H1a02VMAAn5Yd|sWxK6i>U{$DMUheQ944yn)E_q0!f(yRR~NTr#rde@uXl5p zQ2Pu^lU~C(bay2crud0T`B?~2Cxh37zI$*}(dmXXm#oEsko6C@>-aT06`%&C4i1(| zqoc=<4$$b-oC2K?@g27+L~SPAv~nP36xrm44Sa^Xs45$Y-aJtN^oYEeRHvwwRz5XP zk72^YP}9C|n$f0kN(t|B&}wv`jZOE%w3T#|PE5UHTdG4t)gd}kopKlz*D6dHq&{Ef zlO{LKP*|J7P*@x_n1iP-78BVBKhjI-oaDAvobj>AgPK(}_9HUQ%v2lq)|@=*Y;*!g z^FA4{!4kSuD7v!o8Vqo%zm8u&Xl>;jy##9( ztlMydKlT%DpgHkIzp+5c;6eLp9fDbDBsybHPD~-b3@8mI$7_KXH(D7bqEP_Qx)W8F z{IrUbv695Avk*d#ESOW;f52%*`@YKsz>n9Y?lb{ zQhuRt5yn%LT%x~DOfJ!De%CBhRgT{tE46IsmM>8>uc2Z{VoxR_u#n)@|JUBT2g#XT z_hAhzXs^tbv@~gpvM76&=2H8>Vy6c)gT@Rc?qJZsj)-~eJa$1n+#Jj_fY#1*54(G? zfNE{iCM9dLvR$$KEHN=FrIck^WvlWkrD7+Fr6g2}sVI)@QdJm-iR?snF%>zbQj|yu z^ZT90eSKdK21}9UKZM%Z+uwbhd+xdCo_p@O=iX}p)}GGdolXga`Pj|%r9=I((#6I> zLJcrRqz0DPDK+p}OAYb^tAi3MwL(ZpgZCJB$Ms29=$LgYar)l`D0H0QSxgyy`g z5~DSAF0B-ngq$cTAVG`6C`HptG7M-43ep)R<31R@IOYww;2yk&vOhTN;A+QRicmD2 z3yRT9>-G^JOXasndX*7I!z9Vb2p>Fds%v&zDH3#t^%(o9<}I=NUEDN4Ax85OKrBk^ zh_^~=teU8CUvDR^%WTjE2b+4wx9>$8jZk{g`ey{d zXqDk1AnS@t!&Mm|3d!qWnY6_98-u-Fmx^vh`Q4C~9n6o1N4vw$?Y>J*8Xk$4niKg0 zJ*uYWT!!xsk9Hw{yW@u;<_IGzv!l#uON_?*SZJL#_pRP|^x_+P*o!GkXbN92z`Yqr zJOO!K9A~Z8Or|AuDtW9aOiOAs`b3yXeM%m0W?rF@$2v~1NorIA zc{_?Kl|pG^tdn9cv=Q>wdzg}q4OF_N2}%?|15pll?}q|P5ko#r2+WcVbKz2-iD2sy z1(J&C<%_8GJD&ls&ae(wAtixzBLPo}Spw%R6hPghS=Bc22F68{)Y&SXEfB5p4?jfw zNjaB(mkc0YpSNEk@w^&=sc{7EEuvH{wfW-)G zEXlWrnCpPPtSmd75?VccDUS}A40s|yL2t31RBjRi^P%H}0&z(hT8X^}CxP*o@rAV4 ztV3uh6$BxxE+ex-Y=BMWnlo@&b%5a(LOhl1GsHVtVhGGfD0sk=pkYztv*9RFJBial zSEpb&y6I@=E!&MwNAXq7ar_KZ!Pwoc_2rLswzfChORIQ>Prj~R-^9m^erztihG+LA ze)H<;!mR$>YQM6vwEVI5_QIS-*jIBJ&yUrUiVIs1YqSgbde}R7Ai|CAaViZlix5HP z;y@;PlstH>%nK|alDWwv^NWZU9wQXk>KC zn~iRSEIz>LRd6*$lPwe^{s`2=ew89v$~Zne#4hJ50(-y}s!+@snw)`^F2KeKgPySd zag?EmR^l>Jc}63dS*1xmb$pOttYH!)As-h?j@TSBtE6Zgj_%xzyu!r_R#%AZ^x>#M zT^&*btb#?5M`B9KF7Wiw11bWjO@|hn2mc#%OFSs&kZ9 ziN5nnEhVkjoP=2J_Q5XC9tyOY6&O)6!6!dO@(G^tMZ%vt`+NH(-iB49mx{-t>17XBb z%mq&}j9+MyVL2;fSyts&x$6qV3uFk)kD$ToT*kVRsR{8^xmC~7 zX$TXR>$`*y>|~7n!R~ONI!ZnOMAIbygFOn11Ch`^<1b-<#mgzvY$j+!yr!Q*r6d_+ zfrXXU_@yJk;nftD7ARpgdmXLjjaWQb1j&qs&QGG&d_B349}f)l`nO3Pu1sv?2q;k$ zhzM*upbw-jDLmg9?SY9p99vqXMzRuT?P~s#k;NZzU^QnF>db=x*>_34d8MBXH$sxp z<{;~e3jss_r9-ZFbZ@?MIP8AXsmJj%GtKD3VdoH=vc0tB7*A-prDD<89p09AQ)Jm; zGf_IIh*aoj9a4;nBkFQM05<;cTbdlcw7oX6P+SL=!zs(Ltv0eWZ(x>8yvUkW3_86d zZsUoU4bE6lJsi!>B66Rnd{~Q#RL_U8cZsTOGId;ZT}MQ1Id@#5W+?;x5|^!%Dy+tb z&LsoP)u*Cj%~!NI)-MGe@)C7Os(%V0FC)<|TA4F69e}-)SyY5%uhPMxrASFSS7}hS zvlEgFOrT=|lhD(qv0g=!zr1OJjW&&nqfv)Cp*Bt>CJExDw9S!{$Xhl~#ocmb#oqyXZr7!XQwu>cyVz`x2n= zhe8e%Y9kji0+eUGQnWF244H^J2m=yXqK%EaxcLtK%%Jmn*Q@|A8$jqN%9@1J zEO3C|9X`l1fhNc__iW>37RNE!%8Z0!n`swOEfMATxb(3qNLFw17!?AfW`(D)WhHkP%FhJ zK$Id%To%^#V5P7ToV7cZz=YtjyVCLz{Q|YPF__Uw99hFM1DN)MNV?(HPmV~4#vcWa_4m>iZ46L1W*g`J*(`&j$qog3|~t~-BV zwRBY>1f0zV0z|kYWYOtT6L7&67EG3;oor;@Pq8?}8mz87q$XTt$DNw=#m36oHth4` zXGF%LX|RUw#JC_+5Y>`gU@Q^<`%lcKao{4xVb2Cp*cc!*a0BA7%?U=c!Z6?ugBXILv0sWIwiq(!B@H#zitmywb8uduo|OzV`+P z@~TNqI;_R;Z>buQu|Ya05y7r0T2RIw;_C4-E-`RR~%d(AZN3+lF6VXi%;w<6SitL5hBJf;FUF z5OZ~=)4WwA+y$^w)1vX=V%{0b#%MbEc(z5w5-rf_os22^)v$X6Z#!7Qk1^-^AsAh! z$eNWrtI;R{H9ua3VjbGLc*ILSyj1LB*-p_h%0nA510I?ZTuNs2y$SLMc;DDD z8F3dg9Kb($M3oDT0cCrZQ#t^vIl88#ChN)~ z%Natp#`H*wx&}c;-QP#En3K4uZgmCV+%%oDzaS=FpGFW)Z zz_`{&(IDx7Qe?s54)vwq)LWag@|mV*6^y!p7;|#63>z0kf8LwF`x$caQ#i zhLab<4Ipxhq{q9e$T6zOa`N8n@|+I0^8#fw@#=ud9Eg^{V2h$u!^SFa zXff_?Y_4ywFRw%ITv=^zc2?H5wwG?)!0c8etgc^Ox%P3)bw&JUdv*QQHfFv;R#b$h>oM< zn$TPDsqbS`DkP91Y4I#|7IeHHycrtals@D zlL^R*0DvbfscnebDFc%L8`oxXu^L%_f1?U?WBFd^7XIgQVFx{Qm5#bPp3JcAuq_|X z8<+VKO@zxa0cSM}XF9x0T5Z(H+0Cx8Q&!Gpn5;I-a(b=`LIS(gSt*<`Eb|IxDmHMv zoVC&=9EQjVQU_);ZP?4Z2QF~JVF^LZfb-ch!6Q_lx}XeuR6qg6tGJja$}R!VB)z}b zaf=e|s8~iF?;maXCS;igm(Vu`!`GM8j$E!bQa71wNr7Vfl|{~^1fu)+<;J%u8aVjh7Q`8FJe}w1f3b0|AQ_sM3))TVFYpy&IZgU(kxwlpvbY90CYgs z*qBjDnV99lmJmHsp{w2ViV3z<^@s3;VW*WUx3QG2BdmBmOI$g1ohE>!7=Fm@Q(Mzl0vSpiWLL0|?PO0KmN18tlC@Cy?j znSphOh8tFBg#D6+JFePrw~3-8ZKC7|g^`z%XfDS%-D=*@c)>GiJy%0?MX5IbXaZXt zd0UhA=X?+BoOAJ0l@OwlTp{6TV86_NN_+i-GM+_Z2Xf`44Hh^>LCqa0jNGBgn%|yx)?g%`?el4yozvv-An5@TI%$OV{$aHD z{XN1__q!~+A{6^s@(7$l5_3u=`Km5KlcrCWIIbERPs+}|sW9?Ba=`_CBH~-Ug@NT zS%#Z`X+B9eQhRzz>UW|RWZI3`+d?M|_>0`2B2rA9xsNT4*J>1s9<~?7l9~8dsD~{C zl~|>kGr2PQx3oHZ38x10BZ`~apm54^;8?%yfxi+*i;NbS zfhU|Bns@{$97YY7$84>vE?vF4$u;$i(2wU`u`l41#eSLYN+K1!q9=O62a;JiPMyPq z(%?2&^|FDEj7C2E0&9TV1eA~R#2X1!VPO`%lkhEM%y7}VO#uTWrKEwZD=zh!9wzyN zyj{yQ>}{SwR5Cz3ch}n6udQ!>3?_wFmu|pfPAXwn45o#ZjeE1OBp{flTlvbW*voTZ z!jPzq<8)?L1v1+$CQ4ZBWQ1~O%%*B#Vi||(7Qq#MNk*Sk0me2OrGh}n*&GGXqKFY< z=9$QtglBUwf1`-CmSi3UGtl%TOhkAQ0$zW7*IJkah8RJK${4ol%oHQz2Qm zT?NX@XqA3xdpL~cAY|2uac0Fl1=VrNV0j`wFj+xkl;qWhkU#3|BXse(L z*d)QIE4Yi^#ff;ILJ3B+Y!1T)7iTz)-MvpUqr~bSOV}I$3VMq@kpU!MH2U6vro&W+ zgo^NSYaKJGqO3q%`m`KXC~N*x_}xFOAKWIVm5Cpk=U{dr6peL8GawAtgpqOjI-cV^ zQ2zy%4_(TE6Ef+es>-(J9U8`q!J*~G@G;YykR+JBm<~C1?A#x0Xw;DmhIWpaYpIk0 zA3BbXTI0cy*m<>qIz!$XjK_n+G`dDDBZOq83{xe*W<+L{Mr8gBL>5%E+2$Q4NQL=P z;1mTF??`Ug!ocOQU}*vg>`jaIP9n-vbr`dmy;J&KP@)M8H)cACRRTv6Gg$1+W>c9WC{CcuIHN?qQ&naxrXCn@^f*$M$J`vEsVoE;3uPq*Sj)(J#fY zCnRd@xcxUe3Sj+BIg&dHa8Yjb5@Q>Cnfbhvo3lA2s%?3Uba@^nv=NJ)bVSQlX$4#l zLh5x+VTqtj2iSP&^Pc-~QZTk?Pm++z@H_Vm03d3>(KUb(YFy$k?_nw%Ds@kG4G8$! zjv`@z#@(H(8y%hxZEq*LiAkjG7&ftE7(m!IiSHH?6u3m%OskUI5`o>qkY65gb6&up zO)G70Zo-*vZGEkcrAAV(#ADe}Ms?`LgLOzH{>sYQRjf=Z@yqLLYwhK2oPDh%tu3v# zaRjy!ztsk)`D*-Td->IcOV#+*<)y9dg;q6Td;7-1V(ZzXnNmhxANVBKl-WzOOGgjt1 zow$!ET3pS7T*jgoE9yQj-&Bz;A`4G*p2sRKxPgEzFm0+C8hgI?I$EJ_-6D}@Hs%Q% zw{hE+9A!dTc*~thY@e;NA!`=5+Ra+KI6Yro0;%I*3Q4i0N=VR3@)p(x)O)vaBnkL4b4FXr>IkW-VD`#nP{ zjB8h0Xciw3%u8V&7hvB#==HA;2XdVt>1D^;go}t~b@{fVs=G&9 zxnv}9b;?N+ln$(mk8a`VNzWMjQ)s9DKjvaXjN zTNyz~GV!OlBa?P!P_dx6INxv?CFM*jvLebg(iG3I-UrfMyiw~oJC7_j|0|h9p%>O;0aE2JSwvX$%8|@8Mu! z1mh~tJEIboS#7b9GZAHA~zndhClk%&mM)rnT6`Q_*@svd^#f zQjCj#UJERdV8rr2s;`gNlq`uxk(0yYBtLN z=BtXhAQL3i3|xZ;NDw^Z>0Vj`SVyj#p9{cd+(}1{ZH9P(X6~*l=y)xOqc8Xk)*x6d zCNCXP^rMSXlhqnrgxQKmbqf|~o8WKdR=d)JW(-rmx42nhtXxnjBfPStH}^!Dq1}+E z57_WoI7!zKn=SKmW#aZ|@>xEMi$GGNEi-Qq(yR}yLiYgfgOgoB4kS^_JTnmPl?T;4 zacptjqGxWC{O)QElzq8PRhY|E(Mh;|XP3I0u)M0CEZu`^)nxnnhfyqvJEjU7R5!-t>1j@~k9g^91a&$G<4&hS zm^vI#kHh+|2n!m;op%7?J4HW1iy<`tg6|bPDiTlSAo|@$2l2eO!}Ma}1G?d0Cl<4h ziq8uX7$6+>9v#B-p+%YE44sYml+26!4XB5BpM=R@hp*y>??bX2Wb zT|d0y=kj*H1IoRaLD;`{dde+|JXymlDJyd3j*Z6ngABo5Rvg$!bU}kbIGVxfM>tMo zvc_8zx*!fp+iM_f8oW(~o=L%Dyky^nU_t#VR&43YL2ug*4mC&?U!!Rj_vH|wX7iWbAhu1#LDraRH3{dqC;%LkR#9;J(m@PpaTNon zFCIjI7FQ8)`rC0a;UCv&AVg zY}{x<6hM-oa!QjD=|b^Ri?p% ziM-wF-r3wse$3e4NTRN-W1Ay z!Y)SmUS6dGLHJz571`L>2;cmii=UhAjrGlKxC!XI0bG1ukUP&>cz@6yVSBNuSc8`n zFZ@>XvRsiX19N_E?jm$l_4h@7(OV^wy6Y}o{s{&CPux|Qo5S<%*pUeHwNEs5wv!`I zABdG-acAKY$1*U*u_XHyhA{Crbv5divydV>eC>9$w3sx?T-l_au+4=Ppg-TNAk7EO zR@8BiPxs&lYp!+nFYqGYB_`!O{K znP}wc^>^*PIihclWd;u8yl!I`J z9m%u{Fx(T3ZH(%LcpZo~hD_Bqx)&NSr?VNI9>`H<9wFSp@h}h=xpmtT30bS>wXYl{ z8nE{Euq4#iz>SS?Z9QiJaAL5(Xs!$toZ)oYp=mILxi33$v}plIVxP>gTx|I6By(U) zb+6r6y57NXTw2-0@RqOk&5au?%PZWZl<$=_4EQYPi<35XI9~R)5h{RdVF-eF_1-6ClrdV8k}D%HwE@25gV1+Fa)fkSfx138 z;`(qLifLEpoGyL|vM$#FehEk0oind>_ktRm#+$@XODPLDt7+BAn%`hYH9`4A^P!q{ zA20vm8V>jEM_1ux@D!1?JVQ(zntJP#tZNRclp`@^)5gQT}64C9Yv2r^J$?RG* z9-deIP~@D8HeqRNi;j1bLohzp_5sEmx-IPkibJNrgwE|E&`I12y6J1u&SZQXLrg=|uBnrqX z0Hw5hK}*Ke12@qunz?sPR31vzf*DiYLVHf-UDK?UHtM0)Bn_I^PL_6o#k0CqK%J~w zHevq^L&q4#tBsRMQ|_mb5tgkXnly!~;Xt?tJcRW`d!1gCI+LLjt{ohY?xtNWuZWbZ z03mj%5j4##DsqwLL7mZ>fDrn6&brg9L4TD|X|H=W@A|}8h!vera9YCzP}={A^^XN@ zo)e|jVF$m4NeXMz?0vCoOVn-M0BM@JBa!J)R-AH*_Qw*KKMEDDY5vhszpUzTC{>+H zayJIhCACTkd9_$Mu>?l~p@7X+)Y#LdQ`1g*+Vn!Ffr)Jbk5}514Bjvfk*}r|QMZdd zra3E!=L(+AX1QpF)sM5n1xQ*$cMC?AgT0K`Y1FHzZuPuwJNUxuJRoYk4hVtQ#fF3- z6)QqlR3lS6%i{L~zRUep=LuM%}LLll-Q%>orS-S~jz{1&lqf zpO(w`5P@tQq+BK=aitGCes5pV^JA}qeWpYT!StMIR>DbUraEKLqB&6qE2-e;M_J*? zoaqNhk@I1kX|_Ap)s!i16%bmx=@?i&t@gZT;G{ zExf0Qvs*ZCwoR{mkD-D2Y8#hTZ-qD*jTRfP;cn64;qLJD5;7z~cm2wDbIUbiV!CPIPYLk0I6PRJtgG!s-Po@(bkG!68&?jca zrr8yx*OFbNd6O=o9t_e-Tsq`KMrU6$I!XuAwrC?BXv*GCEifJMZbl?dC>aNQWDV06 zXiiMVU@)Q}V{$51;P_%fM(c*=l+lX3v^QZJIUTBI3pAmS4Aq!DG*ktFj*V!s4*U}G zWuw0s&$-yK!2odu-jMS&aL{r%G$F=DV{2fHL`WP#5(BP6rNorj#9%UdYR-a8W>8Fx zz?#oUl#Oy&0*Od26e4}%Sce6V8HE*S-iX4hLZnMsk8b8Nd^3~H5lS7Avi{!8Xq0P{ zH1Mf585b3*Qr=iug+)41DslhtZg04>e=i+(m(cyE9Cw$L161ar7j3cygfQ7&wsDtM zg`rm$osPaVyYq>e^|Urj?KS|Vwi7^VH_?9PT?QMR@k|gu%gZSVXhS6BKvWKLe$Zm;4X^!d%Dwd?IIo;2HB+SlqXF}yWYtq@ z1GS^M-w#xU@+L5Vo0*Q?HA7pQcUK&tSSX0@o7K^RZ?0e~=d(^om(KId)?YKzajebI ze&+q9m45XVO1I9Kxs9C2TF8D60bSEj!9QJY&cF)BaTgbp4{JRyuTkRBp*he%hi_bs zod9MdpA|N;Sz#k6l2uMHRuWFZ%S^Y)&|nB1oZ@&2hnw57`GaWS;&%`o*harOsOpJ& z_ZFDWE-Teox#G}yO%K0Tb{=M?RdLYj6Wy!YbO^9&WDo=yD+Yo_$p+hvGZmXI>Q_BD zbM>GO+@REw3O`OD0r+{8F|wn!vOmMw2SdquE5) zfsuBhVjo_z!|M~7ulewu+C+@?Q&^iMa{!hNvWEiOZf?(HAgy0gC2$7D(26TpJz5*P zdN`c|i1toUJW(BY`bT+sSU_eS9l5$``#jh>W>2NI9qjd6hx)$wMAGZVbH-&eSR$mz zAqx(6413*s-D>}YbAM9P&gh}CX&T^EzgRMTV7 zu2d*nSPkAC_8H`cBVat9m_27`_S9)eq1J#*WSt!biEnW^i3~PWHnmOYbi_0V$ zk1?Sf4RG+$`7}SnRjhC@GH(nnzNs==<%)c4VJAlElC9rbmMG%D%%SjvY7{T)v?JQAPA8QM2( zcmO~c9 zX)ns?k6wc_01%odE-2^%tpf<=5uuSGi;p}rYAS3H+}bIyYr$w!6R*M<_#kFY#eJ8D zcuX~ID@~NrR--Nrk;mMdxueH4HJS!%=gH+p|cl97E*p(UHYs5fx&2kC$RsZXwN>y;SH(OQ*3Q z8Xo#ccJ3o_K;0rH^is^^-Q<_*aVn){)4EB(*8{SC($TDF1vd#DZgxh@*IP5ZWsjZI z(w5KAzF6%fFnOfHtslS*Q<@#FHX*)87ZpJOR)r9|Pqzt)#kx4e?&19;!&Pp$Qu_AM znHG&nE@Y0e7>FSj-OKB*6bd*uB*gv-w}Oy>w1zqX!3a_#PhN++K`awf^o_JiciT;X zLRZ>7y}RSGQ72@hF}g_9Hv0@6?OkBFn@6TXHNCwor>#sZrfg0&{tW0V$6XwI@f8pa zr=u1g1JxTGy@hg#{uuWY`m%`7>>k!?qZ2tqbE!2NpaYMKkvByEEw99t%~wpF262rv zGam(#R5A&e(!|W9N(PUNeTvUOqpJ#S$I4P z*I9)VKxQ#DCB+u5DHo{f?!%m}mNl8;jiOTPNtx(e5}+mXQDhQ3Y0Ex0lB>TZws>%* z@!?VMvz!U0Sy6SRd5e`ZsJb$#If82bi8U$K{+g64z477h=yjxg6=zTRQV`!-<+N)6 zWf;&UAA_Jd&nn38a3g}Y0BM1-&p2Ph^KNAu19-Wo!z=!oU8413quagVXxtgWP0Qo; zW`}IZpgvO~@CzyghKryO28*ubw8<_Z4B;?DyaDJc6t)hZ`C>JAP@-RAxO~8P~SnQ43-NC>sMgsskTFeu~SGzgTHHDaGLuHmo zBU?nnoR_r4h9o4@y)1`g&!Q=|G+tPl<-w77T@xu~CPS)*<9J7g6HiS971r50z9sKS zKP21DyjoftDxlxyV`6wVNbRWoq!WBli|%SmP_>|m)EV%%6f-~4@~$`q-J+JimG7n2*n_PQ#nA-FKcsmwT|DB*}@O z$-ToSuy7*dc#X`YT!oxFDQhb0PRduvyOXj;nRimIT<+P)+j_ZowR`JW4C5HI;Q+3v zyxNPpoHTt;uE7zxhzGSP7F(n49X#5D>pw(_$?}n+4zYXjQENo&O~#BL5AYgk$gV0E z;}7iVS@Bjd@rY-IU+*0AvXh{Txs=3Ii_5g4?&fk*#P(1|8b_aflQBi=#I1!-GJp*i0<`nh>AP|jswKgdd6uNo2ghbMLnU?e+ z);7qlOPvr2yf5G$G?~OAHiBUgob!cJ*W1z4z^Uk*uTkq@B(!FkR+oB|Z0IF>z$hqd z!anAelPo)!#kDn&B7)uW@ooT!n@^)nTe5M3K!gd&HEXg;EF+z)#gT3C~! zV+do>A=u)M2NjjC<-CK=^vHv*8qH7l;woC~5aeg$a2ig@%GhiM%kp@1XUxn0$!Se} z-qF?&vLPj70zG7Nz3zj>fg#XrI}wk_;9xdm<7cDI+23ioe$1xEq1T>b8>nEbJ zD8~czqFKj^)Qj+eV`?ATq2Nvmf&Vzq3mtHxQk9Wn$hem~b3rfSsz>tss1eT=Au2B4 zmNgnLMhN3X$!(qN8jMs;#Ud{tpl$0%YHyI63QNSw}(82CsMNN@hwj#s{BCwQTL?IYVx} z^x7KbPBc0KgI9ElDFKxCg(aQ)}(R62}=g&Y7B1SYC@_uA^@sMa+iHk?sJrrttvZq?~8zJ zm?E3%`5Z7=o8#CNA;$|vLPv_Koo7#IE$f#&H8cX%Wc9~dXxzv`5Jo6cH%5S>@D@&fQ(ABmC^Qaw?8`U6{HIpxNsU1(V&$S<28NAS+h1pY6yCnKjNfxr2f|qO5HY*PDkD*W&= zB=PZ?LdD>L^=3X@=Qo+?d>;+!gyMZFP$WZCo4KoJtw(3shE7zR=d!7OBhK^!hAM8M z<8n7o>>oivE?g{$5GW2tiGFRQBX`GCY2zlB`d3QlnKPL3(Xh*jNVQk2p1?(|MO<0h zmB;>M3kHg@D!S#G%FhX~E>)-$6=xtgTM4%(mFDi26H}Gu=9y`8gzuEGwy@5UGYNWw zeb7L)d5=(&a2K{cz1wUb>}2#SO>;J&Op+Em-rC{p5#fiEzzU3Li9$=g^F?|*v}htn ztdng4PY&R7v0(v2DuQWj5#U1Q?(XPrSY3!oQl$!{@SQFa>;*_HN0lQaUYQ4Ry7bY! z2LoAaaGMMKV+(mry4%Z=^cjefw|ox}c=8ZL;A7{l3i*AtoiLkFvtSGB6x;A)r*LOa z$p9inn`Z$>c*IXbXsJX<$p)LUQ4mKeqyW7}NR^DQ{NC^R()&`b=lR}1E~aYkH^E)M zzAJ%-Iq!&+DkY8X9y&QxV%yZbQLN5j+tiIvwA-#G3ET{_t4Uuk#goZ)QyfI)ZVQ$$ z$*!zm@=&t8+_UPDMW@evRt-0ZL@|qvo2-p?*ufCbkKza}9F1NJX8POGNf?IeLYwZeW)CyotV`~)d}3L+1bYER@jQGY<69{*Vc+ZS$(ETL3Up+_Z)RhF z-4DdtE~btwTqi9rOUC$`jY-sIb}jr;2`mVNK_Xpa_<2v+XBZrgKIU!|R&9vCRK^;- zn!=LZgr;Qo@qiDCJV{SSgJh~13@MV?KtO?vg?kkX7IHY}ih$JOSYp@`80leeGBT02 zm|VehWluMlF7IvF0x-J_hZ%!8%O~|$;WINBN~anuQCzwGi6!*7E9V(`5`sP49~}1d zZDK{r-|Zv^VV#3RYsl59w8RDr@+uqM*gd?pzgy)ZpeVwYP|>x|Ig=SP z<;gCltGwq1*VIBpkg5j>8Nmq<(u#e~f7Uu@A^Qyc=Z!ZK+H6Tc=zIuf2U$~I6O{ub zA2+QCSQ5mZU8Gj2~*e@Op zM!oyEm3Ca4I>v2~p&Tzyzq7aZ0>j?L*8B^@?*84~F~0l9_g|d3aC!d4UVrc4c)$B1 zu)w9{wIKHf{o8#!3|=PQ>Ebbg3Ksi4T9V7WC$=cV*%6Yq24MO_M0smYMET5oYroU$ z!wn9qjJOcOM!esFfmUKLz3T|vhs)yL0d6k?qp8LWgYABYbutQKMqh2*@34(vcEE>3 zNv^4!nVVZAmm(uKdS{Cj8}dUgaAkpC-~)^{LD2`>yyCvOY(HdxS$niP8?7jX!HSV( z#j_F;YhzF{8iy`6uHw!=?2@-IFnKvBLZE+tgeRovqyd3wB-n5`w&s^Pen9$4s|Jl=ws~>%s{f8 z$X%gv=Z-~UblWSah#V#u@%>XgB_ue_7*4YLf*m9{nQ>oSK@J`mEr^DEk^6kO{a6*KgOHhLI`ltEZ&FJ+=prIIXr8N zXwNMuBKI72o-{Yiz!A2+v*)=lvO&mg5AAvMtVhpEl^#9k(U`OVJ7?%i97q&omoJgE zKWts>3_Z)9-&~eJ*jYG%+G$8i8w*W8?Rbp^mnuW5|AZQefURUu8?>BP9r2iAc$RcB}c z7AE)cOXWU(#7p{G)F4?sUQ_p<(sch21w|f&B+XG)rN03wN=XLS!Q6l@3Y9$Gr~tru zoT$(;gICq2a#Yrj^jc5U>H4}TQtvOY83XJzDQw9cU3IvK$~5eQ!QPk zWmwDZZ8wD+0K&0unUth4lVk}`(GrcgCQY!CFSYtcoKatuW^?Ayo_)*+H3L<=)pqd|KtfC)l;XMo;E zi;~otZ$G1;>0aYmbQfyd?y>*`5TCFzP%9&LBi^aj>ZlRZj|V*+K(+5f50woj!blQU zMx>LNr$XsaN{850Xnb67JqY6roj97s#f3+Bq=w6!c}PVaZI|X-+)9UrsLLNzhM~jN z1b)eEj2x6q`$uDaBaR6=LyCrjG`Kt7^G_I!a z&kOin=9uha-*(vho!GdWqzmv=?u$5Qk0|~i*{Y;OlXwgo=jQ!wyVl?I_MAd)Gq7w$uQ2S7$F;xk@7uK zvyUMQjjN*gD8C`wf+a?!NPIrH3&D!m*B58TcYC9bsG5-3GKFYo-|E3*kW);E*4{pV zlYfz*OQ+)_97XGnI)lDiP-U*+HbiSN?3uFKpzv_Z^Ztk^qdyN9d?66dO=)Id2=r+? zCN3VWXUK{jGp+@Sr8@IR2H|$7a-weu zpw>YU#Qmx`Eut$sO<=Ekw`APLg-C_lY7k6qQ$R?yZ!@Gy3leias}I_d`-?-$VDqsU zT7aXbFPh&fdWY?7RZ;cAF>t#%FJ2Mn zs32sy^(x!5GI#>}=L=*z$m9!)c`+?@&lj(7?Vb=TcTL& zfjY<-T{fz-7REH4D4^?O6vCd7t>J)^b~mv@zk?wok}?^xFv=mJz6%m!s_Lo zoe62mG=d~*2S5|+_=Qddw}(33IeYUO&}d|T97C2k=+K-yxvoi+!o>Ot^ZI3)YV?r`7yp23Y=9u}eoO+bRIruh?NTDyW1 zi%J^bG){1-*&DA*)$T}l;ZZ~EcCL1`PqebHt|I9dY|MqG=ZD2ejK zR)Nzvp~h`}$O@epm@4*3EMdMD+xlV~w4~#6}C%_Rl^2HyKa(^66 zAb%;R1ucyp0Ftj}Hm$|fH=I~XMU)u=f@5GZgJ%KqCX&StpB1-F zb6JwO7NCe~6v9Wo^`}|PwLr3%Hj#L6AzIurNewxH5({fu|4dH z9hBg}DWlopU?55~Bft&MNk!;tmAfpmpfh7S{+U_(mcKA5d{~M_qo-=4UZE?${vC$B zM?mZnkai(|$rQjJ1X)*H@@r3Oe$$-!Ar?wQC@9XFt7fDN6FVY;8JV6DS02cL0|BXn zkUyzJ*_pYk-GlD9%dFM_o9b=e*bpNtlLf4L#^<>giahEP|FArMH-cd1LpUxDb#X@^ z?CqewL$8zq=-g2YdY`Q0BnDmS9jJZnD=Tep&7xk4sWKox(Q^s``iD;dGjjP)i3Zl+ zwec062zV=RVA7%(l-khm%BND>Nf4C_p!vn3_57qu9qUF;(Q(Znn3>L!^o-4aN+J_8 zk@CK*W-_P8cqNDyP&)g<*5azh6Dh8OSANExin@FpYw1#znDRTLt6-({w3$+%qW6@Z zz%(b1Lr^-#E(jNrX#xkL{(|k+4I>8nq#@_v#eB z<*W|1V({|tZUG}s=K{p1*1Dm_rUaI?p~mTT)H*;4%@hIxeQ84tl#=D#sm7Ykqyi#{ zTY#^QwG@wZWTf#Go(PC5Z`fQawFOBmwVeb}8yICWrr~;K%Ep>m+fuVcb*$N$1wjZv z1^=pJ&FSaXa_&9eKicBf2)D@F_qZ#8b^RQaXRhmScJOe{PP@}y-Pry(7WbPsJD6U1 zDZS3T@mp5^Gf9S>4cHJk0mnRHR_21As_zy~d+`?JJ9Z(jTxK%DQ#drW4LgU3r;V#A zN)cEF+a8M}K^FVy*m^xUfM*(WP}#?d!nQ^7EFHNM%He?ZS}JSc(}R0<0thr&bO5$J z=m5OG&+?pasbLQeKe&`d@Rq8P9g+!)Hv9m4s_tM7TR7jUag9VqQ`qneJVt$uEq8|2 zE4w0XpaaMyUNI&xag5^X4N#N?`HPx>Vr6lNzc#wFtbBIN`qtC{84eVG#s%ZvZ8=j(8f*6(7FfI6P*B$+F`pyGYd__Q5W0CF&2z@uI{IZ zjdFZnc3{F>ft83jtF&epk))GBm?hwyMbD&}7S5#CC0@doBohyT4DNgA;3i1iaL3t$ zhXio^27(p4DliIpjw4*kP-K9UBr*);%D^ga7{J&mH0*fbkM4P>fi5lZ(*Oc=(HrdT z9(4K`j~)1SS^e}aYx3D|7|!?{IB=9CeGwccEKmgxf{(|75O#Wyowp0NNC-IXOEogY zFkNaZXvu}EPv`b7{TB$$)3p)U4#T+(o*dRB(6)IQ__xB%1L7kV(7z1qfPUX6U2W85 z_>M6aeh++O*K_0IaMXsU&5y?ec*_-O>Z{1!O>KybSVPFJq^3tTfhEt{WEKW}xbHOz|NBWlp}ULU4?ygp-0hgQteLl79|>5&$g zHjw9%w4%)8#E?Am@CF2&JGjBwiJ%-mdkGBVDf>L$jombOb>iZ#cpt_z$JDGj=m-o9 zLcL}>wv=jUOKULa0dVQATnA`XQ3ds?4p9$0-Q6XR@ToO=GA@+Gxv3spC=+5_hC8Td z_~h|!e~d1bbQ$w=lu$rGJGugXZd#o|HN6f#Xsz%CLQMrtu#{K8Yg7vY&0&j;GD>TZ z47>Ld;WIY_5ls9A{@wr%7Vz&J^ea+m?xOa30tg)fjYx?NTFqV5K8KvzJLs8H6{w8S z3+GYxgNm zq_C1mKTUxQ>oC(ZX|sU|Nb8klY^4yK>2?pOEQ*y->QC=;OS>$WMuTIH;yD&2UuD$kWE$g;5Nj^7vzU$<;h z3YTU074VS$YCw~5>-u?joM*{QG5Z4JicYK7bXHXqgJB=H9n_}72&1KuVMn=*r~UOs z2~PT{Y3`QPQ=}LIKtb+ZVVt z931+wdvvOApok6?t`f!r2*<;|m54VJFh5u*S|Mw0tCj0~t)#btx2`FFTY~ZcuRGM$ zjc3dl5Vu>#J59E*y3g{08aGYN0nTX6$mIZcZ zPB~MY=W{NS{ysk3Q2i``Nt2hLz@3y1R;*^pR^iqhZA|{AunfYqHbAT7NX8EFRePAX zIp?N$j?!v_%O$o1gaTedStJ}KF18-a!@O7{D=6HhPyv<6v-D^$m)02u$(28312s9@ zWg(A;cgd*%qVJvinKM5LCrIxstS-iG64oldpC&AUq`H{Amw@)#{yt^i&IRAG1NC#9DYn3=GgO$W>OeXa7o)RY1dDCUHJ)MZ9zfvAyq!Gr1l5vUinKmcgUA`qA zz-l_%zMqxiWgw}rmpwAgA%)ia+xKaE8t4swKwG9bac{PZ$0jD*D+d$PH@F;`n2KwK zaOwYqWK0HPimvu5Z?upGDH)=TQkt`QQaPOc4;D?yt^9&Ix^Q^TCp^33#D;=h4`f-C z+&a({1T~4Gd^YXp*9%GAT3r1c`SMHlIu141wBM2kbpp=}g-NzPDQ`&|1Mz~CNSs2f zCDlq%dE&g}yd15;@lhf}L;|sXl|R~a<)xzWt^~-2R!%&@A_ZVvxR}k-hC=g6%JpC(|k53LI>MdoxQumJ3<~|M%+q=o~}>cK}}zuBEQ!jVf(KSv45|YVR}8q?+rJ18F-k~`8~$@Ix)`a#i8k6 zbv-w6N;siRHV5-m{O}oii8gr83Fl2rfIL4tYPs$|okU7rG| zcpnmbG5b=9Or3I#iWH~G7Wy2LuwQ~%Rjq7geGfITxl_!eY}D5)8aSr|1?@;k018gX@J@Ji{VeUo({qw5SC7O> zp%4|xvMyaA)WJIm!+-rsWy+{_K86KW%;1GF3?zl_`%-X;500$O*?5yK5 zJKck`{7}rm4Ksu7u@NEe?{gHLZ^i~hmdJ? z_?YMrLa!BUT8`|&iIJU@(u%m@o4b8rZ?=gCk5N_T^&!!+FI#c?&?GqhVCx)Kon#l5 z8~1P!kUeSm7u4<2HZST`LD*BRPsTPe$b-@<2NUS@>6xA8{#ue;^+0HHI~fE32>6TO zdb)Je8-5T?+b53cH88kY3>cnLb%YQn9&rwW2|M$3YSM`&Y$?YbaCim_OxsOP6O>}< z=@oF58z;G)_+jPk;CY(aE?|nzzQvfGjSb$U6wXn}p|iR&%oI$8Zrf)hErE(aa4*GB ztF=j&h&*>u?xVvdmiw3jUS`t!nt7NC8FJeS41=iNH+t0S%RM?Z|7RPJ$fx@{wv`gI zgdG#fV*6d#VNu|2S&#nriAF0~dpcJ z=1_^ilB3xR_ioz`$9KC5mFpO|Q^Oag(NmvtaiK;mj}ahet%{18p`D@~ zw~KbrxXq18N=oiM-XD#*gp}8(E6WJSAE0>DwN#A0W+JIx;~LXn+lChQc#Q%y_9h7h zaYnMGl@(Io^hTCqr64-6fP4p<-NTQ7z9XQSYMQnZ&BwZj3{cm6O9F3XjnqODn=k4* zns1eIT;R+FI$jPkaMeV{&L*zS!cTrezulD`N$)W@{k}627l_FqEe-@=m|nifO&ewX zGS!&>gzBH)=evE4=MM&V8ZU?sT;uuieR{Puo*!Chx+)UJ2Z)7lRpWWnG8@nL`bWp( z#`D}>6Qy}INaK0+7;ZdIO|tR4_z1Fg)&E$(|3Z%p$Bh?8m`xkc zkKja3cbbNxPRtc?!`skytckSX>?j9(o<6wUFf1K2Ma=LLv=pm=)YAvYjewV;rRJ8# zp14~y1jm44&OVKDnrZ02-;dx4{Jj@{--f?Gj=%HxYvS(`{$9f0b^NX4@8kH}#oqz` zj`8;<{yu}he*=I2CjNdLfB!E2ej0!O9{$RIA3{Z+z~BFhzt37&=1ISEeKA*?o_Tuo z?9^co=C1yosomk7W4?m)2h!4wd@Z(6lsB?JJJsuBo2s`zC0#mo0L#b0Qwr$B`nAVn z@v(URSxLp?#PZ52@4US6*t1hO`2>$jy1-6hn-zZePmhcaC)OXzI}y41o5FkIx4OgA zv_F)0`aNQKjE&fM^uTd9qa7)DBK{m#5kL*~A5z|iW!Q*8>=Dfl2AcRA055w5WF@>_AO?j@fB z;kh3&1t2;{Lt2BL`wxnqaE$Ws6!4S2HGw>J2hLvo3xt%HU+)Ws$49_U_`s2qS);Bu z^YVH7`rS7IS4{#?Q(g@fUpxAg@E?Ib(!Nk$DP)~Cp@Ap0?= z=lhxW(_en_?e^neIey~PKX-1!LH**-zxA6|_cs#i5B=it9|9tG`xEN?|A5xH#u?Ew z-$Wk%@gKv#Zh!nnwy(U8Kk)BNSFmrKdGqln&Yk;_GcO_T15CibpG28+gz-&}|IpcU zXYu13-zDFF?lB2|_?!fve%HAtk#^z9bAJ|@F)~{C`UL)1eh*xGw)i6Qe-`+B4v9RzgCdO-!sgFe}DS9vyDIPiTLJE{(YZ?CMH0bvyIOo&i_r>kDCZn z{GNn7{x!rAKProk-#&h)^?nZUzC^_B-xNZolNlKPLHzOGr{>N!p3$H3e-`2258;pb zyv;v}e3Wng&q4ngM5Orr7Vw)w9P#@m{5gJq8@~z9`ReBY&tV{1@-h5>@R#QMJo1qT z{U71+-*4hCt@r(}o|QS0^^(QB-fsu66yEOu9_JSS=kWe!k?)!LvtiC=UeEXA$d}f; zfqK~v)=NJ0dcRb_y9#)9^>SWG;oSthXABLu3+2olxNH0zsH1X={GH(@!w;a1Z4f> zocQlC^t_LEjZH@~^Z${?hhuj9V$a5Zk2OH}&T*+?e54~Sg#B5ch|_Cfvi z4}!k0*f%kG3hnROHv{IsUwr>t#rMw_-~Xcc{%w4_j>-Bc2mBx5AKEut_CeJ6Q$_lJ zT6}*2->!QT-ajqU|BvGPtTFFC;J*j_z;WdIH_Lw%-*3KKZPEPx6PBMi62IHU_x`2I@q{jOwCvOV8`Z`a}3-ly^HdN{wA z@a_6Fzh5oN5BZG=n`;fG|I;GGy*$VB2Jdt71vmOi4LPW$F^dt-ZLeXX;-bmd07(?Q&|&85|L=Sutf%9?x+ zyW_jV!5c^^_G;=D0LWx#=-tH^XM57)mi1R%P0Ezt!ZayYekl_X&Alla6{A_DEwY(8 zd7bj8SGj#v`~MA|u_~zuQ$23Q!EORN=$_=r_Fd8FTgSbFF-~3K`Vib(D_IIM6df2` z_fB(;>LKps4Xh~pdxzvHE=&gp@E-um{oR8G&Lm&u7OFg{0PJ^q4PJJkaf3UE!9guA z$JaisTx`+3?p_C$D&&EQt3SY%BD#~rO2z2KAA3g+B)B_lj7EFA{o9SJt6SHlKEUrb zunXFpZXAJ;6BA*D4-)fQO}H$aZt8*i2Ny|?;OCgDR+7mWdEADu9Tbd)%yj@ zF|i~@av)uK4g#2O!ung6g!9`6jjLVU`O&*4yjVhww+y0q!NceV-Y{6uKc|`ARDmkc zqk{)k7+P5EpGeIMBhfFnDAFXQwiK%3a510gYK8d;1+ZYn{(u)Jak)|FUT-))2D^(B zLOC{3s9erVzG@^6A@|_9`Sta#DY+*@zjWz93k6BMc{${?!ZqKSZ&e-rEGC~n&TkkC zp0#j+A&j9fSx7zb%(pihuUUwxy>S+cm2X6N2Fm_2!>LANmElv3#t$Pr`@u%zFYp_7 zydUE?76gBTAu9U^j7R;S=l9c%2G>_-q1^rphA8&|=qYD`%*zPRz>x4g3^8|h82=n} z6oxQHe8$rM3coQ4{}jKG@9*&&`Tlc;FEkqe#Nz)4i~lC*CCv9;gv|FH3}L_BWIXEo zOP2oEEdB3T`aiMsKMW;+<-ZLf%dawg8h#86Uu-l!$#A;Sc)+j;8!AG={Vcy>bohG= zFE$##z!2sBv88{R-)PqpfX(=CN62#D#SnkX_+x$4OPFuKc%=UYhWPt2#v}c2F+_X* z5#y2mn+);y+l)v0d!c7A{ksq{eZUY(%U@-PJU`BKKIR_>KHG7y_>!u=s(+f5zhfD&z6@A2CF^|Ag@<_e%^>?tf!E{(c{DVfwofGJT36 z(q|Zt^i4~D&C(An{ZCo?-?j9A-_rjZ`~JuLMmv6$-+=R8EKJYBxU|X;X82bTk`8~< zzW*V=fyaNvZ{YDi+V?-_H{kqV{3gDCK=ZvDA@gz1i{+l>H}Wm=8~MK5;@27fT%+;5 z_WdXM-D))6vhTmc@5_zG&+;4P{%ec>ZyEnwqwzm8#321mi~ntlf6u$M-fu<7dcT9; zD1V*bi;c!7?E4PC(SM)g_jfcJ|AB>{XFT9fLBTzPKK}^9vl!p6G5k*OGsEv{G=7BX zfcszY`=gDg<(9;|CG4{GZ`B%D=^MxzYH~7^2_)Z-(f%vyd??_c4V0zRPcv8}J+De$2jq%EDh} zJnH+G{6@L&g|dD2YNPQ32wCov{6@J43{l^YSo~kK`2Uz;yV3aX7+wQEF}x02BjwlA zYGXPBSu@oDJcPgW3_>oVpVja8TgZI^rf*yL84Eva;jdWuIQrzwhp?B$5U^&S*K#*3 z{Gx@wVc`?!HT^>te#F9U3lA*(jD`Qq!r!s*eJ^PF^A=vU@cS$rTlk|Ee%Zov7qndC zMGfC?;k1P>TlfhJ?_2mQ7XG@0k56m)DGTpf_$dqjx`jVu;pZ*L2 z)55=O;ZIxm3l@IS!jC{kOTSq7a~A%Nh3|vxX8L&x#}@vmg+F28-?#7_#v=27z`}hC z-?Z?H7XF5X9|7-4{T3cr_!$d-#ljx}ub%;r{4>aoGamupLuRntr!4#l3;(`_zii=` zEj$Mq!u-!zc+%f{v^gY=l{o1H{qPO@KX$d(+40k82_e)KWpKy zS=fLqVfu$Gyl>$b7y`$;kS$FAyoKD!=J#(|_$Fk@8Q}8M3=#he7Jkvf->~pL#`>A3 zU>jlxS@{KqFGFrId^hmFf^ZG?euhsp8eH$%v#k1#yjkbj&H&fynBh#Uz~m4r`P_&y6MDwyvB7EW3CAqyb_Bpt(5La@Dr zU>^xVdI`}D624?1(2;L6UBVj{ZdkZ&;Y|y_&%z^yXvf&X`xd@w;ioK=_5rxG50y*% zfV8x4#p1tP?aOCDr*C07An^f)XfL{`aR%uhMtF7=d#LvP9)5#o1TOMBoxYXn`2M3z zM3xC7H^Jf&^_gP3=BjJC^!YA>Y H{P6z+LpE=k literal 0 HcmV?d00001 diff --git a/common/rules.txt b/common/rules.txt index 9acb0dc..8e684b8 100644 --- a/common/rules.txt +++ b/common/rules.txt @@ -83,3 +83,81 @@ for which the receiver is maintaining state. That range may be application specific and senders/receivers are anticipated to use relatively compatible buffer ranges/sizes based on application needs. + +SENDER/RECEIVER RESOURCE MANAGEMENT + +The NORM implementation restricts the server (sender) to a +user-defined, fixed amount of buffer space (memory) for storing +calculated parity segments and repair state (i.e. which +blocks/segments are pending transmission, and any pending repairs). +The client (receiver) is similarly limited, but on a "per-sender" +basis (i.e. the client allocates a fixed pool of buffer space per +sender). However, the code is designed to successfully work even +when the buffer limits are exceeded. It does this via resource +management (i.e. stealing less critical buffer resources to cache +transmit or receive repair state). The buffer space consists of a +pool of NormBlock objects (keeps send/receive state on FEC coding +blocks) and a pool of segments (vectors for buffering NORM data or +parity payloads). The server (sender) keeps pools on a "per-session" +basis and the client (receiver) keeps pools on a "per-server" basis. +The policies for managing these pools are as follows: + +NORM Server (sender) resource management. + +When the NORM server is transmitting ordinally new objects/ blocks (or +is required to store repair state for previously transmitted blocks +when receiving a NACK) and its block or segment pools are empty, it +will attempt to "steal" block and/or segment resources from +non-pending blocks (oldest first) of objects in its "holding queue". +The presumption is that requests for repair of oldest objects/blocks +is less likely since clients (receivers) request repair of oldest +objects first and older objects/blocks in the queue are more likely to +have already been successfully received by all of the clients. This +results in most efficient use of CPU time on the sender side when +BANDWIDTH*DELAY*LOSS product of the network topology falls well within +the buffer space limits defined for the server. NORM will still +operate successfully at extremes of bandwidth*delay*loss, but a cost +of additional CPU demand for recalculating parity repair segments. +When there are no "non-pending" blocks to steal, repairs may be +ignored, or tranmission paused, until the repair process, and +subsequent transmissions, cause blocks to be marked as non-pending (no +pending segment transmissions or repairs). + +Also note the oldest object is removed from the "holding queue" when +the count of objects in the holding queue exceeds the "transmit +hold" buffer limits when a new object is queued for transmission by +the server (sender) application. When these objects are removed, any +block or segment resources used by those objects are returned to the +respective server pools. The "transmit hold" buffer limits will be +defined as the minimum of 1) total size of objects in the queue +exceeding a "maxSize" parameter, or 2) total count of objects in the +queue exceeding a "maxCount" parameter. The "maxSize" limit is +overridden if a "minCount" value of greater than one is defined. The +"transmit hold" buffer limits define how far back the NORM server is +willing/able to go to provide repair information previous to its +current oridinal transmission point. + +NORM Client (receiver) resource management. + +The client portion of NORM similarly has pre-determined limits on the +quantity of buffering resources (memory) it will use on a per-server +basis. Again, NORM will still successfully operate when the +bandwidth*delay*loss product of the network topology exceed these +expected/allocated buffer limits. The cost, when this occurs, is a +decrease in the repair _efficiency_ of the NORM protocol. When the +client (receiver) becomes buffer limited, it attempts to "steal" +buffer/segment resources from ordinally _newer_ objects/blocks for +which it has accumulated state (Thus giving priority to the repair of +ordinally oldest objects/blocks). When the bandwidth*delay*loss +product of the network topology causes this to occur, the result is +less efficient repair since the client will either "forget" about some +segments it has already succesfully received (when newer blocks are +"stolen") or ignore received segments for the most ordinally recent +blocks in favor of holding state for older objects/blocks. + +In summary, the NORM protocol will operate most efficiently when there +is ample buffer space, but will still successfully converge when +bandwidth*delay*loss of the network reaches extremes. The amount of +memory used by NORM can be controlled by the application. + + diff --git a/common/version.h b/common/version.h new file mode 100644 index 0000000..572cb6f --- /dev/null +++ b/common/version.h @@ -0,0 +1,37 @@ +/********************************************************************* + * + * AUTHORIZATION TO USE AND DISTRIBUTE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: + * + * (1) source code distributions retain this paragraph in its entirety, + * + * (2) distributions including binary code include this paragraph in + * its entirety in the documentation or other materials provided + * with the distribution, and + * + * (3) all advertising materials mentioning features or use of this + * software display the following acknowledgment: + * + * "This product includes software written and developed + * by Brian Adamson and Joe Macker of the Naval Research + * Laboratory (NRL)." + * + * The name of NRL, the name(s) of NRL employee(s), or any entity + * of the United States Government may not be used to endorse or + * promote products derived from this software, nor does the + * inclusion of the NRL written and developed software directly or + * indirectly suggest NRL or United States Government endorsement + * of this product. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + ********************************************************************/ + +#ifndef _NORM_VERSION +#define _NORM_VERSION +#define VERSION "1.0x1" +#endif // _NORM_VERSION + diff --git a/ns/README.txt b/ns/README.txt new file mode 100644 index 0000000..42b8f3c --- /dev/null +++ b/ns/README.txt @@ -0,0 +1,55 @@ + NORM ns-2 Support + +This directory contains files for incorporating the NORM protocol +into the ns-2 network simulation tool. + +FILES: + +nsNormAgent.h +nsNormAgent.cpp - C++ files which implement an ns-2 derivative of + the NormSimAgent class defined in the + ../common/normSimAgent.h files. + +nackCount.cpp - Source code to count sent & suppressed NACKs + from the reports in NORM debug log files. + (build with "gcc -o nc nackCount.cpp") + +example.tcl - Very simple ns-2 TCL script illustrating the + use of a NORM agent. + +simplenorm.tcl - Parameterized ns-2 TCL script which can be + used to evaluate NORM in a hub & spoke + topology. + +suppress.tcl - Executable TCL script which iteratively + invokes "ns" with the "simplenorm.tcl" + script and "nc" (nackCount) to evaluate + NORM NACK suppression performance. + + + +ns-2.1b7a-Makefile.in - Patched version of ns-2.1b7a Makefile.in + with NORM & PROTOLIB stuff included. + + +TO BUILD NS-2 with NORM Agent capabilities: + +1) Put "protolib" and "norm" source tree directories into the ns-2 + source code directory. + +2) Add PROTOLIB and NORM paths, macros, and, source objects to + the ns-2 "Makefile.in" (See the included patched " + ns-2.1b7-Makefile.in" as an example) + + (A future version of this README will include detailed instructions + to patch ns-2 for PROTOLIB and NORM support) + +3) Run "./configure" in the ns-2 source directory. + +4) Use "make ns" to rebuild ns-2 with the NORM/PROTOLIB support. + + + +Brian Adamson + +11 July 2002 diff --git a/ns/example.tcl b/ns/example.tcl new file mode 100644 index 0000000..3618dbe --- /dev/null +++ b/ns/example.tcl @@ -0,0 +1,96 @@ + +# This script describes and illustrates the commands and usage of the +# NORM ns-2 simulation agent. + +# 1) An ns-2 simulator instance is created an configured +# for multicast operation with a dense mode (DM) multicast +# routing protocol: +set ns_ [new Simulator -multicast on] +$ns_ multicast + + +# 2) Trace files are opened and ns and nam +# tracing is enabled: +set f [open norm.tr w] +#$ns_ trace-all $f +set nf [open norm.nam w] +#$ns_ namtrace-all $nf + +set numNodes 10 + +# 3) A simple hub and spoke topology is created with +# a NORM agent attached to each node + +# Node 0 is the hub +set n(0) [$ns_ node] + +puts "Creating nodes and norm agents ..." +for {set i 1} {$i <= $numNodes} {incr i} { + set n($i) [$ns_ node] + set norm($i) [new Agent/NORM] + $ns_ attach-agent $n($i) $norm($i) +} + +puts "Creating topology ..." +for {set i 1} {$i <= $numNodes} {incr i} { + $ns_ duplex-link $n(0) $n($i) 64kb 100ms DropTail + $ns_ queue-limit $n(0) $n($i) 100 + #$ns_ duplex-link-op $n(0) $n($i) orient right + $ns_ duplex-link-op $n(0) $n($i) queuePos 0.5 +} + +# 4) Configure multicast routing as needed +set mproto DM +set mrthandle [$ns_ mrtproto $mproto {}] +if {$mrthandle != ""} { + $mrthandle set_c_rp [list $n(0)] +} + +# 4) Allocate a multicast address to use +set group [Node allocaddr] + +puts "Configuring NORM agents ..." + +# 5) Configure global NORM agent parameters (debugging/logging is global) +# (Uncomment the "log" command to direct NORM debug output to a file) +$norm(1) debug 2 +#$norm(1) log normLog.txt + +# 6) Configure NORM server agent at node 1 +$norm(1) address $group/5000 +$norm(1) rate 32000 +$norm(1) block 1 +$norm(1) parity 0 +$norm(1) repeat -1 +$norm(1) interval 0.0 +$norm(1) txloss 10.0 +# Enabl NORM message tracing at the server +#$norm(1) trace + +# 7) Configure NORM client agents at other nodes +for {set i 2} {$i <= $numNodes} {incr i} { + $norm($i) address $group/5000 + $norm($i) rxbuffer 100000 + $norm($i) txloss 10.0 + #$norm($i) trace +} + +# 8) Start server and clients +$ns_ at 0.0 "$norm(1) start server" +for {set i 2} {$i <= $numNodes} {incr i} { + $ns_ at 0.0 "$norm($i) start client" +} +$ns_ at 0.0 "$norm(1) sendFile 64000" + +$ns_ at 3000.0 "finish $ns_ $f $nf" + +proc finish {ns_ f nf} { + $ns_ flush-trace + close $f + close $nf + $ns_ halt + delete $ns_ +} + +$ns_ run + diff --git a/ns/nackCount.cpp b/ns/nackCount.cpp new file mode 100644 index 0000000..99010b9 --- /dev/null +++ b/ns/nackCount.cpp @@ -0,0 +1,266 @@ + #include + #include + #include + #include // for PATH_MAX + #include + + #define MIN(X,Y) ((XY)?X:Y) + + class Client + { + friend class ClientList; + + public: + Client(unsigned long theId); + unsigned long Id() {return id;} + unsigned long Sent() {return sent;} + void SetSent(unsigned long count) {sent = count;} + unsigned long Suppressed() {return suppressed;} + void SetSuppressed(unsigned long count) {suppressed = count;} + Client* Next() {return next;} + + private: + unsigned long id; + unsigned long sent; + unsigned long suppressed; + Client* next; + }; + + + + class ClientList + { + public: + ClientList(); + ~ClientList(); + void AddClient(Client* theClient); + void Destroy(); + Client* FindClientById(unsigned long theId); + Client* Top() {return top;} + + private: + Client* top; + }; + + + Client::Client(unsigned long theId) + : id(theId), sent(0), suppressed(0), next(NULL) + { + } + + ClientList::ClientList() + : top(NULL) + { + } + + ClientList::~ClientList() + { + Destroy(); + } + + void ClientList::AddClient(Client* theClient) + { + theClient->next = top; + top = theClient; + } // end ClientList::AddClient() + + void ClientList::Destroy() + { + while (top) + { + Client* next = top->next; + delete top; + top = next; + } + } // end ClientList::Destroy() + + Client* ClientList::FindClientById(unsigned long theId) + { + Client* next = top; + while(next) + { + if (theId == next->id) + return next; + else + next = next->next; + } + return NULL; + } // end ClientList::FindClientById() + + + + + const int MAX_LINE = 256; + + class FastReader + { + public: + FastReader(); + bool Read(FILE* filePtr, char* buffer, unsigned int* len); + bool Readline(FILE* filePtr, char* buffer, unsigned int* len); + + private: + char savebuf[256]; + char* saveptr; + unsigned int savecount; + }; // end class FastReader + + FastReader::FastReader() + : savecount(0) + { + + } + + bool FastReader::Read(FILE* filePtr, char* buffer, unsigned int* len) + { + unsigned int want = *len; + if (savecount) + { + unsigned int ncopy = MIN(want, savecount); + memcpy(buffer, saveptr, ncopy); + savecount -= ncopy; + saveptr += ncopy; + buffer += ncopy; + want -= ncopy; + } + while (want) + { + unsigned int result = fread(savebuf, sizeof(char), 256, filePtr); + if (result) + { + unsigned int ncopy= MIN(want, result); + memcpy(buffer, savebuf, ncopy); + savecount = result - ncopy; + saveptr = savebuf + ncopy; + buffer += ncopy; + want -= ncopy; + } + else // end-of-file + { + *len -= want; + if (*len) + return true; // we read something + else + return false; // we read nothing + } + } + return true; + } // end FastReader::Read() + + // An OK text readline() routine (reads what will fit into buffer incl. NULL termination) + // if *len is unchanged on return, it means the line is bigger than the buffer and + // requires multiple reads + +bool FastReader::Readline(FILE* filePtr, char* buffer, unsigned int* len) +{ + unsigned int count = 0; + unsigned int length = *len; + char* ptr = buffer; + unsigned int one = 1; + while ((count < length) && Read(filePtr, ptr, &one)) + { + if (('\n' == *ptr) || ('\r' == *ptr)) + { + *ptr = '\0'; + *len = count; + return true; + } + count++; + ptr++; + } + // Either we've filled the buffer or hit end-of-file + if (count < length) *len = 0; // Set *len = 0 on EOF + return false; +} // end FastReader::Readline() + +int main(int argc, char* argv[]) +{ + ClientList list; + FastReader reader; + char buffer[512]; + unsigned long line = 0; + unsigned int len = 512; + while (reader.Readline(stdin, buffer, &len)) + { + line++; + if (!strncmp(buffer, "Report", 6)) + { + char* ptr = strstr(buffer, "node>"); + if (!ptr) break; + unsigned long nodeId; + if (1 != sscanf(ptr, "node>%lu", &nodeId)) + { + fprintf(stderr, "nackCount: Error parsing log file " + "at line: %lu\n", line); + exit(-1); + } + // Update NACKs transmitted: + len = 512; + while (reader.Readline(stdin, buffer, &len)) + { + line++; + if ('*' == buffer[0]) break; // end of report + len = 512; + //fprintf(stderr, "Searching: \"%s\"\n", buffer); + ptr = strstr(buffer, "nacks>"); + if (ptr) + { + Client* client = list.FindClientById(nodeId); + if (!client) + { + if (!(client = new Client(nodeId))) + { + perror("nackCount: Error creating Client:"); + exit(-1); + } + list.AddClient(client); + } + unsigned long count; + if (1 == sscanf(ptr, "nacks>%lu", &count)) + { + client->SetSent(count); + } + else + { + fprintf(stderr, "Error parsing log file " + "at line: %lu \n", line); + exit(-1); + } + if ((ptr = strstr(buffer, "suppressed>"))) + { + if (1 == sscanf(ptr, "suppressed>%lu", &count)) + { + client->SetSuppressed(count); + break; + } + } + else + { + fprintf(stderr, "Error parsing log file " + "at line: %lu \n", line); + exit(-1); + } + } + } // end while (reader.Readline("nacks/suppressed" search)) + } + len = 512; + } // end while (reader.Readline("Report" search)) + + unsigned long numClient = 0; + Client* next = list.Top(); + unsigned long totalSent = 0; + unsigned long totalSuppressed = 0; + while (next) + { + totalSent += next->Sent(); + totalSuppressed += next->Suppressed(); + next = next->Next(); + numClient++; + } + + fprintf(stdout, "Receivers:%lu Sent:%lu Suppressed:%lu alpha:%f\n", + numClient, totalSent, totalSuppressed, + (double)totalSent / (double) (totalSuppressed+totalSent)); +} // end main() + diff --git a/ns/ns-2.1b7a-Makefile.in b/ns/ns-2.1b7a-Makefile.in new file mode 100644 index 0000000..66124d8 --- /dev/null +++ b/ns/ns-2.1b7a-Makefile.in @@ -0,0 +1,432 @@ +# Copyright (c) 1994, 1995, 1996 +# The Regents of the University of California. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that: (1) source code distributions +# retain the above copyright notice and this paragraph in its entirety, (2) +# distributions including binary code include the above copyright notice and +# this paragraph in its entirety in the documentation or other materials +# provided with the distribution, and (3) all advertising materials mentioning +# features or use of this software display the following acknowledgement: +# ``This product includes software developed by the University of California, +# Lawrence Berkeley Laboratory and its contributors.'' Neither the name of +# the University nor the names of its contributors may be used to endorse +# or promote products derived from this software without specific prior +# written permission. +# THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. +# +# @(#) $Header: /cvsroot/norm/norm/ns/ns-2.1b7a-Makefile.in,v 1.1 2002/07/11 15:25:11 adamson Exp $ (LBL) + +# +# Various configurable paths (remember to edit Makefile.in, not Makefile) +# + +# Top level hierarchy +prefix = @prefix@ +# Pathname of directory to install the binary +BINDEST = @prefix@/bin +# Pathname of directory to install the man page +MANDEST = @prefix@/man + +BLANK = # make a blank space. DO NOT add anything to this line + +# The following will be redefined under Windows (see WIN32 lable below) +CC = @CC@ +CPP = @CXX@ +LINK = $(CPP) +MKDEP = ./conf/mkdep +TCLSH = @V_TCLSH@ +TCL2C = @V_TCL2CPP@ +AR = ar rc $(BLANK) + +RANLIB = @V_RANLIB@ +INSTALL = @INSTALL@ +LN = ln +TEST = test +RM = rm -f +PERL = @PERL@ + + +CCOPT = @V_CCOPT@ +STATIC = @V_STATIC@ +LDFLAGS = $(STATIC) +LDOUT = -o $(BLANK) + +DEFINE = -DTCP_DELAY_BIND_ALL -DNO_TK @V_DEFINE@ @V_DEFINES@ @DEFS@ + +PROTOLIB_INCLUDES = -Iprotolib/common -Iprotolib/ns +MDP_INCLUDES = -Imdp/common -Imdp/ns -Imdp/unix +NORM_INCLUDES = -Inorm/common -Inorm/ns + +INCLUDES = \ + -I. @V_INCLUDE_X11@ \ + @V_INCLUDES@ \ + $(PROTOLIB_INCLUDES) $(MDP_INCLUDES) $(NORM_INCLUDES) + +LIB = \ + @V_LIBS@ \ + @V_LIB_X11@ \ + @V_LIB@ \ + -lm @LIBS@ +# -L@libdir@ \ + +# These flags work on Linux +PROTOLIB_CFLAGS = -DUNIX -DNS2 -DPROTO_DEBUG -DSIMULATE -DHAVE_ASSERT -DHAVE_DIRFD + +CFLAGS = $(CCOPT) $(DEFINE) -g $(PROTOLIB_CFLAGS) + +# Explicitly define compilation rules since SunOS 4's make doesn't like gcc. +# Also, gcc does not remove the .o before forking 'as', which can be a +# problem if you don't own the file but can write to the directory. +# PROTOLIB/MDP/NORM code uses .cpp files so we added a suffix and rule +.SUFFIXES: .cc .cpp # $(.SUFFIXES) + +.cpp.o: + @rm -f $@ + $(CPP) -c $(CFLAGS) $(INCLUDES) -o $@ $*.cpp + +.cc.o: + @rm -f $@ + $(CPP) -c $(CFLAGS) $(INCLUDES) -o $@ $*.cc + +.c.o: + @rm -f $@ + $(CC) -c $(CFLAGS) $(INCLUDES) -o $@ $*.c + + +GEN_DIR = gen/ +LIB_DIR = lib/ +NS = ns +NSX = nsx +NSE = nse + +# To allow conf/makefile.win overwrite this macro +OBJ_STL = @V_STLOBJ@ + +# WIN32: uncomment the following line to include specific make for VC++ +# !include + +OBJ_CC = \ + random.o rng.o ranvar.o misc.o timer-handler.o \ + scheduler.o object.o \ + packet.o ip.o route.o connector.o ttl.o \ + trace.o trace-ip.o \ + classifier.o classifier-addr.o classifier-hash.o classifier-virtual.o \ + classifier-mcast.o classifier-bst.o classifier-mpath.o replicator.o \ + classifier-mac.o classifier-port.o ump.o \ + app.o telnet.o tcplib-telnet.o \ + trafgen.o traffictrace.o pareto.o expoo.o cbr_traffic.o \ + tbf.o resv.o sa.o saack.o \ + measuremod.o estimator.o adc.o ms-adc.o timewindow-est.o acto-adc.o \ + pointsample-est.o salink.o actp-adc.o hb-adc.o expavg-est.o\ + param-adc.o null-estimator.o \ + adaptive-receiver.o vatrcvr.o consrcvr.o \ + agent.o message.o udp.o session-rtp.o rtp.o rtcp.o ivs.o \ + tcp.o tcp-sink.o tcp-reno.o tcp-newreno.o \ + tcp-vegas.o tcp-rbp.o tcp-full.o \ + scoreboard.o tcp-sack1.o tcp-fack.o \ + tcp-asym.o tcp-asym-sink.o tcp-fs.o tcp-asym-fs.o \ + tcp-int.o chost.o tcp-session.o nilist.o \ + integrator.o queue-monitor.o flowmon.o loss-monitor.o \ + queue.o drop-tail.o simple-intserv-sched.o red.o \ + semantic-packetqueue.o semantic-red.o ack-recons.o \ + sfq.o fq.o drr.o cbq.o \ + hackloss.o errmodel.o \ + delay.o snoop.o \ + dynalink.o rtProtoDV.o net-interface.o \ + ctrMcast.o mcast_ctrl.o srm.o \ + sessionhelper.o delaymodel.o srm-ssm.o \ + srm-topo.o \ + mftp.o mftp_snd.o mftp_rcv.o codeword.o \ + alloc-address.o address.o \ + $(LIB_DIR)int.Vec.o $(LIB_DIR)int.RVec.o \ + $(LIB_DIR)dmalloc_support.o \ + webcache/http.o webcache/tcp-simple.o webcache/pagepool.o \ + webcache/inval-agent.o webcache/tcpapp.o webcache/http-aux.o \ + webcache/mcache.o webcache/webtraf.o \ + realaudio/realaudio.o \ + lanRouter.o filter.o pkt-counter.o \ + Decapsulator.o Encapsulator.o encap.o \ + channel.o mac.o ll.o mac-802_11.o mac-802_3.o mac-tdma.o \ + mip.o mip-reg.o gridkeeper.o \ + propagation.o tworayground.o antenna.o omni-antenna.o \ + shadowing.o bi-connector.o node.o mobilenode.o \ + arp.o god.o dem.o topography.o modulation.o priqueue.o \ + phy.o wired-phy.o wireless-phy.o \ + mac-timers.o cmu-trace.o varp.o \ + dsdv/dsdv.o dsdv/rtable.o rtqueue.o rttable.o \ + imep/imep.o imep/dest_queue.o imep/imep_api.o \ + imep/imep_rt.o imep/rxmit_queue.o imep/imep_timers.o \ + imep/imep_util.o imep/imep_io.o \ + tora/tora.o tora/tora_api.o tora/tora_dest.o tora/tora_io.o \ + tora/tora_logs.o tora/tora_neighbor.o \ + dsr/dsragent.o dsr/hdr_sr.o dsr/mobicache.o dsr/path.o \ + dsr/requesttable.o dsr/routecache.o \ + aodv/aodv_logs.o aodv/aodv.o \ + ns-process.o \ + satgeometry.o sathandoff.o satlink.o satnode.o \ + satposition.o satroute.o sattrace.o \ + rap/raplist.o rap/rap.o rap/media-app.o rap/utilities.o \ + fsm.o tcp-abs.o \ + diffusion/diffusion.o diffusion/diff_rate.o diffusion/diff_prob.o \ + diffusion/diff_sink.o diffusion/flooding.o diffusion/omni_mcast.o \ + diffusion/hash_table.o diffusion/routing_table.o diffusion/iflist.o \ + tfrc.o tfrc-sink.o energy-model.o ping.o tcp-rfc793edu.o \ + rio.o semantic-rio.o tcp-sack-rh.o scoreboard-rh.o \ + plm/loss-monitor-plm.o plm/cbr-traffic-PP.o \ + linkstate/hdr-ls.o \ + mpls/classifier-addr-mpls.o mpls/ldp.o mpls/mpls-module.o \ + rtmodule.o classifier-hier.o addr-params.o \ + $(OBJ_STL) + +# don't allow comments to follow continuation lines + +# mac-csma.o mac-multihop.o\ +# sensor-nets/landmark.o mac-simple-wireless.o \ +# sensor-nets/tags.o sensor-nets/sensor-query.o \ +# sensor-nets/flood-agent.o \ + +# what was here before is now in emulate/ +OBJ_C = + +OBJ_COMPAT = $(OBJ_GETOPT) win32.o +#XXX compat/win32x.o compat/tkConsole.o + +OBJ_EMULATE_CC = \ + emulate/net-ip.o \ + emulate/net.o \ + emulate/tap.o \ + emulate/ether.o \ + emulate/internet.o \ + emulate/ping_responder.o \ + emulate/arp.o \ + emulate/icmp.o \ + emulate/net-pcap.o \ + emulate/nat.o + +OBJ_EMULATE_C = \ + emulate/inet.o + +OBJ_PROTOLIB_CPP = \ + protolib/ns/nsProtoAgent.o protolib/common/protoSim.o \ + protolib/common/networkAddress.o protolib/common/protocolTimer.o \ + protolib/common/debug.o + +OBJ_MDP_CPP = \ + mdp/ns/nsMdpAgent.o mdp/common/mdpSimAgent.o \ + mdp/common/mdpBitMask.o mdp/common/mdpMessage.o \ + mdp/common/mdpEncoder.o mdp/common/galois.o \ + mdp/common/mdpSession.o mdp/common/mdpMsgHandler.o \ + mdp/common/mdpNode.o mdp/common/mdpObject.o \ + mdp/unix/mdpFile.o + +OBJ_NORM_CPP = \ + norm/ns/nsNormAgent.o norm/common/normSimAgent.o \ + norm/common/normMessage.o norm/common/normSession.o \ + norm/common/normNode.o norm/common/normObject.o \ + norm/common/normSegment.o norm/common/normBitmask.o \ + norm/common/normEncoder.o norm/common/galois.o \ + norm/common/normFile.o + +OBJ_GEN = $(GEN_DIR)version.o $(GEN_DIR)ns_tcl.o $(GEN_DIR)ptypes.o + +SRC = $(OBJ_C:.o=.c) $(OBJ_CC:.o=.cc) \ + $(OBJ_EMULATE_C:.o=.c) $(OBJ_EMULATE_CC:.o=.cc) \ + tclAppInit.cc tkAppInit.cc \ + $(OBJ_PROTOLIB_CPP:.o=.cpp) $(OBJ_MDP_CPP:.o=.cpp) \ + $(OBJ_NORM_CPP:.o=.cpp) + +OBJ = $(OBJ_C) $(OBJ_CC) $(OBJ_GEN) $(OBJ_COMPAT) \ + $(OBJ_PROTOLIB_CPP) $(OBJ_MDP_CPP) $(OBJ_NORM_CPP) + +CLEANFILES = ns nsx ns.dyn $(OBJ) $(OBJ_EMULATE_CC) \ + $(OBJ_EMULATE_C) tclAppInit.o \ + $(GEN_DIR)* $(NS).core core core.$(NS) core.$(NSX) core.$(NSE) \ + ptypes2tcl ptypes2tcl.o + +SUBDIRS=\ + indep-utils/cmu-scen-gen/setdest \ + indep-utils/webtrace-conv/dec \ + indep-utils/webtrace-conv/epa \ + indep-utils/webtrace-conv/nlanr \ + indep-utils/webtrace-conv/ucb + +all: $(NS) all-recursive + +all-recursive: + for i in $(SUBDIRS); do ( cd $$i; $(MAKE) all; ) done + +$(NS): $(OBJ) tclAppInit.o Makefile + $(LINK) $(LDFLAGS) $(LDOUT)$@ \ + tclAppInit.o $(OBJ) $(LIB) + +Makefile: Makefile.in + @echo "Makefile.in is newer than Makefile." + @echo "You need to re-run configure." + false + +$(NSE): $(OBJ) tclAppInit.o $(OBJ_EMULATE_CC) $(OBJ_EMULATE_C) + $(LINK) $(LDFLAGS) $(LDOUT)$@ \ + tclAppInit.o $(OBJ) \ + $(OBJ_EMULATE_CC) $(OBJ_EMULATE_C) $(LIB) -lpcap + +ns.dyn: $(OBJ) tclAppInit.o + $(LINK) $(LDFLAGS) -o $@ \ + tclAppInit.o $(OBJ) $(LIB) + +PURIFY = purify -cache-dir=/tmp +ns-pure: $(OBJ) tclAppInit.o + $(PURIFY) $(LINK) $(LDFLAGS) -o $@ \ + tclAppInit.o $(OBJ) $(LIB) + +NS_TCL_LIB = \ + tcl/lib/ns-compat.tcl \ + tcl/lib/ns-default.tcl \ + tcl/lib/ns-errmodel.tcl \ + tcl/lib/ns-lib.tcl \ + tcl/lib/ns-link.tcl \ + tcl/lib/ns-mobilenode.tcl \ + tcl/lib/ns-sat.tcl \ + tcl/lib/ns-cmutrace.tcl \ + tcl/lib/ns-node.tcl \ + tcl/lib/ns-rtmodule.tcl \ + tcl/lib/ns-hiernode.tcl \ + tcl/lib/ns-packet.tcl \ + tcl/lib/ns-queue.tcl \ + tcl/lib/ns-source.tcl \ + tcl/lib/ns-nam.tcl \ + tcl/lib/ns-trace.tcl \ + tcl/lib/ns-agent.tcl \ + tcl/lib/ns-random.tcl \ + tcl/lib/ns-namsupp.tcl \ + tcl/lib/ns-address.tcl \ + tcl/lib/ns-intserv.tcl \ + tcl/lib/ns-autoconf.tcl \ + tcl/rtp/session-rtp.tcl \ + tcl/lib/ns-mip.tcl \ + tcl/rtglib/dynamics.tcl \ + tcl/rtglib/route-proto.tcl \ + tcl/rtglib/algo-route-proto.tcl \ + tcl/rtglib/ns-rtProtoLS.tcl \ + tcl/interface/ns-iface.tcl \ + tcl/mcast/BST.tcl \ + tcl/mcast/ns-mcast.tcl \ + tcl/mcast/McastProto.tcl \ + tcl/mcast/DM.tcl \ + tcl/mcast/srm.tcl \ + tcl/mcast/srm-adaptive.tcl \ + tcl/mcast/srm-ssm.tcl \ + tcl/mcast/timer.tcl \ + tcl/mcast/McastMonitor.tcl \ + tcl/mcast/mftp_snd.tcl \ + tcl/mcast/mftp_rcv.tcl \ + tcl/mcast/mftp_rcv_stat.tcl \ + tcl/mobility/dsdv.tcl \ + tcl/mobility/dsr.tcl \ + tcl/ctr-mcast/CtrMcast.tcl \ + tcl/ctr-mcast/CtrMcastComp.tcl \ + tcl/ctr-mcast/CtrRPComp.tcl \ + tcl/rlm/rlm.tcl \ + tcl/rlm/rlm-ns.tcl \ + tcl/session/session.tcl \ + tcl/lib/ns-route.tcl \ + tcl/emulate/ns-emulate.tcl \ + tcl/lan/vlan.tcl \ + tcl/lan/ns-ll.tcl \ + tcl/lan/ns-mac.tcl \ + tcl/webcache/http-agent.tcl \ + tcl/webcache/http-server.tcl \ + tcl/webcache/http-cache.tcl \ + tcl/webcache/http-mcache.tcl \ + tcl/webcache/webtraf.tcl \ + tcl/plm/plm.tcl \ + tcl/plm/plm-ns.tcl \ + tcl/plm/plm-topo.tcl \ + tcl/mpls/ns-mpls-classifier.tcl \ + tcl/mpls/ns-mpls-ldpagent.tcl \ + tcl/mpls/ns-mpls-node.tcl \ + tcl/mpls/ns-mpls-simulator.tcl + +$(GEN_DIR)ns_tcl.cc: $(NS_TCL_LIB) + $(TCLSH) bin/tcl-expand.tcl tcl/lib/ns-lib.tcl tcl/lib/ns-stl.tcl | $(TCL2C) et_ns_lib > $@ + +$(GEN_DIR)version.c: VERSION + $(RM) $@ + $(TCLSH) bin/string2c.tcl version_string < VERSION > $@ + +$(GEN_DIR)ptypes.cc: ptypes2tcl packet.h + ./ptypes2tcl > $@ + +ptypes2tcl: ptypes2tcl.o + $(LINK) $(LDFLAGS) $(LDOUT)$@ ptypes2tcl.o + +ptypes2tcl.o: ptypes2tcl.cc packet.h + +install: force install-ns install-man install-recursive + +install-ns: force + $(INSTALL) -m 555 -o bin -g bin ns $(DESTDIR)$(BINDEST) + +install-man: force + $(INSTALL) -m 444 -o bin -g bin ns.1 $(DESTDIR)$(MANDEST)/man1 + +install-recursive: force + for i in $(SUBDIRS); do cd $$i; $(MAKE) install; done + +clean: + $(RM) $(CLEANFILES) + +AUTOCONF_GEN = tcl/lib/ns-autoconf.tcl tcl/lib/ns-stl.tcl +distclean: + $(RM) $(CLEANFILES) Makefile config.cache config.log config.status \ + gnuc.h os-proto.h $(AUTOCONF_GEN) + +tags: force + ctags -wtd *.cc *.h webcache/*.cc webcache/*.h dsdv/*.cc dsdv/*.h \ + dsr/*.cc dsr/*.h webcache/*.cc webcache/*.h lib/*.cc lib/*.h \ + ../Tcl/*.cc ../Tcl/*.h + +TAGS: force + etags *.cc *.h webcache/*.cc webcache/*.h dsdv/*.cc dsdv/*.h \ + dsr/*.cc dsr/*.h webcache/*.cc webcache/*.h lib/*.cc lib/*.h \ + ../Tcl/*.cc ../Tcl/*.h + +tcl/lib/TAGS: force + ( \ + cd tcl/lib; \ + $(TCLSH) ../../bin/tcl-expand.tcl ns-lib.tcl | grep '^### tcl-expand.tcl: begin' | awk '{print $$5}' >.tcl_files; \ + etags --lang=none -r '/^[ \t]*proc[ \t]+\([^ \t]+\)/\1/' `cat .tcl_files`; \ + etags --append --lang=none -r '/^\([A-Z][^ \t]+\)[ \t]+\(instproc\|proc\)[ \t]+\([^ \t]+\)[ \t]+/\1::\3/' `cat .tcl_files`; \ + ) + +depend: $(SRC) + $(MKDEP) $(CFLAGS) $(INCLUDES) $(SRC) + +srctar: + @cwd=`pwd` ; dir=`basename $$cwd` ; \ + name=ns-`cat VERSION | tr A-Z a-z` ; \ + tar=ns-src-`cat VERSION`.tar.gz ; \ + list="" ; \ + for i in `cat FILES` ; do list="$$list $$name/$$i" ; done; \ + echo \ + "(rm -f $$tar; cd .. ; ln -s $$dir $$name)" ; \ + (rm -f $$tar; cd .. ; ln -s $$dir $$name) ; \ + echo \ + "(cd .. ; tar cfh $$tar [lots of files])" ; \ + (cd .. ; tar cfh - $$list) | gzip -c > $$tar ; \ + echo \ + "rm ../$$name; chmod 444 $$tar" ; \ + rm ../$$name; chmod 444 $$tar + +force: + +test: force + ./validate + +# Create makefile.vc for Win32 development by replacing: +# "# !include ..." -> "!include ..." +makefile.vc: Makefile.in + $(PERL) bin/gen-vcmake.pl < Makefile.in > makefile.vc +# $(PERL) -pe 's/^# (\!include)/\!include/o' < Makefile.in > makefile.vc diff --git a/ns/nsNormAgent.cpp b/ns/nsNormAgent.cpp new file mode 100644 index 0000000..5d4afa2 --- /dev/null +++ b/ns/nsNormAgent.cpp @@ -0,0 +1,97 @@ +/********************************************************************* + * + * AUTHORIZATION TO USE AND DISTRIBUTE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: + * + * (1) source code distributions retain this paragraph in its entirety, + * + * (2) distributions including binary code include this paragraph in + * its entirety in the documentation or other materials provided + * with the distribution, and + * + * (3) all advertising materials mentioning features or use of this + * software display the following acknowledgment: + * + * "This product includes software written and developed + * by Brian Adamson and Joe Macker of the Naval Research + * Laboratory (NRL)." + * + * The name of NRL, the name(s) of NRL employee(s), or any entity + * of the United States Government may not be used to endorse or + * promote products derived from this software, nor does the + * inclusion of the NRL written and developed software directly or + * indirectly suggest NRL or United States Government endorsement + * of this product. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + ********************************************************************/ + +#include "ip.h" // for hdr_ip def +#include "flags.h" // for hdr_flags def + +#include "nsNormAgent.h" + +static class NsNormAgentClass : public TclClass +{ + public: + NsNormAgentClass() : TclClass("Agent/NORM") {} + TclObject *create(int argc, const char*const* argv) + {return (new NsNormAgent());} +} class_norm_agent; + + +NsNormAgent::NsNormAgent() +{ + NormSimAgent::Init(ProtoSimAgent::TimerInstaller, + static_cast(this), + ProtoSimAgent::SocketInstaller, + static_cast(this)); +} + +NsNormAgent::~NsNormAgent() +{ +} + +int NsNormAgent::command(int argc, const char*const* argv) +{ + // Process commands, passing unknown commands to base Agent class + int i = 1; + while (i < argc) + { + NormSimAgent::CmdType cmdType = CommandType(argv[i]); + switch (cmdType) + { + case NormSimAgent::CMD_NOARG: + if (!ProcessCommand(argv[i], NULL)) + { + DMSG(0, "NsNormAgent::command() ProcessCommand(%s) error\n", + argv[i]); + return TCL_ERROR; + } + i++; + break; + + case NormSimAgent::CMD_ARG: + if (!ProcessCommand(argv[i], argv[i+1])) + { + DMSG(0, "NsNormAgent::command() ProcessCommand(%s, %s) error\n", + argv[i], argv[i+1]); + return TCL_ERROR; + } + i += 2; + break; + + case NormSimAgent::CMD_INVALID: + return NsProtoAgent::command(argc, argv); + } + } + return TCL_OK; +} // end NsNormAgent::command() + + + + diff --git a/ns/nsNormAgent.h b/ns/nsNormAgent.h new file mode 100644 index 0000000..10a7154 --- /dev/null +++ b/ns/nsNormAgent.h @@ -0,0 +1,57 @@ +/********************************************************************* + * + * AUTHORIZATION TO USE AND DISTRIBUTE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that: + * + * (1) source code distributions retain this paragraph in its entirety, + * + * (2) distributions including binary code include this paragraph in + * its entirety in the documentation or other materials provided + * with the distribution, and + * + * (3) all advertising materials mentioning features or use of this + * software display the following acknowledgment: + * + * "This product includes software written and developed + * by Brian Adamson and Joe Macker of the Naval Research + * Laboratory (NRL)." + * + * The name of NRL, the name(s) of NRL employee(s), or any entity + * of the United States Government may not be used to endorse or + * promote products derived from this software, nor does the + * inclusion of the NRL written and developed software directly or + * indirectly suggest NRL or United States Government endorsement + * of this product. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + ********************************************************************/ + +#ifndef _NS_NORM_AGENT +#define _NS_NORM_AGENT + +#include "nsProtoAgent.h" // from ProtoLib +#include "normSimAgent.h" + +// The "NsNormAgent" is based on the ns agent class. +// This lets us have a send/recv attachment to the NS +// simulation environment + +// IMPORTANT NOTE! NsProtoAgent must be listed _first_ here +// (because we can't dynamic_cast install_data void* pointers) +class NsNormAgent : public NormSimAgent, public NsProtoAgent +{ + public: + NsNormAgent(); + ~NsNormAgent(); + + // NsProtoAgent base class overrides + int command(int argc, const char*const* argv); + unsigned long GetAgentId() {return (unsigned long)addr();} +}; // end class NsNormAgent + + +#endif // _NS_NORM_AGENT diff --git a/ns/simplenorm.tcl b/ns/simplenorm.tcl new file mode 100644 index 0000000..5d6d9b4 --- /dev/null +++ b/ns/simplenorm.tcl @@ -0,0 +1,167 @@ + +# This script describes and illustrates the commands and usage of the +# NORM ns-2 simulation agent. + +proc SimpleNORM {optionList} { + +#Some default parameters for NORM +set groupSize "10" +set backoffFactor "4.0" +set sendRate "64kb" +set duration "120.0" +set nsTracing 0 + +#Parse optionList for parameters +set state flag +foreach option $optionList { + switch -- $state { + "flag" { + switch -glob -- $option { + "gsize" {set state "gsize"} + "rate" {set state "rate"} + "backoff" {set state "backoff"} + "duration" {set state "duration"} + "trace" {set nsTracing 1} + default {error "simplenorm: Bad option argument $option"} + } + + } + "gsize" { + set groupSize $option + set state "flag" + } + "backoff" { + set backoffFactor $option + set state "flag" + } + "rate" { + set sendRate $option + set state "flag" + } + "duration" { + set duration $option + set state "flag" + } + default { + error "simplenorm: Bad option parse state!" + } + } +} + + + +# 1) An ns-2 simulator instance is created an configured +# for multicast operation with a dense mode (DM) multicast +# routing protocol: +set ns_ [new Simulator -multicast on] +$ns_ multicast + + +# 2) Trace files are opened and ns and nam +# tracing is enabled: +if {$nsTracing} { + set f [open simplenorm.tr w] + $ns_ trace-all $f + set nf [open simplenorm.nam w] + $ns_ namtrace-all $nf +} else { + set f 0 + set nf 0 +} + +set numNodes [expr $groupSize + 1] + +# 3) A simple hub and spoke topology is created with +# a NORM agent attached to each spoke node + +# Node 0 is the hub of our hub & spoke topology +# Note there is _not_ a NORM agent at the hub. +set n(0) [$ns_ node] + +puts "simplenorm: Creating $numNodes nodes with norm agents ..." +for {set i 1} {$i <= $numNodes} {incr i} { + set n($i) [$ns_ node] + set norm($i) [new Agent/NORM] + $ns_ attach-agent $n($i) $norm($i) +} + +puts "simplenorm: Creating spoke links ..." +set linkRate [expr $groupSize * [bw_parse $sendRate]] +for {set i 1} {$i <= $numNodes} {incr i} { + $ns_ duplex-link $n(0) $n($i) $linkRate 100ms DropTail + $ns_ queue-limit $n(0) $n($i) 100 + #$ns_ duplex-link-op $n(0) $n($i) orient right + $ns_ duplex-link-op $n(0) $n($i) queuePos 0.5 +} + +# 4) Configure multicast routing for topology +set mproto DM +set mrthandle [$ns_ mrtproto $mproto {}] + if {$mrthandle != ""} { + $mrthandle set_c_rp [list $n(0)] +} + +# 5) Allocate a multicast address to use +set group [Node allocaddr] + +puts "simplenorm: Configuring NORM agents ..." + +# 6) Configure global NORM agent commands (using norm(1)) +$norm(1) debug 2 +$norm(1) log normLog.txt + +# 7) Configure NORM server agent at node 1 +$norm(1) address $group/5000 +$norm(1) rate [bw_parse $sendRate] +$norm(1) backoff $backoffFactor +$norm(1) parity 0 +$norm(1) repeat -1 +$norm(1) interval 0.0 +$norm(1) txloss 10.0 +$norm(1) gsize $groupSize +#$norm(1) trace + +# 8) Configure NORM client agents at other nodes +for {set i 2} {$i <= $numNodes} {incr i} { + $norm($i) address $group/5000 + $norm($i) backoff $backoffFactor + $norm($i) rxbuffer 100000 + $norm($i) txloss 10.0 + $norm($i) gsize $groupSize + #$norm($i) trace +} + +# 9) Start server and clients +$ns_ at 0.0 "$norm(1) start server" +for {set i 2} {$i <= $numNodes} {incr i} { + $ns_ at 0.0 "$norm($i) start client" +} + +$ns_ at 0.0 "$norm(1) sendFile 64000" + +$ns_ at $duration "finish $ns_ $f $nf $nsTracing" + +puts "simplenorm: Running simulation (gsize:$groupSize rate:$sendRate backoff $backoffFactor duration:$duration) ..." +$ns_ run + +} + +proc finish {ns_ f nf nsTracing} { + puts "simplenorm: Done." + $ns_ flush-trace + if {$nsTracing} { + close $f + close $nf + } + $ns_ halt + delete $ns_ +} + +# Run a set of trials with optional command-line parameters + +#Usage: +#ns simplenorm.tcl [gsize ][rate ][backoff ][duration ][trace] + +puts "Running simplenorm: $argv" +SimpleNORM $argv + diff --git a/ns/sizeAve.cpp b/ns/sizeAve.cpp new file mode 100644 index 0000000..4b674a6 --- /dev/null +++ b/ns/sizeAve.cpp @@ -0,0 +1,132 @@ + #include + #include + #include + #include // for PATH_MAX + #include + + #define MIN(X,Y) ((XY)?X:Y) + + const int MAX_LINE = 256; + + class FastReader + { + public: + FastReader(); + bool Read(FILE* filePtr, char* buffer, unsigned int* len); + bool Readline(FILE* filePtr, char* buffer, unsigned int* len); + + private: + char savebuf[MAX_LINE]; + char* saveptr; + unsigned int savecount; + }; // end class FastReader + + FastReader::FastReader() + : savecount(0) + { + + } + + bool FastReader::Read(FILE* filePtr, char* buffer, unsigned int* len) + { + unsigned int want = *len; + if (savecount) + { + unsigned int ncopy = MIN(want, savecount); + memcpy(buffer, saveptr, ncopy); + savecount -= ncopy; + saveptr += ncopy; + buffer += ncopy; + want -= ncopy; + } + while (want) + { + unsigned int result = fread(savebuf, sizeof(char), MAX_LINE, filePtr); + if (result) + { + unsigned int ncopy= MIN(want, result); + memcpy(buffer, savebuf, ncopy); + savecount = result - ncopy; + saveptr = savebuf + ncopy; + buffer += ncopy; + want -= ncopy; + } + else // end-of-file + { + *len -= want; + if (*len) + return true; // we read something + else + return false; // we read nothing + } + } + return true; + } // end FastReader::Read() + + // An OK text readline() routine (reads what will fit into buffer incl. NULL termination) + // if *len is unchanged on return, it means the line is bigger than the buffer and + // requires multiple reads + +bool FastReader::Readline(FILE* filePtr, char* buffer, unsigned int* len) +{ + unsigned int count = 0; + unsigned int length = *len; + char* ptr = buffer; + unsigned int one = 1; + while ((count < length) && Read(filePtr, ptr, &one)) + { + if (('\n' == *ptr) || ('\r' == *ptr)) + { + *ptr = '\0'; + *len = count; + return true; + } + count++; + ptr++; + } + // Either we've filled the buffer or hit end-of-file + if (count < length) *len = 0; // Set *len = 0 on EOF + return false; +} // end FastReader::Readline() + +int main(int argc, char* argv[]) +{ + FastReader reader; + char buffer[MAX_LINE]; + unsigned long line = 0; + unsigned int len = MAX_LINE; + const char* header = "NormSession::ServerUpdateGroupSize"; + unsigned int headerLen = strlen(header); + + double totalSize = 0.0; + unsigned long sizeCount = 0; + while (reader.Readline(stdin, buffer, &len)) + { + line++; + if (!strncmp(buffer, header, headerLen)) + { + char* ptr = strstr(buffer, "size:"); + if (!ptr) break; + double size; + if (1 != sscanf(ptr, "size: %lf", &size)) + { + fprintf(stderr, "sizeAve: Error parsing log file " + "at line: %lu\n", line); + exit(-1); + } + totalSize += size; + sizeCount++; + } + len = MAX_LINE; + } // end while (reader.Readline("Report" search)) + + double sizeAve = 0.0; + if (sizeCount) + { + sizeAve = totalSize / (double)sizeCount; + } + + fprintf(stdout, "Average group size estimate:%lf\n", sizeAve); +} // end main() + diff --git a/ns/suppress.tcl b/ns/suppress.tcl new file mode 100755 index 0000000..9a573c0 --- /dev/null +++ b/ns/suppress.tcl @@ -0,0 +1,55 @@ +#!/usr/bin/tclsh + +# Runs ns simplenorm.tcl for Nack stats vs. group size + +set sizeList {10 20 35 50 75 100 150 250 350 500 750 1000} + +set rate 32kb +set duration 120.0 +set backoff 4.0 + +set fileName "results.gp" + +exec rm -f $fileName + +# Gnuplot header +set outFile [open $fileName "w"] +puts $outFile "set title 'NACK Suppression Performance'" +puts $outFile "set data style lines" +puts $outFile "set xlabel 'Number of Receivers'" +puts $outFile "set ylabel 'NACK Transmission Fraction'" +puts $outFile "plot \\" +puts $outFile "'-' using 1:2 \\" +puts $outFile "'Receivers:%lf alpha:%lf' t 'Theoretical Multicast Nacking', \\" +puts $outFile "'-' using 1:2 \\" +puts $outFile "'Receivers:%lf Sent:%lf Suppressed:%lf alpha:%lf' t 'Measured Multicast Nacking'\n" +close $outFile + +puts "Computing theoretical results ..." +# T = number of GRTT for NACK backoff timers ... +set T $backoff + +# Theoretical Multicast NACK results +set outFile [open $fileName "a"] +puts $outFile "#Theoretical Multicast NACK Transmission Fraction" +foreach groupSize $sizeList { + set lambda [expr log($groupSize) + 1.0] + set N [expr exp((1.2/(2.0 * $T)) * ($lambda))] + set alpha [expr $N / $groupSize] + puts $outFile "Receivers:$groupSize alpha:$alpha" +} +puts $outFile "e\n" +close $outFile + +# Measured Multicast NACK results +set outFile [open $fileName "a"] +puts $outFile "#Measured Multicast NACK Transmission Fraction" +close $outFile +foreach groupSize $sizeList { + puts "Starting simulation run for groupSize:$groupSize ..." + puts " ns simplenorm.tcl gsize $groupSize backoff $backoff rate $rate duration $duration" + catch {eval exec ns simplenorm.tcl gsize $groupSize backoff $backoff rate $rate duration $duration} + puts " cat normLog.txt | nc >> $fileName" + catch {eval exec cat normLog.txt | nc >> $fileName} +} +puts "Finished." diff --git a/unix/Makefile.common b/unix/Makefile.common index 4d577a0..898eaa3 100644 --- a/unix/Makefile.common +++ b/unix/Makefile.common @@ -4,11 +4,11 @@ SHELL=/bin/sh -.SUFFIXES: .cpp $(.SUFFIXES) +.SUFFIXES: .cpp -sim.o $(.SUFFIXES) PROTOLIB = ../protolib COMMON = ../common -UNIX = ./ +UNIX = ../unix NS = ../ns INCLUDES = $(TCL_INCL_PATH) $(SYSTEM_INCLUDES) -I$(UNIX) -I$(COMMON) -I$(PROTOLIB)/common @@ -54,26 +54,37 @@ $(PROTOLIB)/unix/libProto.a: NORM_SRC = $(COMMON)/normMessage.cpp $(COMMON)/normSession.cpp \ $(COMMON)/normNode.cpp $(COMMON)/normObject.cpp \ $(COMMON)/normSegment.cpp $(COMMON)/normBitmask.cpp \ - $(COMMON)/normEncoder.cpp $(COMMON)/galois.cpp + $(COMMON)/normEncoder.cpp $(COMMON)/galois.cpp \ + $(COMMON)/normFile.cpp NORM_OBJ = $(NORM_SRC:.cpp=.o) LIB_SRC = $(NORM_SRC) LIB_OBJ = $(LIB_SRC:.cpp=.o) -libNorm.a: $(LIB_OBJ) +libnorm.a: $(LIB_OBJ) ar rcv $@ $(LIB_OBJ) ranlib $@ + +.cpp-sim.o: + $(CC) -c $(CFLAGS) -DSIMULATE -o $*-sim.o $*.cpp + +SIM_SRC = $(NORM_SRC) $(COMMON)/normSimAgent.cpp +SIM_OBJ = $(SIM_SRC:.cpp=-sim.o) + +libnormSim.a: $(SIM_OBJ) + ar rcv $@ $(SIM_OBJ) + ranlib $@ # (mdp) command-line file broadcaster/receiver APP_SRC = $(COMMON)/normApp.cpp APP_OBJ = $(APP_SRC:.cpp=.o) -norm: $(APP_OBJ) libNorm.a $(LIBPROTO) - $(CC) $(CFLAGS) -o $@ $(APP_OBJ) $(LDFLAGS) $(LIBS) libNorm.a $(LIBPROTO) +norm: $(APP_OBJ) libnorm.a $(LIBPROTO) + $(CC) $(CFLAGS) -o $@ $(APP_OBJ) $(LDFLAGS) $(LIBS) libnorm.a $(LIBPROTO) clean: - rm -f $(COMMON)/*.o $(UNIX)/*.o $(UNIX)/libNorm.a $(UNIX)/norm; + rm -f $(COMMON)/*.o $(UNIX)/*.o $(UNIX)/libnorm.a $(UNIX)/norm $(NS)/*.o; make -C $(PROTOLIB)/unix -f Makefile.common clean # DO NOT DELETE THIS LINE -- mkdep uses it.