From a8ab40d74933c24e95adef19fb980b575f87484b Mon Sep 17 00:00:00 2001 From: Jeff Weston Date: Wed, 11 Sep 2019 12:21:00 -0400 Subject: [PATCH] v1.5.7 --- README-Java.txt | 47 +- VERSION.TXT | 40 + doc/NormDeveloperGuide.xml | 6 +- examples/java/NormMsgr.java | 2 +- examples/normClient.cpp | 163 +- examples/normFileRecv.cpp | 3 + examples/normFileSend.cpp | 5 +- examples/normMsgr.cpp | 542 ++-- examples/normServer.cpp | 422 +++- examples/normSocket.cpp | 793 ++++-- examples/normSocket.h | 52 +- examples/normSocketNotes.txt | 155 ++ examples/normStreamer.cpp | 2178 +++++++++++++++++ examples/win32InputHandler.cpp | 243 ++ examples/wintest.cpp | 57 + include/normApi.h | 51 +- include/normMessage.h | 204 +- include/normNode.h | 110 +- include/normObject.h | 174 +- include/normSegment.h | 68 +- include/normSession.h | 88 +- include/normVersion.h | 2 +- makefiles/Makefile.common | 9 + makefiles/Makefile.linux | 9 +- makefiles/android/AndroidManifest.xml | 4 +- makefiles/android/jni/Application.mk | 2 +- makefiles/android/project.properties | 2 +- makefiles/win32/NormDll.vcxproj | 7 +- makefiles/win32/NormLib.vcxproj | 7 +- makefiles/win32/norm/norm.vcxproj | 13 +- makefiles/win32/normTest.vcxproj | 7 +- makefiles/win32/npc.vcxproj | 7 +- src/common/normApi.cpp | 256 +- src/common/normApp.cpp | 189 +- src/common/normFile.cpp | 3 +- src/common/normMessage.cpp | 46 +- src/common/normNode.cpp | 772 +++--- src/common/normObject.cpp | 1602 +++++++----- src/common/normPrecode.cpp | 125 +- src/common/normSegment.cpp | 307 ++- src/common/normSession.cpp | 979 +++++--- src/common/pcap2norm.cpp | 56 +- src/common/raft.cpp | 10 +- src/java/jni/normInstanceJni.cpp | 6 + src/java/jni/normNodeJni.cpp | 2 +- .../navy/nrl/norm/enums/NormEventType.java | 55 +- src/pynorm/core.py | 2 +- src/pynorm/event.py | 6 + src/pynorm/node.py | 9 +- src/pynorm/session.py | 6 + {makefiles => src}/win32/win32PostProcess.cpp | 0 waf | 21 +- wscript | 100 +- 53 files changed, 7910 insertions(+), 2114 deletions(-) create mode 100644 examples/normSocketNotes.txt create mode 100644 examples/normStreamer.cpp create mode 100644 examples/win32InputHandler.cpp create mode 100644 examples/wintest.cpp rename {makefiles => src}/win32/win32PostProcess.cpp (100%) diff --git a/README-Java.txt b/README-Java.txt index cae9aca..1fc2ce6 100644 --- a/README-Java.txt +++ b/README-Java.txt @@ -4,28 +4,42 @@ Java JNI bindings for NORM By: Jason Rush Peter Griffin + updates covering waf build system by Duc Nguyen + +Updated: 2018-03-14 The Java JNI bindings for NORM provide Java bindings for the NORM C API. For documentation about the main NORM API calls, refer to the NORM Developers guide in the regular NORM distribution. +The JNI bindings for NORM can be built using two methods. The original +method uses Make and an Ant build.xml script in makefiles/java. The +newer method uses the waf build system and builds for all platforms with +various configurations via configuration switches. + ------------ Requirements ------------ Java JNI bindings for NORM requires at least Java 1.5; however, it has also -been tested with Java 1.6. +been tested with Java 1.6 and Java 1.8 The NORM library should be built prior to building the Java JNI bindings since they link against it. +### Using Ant and Make ### Apache Ant is required for building the class files and jar file. +### Python waf ### +Tested with Python 2 but Python 3 should be ok. + ------------ Building ------------ +### Using Ant and Make ### + The build files for Java JNI bindings for NORM are located in the makefiles/java directory. @@ -42,6 +56,23 @@ makefiles/java directory with the correct make file for your system: This will produce a libmil_navy_nrl_norm.so file in the lib/ directory. +### Python waf ### + +Reconfigure waf to use java and choose the target on a windows system: + + > cd + > waf configure --build-java --msvc_target=x64 + +The msvc_target flag allows you to choose the target Windows +architecture. The default is "x86" (i.e. 32bit arch) + +Build the NORM libraries: + + > waf build + +The output will be in the build/ directory. Copy both +mil_navy_nrl_norm.* and norm-1.* to the location the JRE libraries. + ------------ Installation ------------ @@ -75,9 +106,23 @@ If you observe an "java.lang.UnsatisfiedLinkError" exception while running your application, you do not have the libmil_navy_nrl_norm.so installed in the correct location. +If you're running on Windows, you also need to have the PATH environment +variable set with along with the -Djava.library.path= + ------------ Examples ------------ Examples using the Java JNI bindings for NORM are provided in the examples/java directory. + +Running on Windows (could be put in a .bat script): + + set PATH=%PATH%; + java -Djava.library.path= ... + +Running on Linux or MacOS: + + java -Djava.library.path= ... + + diff --git a/VERSION.TXT b/VERSION.TXT index 3247c46..65d2e34 100644 --- a/VERSION.TXT +++ b/VERSION.TXT @@ -1,5 +1,45 @@ NORM Version History +Version 1.5.7 +============= + - Fixed issue with NormSession::SenderRemoveAckingNode() method where + the removed NormAckingNode instance was not being properly deleted. + (Thanks to Chad Gioia) + - Fixed bug in normPrecoder (npc) when building on 64-bit architectures + (Thanks to Leon Meyer) + - Fixed bug in JNI NormNode.getAddress() method where an "unsigned short" + type was used instead of "UINT16" to receive the port portion of the + address. (Thanks to Jame Nguyen) + - Reduced warnings, added gnu and other build options to 'waf' wscripts + (Thanks to Luca Buccassi) + - Greatly improved high speed performance via enhancements to Protolib + - Fixed bugs related to coding block id wrapping for very long-lived + (or very high speed) NORM sessions. + - Added NormSocket API extension in "norm/examples" that provides a + sockets-like API and client-server connection-oriented paradigm + for using NORM. This currently supports a byte-stream service + for both multicast and unicast using NORM_OBJECT_STREAM. This API + extension is still work in progress and will be expanded to include + message-stream service support as well as NORM's file/data object + modes. The "norm/examples/normClient.cpp" (and "normServer.cpp) + illustrates use of the the API extension defined in + "norm/examples/normSocket.h" + - Added "normStreamer" example that provides the same function as the + "normMsgr" example except using NORM_OBJECT_STREAM. + - The bug fixes and performance enhancements make this a key update. + The NORM Developer's Guide is not yet updated to described some of + the new API features added. + - Added APIs to iterate through sender "Acking Node List" to determine + who has and who has not acknowledged a watermark acknowledgement + request. + - Added NormSetAutoAckingNodes() call with NormTrackingStatus to + automate detection of new group members. The Acking Node List + iterator function NormGetNextAckingNode() can be used to learn + who has joined and the NORM_ACKING_NODE_NEW provides notification + to the application of new members. + - The 'waf' build scripts assign the current version number to + the NORM shared library built for better version control/tracking. + Version 1.5r6 ============= - Fixed Protolib ProtoBitmask bug for NDEBUG (release) builds. diff --git a/doc/NormDeveloperGuide.xml b/doc/NormDeveloperGuide.xml index c4b5229..22a0dda 100644 --- a/doc/NormDeveloperGuide.xml +++ b/doc/NormDeveloperGuide.xml @@ -1181,7 +1181,7 @@ bool NormSetCacheDirectoryNormGetNextEvent(NormInstanceHandle instanceHandle, - NormEvent* theEvent); + NormEvent* theEvent); @@ -3763,7 +3763,7 @@ void NormSetTxRobustFactor#include <normApi.h> -boolNormRequeueObject(NormSessionHandle sessionHandle, +bool NormRequeueObject(NormSessionHandle sessionHandle, NormObjectHandle objectHandle); @@ -6596,7 +6596,7 @@ bool NormNodeGetAddress#include <normApi.h> -double NormNodeGetId(NormNodeGetGrtt(NormNodeHandle nodeHandle); diff --git a/examples/java/NormMsgr.java b/examples/java/NormMsgr.java index 6ed9680..4cdcd68 100644 --- a/examples/java/NormMsgr.java +++ b/examples/java/NormMsgr.java @@ -239,7 +239,7 @@ public class NormMsgr normTxLock.release(); return false; } - //System.err.println("caching msg for object " + obj + "\n"); + // System.err.println("caching msg for object " + obj + "\n"); // Cache the msg associated with the resultant tx object. We use the // input_msg_list HashMap so we can remove the msg upon NORM_TX_OBJECT_PURGED input_msg_list.put(obj, msg); diff --git a/examples/normClient.cpp b/examples/normClient.cpp index 9043b2c..7491cbc 100644 --- a/examples/normClient.cpp +++ b/examples/normClient.cpp @@ -1,9 +1,15 @@ #include "normSocket.h" #include #include + +#ifdef WIN32 +#include "win32InputHandler.cpp" // brings in the entire implementation +#else #include #include // for, well, fnctl() #include // obvious child +#endif // if/else WIN32/UNIX + // BUILD: (assumes "normApi.h" in "include" ... // g++ -I../include -o normClient normClient.cpp normSocket.cpp ../lib/libnorm.a ../protolib/lib/libprotokit.a -lresolv @@ -19,6 +25,11 @@ // To deny a connection, the "server" can send a stream close??? Which means the server will have // to establish state just to deny a client a connection (yuck!) +void Usage() +{ + fprintf(stderr, "Usage: normClient [connect [/][,]][debug ][trace]\n"); +} + const unsigned int MSG_LENGTH_MAX = 64; int main(int argc, char* argv[]) @@ -33,18 +44,19 @@ int main(int argc, char* argv[]) bool trace = false; unsigned int debugLevel = 0; - for (int i = 1; i < argc; i++) + for (int i = 1; i < argc; i++) { const char* cmd = argv[i]; unsigned int len = strlen(cmd); if (0 == strncmp(cmd, "connect", len)) { - // connect /[, + // connect /[,] const char* val = argv[++i]; const char* portPtr = strchr(val, '/'); if (NULL == portPtr) { fprintf(stderr, "normClient error: missing number\n"); + Usage(); return -1; } portPtr++; @@ -58,6 +70,7 @@ int main(int argc, char* argv[]) else { fprintf(stderr, "normClient error: missing \n"); + Usage(); return -1; } char portText[32]; @@ -76,6 +89,7 @@ int main(int argc, char* argv[]) if (1 != sscanf(portText, "%hu", &serverPort)) { fprintf(stderr, "normClient error: invalid \n"); + Usage(); return -1; } if (NULL != groupPtr) @@ -96,7 +110,8 @@ int main(int argc, char* argv[]) { if (1 != sscanf(argv[++i], "%u", &debugLevel)) { - fprintf(stderr, "normServer error: invalid debug level\n"); + fprintf(stderr, "normClient error: invalid debug level\n"); + Usage(); return -1; } } @@ -107,13 +122,15 @@ int main(int argc, char* argv[]) if (1 != sscanf(val, "%lu", &id)) { fprintf(stderr, "normClient error: invalid 'id' value\n"); + Usage(); return -1; } clientId = id; } else { - fprintf(stderr, "normServer error: invalid command \"%s\"\n", cmd); + fprintf(stderr, "normClient error: invalid command \"%s\"\n", cmd); + Usage(); return -1; } } @@ -122,23 +139,39 @@ int main(int argc, char* argv[]) // Initate connection to server ... fprintf(stderr, "normClient: connecting to %s/%hu ...\n", serverAddr, serverPort); - NormSocketHandle normSocket = NormConnect(instance, serverAddr, serverPort, groupAddrPtr, clientId); + NormSocketHandle normSocket = NormOpen(instance); + + // setting 'localPort' param here to zero lets an ephemeral port be picked + NormConnect(normSocket, serverAddr, serverPort, 0, groupAddrPtr, clientId); + /* // Optional code to test NormWrite() immediately after NormConnect() call + // (Note this is _not_ compatible with newer stale connection reject code) + const char* helloStr = "Hello\n"; + unsigned int helloLen = strlen(helloStr) + 1; + NormWrite(normSocket, helloStr, helloLen); + NormFlush(normSocket);*/ if (trace) { - NormSetMessageTrace(NormGetSession(normSocket), true); + NormSetMessageTrace(NormGetSocketSession(normSocket), true); if (NULL != groupAddrPtr) - NormSetMessageTrace(NormGetMulticastSession(normSocket), true); + NormSetMessageTrace(NormGetSocketMulticastSession(normSocket), true); } if (0 != debugLevel) NormSetDebugLevel(debugLevel); //NormSetDebugLevel(3); - //NormSetMessageTrace(NormGetSession(normSocket), true); + //NormSetMessageTrace(NormGetSocketSession(normSocket), true); - // We use a select() call to multiplex input reading and NormSocket handling +#ifdef WIN32 + HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE); + Win32InputHandler inputHandler; + inputHandler.Open(); + HANDLE handleArray[2]; + handleArray[0] = NormGetDescriptor(instance); + handleArray[1] = inputHandler.GetEventHandle(); +#else + // On Unix, use a select() call to multiplex input reading and NormSocket handling fd_set fdset; FD_ZERO(&fdset); - // Get our input (STDIN) descriptor and set non-blocking FILE* inputFile = stdin; int inputfd = fileno(inputFile); @@ -146,15 +179,45 @@ int main(int argc, char* argv[]) perror("normClient: fcntl(inputfd, O_NONBLOCK) error"); // Get our NormInstance descriptor int normfd = NormGetDescriptor(instance); +# endif // if/else WIN32 bool keepGoing = true; bool writeReady = false; - unsigned int inputLength = 0; + int inputLength = 0; unsigned int bytesWritten = 0; const unsigned int BUFFER_LENGTH = 2048; char inputBuffer[BUFFER_LENGTH]; bool inputNeeded = false; // will be set to "true" upon CONNECT while (keepGoing) { + bool normEventPending = false; + bool inputEventPending = false; +#ifdef WIN32 + DWORD handleCount = inputNeeded ? 2 : 1; + DWORD waitStatus = + MsgWaitForMultipleObjectsEx(handleCount, // number of handles in array + handleArray, // object-handle array + INFINITE, // time-out interval + QS_ALLINPUT, // input-event type + 0); + if ((WAIT_OBJECT_0 <= waitStatus) && (waitStatus < (WAIT_OBJECT_0 + handleCount))) + { + if (0 == (waitStatus - WAIT_OBJECT_0)) + normEventPending = true; + else + inputEventPending = true; + } + else if (-1 == waitStatus) + { + perror("normClient: MsgWaitForMultipleObjectsEx() error"); + break; + } + else + { + // TBD - any other status we should handle? + // (e.g. WAIT_TIMEOUT, WAIT_ABANDONED or WAIT_IO_COMPLETION) + continue; // ignore for now + } +#else FD_SET(normfd, &fdset); int maxfd = normfd; if (inputNeeded) @@ -173,8 +236,36 @@ int main(int argc, char* argv[]) break; } if (FD_ISSET(inputfd, &fdset)) + inputEventPending = true; + if (FD_ISSET(normfd, &fdset)) + normEventPending = true; +#endif // if/else WIN32/UNIX + + if (inputEventPending) { // Read input into our txBuffer +#ifdef WIN32 + inputLength = inputHandler.ReadData(inputBuffer, BUFFER_LENGTH); + if (inputLength > 0) + { + // We got our input + bytesWritten = 0; + inputNeeded = false; + } + else if (inputLength < 0) + { + // Input stream has likely closed, initiate client shutown + // TBD - initiate client shutdown + if (NULL == groupAddrPtr) + { + fprintf(stderr, "normClient: CLOSING connection to server ...\n"); + NormShutdown(normSocket); + } + inputNeeded = false; // TBD -should we also fclose(inputFile)??? + inputHandler.Close(); + } + // else zero bytes read, still need input +#else inputLength = fread(inputBuffer, 1, BUFFER_LENGTH, inputFile); if (inputLength > 0) { @@ -191,7 +282,11 @@ int main(int argc, char* argv[]) NormShutdown(normSocket); } inputNeeded = false; // TBD -should we also fclose(inputFile)??? - + if (stdin != inputFile) + { + fclose(inputFile); + inputFile = NULL; + } // else stick around to receive stuff from the server } else if (ferror(inputFile)) @@ -205,14 +300,15 @@ int main(int argc, char* argv[]) // input starved, wait for next notification break; default: - perror("normClient: error reading input"); + perror("normClient: error reading input?!"); break; } } - } - if (FD_ISSET(normfd, &fdset)) +#endif // if/else WIN32/UNIX + } // end if inputEventPending + + if (normEventPending) { - // There's a NORM event pending NormSocketEvent event; if (NormGetSocketEvent(instance, &event)) @@ -225,7 +321,11 @@ int main(int argc, char* argv[]) case NORM_SOCKET_CONNECT: { - fprintf(stderr, "normClient: CONNECTED to server ...\n"); + char remoteAddr[16]; + unsigned int addrLen = 16; + UINT16 remotePort; + NormGetPeerName(normSocket, remoteAddr, &addrLen, &remotePort); + fprintf(stderr, "normClient: CONNECTED to server %s/%hu\n", serverAddr, remotePort); inputNeeded = true; writeReady = true; break; @@ -246,7 +346,17 @@ int main(int argc, char* argv[]) continue; } if (bytesRead > 0) + { +#ifdef WIN32 + // Use Win32 WriteFile() so there is no buffer delay + DWORD dwWritten; + WriteFile(hStdout, buffer, bytesRead, &dwWritten, NULL); +#else fwrite(buffer, sizeof(char), bytesRead, stdout); +#endif // if/else WIN32 + } + // If less bytes read than request then need + // to wait for next NORM_SOCKET_READ event if (bytesRead < 1024) rxReady = false; } break; @@ -256,13 +366,16 @@ int main(int argc, char* argv[]) break; case NORM_SOCKET_CLOSING: fprintf(stderr, "normClient: server CLOSING connection ...\n"); + writeReady = false; inputNeeded = false; break; - case NORM_SOCKET_CLOSED: + case NORM_SOCKET_CLOSE: { fprintf(stderr, "normClient: connection to server CLOSED.\n"); + writeReady = false; inputNeeded = false; keepGoing = false; + break; } case NORM_SOCKET_NONE: @@ -273,7 +386,7 @@ int main(int argc, char* argv[]) { fprintf(stderr, "normClient: NormGetSocketEvent() returned false\n"); } - } // end if FD_ISSET(normfd) + } // end if normEventPending // If we have data in our inputBuffer and the NormSocket is "writeReady", then send it if (writeReady && (inputLength > 0)) @@ -293,4 +406,16 @@ int main(int argc, char* argv[]) } } } // end while(keepGoing) + NormClose(normSocket); +#ifdef WIN32 + inputHandler.Close(); +#else + if ((stdin != inputFile) && (NULL != inputFile)) + { + fclose(inputFile); + inputFile = NULL; + } +#endif // if/else WIN32 + fprintf(stderr, "normClient: Done.\n"); + return 0; } diff --git a/examples/normFileRecv.cpp b/examples/normFileRecv.cpp index e980b85..97b04d6 100644 --- a/examples/normFileRecv.cpp +++ b/examples/normFileRecv.cpp @@ -58,6 +58,9 @@ int main(int argc, char* argv[]) 6003, NORM_NODE_ANY); + NormSetRxPortReuse(session, true); + NormSetMulticastLoopback(session, true); + // NOTE: These are debugging routines available // (not necessary for normal app use) // (Need to include "protolib/common/protoDebug.h" for this diff --git a/examples/normFileSend.cpp b/examples/normFileSend.cpp index a1a3cc5..84bcf1c 100644 --- a/examples/normFileSend.cpp +++ b/examples/normFileSend.cpp @@ -64,7 +64,10 @@ int main(int argc, char* argv[]) NormSessionHandle session = NormCreateSession(instance, "224.1.2.3", 6003, - NORM_NODE_ANY); + 1); + + NormSetRxPortReuse(session, true); + NormSetMulticastLoopback(session, true); // NOTE: These are some debugging routines available // (not necessary for normal app use) diff --git a/examples/normMsgr.cpp b/examples/normMsgr.cpp index cf4e9b8..7e6dc67 100644 --- a/examples/normMsgr.cpp +++ b/examples/normMsgr.cpp @@ -1,8 +1,6 @@ #include "normApi.h" - -#include "protoCheck.h" - +#include // for read() and write() #include // for printf(), etc #include // for srand() #include // for strrchr(), memset(), etc @@ -12,6 +10,10 @@ #include // obvious child #include // embarrassingly obvious +#define SHOOT_NOW 1 + +#include "protoCheck.h" + class NormMsgr { public: @@ -87,11 +89,17 @@ class NormMsgr void Destroy(); + unsigned int GetCount() const + {return msg_count;} + private: - Message* head; - Message* tail; + Message* head; + Message* tail; + unsigned int msg_count; }; // end class NormMsgr::MessageQueue Message* NewMessage(unsigned int size); + + void Destroy(); bool OpenNormSession(NormInstanceHandle instance, const char* addr, @@ -110,6 +118,12 @@ class NormMsgr assert(NORM_SESSION_INVALID != norm_session); NormSetMulticastInterface(norm_session, ifaceName); } + void SetLoopback(bool state) + { + loopback = state; + if (NORM_SESSION_INVALID != norm_session) + NormSetMulticastLoopback(norm_session, state); + } void SetNormMessageTrace(bool state) { assert(NORM_SESSION_INVALID != norm_session); @@ -121,36 +135,57 @@ class NormMsgr NormAddAckingNode(norm_session, ackId); norm_acking = true; // invoke ack-based flow control } + void SetFlushing(bool state) + {norm_flushing = state;} bool Start(bool sender, bool receiver); void Stop() {is_running = false;} bool IsRunning() const {return is_running;} + + unsigned long GetSentCount() + {return sent_count;} + void HandleNormEvent(const NormEvent& event); // Sender methods FILE* GetInputFile() const {return input_file;} + int GetInputDescriptor() const + {return input_fd;} + bool InputReady() const + {return input_ready;} + void SetInputReady() + {input_ready = true;} bool InputNeeded() const {return input_needed;} bool InputMessageReady() const {return ((NULL != input_msg) && !input_needed);} bool ReadInput(); - bool NormTxReady() const; + bool TxReady() const; bool SendMessage(); bool EnqueueMessageObject(); // Receiver methods + void SetOutputFile(FILE* filePtr) + { + output_file = filePtr; + output_fd = fileno(filePtr); + } FILE* GetOutputFile() const {return output_file;} + int GetOutputDescriptor() const + {return output_fd;} void SetOutputReady() {output_ready = true;} bool OutputReady() const {return output_ready;} bool OutputPending() const - {return output_pending;} + {return (NULL != output_msg);} + bool RxNeeded() const + {return rx_needed;} bool WriteOutput(); void OmitHeader(bool state) @@ -164,24 +199,34 @@ class NormMsgr private: bool is_running; - FILE* input_file; // stdin by default - bool input_needed; // + FILE* input_file; // stdin by default + int input_fd; + bool input_ready; + bool input_needed; // + bool input_finished; Message* input_msg; // current input message being read/sent* MessageQueue input_msg_list; // list of enqueued messages (in norm sender cache) NormSessionHandle norm_session; bool is_multicast; + bool loopback; unsigned int norm_tx_queue_max; // max number of objects that can be enqueued at once unsigned int norm_tx_queue_count; // count of unacknowledged enqueued objects (TBD - optionally track size too) - bool norm_tx_watermark_pending; + bool norm_flow_control_pending; bool norm_tx_vacancy; bool norm_acking; + bool norm_flushing; + NormObjectHandle norm_flush_object; + NormObjectHandle norm_last_object; + unsigned long sent_count; + bool rx_needed; FILE* output_file; + int output_fd; bool output_ready; - bool output_pending; Message* output_msg; MessageQueue output_msg_queue; + MessageQueue temp_msg_queue; // These are some options mainly for testing purposes bool omit_header; // if "true", receive message length header is _not_ written to output bool rx_silent; @@ -190,22 +235,39 @@ class NormMsgr }; // end class NormMsgr NormMsgr::NormMsgr() - : input_file(stdin), input_needed(false), input_msg(NULL), - norm_session(NORM_SESSION_INVALID), is_multicast(false), norm_tx_queue_max(2048), norm_tx_queue_count(0), - norm_tx_watermark_pending(false), norm_tx_vacancy(true), norm_acking(false), - output_file(stdout), output_ready(true), output_pending(false), output_msg(NULL), + : input_file(stdin), input_fd(fileno(stdin)), input_ready(false), input_needed(false), input_finished(false), + input_msg(NULL), norm_session(NORM_SESSION_INVALID), is_multicast(false), loopback(false), + norm_tx_queue_max(8192), norm_tx_queue_count(0), + norm_flow_control_pending(false), norm_tx_vacancy(true), norm_acking(false), + norm_flushing(true), norm_flush_object(NORM_OBJECT_INVALID), norm_last_object(NORM_OBJECT_INVALID), + sent_count(0), rx_needed(false), output_file(stdout), output_fd(fileno(stdout)), + output_ready(true), output_msg(NULL), omit_header(false), rx_silent(false), tx_loss(0.0) { } NormMsgr::~NormMsgr() { - if (NULL != input_msg) delete input_msg; + Destroy(); +} + +void NormMsgr::Destroy() +{ + if (NULL != input_msg) + { + delete input_msg; + input_msg = NULL; + } input_msg_list.Destroy(); - if (NULL != output_msg) delete output_msg; + if (NULL != output_msg) + { + delete output_msg; + delete output_msg; + } output_msg_queue.Destroy(); } + bool NormMsgr::OpenNormSession(NormInstanceHandle instance, const char* addr, unsigned short port, NormNodeId nodeId) { if (NormIsUnicastAddress(addr)) @@ -222,21 +284,24 @@ bool NormMsgr::OpenNormSession(NormInstanceHandle instance, const char* addr, un if (is_multicast) { NormSetRxPortReuse(norm_session, true); - // TBD - make full loopback a command line option? - NormSetMulticastLoopback(norm_session, true); + if (loopback) + NormSetMulticastLoopback(norm_session, true); } // Set some default parameters (maybe we should put parameter setting in Start()) + fprintf(stderr, "setting rx cache limit to %u\n", norm_tx_queue_max); + if (norm_tx_queue_max > 65535/2) norm_tx_queue_max = 65535/2; NormSetRxCacheLimit(norm_session, norm_tx_queue_max); NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); - NormSetDefaultUnicastNack(norm_session, true); + if (!is_multicast) + NormSetDefaultUnicastNack(norm_session, true); NormSetTxCacheBounds(norm_session, 10*1024*1024, norm_tx_queue_max, norm_tx_queue_max); //NormSetMessageTrace(norm_session, true); - NormSetTxRobustFactor(norm_session, 2); + //NormSetTxRobustFactor(norm_session, 20); return true; } // end NormMsgr::OpenNormSession() @@ -274,13 +339,27 @@ void NormMsgr::SetNormCongestionControl(CCMode ccMode) bool NormMsgr::Start(bool sender, bool receiver) { + fprintf(stderr, "enter NormMsgr::Start() ...\n"); + // TBD - make these command-line accessible + unsigned int bufferSize = 64*1024*1024; + unsigned int segmentSize = 1400; + unsigned int blockSize = 64; + unsigned int numParity = 0; + unsigned int txSockBufferSize = 4*1024*1024; + unsigned int rxSockBufferSize = 6*1024*1024; + if (receiver) { - if (!NormStartReceiver(norm_session, 10*1024*1024)) + if (!NormStartReceiver(norm_session, bufferSize)) { fprintf(stderr, "normMsgr error: unable to start NORM receiver\n"); return false; } + // Note: NormPreallocateRemoteSender() MUST be called AFTER NormStartReceiver() + NormPreallocateRemoteSender(norm_session, bufferSize, segmentSize, blockSize, numParity, bufferSize); + NormSetRxSocketBuffer(norm_session, rxSockBufferSize); + rx_needed = true; + fprintf(stderr, "normMsgr: receiver ready.\n"); } if (sender) { @@ -290,19 +369,26 @@ bool NormMsgr::Start(bool sender, bool receiver) // so disable timer-based flow control NormSetFlowControl(norm_session, 0.0); } + NormSetGrttEstimate(norm_session, 0.001); + //NormSetGrttMax(norm_session, 0.100); + NormSetBackoffFactor(norm_session, 0); + // Pick a random instance id for now struct timeval currentTime; gettimeofday(¤tTime, NULL); srand(currentTime.tv_usec); // seed random number generator NormSessionId instanceId = (NormSessionId)rand(); - if (!NormStartSender(norm_session, instanceId, 10*1024*1024, 1400, 16, 4)) + if (!NormStartSender(norm_session, instanceId, bufferSize, segmentSize, blockSize, numParity)) { fprintf(stderr, "normMsgr error: unable to start NORM sender\n"); if (receiver) NormStopReceiver(norm_session); return false; } + //NormSetAutoParity(norm_session, 2); + NormSetTxSocketBuffer(norm_session, txSockBufferSize); input_needed = true; } + //ProtoCheckResetLogging(); is_running = true; return true; } // end NormMsgr::Start(); @@ -337,11 +423,11 @@ bool NormMsgr::ReadInput() readLength = input_msg->GetSize() - offset; bufferPtr = input_msg->AccessBuffer() + offset; } - size_t result = fread(bufferPtr, 1, readLength, input_file); + ssize_t result = read(input_fd, bufferPtr, readLength); if (result > 0) { input_msg->IncrementIndex(result); - if (result < readLength) + if ((size_t)result < readLength) { // Still need more input // (wait for next input notification to read more) @@ -368,27 +454,37 @@ bool NormMsgr::ReadInput() input_needed = false; } } - else + else if (0 == result) { - if (feof(input_file)) + // end-of-file reached, TBD - trigger final flushing and wrap-up + fprintf(stderr, "normMsgr: input end-of-file detected (last:%p)...\n", norm_last_object); + delete input_msg; + input_msg = NULL; + input_ready = false; + input_needed = false; + input_finished = true; + if (norm_acking) { - // end-of-file reached, TBD - trigger final flushing and wrap-up - fprintf(stderr, "normMsgr: input end-of-file detected ...\n"); + if (NORM_OBJECT_INVALID == norm_last_object) + is_running = false; // everything sent and acked + else if (!norm_flushing) + NormSetWatermark(norm_session, norm_last_object, true); } - else if (ferror(input_file)) + } + else // result < 0 + { + switch (errno) { - switch (errno) - { - case EINTR: - continue; // interupted, try again - case EAGAIN: - // input starved, wait for next notification - break; - default: - perror("normMsgr error reading input"); - break; - } + case EINTR: + continue; // interupted, try again + case EAGAIN: + // input starved, wait for next notification + break; + default: + perror("normMsgr error reading input"); + break; } + input_ready = false; return false; } } @@ -397,7 +493,7 @@ bool NormMsgr::ReadInput() bool NormMsgr::WriteOutput() { - while (output_pending) + while (NULL != output_msg) { size_t writeLength; const char* bufferPtr; @@ -412,38 +508,40 @@ bool NormMsgr::WriteOutput() writeLength = output_msg->GetSize() - offset; bufferPtr = output_msg->GetBuffer() + offset; } - size_t result = fwrite(bufferPtr, 1, writeLength, output_file); - - output_msg->IncrementIndex(result); - if (result < writeLength) + ssize_t result = write(output_fd, bufferPtr, writeLength); + if (result >= 0) { - if (feof(output_file)) + output_msg->IncrementIndex(result); + if ((size_t)result < writeLength) + output_ready = false; // blocked, wait for output notification + } + else + { + switch (errno) { - // end-of-file reached, TBD - stop acting as receiver, signal sender we're done? - fprintf(stderr, "normMsgr: output end-of-file detected ...\n"); - } - else if (ferror(input_file)) - { - fprintf(stderr, "normMsgr: output error detected ...\n"); - switch (errno) - { - case EINTR: - continue; // interupted, try again - case EAGAIN: - // input starved, wait for next notification - output_ready = false; - break; - default: - perror("normMsgr error writing output"); - break; - } + case EINTR: + continue; // interupted, try again + case EAGAIN: + // input starved, wait for next notification + output_ready = false; + break; + default: + perror("normMsgr error writing output"); + break; } return false; } if (output_msg->IsComplete()) { fflush(output_file); - delete output_msg; + //delete output_msg; + + temp_msg_queue.Append(*output_msg); // cache for debugging purposes + if (temp_msg_queue.GetCount() > 32) + { + //fprintf(stderr, "deleting cached recv'd message ...\n"); + delete temp_msg_queue.RemoveHead(); + } output_msg = output_msg_queue.RemoveHead(); if (NULL != output_msg) { @@ -454,7 +552,7 @@ bool NormMsgr::WriteOutput() } else { - output_pending = false; + rx_needed = true; } break; } @@ -462,7 +560,7 @@ bool NormMsgr::WriteOutput() return true; } // end NormMsgr::WriteOutput() -bool NormMsgr::NormTxReady() const +bool NormMsgr::TxReady() const { // This returns true if new tx data can be enqueued to NORM // This is based on the state with respect to prior successful data @@ -484,8 +582,6 @@ bool NormMsgr::NormTxReady() const bool NormMsgr::SendMessage() { - // TBD - call EnqueueMessageObject() or WriteMessageStream() - // depending upon on configured mode if (EnqueueMessageObject()) { // Our buffered message was sent, so reset input indices @@ -493,6 +589,7 @@ bool NormMsgr::SendMessage() input_msg_list.Append(*input_msg); input_msg = NULL; input_needed = true; + sent_count++; return true; } // else will be prompted to retry by NORM event (queue vacancy, watermark completion) @@ -515,23 +612,42 @@ bool NormMsgr::EnqueueMessageObject() // This might happen if a non-acking receiver is present and // has nacked for the oldest object in the queue even if all // of our acking receivers have acknowledged it. + fprintf(stderr, "NO VACANCY count:%u max:%u\n", norm_tx_queue_count, norm_tx_queue_max); norm_tx_vacancy = false; return false; } + //NormObjectRetain(object); NormObjectSetUserData(object, input_msg); // so we can remove/delete upon purge if (norm_acking) { // ack-based flow control has been enabled norm_tx_queue_count++; - if (!norm_tx_watermark_pending && (norm_tx_queue_count >= (norm_tx_queue_max / 2))) + if (!norm_flow_control_pending && (norm_tx_queue_count >= (norm_tx_queue_max / 2))) { - NormSetWatermark(norm_session, object); - norm_tx_watermark_pending = true; + NormSetWatermark(norm_session, object, true); // overrideFlush == true + norm_last_object = object; + norm_flow_control_pending = true; + } + else if (norm_flushing) // per-message acking + { +#ifdef SHOOT_FIRST + NormSetWatermark(norm_session, object, true); + norm_last_object = object; +#else // ACK_LATER + if (norm_flow_control_pending) + { + norm_flush_object = object; // will be used as watermark upon flow control ack + } + else + { + NormSetWatermark(norm_session, object, true); + norm_last_object = object; + } +#endif // SHOOT_FIRST/ACK_LATER } else { - // TBD - make non-flow control acking separable option? - NormSetWatermark(norm_session, object); + //norm_last_object = object; } } return true; @@ -539,7 +655,6 @@ bool NormMsgr::EnqueueMessageObject() void NormMsgr::HandleNormEvent(const NormEvent& event) { - bool logAllocs = false; switch (event.type) { case NORM_TX_QUEUE_EMPTY: @@ -555,18 +670,37 @@ void NormMsgr::HandleNormEvent(const NormEvent& event) if (NORM_ACK_SUCCESS == NormGetAckingStatus(norm_session)) { //fprintf(stderr, "WATERMARK COMPLETED\n"); - if (norm_tx_watermark_pending) + norm_last_object = NORM_OBJECT_INVALID; + if (norm_flow_control_pending) { norm_tx_queue_count -= (norm_tx_queue_max / 2); - norm_tx_watermark_pending = false; + norm_flow_control_pending = false; + if (NORM_OBJECT_INVALID != norm_flush_object) + { + NormSetWatermark(norm_session, norm_flush_object, true); + norm_last_object = norm_flush_object; + norm_flush_object = NORM_OBJECT_INVALID; + } } + if (input_finished && (NORM_OBJECT_INVALID == norm_last_object)) + is_running = false; } else { - // TBD - we could see who didn't ACK and possibly remove them - // from our acking list. For now, we are infinitely - // persistent by resetting watermark ack request - NormResetWatermark(norm_session); + // TBD - we could see who did and how didn't ACK and possibly remove them + // from our acking "membership". For now, we are infinitely + // persistent by resetting watermark ack request without clearing + // flow control + if (NORM_OBJECT_INVALID == norm_flush_object) + { + NormResetWatermark(norm_session); + } + else // might as request ack for most recent enqueued object + { + NormSetWatermark(norm_session, norm_flush_object, true); + norm_last_object = norm_flush_object; + norm_flush_object = NORM_OBJECT_INVALID; + } } break; @@ -574,11 +708,16 @@ void NormMsgr::HandleNormEvent(const NormEvent& event) { NormDataDetachData(event.object); Message* msg = (Message*)NormObjectGetUserData(event.object); + if(event.object == norm_flush_object) + norm_flush_object = NORM_OBJECT_INVALID; if (NULL != msg) { input_msg_list.Remove(*msg); delete msg; } + //NormObjectRelease(event.object); + //fprintf(stderr, "normMsgr LOGGING ALLOCATIONS\n"); + //ProtoCheckLogAllocations(stderr); break; } @@ -590,11 +729,11 @@ void NormMsgr::HandleNormEvent(const NormEvent& event) case NORM_RX_OBJECT_ABORTED: //fprintf(stderr, "NORM_RX_OBJECT_ABORTED\n");// %hu\n", NormObjectGetTransportId(event.object)); - logAllocs = true; break; case NORM_RX_OBJECT_COMPLETED: { + sent_count++; char* data = NormDataDetachData(event.object); if (NULL != data) { @@ -613,12 +752,12 @@ void NormMsgr::HandleNormEvent(const NormEvent& event) output_msg->ResetIndex(MSG_HEADER_SIZE); else output_msg->ResetIndex(); - output_pending = true; } else { output_msg_queue.Append(*msg); } + rx_needed = false; } // else TBD - "termination" info-only object? break; @@ -628,13 +767,6 @@ void NormMsgr::HandleNormEvent(const NormEvent& event) break; } //NormReleasePreviousEvent(NormGetInstance(norm_session)); - - if (logAllocs) - { -#ifdef USE_PROTO_CHECK - ProtoCheckLogAllocations(stderr); -#endif // USE_PROTO_CHECK - } } // end NormMsgr::HandleNormEvent() @@ -643,7 +775,7 @@ void Usage() { fprintf(stderr, "Usage: normMsgr id {send &| recv} [addr [/]][ack [,,...]\n" " [cc|cce|ccl|rate ][interface ][debug ][trace]\n" - " [omit][silent][txloss ]\n"); + " [flush {none|active}][omit][silent][txloss ]\n"); } int main(int argc, char* argv[]) { @@ -658,16 +790,21 @@ int main(int argc, char* argv[]) NormNodeId ackingNodeList[256]; unsigned int ackingNodeCount = 0; + bool flushing = false; double txRate = 0.0; // used for non-default NORM_FIXED ccMode NormMsgr::CCMode ccMode = NormMsgr::NORM_CC; const char* mcastIface = NULL; int debugLevel = 0; + const char* debugLog = NULL; // stderr by default bool trace = false; bool omitHeaderOnOutput = false; bool silentReceiver = false; double txloss = 0.0; + bool loopback = false; + + NormMsgr normMsgr; // Parse command-line int i = 1; @@ -683,6 +820,10 @@ int main(int argc, char* argv[]) { recv = true; } + else if (0 == strncmp(cmd, "loopback", len)) + { + loopback = true; + } else if (0 == strncmp(cmd, "addr", len)) { if (i >= argc) @@ -708,6 +849,23 @@ int main(int argc, char* argv[]) sessionPort = atoi(portPtr); } } + else if (0 == strncmp(cmd, "output", len)) + { + if (i >= argc) + { + fprintf(stderr, "normMsgr error: missing output 'device' name!\n"); + Usage(); + return -1; + } + FILE* outfile = fopen(argv[i++], "w+"); + if (NULL == outfile) + { + perror("normMsgr output device fopen() error"); + Usage(); + return -1; + } + normMsgr.SetOutputFile(outfile); + } else if (0 == strncmp(cmd, "id", len)) { if (i >= argc) @@ -731,17 +889,47 @@ int main(int argc, char* argv[]) while ((NULL != alist) && (*alist != '\0')) { // TBD - Do we need to skip leading white space? - if (1 != sscanf(alist, "%d", ackingNodeList + ackingNodeCount)) + int id; + if (1 != sscanf(alist, "%d", &id)) { fprintf(stderr, "nodeMsgr error: invalid acking node list!\n"); Usage(); return -1; } + ackingNodeList[ackingNodeCount] = NormNodeId(id); ackingNodeCount++; alist = strchr(alist, ','); if (NULL != alist) alist++; // point past comma } } + else if (0 == strncmp(cmd, "flush", len)) + { + // "none", "passive", or "active" + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'flush' !\n"); + Usage(); + return -1; + } + const char* mode = argv[i++]; + if (0 == strcmp(mode, "none")) + { + flushing = false; + } + else if (0 == strcmp(mode, "passive")) + { + flushing = false; + } + else if (0 == strcmp(mode, "active")) + { + flushing = true; + } + else + { + fprintf(stderr, "normMsgr error: invalid 'flush' mode \"%s\"\n", mode); + return -1; + } + } else if (0 == strncmp(cmd, "rate", len)) { if (i >= argc) @@ -802,12 +990,22 @@ int main(int argc, char* argv[]) { if (i >= argc) { - fprintf(stderr, "nodeMsgr error: missing 'interface' !\n"); + fprintf(stderr, "nodeMsgr error: missing 'debug' !\n"); Usage(); return -1; } debugLevel = atoi(argv[i++]); } + else if (0 == strncmp(cmd, "log", len)) + { + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'log' !\n"); + Usage(); + return -1; + } + debugLog = argv[i++]; + } else if (0 == strncmp(cmd, "trace", len)) { trace = true; @@ -841,8 +1039,11 @@ int main(int argc, char* argv[]) // TBD - should provide more error checking of calls NormInstanceHandle normInstance = NormCreateInstance(); NormSetDebugLevel(debugLevel); + if (NULL != debugLog) + NormOpenDebugLog(normInstance, debugLog); - NormMsgr normMsgr; + normMsgr.SetLoopback(loopback); + normMsgr.SetFlushing(flushing); if (omitHeaderOnOutput) normMsgr.OmitHeader(true); @@ -873,89 +1074,134 @@ int main(int argc, char* argv[]) int normfd = NormGetDescriptor(normInstance); // Get input/output descriptors and set to non-blocking i/o - int inputfd = fileno(normMsgr.GetInputFile()); + int inputfd = normMsgr.GetInputDescriptor(); if (-1 == fcntl(inputfd, F_SETFL, fcntl(inputfd, F_GETFL, 0) | O_NONBLOCK)) perror("normMsgr: fcntl(inputfd, O_NONBLOCK) error"); - int outputfd = fileno(normMsgr.GetOutputFile()); + int outputfd = normMsgr.GetOutputDescriptor(); if (-1 == fcntl(outputfd, F_SETFL, fcntl(outputfd, F_GETFL, 0) | O_NONBLOCK)) perror("normMsgr: fcntl(outputfd, O_NONBLOCK) error"); fd_set fdsetInput, fdsetOutput; FD_ZERO(&fdsetInput); FD_ZERO(&fdsetOutput); + struct timeval lastTime; + gettimeofday(&lastTime, NULL); while (normMsgr.IsRunning()) { - int maxfd = normfd; - FD_SET(normfd, &fdsetInput); - if (normMsgr.InputNeeded()) + int maxfd = -1; + // Only wait on NORM if needed for tx readiness + bool waitOnNorm = true; + if (!(normMsgr.RxNeeded() || normMsgr.InputMessageReady())) + waitOnNorm = false; // no need to wait + else if (normMsgr.InputMessageReady() && normMsgr.TxReady()) + waitOnNorm = false; // no need to wait if already tx ready + if (waitOnNorm) + { + maxfd = normfd; + FD_SET(normfd, &fdsetInput); + } + else + { + FD_CLR(normfd, &fdsetInput); + } + if (normMsgr.InputNeeded() && !normMsgr.InputReady()) { - //fprintf(stderr, "NEED INPUT ...\n"); FD_SET(inputfd, &fdsetInput); - if (inputfd > maxfd) - maxfd = inputfd; + if (inputfd > maxfd) maxfd = inputfd; } else { FD_CLR(inputfd, &fdsetInput); } - int result; if (normMsgr.OutputPending() && !normMsgr.OutputReady()) { FD_SET(outputfd, &fdsetOutput); if (outputfd > maxfd) maxfd = outputfd; - result = select(maxfd+1, &fdsetInput, &fdsetOutput, NULL, NULL); } else { FD_CLR(outputfd, &fdsetOutput); - result = select(maxfd+1, &fdsetInput, NULL, NULL, NULL); } - switch (result) + if (maxfd >= 0) { - case -1: - switch (errno) - { - case EINTR: - case EAGAIN: - continue; - default: - perror("normMsgr select() error"); - // TBD - stop NormMsgr - break; - } - break; - case 0: - // shouldn't occur for now (no timeout) - continue; - default: - if (FD_ISSET(inputfd, &fdsetInput)) - { - normMsgr.ReadInput(); - } - if (FD_ISSET(normfd, &fdsetInput)) - { - NormEvent event; - if (NormGetNextEvent(normInstance, &event)) - normMsgr.HandleNormEvent(event); - } - if (FD_ISSET(outputfd, &fdsetOutput)) - { - normMsgr.SetOutputReady(); - } - break; + struct timeval timeout; + timeout.tv_sec = 1; + timeout.tv_usec = 0; + int result = select(maxfd+1, &fdsetInput, &fdsetOutput, NULL, &timeout); + switch (result) + { + case -1: + switch (errno) + { + case EINTR: + case EAGAIN: + continue; + default: + perror("normMsgr select() error"); + // TBD - stop NormMsgr + break; + } + break; + case 0: + // shouldn't occur for now (no timeout) + //fprintf(stderr, "normMsgr timeout ...\n"); + continue; + default: + if (FD_ISSET(inputfd, &fdsetInput)) + { + normMsgr.SetInputReady(); + } + /*if (FD_ISSET(normfd, &fdsetInput)) + { + NormEvent event; + while (NormGetNextEvent(normInstance, &event, false)) + normMsgr.HandleNormEvent(event); + }*/ + if (FD_ISSET(outputfd, &fdsetOutput)) + { + normMsgr.SetOutputReady(); + } + break; + } } + + NormSuspendInstance(normInstance); + NormEvent event; + while (NormGetNextEvent(normInstance, &event, false)) + normMsgr.HandleNormEvent(event); // As a result of reading input or NORM notification events, - // we may be ready to send a message if it's been read - if (normMsgr.InputMessageReady() && normMsgr.NormTxReady()) - normMsgr.SendMessage(); - // and/or output a received message if we need - if (normMsgr.OutputPending() && normMsgr.OutputReady()) - normMsgr.WriteOutput(); // TBD - implement output async i/o notification as needed + // we may be ready to read input and/or send a message if it's been read + const int LOOP_MAX = 100; + int loopCount = 0; + while (loopCount < LOOP_MAX) + { + loopCount++; + if (normMsgr.InputNeeded() && normMsgr.InputReady()) + normMsgr.ReadInput(); + if (normMsgr.InputMessageReady() && normMsgr.TxReady()) + normMsgr.SendMessage(); + // and/or output a received message if we need + if (normMsgr.OutputPending() && normMsgr.OutputReady()) + normMsgr.WriteOutput(); + } + NormResumeInstance(normInstance); } // end while(normMsgr.IsRunning() - fprintf(stderr, "normMsgr exiting ...\n"); + + + NormCloseDebugLog(normInstance); + + fprintf(stderr, "destroying session ...\n"); + normMsgr.CloseNormSession(); + + fprintf(stderr, "destroying instance ...\n"); NormDestroyInstance(normInstance); + + normMsgr.Destroy(); + + fprintf(stderr, "normMsgr exiting ...\n"); + } // end main() NormMsgr::Message::Message() @@ -980,6 +1226,7 @@ NormMsgr::Message::~Message() // to delete msg_buffer if it came from NORM if (NULL != msg_buffer) { + //fprintf(stderr, "deleting msg_buffer ...\n"); delete[] msg_buffer; msg_buffer = NULL; } @@ -1012,7 +1259,7 @@ bool NormMsgr::Message::Init(unsigned int size) } // end NormMsgr::Message::Init() NormMsgr::MessageQueue::MessageQueue() - : head(NULL), tail(NULL) + : head(NULL), tail(NULL), msg_count(0) { } @@ -1038,6 +1285,7 @@ void NormMsgr::MessageQueue::Prepend(Message& msg) tail = &msg; msg.next = head; head = &msg; + msg_count++; } // end NormMsgr::MessageQueue::Prepend() void NormMsgr::MessageQueue::Append(Message& msg) @@ -1049,6 +1297,7 @@ void NormMsgr::MessageQueue::Append(Message& msg) head = &msg; msg.prev = tail; tail = &msg; + msg_count++; } // end NormMsgr::MessageQueue::Append() void NormMsgr::MessageQueue::Remove(Message& msg) @@ -1062,6 +1311,7 @@ void NormMsgr::MessageQueue::Remove(Message& msg) else msg.next->prev = msg.prev; msg.prev = msg.next = NULL; + msg_count--; } // end NormMsgr::MessageQueue::Remove() NormMsgr::Message* NormMsgr::MessageQueue::RemoveHead() diff --git a/examples/normServer.cpp b/examples/normServer.cpp index cca42c8..f448b54 100644 --- a/examples/normServer.cpp +++ b/examples/normServer.cpp @@ -36,14 +36,27 @@ // socket" approach could be supported for unicast and SSM streams without too much // difficulty. -#include "normSocket.h" -#include // for inet_ntoa #include // for fprintf() #include // for memcmp() #include // for std::map<> +#include // more obvious + +#include "normSocket.h" +#ifdef WIN32 +#include "win32InputHandler.cpp" // to include full implementation +#include // for inet_ntoa() +#include // for inet_ntop() +#else +#include // for inet_ntoa #include #include // for, well, fnctl() #include // obvious child +#endif // if/else WIN32/UNIX + +void Usage() +{ + fprintf(stderr, "Usage: normServer [listen [/]][debug ][trace]\n"); +} // Our "server" indexes clients by their source addr/port class ClientInfo @@ -58,11 +71,11 @@ class ClientInfo UINT16 GetPort() const {return client_port;} - const char* GetAddressString() const; - void Print(FILE* filePtr) const; + const char* GetAddressString(); + void Print(FILE* filePtr); private: - UINT8 addr_version; // 4 or 6 + UINT8 addr_version; // 4 or 6 char client_addr[16]; // big enough for IPv6 UINT16 client_port; @@ -110,7 +123,7 @@ int ClientInfo::GetAddressFamily() const } // end ClientInfo::GetAddressFamily() -const char* ClientInfo::GetAddressString() const +const char* ClientInfo::GetAddressString() { static char text[64]; text[63] = '\0'; @@ -123,7 +136,7 @@ const char* ClientInfo::GetAddressString() const return text; } // end ClientInfo::GetAddressString() -void ClientInfo::Print(FILE* filePtr) const +void ClientInfo::Print(FILE* filePtr) { char text[64]; text[63] = '\0'; @@ -136,9 +149,45 @@ void ClientInfo::Print(FILE* filePtr) const fprintf(filePtr, "%s/%hu", text, client_port); } // end ClientInfo::Print() +class Client +{ + public: + Client(NormSocketHandle clientSocket); + ~Client(); + + NormSocketHandle GetSocket() const + {return client_socket;} + + bool GetWriteReady() const + {return write_ready;} + void SetWriteReady(bool state) + {write_ready = state;} + + unsigned int GetBytesWritten() const + {return bytes_written;} + void SetBytesWritten(unsigned long numBytes) + {bytes_written = numBytes;} + + private: + NormSocketHandle client_socket; + // These are state variables for unicast server -> client communication + bool write_ready; + unsigned int bytes_written; + +}; // end class Client + +Client::Client(NormSocketHandle clientSocket) + : client_socket(clientSocket), + write_ready(true), bytes_written(0) +{ +} + +Client::~Client() +{ +} // C++ map used to index client sessions by the client source addr/port -typedef std::map ClientMap; +typedef std::map ClientMap; ClientInfo NormGetClientInfo(NormNodeHandle client) { @@ -146,32 +195,46 @@ ClientInfo NormGetClientInfo(NormNodeHandle client) unsigned int addrLen = 16; UINT16 port; NormNodeGetAddress(client, addr, &addrLen, &port); - int addrFamily; UINT8 version; if (4 == addrLen) - { - addrFamily = AF_INET; version = 4; - } else - { - addrFamily = AF_INET6; version = 6; - } return ClientInfo(version, addr, port); -} // end NormGetClientInfo() +} // end NormGetClientInfo(NormNodeHandle) -NormSocketHandle FindClientSocket(ClientMap& clientMap, const ClientInfo& clientInfo) +ClientInfo NormGetSocketInfo(NormSocketHandle socket) +{ + char addr[16]; // big enough for IPv6 + unsigned int addrLen = 16; + UINT16 port; + NormGetPeerName(socket, addr, &addrLen, &port); + UINT8 version; + if (4 == addrLen) + version = 4; + else + version = 6; + return ClientInfo(version, addr, port); +} // end NormGetSocketInfo(NormSocketHandle) + +Client* FindClient(ClientMap& clientMap, const ClientInfo& clientInfo) { ClientMap::iterator it = clientMap.find(clientInfo); if (clientMap.end() != it) - return &(it->second); + return it->second; else + return NULL; +} // end FindClient() + +NormSocketHandle FindClientSocket(ClientMap& clientMap, const ClientInfo& clientInfo) +{ + Client* client = FindClient(clientMap, clientInfo); + if (NULL == client) return NORM_SOCKET_INVALID; + else + return client->GetSocket(); } // end FindClientSocket() - - int main(int argc, char* argv[]) { ClientMap clientMap; @@ -185,7 +248,6 @@ int main(int argc, char* argv[]) bool trace = false; unsigned int debugLevel = 0; - for (int i = 1; i < argc; i++) { const char* cmd = argv[i]; @@ -210,11 +272,13 @@ int main(int argc, char* argv[]) if (1 != sscanf(portPtr, "%hu", &serverPort)) { fprintf(stderr, "normServer error: invalid \"%s\"\n", portPtr); + Usage(); return -1; } } else if (0 == strncmp(cmd, "interface", len)) { + // Note the NormSocket code does not yet expose mcast interface mcastInterface = argv[++i]; } else if (0 == strncmp(cmd, "trace", len)) @@ -226,37 +290,42 @@ int main(int argc, char* argv[]) if (1 != sscanf(argv[++i], "%u", &debugLevel)) { fprintf(stderr, "normServer error: invalid debug level\n"); + Usage(); return -1; } } else { fprintf(stderr, "normServer error: invalid command \"%s\"\n", cmd); + Usage(); return -1; } } - // For unicast operation in this demo app, the server only "talks back" - // to one client on a first come, first serve basis. (Multiple clients - // can connect and send data _to_ the server, but in this simple example, - // the server only sends to one at a time. For the multicast server case, - // the server multicasts to the entire group. - NormSocketHandle firstClientSocket = NORM_SOCKET_INVALID; - NormInstanceHandle instance = NormCreateInstance(); - NormSocketHandle serverSocket = NormListen(instance, serverPort, groupAddrPtr); + NormSocketHandle serverSocket = NormOpen(instance); - if (trace) NormSetMessageTrace(NormGetSession(serverSocket), true); + NormListen(serverSocket, serverPort, groupAddrPtr); + + if (trace) NormSetMessageTrace(NormGetSocketSession(serverSocket), true); if (0 != debugLevel) NormSetDebugLevel(debugLevel); //NormSetDebugLevel(8); - //NormSetMessageTrace(NormGetSession(serverSocket), true); + //NormSetMessageTrace(NormGetSocketSession(serverSocket), true); + +#ifdef WIN32 + HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE); + Win32InputHandler inputHandler; + inputHandler.Open(); + HANDLE handleArray[2]; + handleArray[0] = NormGetDescriptor(instance); + handleArray[1] = inputHandler.GetEventHandle(); +#else // We use a select() call to multiplex input reading and NormSocket handling fd_set fdset; FD_ZERO(&fdset); - // Get our input (STDIN) descriptor and set non-blocking FILE* inputFile = stdin; int inputfd = fileno(inputFile); @@ -264,15 +333,48 @@ int main(int argc, char* argv[]) perror("normClient: fcntl(inputfd, O_NONBLOCK) error"); // Get our NormInstance descriptor int normfd = NormGetDescriptor(instance); +#endif // if/else WIN32/UNIX + bool keepGoing = true; bool writeReady = false; - unsigned int inputLength = 0; + int inputLength = 0; unsigned int bytesWritten = 0; const unsigned int BUFFER_LENGTH = 2048; char inputBuffer[BUFFER_LENGTH]; bool inputNeeded = false; // will be set to "true" upon CONNECT + bool inputClosed = false; + unsigned int clientCount = 0; while (keepGoing) { + bool normEventPending = false; + bool inputEventPending = false; +#ifdef WIN32 + DWORD handleCount = inputNeeded ? 2 : 1; + DWORD waitStatus = + MsgWaitForMultipleObjectsEx(handleCount, // number of handles in array + handleArray, // object-handle array + INFINITE, // time-out interval + QS_ALLINPUT, // input-event type + 0); + if ((WAIT_OBJECT_0 <= waitStatus) && (waitStatus < (WAIT_OBJECT_0 + handleCount))) + { + if (0 == (waitStatus - WAIT_OBJECT_0)) + normEventPending = true; + else + inputEventPending = true; + } + else if (-1 == waitStatus) + { + perror("normServer: MsgWaitForMultipleObjectsEx() error"); + break; + } + else + { + // TBD - any other status we should handle? + // (e.g. WAIT_TIMEOUT, WAIT_ABANDONED or WAIT_IO_COMPLETION) + continue; // ignore for now + } +#else FD_SET(normfd, &fdset); int maxfd = normfd; if (inputNeeded) @@ -287,12 +389,33 @@ int main(int argc, char* argv[]) int result = select(maxfd+1, &fdset, NULL, NULL, NULL); if (result <= 0) { - perror("normClient: select() error"); + perror("normServer: select() error"); break; } if (FD_ISSET(inputfd, &fdset)) + inputEventPending = true; + if (FD_ISSET(normfd, &fdset)) + normEventPending = true; +#endif // if/else WIN32/UNIX + + if (inputEventPending) { // Read input into our txBuffer +#ifdef WIN32 + inputLength = inputHandler.ReadData(inputBuffer, BUFFER_LENGTH); + if (inputLength > 0) + { + // We got our input + bytesWritten = 0; + inputNeeded = false; + } + else if (inputLength < 0) + { + inputHandler.Close(); + inputClosed = true; + } + // else zero bytes read, still need input +#else inputLength = fread(inputBuffer, 1, BUFFER_LENGTH, inputFile); if (inputLength > 0) { @@ -302,10 +425,13 @@ int main(int argc, char* argv[]) } else if (feof(inputFile)) { - // TBD - initiate server shutdown if it's job is - // only until the input is closed - // Meanwhile, we just stick around to receive stuff from the clients - inputNeeded = false; // TBD - should also fclose(inputFile)?? + if (stdin != inputFile) + { + fclose(inputFile); + inputFile = NULL; + } + inputClosed = true; + } else if (ferror(inputFile)) { @@ -318,12 +444,35 @@ int main(int argc, char* argv[]) // input starved, wait for next notification break; default: - perror("normClient: error reading input"); + perror("normServer: error reading input?!"); break; } } - } - if (FD_ISSET(normfd, &fdset)) +#endif // if/else WIN32/UNIX + if (inputClosed) + { + inputNeeded = false; + // Gracefully shutdown any connected clients + // TBD - set state variable to indicate exit upon all closed? + if (clientMap.empty()) + { + keepGoing = false; + continue; + } + else + { + ClientMap::iterator it; + for (it = clientMap.begin(); it != clientMap.end(); it++) + { + Client* client = it->second; + NormSocketHandle clientSocket = client->GetSocket(); + NormShutdown(clientSocket); + } + } + } + } // end if inputEventPending + + if (normEventPending) { // There's a NORM event pending @@ -333,6 +482,8 @@ int main(int argc, char* argv[]) ClientInfo clientInfo; if (NORM_NODE_INVALID != event.sender) clientInfo = NormGetClientInfo(event.sender); + else + clientInfo = NormGetSocketInfo(event.socket); switch (event.type) { case NORM_SOCKET_ACCEPT: @@ -344,26 +495,44 @@ int main(int argc, char* argv[]) if (NORM_SOCKET_INVALID != FindClientSocket(clientMap, clientInfo)) { // We think we're already connected to this client - fprintf(stderr, "normServer: duplicative %s client ...\n", - (NORM_REMOTE_SENDER_NEW == event.event.type) ? "new" : "reset"); + fprintf(stderr, "normServer: duplicative %s from client %s/%hu...\n", + (NORM_REMOTE_SENDER_NEW == event.event.type) ? "new" : "reset", + clientInfo.GetAddressString(), clientInfo.GetPort()); continue; } NormSocketHandle clientSocket = NormAccept(serverSocket, event.sender); - // TBD - For multicast, if we are sending a tx_stream, we could flush it here to + Client* client = new Client(clientSocket); + if (NULL == client) + { + perror("normServer: new Client() error"); + NormClose(clientSocket); + continue; + } + + // TBD - For multicast, if we are sending a tx_stream, we could flush it here to // bring the new receiver "up to date" ... probably would be best to // do this on some sort of timer-basis in the case of a bunch of receivers // joining in a short window of time ... if (trace) // note we're already tracing the mcast session - NormSetMessageTrace(NormGetSession(clientSocket), true); - //NormSetMessageTrace(NormGetSession(clientSocket), true); - clientMap[clientInfo] = clientSocket; + NormSetMessageTrace(NormGetSocketSession(clientSocket), true); + //NormSetMessageTrace(NormGetSocketSession(clientSocket), true); + clientMap[clientInfo] = client; + + // ACCEPTED is good as CONNECTED, so enable writing right away + client->SetWriteReady(true); + if (0 == clientCount) + { + // We have at least one client, so lets serve up some juicy input + inputNeeded = true; + writeReady = true; + } + clientCount++; fprintf(stderr, "normServer: ACCEPTED connection from %s/%hu\n", clientInfo.GetAddressString(), clientInfo.GetPort()); - // We have at least one client, so lets serve up some juicy input - inputNeeded = true; - writeReady = true; + // Note that an ACCEPTED socket is essentially CONNECTED, so we could + // go ahead and set writeReady to true, etc here } else { @@ -375,8 +544,8 @@ int main(int argc, char* argv[]) { fprintf(stderr, "normServer: CONNECTED to %s/%hu ...\n", clientInfo.GetAddressString(), clientInfo.GetPort()); - if (NORM_SOCKET_INVALID == firstClientSocket) - firstClientSocket = event.socket; + Client* client = FindClient(clientMap, clientInfo); + assert(NULL != client); break; } case NORM_SOCKET_READ: @@ -395,36 +564,60 @@ int main(int argc, char* argv[]) continue; } if (bytesRead > 0) + + { +#ifdef WIN32 + // Use WriteFile() so there is no buffer delay + DWORD dwWritten; + WriteFile(hStdout, buffer, bytesRead, &dwWritten, NULL); +#else fwrite(buffer, sizeof(char), bytesRead, stdout); +#endif // if/else WIN32 + } if (bytesRead < 1024) rxReady = false; } + // Following lines are test code (to immediately close connection after first reaad) + // (tests server stale connection "reject" command) + //clientMap.erase(clientInfo); + //NormClose(event.socket); break; } case NORM_SOCKET_WRITE: - // We only demo server data transmission for the multicast server case - // (see comment below) - if ((NULL != groupAddrPtr) || (event.socket == firstClientSocket)) + { + if (NULL != groupAddrPtr) + { + // We are a multicast server writeReady = true; - break; + } + else + { + Client* client = FindClient(clientMap, clientInfo); + assert(NULL != client); + client->SetWriteReady(true); + } + break; + } case NORM_SOCKET_CLOSING: + { fprintf(stderr, "normServer: client %s/%hu CLOSING connection ...\n", clientInfo.GetAddressString(), clientInfo.GetPort()); - if (event.socket == firstClientSocket) - firstClientSocket = NORM_SOCKET_INVALID; - - break; - case NORM_SOCKET_CLOSED: + Client* client = FindClient(clientMap, clientInfo); + assert(NULL != client); + client->SetWriteReady(false); + break; + } + case NORM_SOCKET_CLOSE: { fprintf(stderr, "normServer: connection to client %s/%hu CLOSED ...\n", clientInfo.GetAddressString(), clientInfo.GetPort()); - if (event.socket == firstClientSocket) - firstClientSocket = NORM_SOCKET_INVALID; clientMap.erase(clientInfo); - break; + NormClose(event.socket); + if (inputClosed && clientMap.empty()) + keepGoing = false; + break; } case NORM_SOCKET_NONE: break; - break; } // end switch(event.type) } else @@ -433,31 +626,96 @@ int main(int argc, char* argv[]) } } // end if FD_ISSET(normfd) - // For our _multicast_ "normServer" example, the server can send to the group - // (For a unicast "normServer", we would need to do something more complex - // to manage sending data to each individual client that connects to us. - // So, for the moment, the unicast "normServer" only sends to the "firstClientSocket" - NormSocketHandle sendSocket = (NULL != groupAddrPtr) ? serverSocket : firstClientSocket; - if (NORM_SOCKET_INVALID != sendSocket) + // If the normServer app has unsent data from STDIN, send it to the clients. + // Note that a _multicast_ server multicasts to all clients at once while a + // unicast server sends to each connected client individually + if ((inputLength > 0) && !inputNeeded) { - if (writeReady && (inputLength > 0)) + // There is inputBuffer data for the server to send to the client(s) + if (NULL == groupAddrPtr) { - // We have data in our inputBuffer and the NormSocket is "writeReady", so send it - bytesWritten += NormWrite(sendSocket, inputBuffer + bytesWritten, inputLength - bytesWritten); - if (bytesWritten < inputLength) + // Unicast the data to each connected client individually by iterating + // over the clientMap and sending data out to each pending client socket + // (inputNeeded is reset to "true" when _all_ clients are non-pending) + bool clientPending = false; + ClientMap::iterator it; + for (it = clientMap.begin(); it != clientMap.end(); it++) { - // Couldn't write whole inputBuffer, need to wait for NORM_SOCKET_WRITE event - writeReady = false; - } - else + Client* client = it->second; + if (!client->GetWriteReady()) + { + clientPending = true; + continue; + } + unsigned int numBytes = client->GetBytesWritten(); + if (numBytes < inputLength) + { + NormSocketHandle clientSocket = client->GetSocket(); + bytesWritten += NormWrite(clientSocket, inputBuffer + numBytes, inputLength - numBytes); + client->SetBytesWritten(numBytes); + if (bytesWritten < inputLength) + { + // Couldn't write whole inputBuffer, need to wait for NORM_SOCKET_WRITE event + // for this client socket + client->SetWriteReady(false); + clientPending = true; + } + else + { + // inputBuffer has been completely written to this client + NormFlush(clientSocket); + } + } + } + if (!clientPending) { - // inputBuffer has been completely written + // inputBuffer was sent to _all_ clients, so reset inputLength = 0; inputNeeded = true; - NormFlush(sendSocket); + // Reset all client "bytes_written" to zero for next chunk of data + for (it = clientMap.begin(); it != clientMap.end(); it++) + it->second->SetBytesWritten(0); } } - } + else + { + // Multicast the data out the "serverSocket" to all clients + // (Use the 'writeReady' and 'bytesWritten' state variables) + NormSocketHandle sendSocket = serverSocket; + if (NORM_SOCKET_INVALID != sendSocket) + { + if (writeReady && (inputLength > 0)) + { + // We have data in our inputBuffer and the NormSocket is "writeReady", so send it + bytesWritten += NormWrite(sendSocket, inputBuffer + bytesWritten, inputLength - bytesWritten); + if (bytesWritten < inputLength) + { + // Couldn't write whole inputBuffer, need to wait for NORM_SOCKET_WRITE event + writeReady = false; + } + else + { + // inputBuffer has been completely written + inputLength = 0; + inputNeeded = true; + NormFlush(sendSocket); + } + } + } + } + } // end if ((inputLength > 0) && !inputNeeded) + } // end while (keepGoing) +#ifdef WIN32 + inputHandler.Close(); +#else + if ((stdin != inputFile) && (NULL != inputFile)) + { + fclose(inputFile); + inputFile = NULL; + } +#endif // if/else WIN32 + NormClose(serverSocket); + serverSocket = NORM_SOCKET_INVALID; } // end main() diff --git a/examples/normSocket.cpp b/examples/normSocket.cpp index 5b1a35f..ad6767b 100644 --- a/examples/normSocket.cpp +++ b/examples/normSocket.cpp @@ -3,11 +3,21 @@ #include // for stderr #include // for assert() #include // for strlen() + +#ifdef WIN32 +#include // for inet_ntoa() (TBD - change to use Protolib routines?) +#include // for inet_ntop() +#else #include // for inet_ntoa() (TBD - change to use Protolib routines?) +#endif // if/else WIN32/UNIX // COMPILE: (assumes "normApi.h" in "include" ... // g++ -I../include -c normSocket.cpp +#define TRACE(...) fprintf(stderr, __VA_ARGS__) + +// Extra, non-public NORM API functions used by NormSocket stuff +extern void NormSetId(NormSessionHandle sesssionHandle, NormNodeId normId); // This "NormSocket" class is used to maintain tx/rx state for a NORM "socket" connection. // At the moment this "socket" connection represents a single, bi-directional NORM_OBJECT_STREAM @@ -19,6 +29,16 @@ const NormSocketHandle NORM_SOCKET_INVALID = (NormSocketHandle)0; const double NORM_DEFAULT_CONNECT_TIMEOUT = 60.0; +// This is extra stuff defined for NormSocket API extension purposes. As the NormSocket +// extension is finalized, these may be refined/relocated +enum {NORM_SOCKET_VERSION = 1}; +enum NormSocketCommand +{ + NORM_SOCKET_CMD_NULL = 0, // reserved, invalid/null command + NORM_SOCKET_CMD_REJECT, // sent by server-listener to reject invalid connection messages + NORM_SOCKET_CMD_ALIVE // TBD - for NormSocket "keep-alive" option? +}; + // a 'helper' function we use for debugging const char* NormNodeGetAddressString(NormNodeHandle node) { @@ -48,6 +68,7 @@ class NormSocket { public: NormSocket(NormSessionHandle normSession = NORM_SESSION_INVALID); + ~NormSocket(); // These methods identify the role of this socket with respect // to the client / server relationship (a "server socket" is @@ -69,10 +90,13 @@ class NormSocket bool IsClientSide() const {return (NULL == server_socket);} + NormInstanceHandle GetInstance() const + {return NormGetInstance(norm_session);} NormSessionHandle GetSession() const {return norm_session;} NormSessionHandle GetMulticastSession() const {return mcast_session;} + void InitRxStream(NormObjectHandle rxStream) {rx_stream = rxStream;} @@ -88,12 +112,14 @@ class NormSocket tx_stream_buffer_count = 0; tx_stream_bytes_remain = 0; tx_watermark_pending = false; + tx_ready = true; } + bool Open(NormInstanceHandle instance = NORM_INSTANCE_INVALID); - bool Listen(NormInstanceHandle instance, UINT16 serverPort, const char* groupAddr); + bool Listen(UINT16 serverPort, const char* groupAddr, const char* serverAddr); NormSocket* Accept(NormNodeHandle client, NormInstanceHandle instance = NORM_INSTANCE_INVALID); - bool Connect(NormInstanceHandle instance, const char* serverAddr, UINT16 serverPort, const char* groupAddr, NormNodeId clientId); + bool Connect(const char* serverAddr, UINT16 serverPort, UINT16 localPort, const char* groupAddr, NormNodeId clientId); // Write to tx stream (with flow control) @@ -108,11 +134,19 @@ class NormSocket // hard, immediate closure void Close(); + void SetUserData(const void* userData) + {user_data = userData;} + const void* GetUserData() const + {return user_data;} + + void SetTrace(bool state); + void GetSocketEvent(const NormEvent& event, NormSocketEvent& socketEvent); typedef enum State { CLOSED, + OPEN, LISTENING, CONNECTING, ACCEPTING, @@ -134,16 +168,50 @@ class NormSocket } void RemoveAckingNode(NormNodeId nodeId) {NormRemoveAckingNode(norm_session, nodeId);} - + + void GetPeerName(char* addr, unsigned int* addrLen, UINT16* port) + { + if (NULL == addrLen) return; + switch (remote_version) + { + case 4: + if ((*addrLen >= 4) && (NULL != addr)) + memcpy(addr, remote_addr, 4); + *addrLen = 4; + break; + case 6: + if ((*addrLen >= 16) && (NULL != addr)) + memcpy(addr, remote_addr, 16); + *addrLen = 16; + break; + default: + *addrLen = 0; + return; + } + if (NULL != port) *port = remote_port; + } private: + void UpdateRemoteAddress() + { + unsigned int addrLen = 16; + NormNodeGetAddress(remote_node, remote_addr, &addrLen, &remote_port); + if (4 == addrLen) + remote_version = 4; + else + remote_version = 6; + } + State socket_state; NormSessionHandle norm_session; NormSessionHandle mcast_session; // equals norm_session for a multicast server NormSocket* server_socket; // only applies to server-side sockets unsigned int client_count; // only applies to mcast server sockets NormNodeId client_id; // only applies to mcast client socket - NormNodeHandle remote_node; // + NormNodeHandle remote_node; // client socket peer info + UINT8 remote_version; // 4 or 6 + char remote_addr[16]; // big enough for IPv6 + UINT16 remote_port; // Send stream and associated flow control state variables NormObjectHandle tx_stream; bool tx_ready; @@ -154,6 +222,7 @@ class NormSocket bool tx_watermark_pending; // Receive stream state NormObjectHandle rx_stream; + const void* user_data; // for use by user application }; // end class NormSocket @@ -161,11 +230,12 @@ class NormSocket NormSocket::NormSocket(NormSessionHandle normSession) : socket_state(CLOSED), norm_session(normSession), mcast_session(NORM_SESSION_INVALID), server_socket(NULL), - client_count(0), client_id(NORM_NODE_NONE), remote_node(NORM_NODE_INVALID), + client_count(0), client_id(NORM_NODE_NONE), + remote_node(NORM_NODE_INVALID), remote_version(0), remote_port(0), tx_stream(NORM_OBJECT_INVALID), tx_ready(false), tx_segment_size(0), tx_stream_buffer_max(0), tx_stream_buffer_count(0), tx_stream_bytes_remain(0), tx_watermark_pending(false), - rx_stream(NORM_OBJECT_INVALID) + rx_stream(NORM_OBJECT_INVALID), user_data(NULL) { // For now we use the NormSession "user data" option to associate // the session with a "socket". In the future we may add a @@ -175,72 +245,104 @@ NormSocket::NormSocket(NormSessionHandle normSession) NormSetUserData(normSession, this); } +NormSocket::~NormSocket() +{ + Close(); + if (NORM_SESSION_INVALID != norm_session) + { + NormDestroySession(norm_session); + norm_session = NORM_SESSION_INVALID; + } +} -bool NormSocket::Listen(NormInstanceHandle instance, UINT16 serverPort, const char* groupAddr) +bool NormSocket::Open(NormInstanceHandle instance) { if (CLOSED != socket_state) { - fprintf(stderr, "NormSocket::Listen() error: socket already open?!\n"); + fprintf(stderr, "NormSocket::Open() error: socket already open?!\n"); return false; } + // A proper NormNodeId will be set upon NormBind(), NormConnect(), or NormListen() + if (NORM_SESSION_INVALID == (norm_session = NormCreateSession(instance, "127.0.0.1", 0, NORM_NODE_ANY))) + { + perror("NormSocket::Open() error"); + return false; + } + NormSetUserData(norm_session, this); + socket_state = OPEN; + return true; +} // end NormSocket::Open() + +bool NormSocket::Listen(UINT16 serverPort, const char* groupAddr, const char* serverAddr) +{ + if (OPEN != socket_state) + { + fprintf(stderr, "NormSocket::Listen() error: socket not open?!\n"); + return false; + } + // THe code below will be cleaned/tightened up somewhat once all is working + // Note that port reuse here lets us manage our "client" rx-only unicast connections the + // way we need, but does allow a second multicast server to be started on this group which leads + // to undefined behavior. TBD - see if we can prevent via binding wizardry + // (How is it done for TCP servers? - probably because the accept() call is in the network stack + // instead of user-space) Perhaps we could have a semaphore lock to block a second "server" if (NULL != groupAddr) { // TBD - validate that "groupAddr" is indeed a multicast address - norm_session = NormCreateSession(instance, groupAddr, serverPort, NORM_NODE_ANY); - NormSetTxPort(norm_session, serverPort); // can't do this and receive unicast feedback + NormChangeDestination(norm_session, groupAddr, serverPort); + NormSetId(norm_session, 1); // server always uses NormNodeId '1' + // TBD - we _could_ let the server have an independent, ephemeral tx_port + // by _not_ calling NormSetTxPort() here to enable multiple multicast + // servers on same group/port on same host if server instance use unique NormNodeIds? + NormSetTxPort(norm_session, serverPort); // can't do this and distinguish unicast feedback + NormSetMulticastInterface(norm_session, serverAddr); + NormSetRxPortReuse(norm_session, true); mcast_session = norm_session; + + NormSetMulticastLoopback(norm_session, true); // for testing + } else { // For unicast , the "server" has a NormNodeId of '1' and the "clients" are '2' // to obviate need for explicit id management and will allow NAT to work, etc - norm_session = NormCreateSession(instance, "127.0.0.1", serverPort, 1); - } - if (NORM_SESSION_INVALID == norm_session) + NormChangeDestination(norm_session, "127.0.0.1", serverPort); + NormSetId(norm_session, 1); // server always uses NormNodeId '1' + NormSetTxPort(norm_session, serverPort); +#ifdef WIN32 + // UDP socket bind/connect does not work properly on WIN32, so no port reuse + // (so a little different strategy is used for Win32 connections) + NormSetRxPortReuse(norm_session, false, serverAddr); +#else + NormSetRxPortReuse(norm_session, true, serverAddr); +#endif // if/else WIN32 + + } + + // TBD - the next four calls could be combined into a "NormStartListener()" function + + // Set session to track incoming clients by their addr/port + // (instead of NormNodeId as usual) + NormSetServerListener(norm_session, true); + + // Our listener is a "silent" receiver since all actual reception + // (unicast) is handed off to a separate "client" session + NormSetSilentReceiver(norm_session, true); + + // So that the listener can construct (unsent) ACKs without failure + NormSetDefaultUnicastNack(norm_session, true); + + // Note we use a small buffer size here since a "listening" socket isn't + // going to be receiving data (TBD - implement a mechanism to handoff remote + // sender (i.e. "client") from parent + if (!NormStartReceiver(norm_session, 2048)) { - fprintf(stderr, "NormSocket::Listen() error: NormCreateSession() failure\n"); + fprintf(stderr, "NormSocket::Listen() error: NormStartReceiver() failure (perhaps port already in use)\n"); + //NormDestroySession(norm_session); + //norm_session = NORM_SESSION_INVALID; return false; } - NormSetUserData(norm_session, this); - // Note the port reuse here lets us manage our "client" rx-only unicast connections the - // way we need, but does allow a second multicast server to be started on this group which leads - // to undefined behavior. TBD - see if we can prevent via binding wizardry - // (How is it done for TCP servers? - probably because the accept() call is in the network stack - // instead of user-space) Perhaps we could have a semaphore lock to block a second "server" - NormSetRxPortReuse(norm_session, true); - - // use default sync policy so a "serversocket" doesn't NACK the senders it detects - // NORM_SYNC_STREAM tries to get everything the sender has cached/buffered - //NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_STREAM); - //NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); - - if (NULL == groupAddr) - { - // Unicast server - // Note we use a small buffer size here since a "listening" socket isn't - // going to be receiving data (TBD - implement a mechanism to handoff remote - // sender (i.e. "client") from parent - if (!NormStartReceiver(norm_session, 2048)) - { - fprintf(stderr, "NormSocket::Listen() error: NormStartReceiver() failure (perhaps port already in use)\n"); - NormDestroySession(norm_session); - norm_session = NORM_SESSION_INVALID; - } - } - else - { - //NormSetMulticastInterface(norm_session, "lo0"); - NormSetMulticastLoopback(norm_session, true); // for testing - if (!NormStartReceiver(norm_session, 2048)) - { - fprintf(stderr, "NormSocket::Listen() error: NormStartReceiver() failure (perhaps port already in use)\n"); - NormDestroySession(norm_session); - norm_session = NORM_SESSION_INVALID; - } - // TBD - We _could_ go ahead and call NormStartSender(), but for now we'll wait until we hear the application - // makes at least one NormAccept() call ... - } server_socket = this; socket_state = LISTENING; return true; @@ -272,10 +374,16 @@ NormSocket* NormSocket::Accept(NormNodeHandle client, NormInstanceHandle instanc UINT16 serverPort = NormGetRxPort(norm_session); if (NORM_INSTANCE_INVALID == instance) instance = NormGetInstance(norm_session); - +#ifdef WIN32 + NormSessionHandle clientSession = NormCreateSession(instance, clientAddr, 0, 1); +#else NormSessionHandle clientSession = NormCreateSession(instance, clientAddr, serverPort, 1); - - NormSetTxPort(clientSession, serverPort, false); + NormSetTxPort(clientSession, serverPort, true); +#endif // if/else WIN32/UNIX + + // NORM_SYNC_STREAM tries to get everything the sender has cached/buffered + NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_STREAM); + //NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); // This next API call will cause NORM to tightly bind the remote client src addr/port to // our server port so the "clientSession" captures the client packets instead of the "server" session @@ -286,24 +394,34 @@ NormSocket* NormSocket::Accept(NormNodeHandle client, NormInstanceHandle instanc // rx socket buffer may look like a new sender if deleted now, so // we wait for NORM_REMOTE_SENDER_INACTIVE to delete - // NORM_SYNC_STREAM tries to get everything the sender has cached/buffered - //NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_STREAM); - NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); - - NormSetRxPortReuse(clientSession, true, 0, clientAddr, clientPort); // "connects" to remote client addr/port +#ifndef WIN32 + // Enable rx port reuse since it's the server port, and connect + // this socket to client addr/port for unique, tight binding + // TBD - support option to bind to specific server address + //fprintf(stderr, "accepting connection from %s/%d on port %d ...\n", clientAddr, clientPort, serverPort); + NormSetRxPortReuse(clientSession, true, NULL, clientAddr, clientPort); +#endif // WIN32 NormSetDefaultUnicastNack(clientSession, true); - + NormStartReceiver(clientSession, 2*1024*1024); + + // This call immediately inserts the "client" remote sender state + // into the newly-created clientSession by injecting a NORM_CMD(CC) + // message (and, as a result, a NORM_ACK is sent back to the client + // for a quick initial RTT estimate) + NormTransferSender(clientSession, client); NormSocket* clientSocket = new NormSocket(clientSession); clientSocket->server_socket = this; // this is a server-side socket clientSocket->remote_node = client; + clientSocket->UpdateRemoteAddress(); NormNodeSetUserData(client, clientSocket); NormNodeId clientId = NormNodeGetId(client); if (IsUnicastSocket()) { + NormChangeDestination(clientSession, clientAddr, clientPort, false); // point unicast session dest to client port // The clientSession is bi-directional so we need to NormStartSender(), etc NormAddAckingNode(clientSession, 2); //clientId); NormSetFlowControl(clientSession, 0); // disable timer-based flow control since we do explicit, ACK-based flow control @@ -316,15 +434,21 @@ NormSocket* NormSocket::Accept(NormNodeHandle client, NormInstanceHandle instanc // (probably for heavyweight; for lightweight we know the client // has already started his multicast receiver) AddAckingNode(clientId); // TBD - check result - NormNodeHandle node = NormGetAckingNodeHandle(mcast_session, clientId); - NormNodeSetUserData(node, clientSocket); // a way to track mcast client sockets + NormNodeHandle node = NormGetAckingNodeHandle(mcast_session, clientId); + NormNodeSetUserData(node, clientSocket); // a way to track mcast client sockets clientSocket->mcast_session = mcast_session; - clientSocket->client_id = client_id; + clientSocket->client_id = clientId; if (LISTENING == socket_state) { - NormSetFlowControl(norm_session, 0); // disable timer-based flow control since we do explicit, ACK-based flow control - NormStartSender(norm_session, NormGetRandomSessionId(), 2*1024*1024, 1400, 16, 4); - socket_state = CONNECTED; + NormSetFlowControl(norm_session, 0); // disable timer-based flow control since we do explicit, ACK-based flow control + NormStartSender(norm_session, NormGetRandomSessionId(), 2*1024*1024, 1400, 16, 4); + socket_state = CONNECTED; + if (NORM_OBJECT_INVALID == tx_stream) + { + tx_stream = NormStreamOpen(norm_session, 2*1024*1024); + InitTxStream(tx_stream, 2*1024*1024, 1400, 16); + } + } /* The code below would be invoked for "heavyweight" mcast client admission (for the moment we go with a "lightweight" model - this might be invokable upon @@ -341,28 +465,42 @@ NormSocket* NormSocket::Accept(NormNodeHandle client, NormInstanceHandle instanc NormSetWatermark(clientSession, tempStream, true); // future NORM API will add "bool watermark" option to graceful close */ } - clientSocket->socket_state = ACCEPTING; // will transision to CONNECTED when client is detected on new clientSession + // Note that for this lightweight connection mode, ACCEPTING is essentially CONNECTED so + // app can treat this as a CONNECTED socket until otherwise notified + // Should we start the time clientSocket timer and timeout if not connected in a timely fashion? + clientSocket->socket_state = ACCEPTING; // will transition to CONNECTED when client is detected on new clientSession return clientSocket; } // end NormSocket::Accept() // TBD - provide options for binding to a specific local address, interface, etc -bool NormSocket::Connect(NormInstanceHandle instance, const char* serverAddr, UINT16 serverPort, const char* groupAddr, NormNodeId clientId) +bool NormSocket::Connect(const char* serverAddr, + UINT16 serverPort, + UINT16 localPort, + const char* groupAddr, + NormNodeId clientId) { // For unicast connections, the "client" manages a single NormSession for send and receive // (For multicast connections, there are two sessions: The same unicast session that will - // be set to txOnly upon CONNECT and a NormSession for multicast reception) - norm_session = NormCreateSession(instance, "127.0.0.1", 0, clientId); // TBD - use "clientId" here for mcast sockets? - if (NORM_SESSION_INVALID == norm_session) - { - fprintf(stderr, "NormSocket::Connect() error: NormCreateSession() failure\n"); - return false; - } + // be set to txOnly upon CONNECT and a separate NormSession for multicast reception) + // Setting the session port to zero here causes an ephemeral port to be assigned _and_ + // it is also a single socket (tx_socket == rx_socket) session for client->server unicast + NormSetId(norm_session, clientId); // NORM_SYNC_STREAM tries to get everything the sender has cached/buffered - //NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_STREAM); - NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); - - NormSetUserData(norm_session, this); - NormSetRxPortReuse(norm_session, true, NULL, serverAddr, serverPort); + NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_STREAM); + //NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); + +#ifndef WIN32 + // We don't set reuse for the ephemeral port, but do want to 'connect' to + // the server addr/port for this unicast client->server socket + NormSetRxPortReuse(norm_session, false, NULL, serverAddr, serverPort); +#endif // WIN32 + + if (0 != localPort) + { + // Set client session up to use a user-specified (non-ephemeral) port number + NormChangeDestination(norm_session, NULL, localPort, false); + NormSetTxPort(norm_session, localPort); + } // TBD - for a multicast connection, the unicast receiver could be started with minimal buffer // (not that it matters since the buffers aren't activated until a sender starts sending _data_) if (!NormStartReceiver(norm_session, 2*1024*1024)) // to get ephemeral port assigned @@ -370,9 +508,14 @@ bool NormSocket::Connect(NormInstanceHandle instance, const char* serverAddr, UI fprintf(stderr, "NormSocket::Connect() error: unicast NormStartReceiver() failure\n"); return false; } - NormChangeDestination(norm_session, serverAddr, serverPort, false); // "connect" our NORM tx_socket (so we can get ICMP) - NormSessionId sessionId = NormGetRandomSessionId(); - NormAddAckingNode(norm_session, 1); // servers always have NormNodeId '1' + + NormSetSynStatus(norm_session, true); + + // Point our unicast socket at the unicast server addr/port + NormChangeDestination(norm_session, serverAddr, serverPort, false); + NormSessionId sessionId = NormGetRandomSessionId(); // TBD - use ephemeral port as session/instance id? + //NormAddAckingNode(norm_session, 1); // servers always have NormNodeId '1' for unicast sessions + NormSetAutoAckingNodes(norm_session, NORM_TRACK_RECEIVERS); // this way we get informed upon first ACK NormSetFlowControl(norm_session, 0); // since we do explicit, ACK-based flow control if (!NormStartSender(norm_session, sessionId, 2*1024*1024, 1400, 16, 4)) { @@ -383,18 +526,21 @@ bool NormSocket::Connect(NormInstanceHandle instance, const char* serverAddr, UI if (NULL != groupAddr) { // Create the "mcast_session" for multicast reception - mcast_session = NormCreateSession(instance, groupAddr, serverPort, clientId); + mcast_session = NormCreateSession(NormGetInstance(norm_session), groupAddr, serverPort, clientId); //NormSetTxPort(mcast_session, serverPort); // TBD - not sure this is a good idea if multiple clients on a machine? NormSetUserData(mcast_session, this); // NORM_SYNC_STREAM tries to get everything the sender has cached/buffered - //NormSetDefaultSyncPolicy(mcast_session, NORM_SYNC_STREAM); - NormSetDefaultSyncPolicy(mcast_session, NORM_SYNC_ALL); + NormSetDefaultSyncPolicy(mcast_session, NORM_SYNC_STREAM); + //NormSetDefaultSyncPolicy(mcast_session, NORM_SYNC_ALL); NormSetDefaultUnicastNack(mcast_session, true); // we could optionally allow multicast NACKing, too - NormSetMulticastLoopback(norm_session, true); // for testing + NormSetMulticastLoopback(mcast_session, true); // for testing client_id = clientId; // TBD - make this SSM??? ... this would allow for multiple servers using the same groupAddr/port - NormSetRxPortReuse(mcast_session, true, groupAddr); // Should we upgrade rx port reuse and 'connect' to server tx port upon CONNECT? + // Note we 'connect' to server addr/port to make this associated with single, specific mcast server + // TBD - Once we add code to set multicast interface, we can set the port reuse + // here to 'connect' to the specified server addr/port for tighter binding + NormSetRxPortReuse(mcast_session, true, groupAddr);//, serverAddr, serverPort); // For a "lightweight" client->server connection establishment, we go ahead and // stop our unicast receiver and start multicast receiver, assuming the server // will admit us into the group. @@ -412,7 +558,11 @@ bool NormSocket::Connect(NormInstanceHandle instance, const char* serverAddr, UI NormSetUserTimer(norm_session, NORM_DEFAULT_CONNECT_TIMEOUT); } server_socket = NULL; // this is a client-side socket - + if (NORM_OBJECT_INVALID == tx_stream) + { + tx_stream = NormStreamOpen(norm_session, 2*1024*1024); + InitTxStream(tx_stream, 2*1024*1024, 1400, 16); + } socket_state = CONNECTING; return true; @@ -421,9 +571,15 @@ bool NormSocket::Connect(NormInstanceHandle instance, const char* serverAddr, UI unsigned int NormSocket::Write(const char* buffer, unsigned int numBytes) { - // TBD - make sure the socket is CONNECTED first + // Make sure the socket is CONNECTED first + // (TBD - an option for allowing NormWrite() to start sending + // data prior to connection confirmation is being considered + // to accelerate data transfer (most useful for short-lived + // or 'urgent' connections such as transactions) + if (CONNECTED != socket_state) return 0; + if (IsMulticastClient() && IsServerSide()) - { + { // This is multicast server rxonly client socket, so we redirect // the write() to the associated txonly multicast socket return server_socket->Write(buffer, numBytes); @@ -464,10 +620,12 @@ unsigned int NormSocket::Write(const char* buffer, unsigned int numBytes) NormSetWatermark(norm_session, tx_stream); tx_watermark_pending = true; } + // TBD - set "tx_ready" to false if tx_stream_buffer_count == tx_stream_buffer_max here ??? return bytesWritten; } else { + tx_ready = false; return 0; } } // end NormSocket::Write() @@ -477,8 +635,8 @@ void NormSocket::Flush(bool eom, NormFlushMode flushMode) // TBD - make sure the socket is CONNECTED first if (IsMulticastClient() && IsServerSide()) { - // This is multicast server rxOnly client socket, so we redirect - // the flush() to the associated txonly multicast socket + // This is multicast server rx-only client socket, so we redirect + // the flush() to the associated tx-only multicast socket return server_socket->Flush(eom, flushMode); } @@ -512,80 +670,98 @@ void NormSocket::Flush(bool eom, NormFlushMode flushMode) tx_watermark_pending = true; } } - } // end NormSocket::Flush() - - bool NormSocket::Read(char* buffer, unsigned int& numBytes) - { - // TBD - make sure rx_stream is valid! - // TBD - make sure this is not a tx only client socket ... - return NormStreamRead(rx_stream, buffer, &numBytes); - } // end NormSocket::Read() - - void NormSocket::Shutdown() - { - if ((NORM_OBJECT_INVALID == tx_stream) || - (IsServerSide() && IsMulticastClient())) - { - Close(); // close immediately since this socket doesn't control a tx_stream - } - else - { - // It controls a tx_stream, so shutdown the tx_stream gracefully - NormStreamClose(tx_stream, true); // Note our "trick" here to do a graceful close, _then_ watermark to get ack - NormSetWatermark(norm_session, tx_stream, true); // future NORM API will add "bool watermark" option to graceful close - socket_state = CLOSING; - } - } // end NormSocket::Shutdown() - - void NormSocket::Close() - { - if (IsMulticastSocket()) - { - if (IsServerSide()) - { - if (IsServerSocket()) - { - // IsMulticastSocket() guarantees the mcast_session is valid - // Dissociate remaining clients from this session and set their - // timer so that NORM_SOCKET_CLOSED events are dispatched for them - NormNodeId nodeId = NORM_NODE_NONE; - while (NormGetNextAckingNode(mcast_session, &nodeId)) - { - NormNodeHandle node = NormGetAckingNodeHandle(mcast_session, nodeId); - assert(NORM_NODE_INVALID != node); - NormSocket* clientSocket = (NormSocket*)NormNodeGetUserData(node); - NormSetUserTimer(clientSocket->norm_session, 0.0); - } - // for mcast server mcast_session == norm_session so it's destroyed below - } - else - { - // "IsServerSide()" guarantees the "server_socket" is non-NULL - // server-side multicast client socket closing, so we - // need to remove this "client" NormNodeId from the mcast - // session's acking node list - server_socket->RemoveAckingNode(client_id); - } - } - else // client-side multicast socket, so we need to destroy mcast_session, too +} // end NormSocket::Flush() + +bool NormSocket::Read(char* buffer, unsigned int& numBytes) +{ + // TBD - make sure rx_stream is valid! + // TBD - make sure this is not a tx only client socket ... + if (NORM_OBJECT_INVALID != rx_stream) + { + return NormStreamRead(rx_stream, buffer, &numBytes); + } + else + { + numBytes = 0; + return true; + } +} // end NormSocket::Read() + +void NormSocket::Shutdown() +{ + // TBD - should we call NormStopReceiver(norm_session) here + // or have SHUT_RD, SHUT_WR, and SHUT_RDWR flags + // like the sockets "shutdown()" call??? + // For now, we do a "graceful" SHUT_RDWR behavior + if (CONNECTED == socket_state) + { + NormStopReceiver(norm_session); + rx_stream = NULL; + if ((IsServerSide() && IsMulticastClient()) || (NORM_OBJECT_INVALID == tx_stream)) { - NormDestroySession(mcast_session); + // Use a zero-timeout to immediately post NORM_SOCKET_CLOSE notification + NormSetUserTimer(norm_session, 0.0); } - mcast_session = NORM_SESSION_INVALID; - } - if (NORM_SESSION_INVALID != norm_session) - { - NormDestroySession(norm_session); - norm_session = NORM_SESSION_INVALID; - } - server_socket = NULL; - tx_stream = NORM_OBJECT_INVALID; - tx_segment_size = 0; - tx_stream_buffer_max = tx_stream_buffer_count = tx_stream_bytes_remain = 0; - tx_watermark_pending = false; - rx_stream = NORM_OBJECT_INVALID; - socket_state = CLOSED; - } // end NormSocket::Close() + else if (NORM_OBJECT_INVALID != tx_stream) + { + // It controls a tx_stream, so shutdown the tx_stream gracefully + NormStreamClose(tx_stream, true); // Note our "trick" here to do a graceful close, _then_ watermark to get ack + NormSetWatermark(norm_session, tx_stream, true); // future NORM API will add "bool watermark" option to graceful close + } + socket_state = CLOSING; + } +} // end NormSocket::Shutdown() + +void NormSocket::Close() +{ + if (IsMulticastSocket()) + { + if (IsServerSide()) + { + if (IsServerSocket()) + { + // IsMulticastSocket() guarantees the mcast_session is valid + // Dissociate remaining clients from this session and set their + // timers so that NORM_SOCKET_CLOSE events are dispatched for them + NormNodeId nodeId = NORM_NODE_NONE; + while (NormGetNextAckingNode(mcast_session, &nodeId)) + { + NormNodeHandle node = NormGetAckingNodeHandle(mcast_session, nodeId); + assert(NORM_NODE_INVALID != node); + NormSocket* clientSocket = (NormSocket*)NormNodeGetUserData(node); + NormSetUserTimer(clientSocket->norm_session, 0.0); + } + // for mcast server mcast_session == norm_session so it's destroyed below + } + else + { + // "IsServerSide()" guarantees the "server_socket" is non-NULL + // server-side multicast client socket closing, so we + // need to remove this "client" NormNodeId from the mcast + // session's acking node list + server_socket->RemoveAckingNode(client_id); + } + } + else // client-side multicast socket, so we need to destroy mcast_session + { + NormDestroySession(mcast_session); + } + mcast_session = NORM_SESSION_INVALID; + } + if (NORM_SESSION_INVALID != norm_session) + { + NormStopSender(norm_session); + NormStopReceiver(norm_session); + } + server_socket = NULL; + remote_node = NORM_NODE_INVALID; + tx_stream = NORM_OBJECT_INVALID; + tx_segment_size = 0; + tx_stream_buffer_max = tx_stream_buffer_count = tx_stream_bytes_remain = 0; + tx_watermark_pending = false; + rx_stream = NORM_OBJECT_INVALID; + socket_state = CLOSED; +} // end NormSocket::Close() void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketEvent) @@ -593,6 +769,7 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE socketEvent.socket = (NormSocketHandle)this; socketEvent.type = NORM_SOCKET_NONE; // default socket event type if no socket-specific state change occurs socketEvent.event = event; + //fprintf(stderr, "NormSocket::GetSocketEvent() norm event type:%d session:%p\n", event.type, event.session); switch (event.type) { case NORM_TX_QUEUE_EMPTY: @@ -613,6 +790,7 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE { switch (socket_state) { + /* case ACCEPTING: { // This only comes into play for the "confirmed connection" @@ -629,17 +807,17 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE else { // Client didn't acknowledge, so we cull him from our server - Close(); - socketEvent.type = NORM_SOCKET_CLOSED; + socketEvent.type = NORM_SOCKET_CLOSE; } break; } + */ case CLOSING: { // Socket that was shutdown has either been acknowledged or timed out // TBD - should we issue a different event if ACK_FAILURE??? Close(); - socketEvent.type = NORM_SOCKET_CLOSED; + socketEvent.type = NORM_SOCKET_CLOSE; break; } default: @@ -656,10 +834,10 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE if (IsUnicastSocket() || IsMulticastClient()) { // We could be infinitely persistent w/ NormResetWatermark() - //NormResetWatermark(event.session); - // For now, we'll just declare the connection broken/closed - Close(); - socketEvent.type = NORM_SOCKET_CLOSED; + // (TBD - provide a NormSocket "keep alive" option + NormResetWatermark(event.session); + // Or just declare the connection broken/closed + //socketEvent.type = NORM_SOCKET_CLOSE; } else { @@ -680,10 +858,10 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE assert(NORM_NODE_INVALID != node); NormSocket* clientSocket = (NormSocket*)NormNodeGetUserData(node); assert(NULL != clientSocket); - // We use the session timer to dispatch a NORM_SOCKET_CLOSED per failed client + // We use the session timer to dispatch a NORM_SOCKET_CLOSE per failed client // (This will also remove the client from this server's acking list) - NormSetUserTimer(clientSocket->norm_session, 0.0); clientSocket->socket_state = CLOSING; + NormSetUserTimer(clientSocket->norm_session, 0.0); } } // TBD - what do we if all clients failed ... issue a NORM_SOCKET_DISCONNECT event, @@ -706,6 +884,7 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE } break; } + case NORM_ACKING_NODE_NEW: // This means we have received an ACK from the server case NORM_REMOTE_SENDER_RESET: case NORM_REMOTE_SENDER_NEW: { @@ -716,14 +895,29 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE break; case ACCEPTING: if (IsServerSide() && IsClientSocket() && (NORM_NODE_INVALID != remote_node)) + { NormNodeDelete(remote_node); + } case CONNECTING: // TBD - We should validate that it's the right remote sender // (i.e., by source address and/or nodeId) NormCancelUserTimer(norm_session); socketEvent.type = NORM_SOCKET_CONNECT; + NormSetSynStatus(norm_session, false); socket_state = CONNECTED; + // Since UDP connect/bind doesn't really work properly on + // Windows, the Windows NormSocket server farms out client connections + // to new ephemeral port numbers, so we need to update + // the destination port upon connection (Yuck!) remote_node = event.sender; + UpdateRemoteAddress(); + NormChangeDestination(norm_session, NULL, remote_port); + if (NORM_OBJECT_INVALID == tx_stream) + { + tx_stream = NormStreamOpen(norm_session, 2*1024*1024); + InitTxStream(tx_stream, 2*1024*1024, 1400, 16); + } + break; case CONNECTED: if (IsMulticastSocket()) @@ -735,30 +929,58 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE } else { + // TBD - validate if this same server or not (e.g. by source addr/port) // Different sender showing up in multicast group!? - fprintf(stderr, "NormSocket warning: multicast sender %s reset?!\n", NormNodeGetAddressString(event.sender)); - // TBD - should Close() the socket and issue a NORM_SOCKET_CLOSED event + if (event.sender != remote_node) + { + char senderAddr[16]; + unsigned int addrLen = 16; + UINT16 senderPort; + NormNodeGetAddress(event.sender, senderAddr, &addrLen, &senderPort); + unsigned int senderVersion = (4 == addrLen) ? 4 : 6; + if ((senderVersion != remote_version) || + (senderPort != remote_port) || + (0 != memcmp(senderAddr, remote_addr, addrLen))) + { + //fprintf(stderr, "NormSocket warning: multicast sender %s reset?!\n", NormNodeGetAddressString(event.sender)); + } + } + // TBD - should Close() the socket and issue a NORM_SOCKET_CLOSE event // and leave it up to the application to reconnect? Or should we // provides some sort of NORM_SOCKET_DISCONNECT event - socketEvent.type = NORM_SOCKET_CLOSED; - Close(); + //socketEvent.type = NORM_SOCKET_CLOSE; } } else // unicast { // Eemote sender reset? How do we tell? - fprintf(stderr, "NormSocket warning: unicast sender %s reset?!\n", NormNodeGetAddressString(event.sender)); - socketEvent.type = NORM_SOCKET_CLOSED; - Close(); + // TBD - validate if this same server or not (e.g. by source addr/port) + if (event.sender != remote_node) + { + char senderAddr[16]; + unsigned int addrLen = 16; + UINT16 senderPort; + NormNodeGetAddress(event.sender, senderAddr, &addrLen, &senderPort); + unsigned int senderVersion = (4 == addrLen) ? 4 : 6; + if ((senderVersion != remote_version) || + (senderPort != remote_port) || + (0 != memcmp(senderAddr, remote_addr, addrLen))) + { + fprintf(stderr, "NormSocket warning: unicast sender %s reset?!\n", NormNodeGetAddressString(event.sender)); + } + } + // Close(); + //socketEvent.type = NORM_SOCKET_CLOSE; } break; - default: // CLOSING, CLOSED + default: // CLOSING, CLOSE // shouldn't happen break; } break; } + case NORM_SEND_ERROR: { switch (socket_state) @@ -769,9 +991,12 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE case CLOSING: if (IsMulticastServer()) fprintf(stderr, "SEND_ERROR on a multicast server socket?!\n"); - socketEvent.event.sender = remote_node; - socketEvent.type = NORM_SOCKET_CLOSED; + /*else + fprintf(stderr, "SEND_ERROR session:%p sender:%p remote_node:%p (%s)\n", + event.session, event.sender, remote_node, + NormNodeGetAddressString(remote_node));*/ Close(); + socketEvent.type = NORM_SOCKET_CLOSE; break; default: // shouldn't happen @@ -779,7 +1004,6 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE } break; } - case NORM_USER_TIMEOUT: { switch (socket_state) @@ -788,9 +1012,8 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE case ACCEPTING: // accepted client didn't follow through case CONNECTED: // multicast client ack failure case CLOSING: - socketEvent.event.sender = remote_node; - socketEvent.type = NORM_SOCKET_CLOSED; Close(); + socketEvent.type = NORM_SOCKET_CLOSE; break; default: // shouldn't happen @@ -799,6 +1022,35 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE } break; } + case NORM_RX_CMD_NEW: + { + char buffer[4096]; + unsigned int buflen = 4096; + if (NormNodeGetCommand(event.sender, buffer, &buflen)) + { + if ((buflen < 2) || (NORM_SOCKET_VERSION == buffer[0])) + { + if (NORM_SOCKET_CMD_REJECT == buffer[1]) + { + Close(); + socketEvent.type = NORM_SOCKET_CLOSE; + } + else + { + fprintf(stderr, "NormSocket warning: received unknown command\n"); + } + } + else + { + fprintf(stderr, "NormSocket warning: received command with invalid version\n"); + } + } + else + { + fprintf(stderr, "NormSocket warning: unable to get received command\n"); + } + break; + } case NORM_REMOTE_SENDER_INACTIVE: { switch (socket_state) @@ -854,7 +1106,7 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE else { // Stream reset - fprintf(stderr, "NormSocket::GetSocketEvent(): client stream reset?!\n"); + fprintf(stderr, "NormSocket::GetSocketEvent(NORM_RX_OBJECT_NEW) warning: client stream reset?!\n"); } break; default: // CONNECTING, ACCEPTING, CLOSING, CLOSED @@ -868,11 +1120,12 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE switch (socket_state) { case CONNECTED: - case CLOSING: // we allow reading during graceful closure // TBD - use an rx_ready indication to filter this event a little more if (IsServerSocket()) break; // we don't receive data on server socket - assert(event.object == rx_stream); - socketEvent.type = NORM_SOCKET_READ; + if (event.object == rx_stream) + socketEvent.type = NORM_SOCKET_READ; + else + fprintf(stderr, "NormSocket::GetSocketEvent(NORM_RX_OBJECT_UPDATED) warning: non-matching rx object\n"); break; default: // shouldn't happen @@ -880,20 +1133,30 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE } break; } + case NORM_RX_OBJECT_ABORTED: case NORM_RX_OBJECT_COMPLETED: { + if (event.object != rx_stream) + break; // not our stream, so ignore rx_stream = NORM_OBJECT_INVALID; switch (socket_state) { case CONNECTED: // Initiate graceful closure of our tx_stream to allow at least some time to // acknowledge the remote before closing everything down - NormStreamClose(tx_stream, true); // Note our "trick" here to do a graceful close, _then_ watermark to get ack - NormSetWatermark(norm_session, tx_stream, true); // future NORM API will add "bool watermark" option to graceful close - socket_state = CLOSING; + if (NORM_OBJECT_INVALID != tx_stream) + { + NormStreamClose(tx_stream, true); // Note our "trick" here to do a graceful close, _then_ watermark to get ack + NormSetWatermark(norm_session, tx_stream, true); // future NORM API will add "bool watermark" option to graceful close + socket_state = CLOSING; + } + else + { + // This still allows at least a chance of an ACK to be sent upon completion + NormSetUserTimer(norm_session, 0.0); + } socketEvent.type = NORM_SOCKET_CLOSING; break; - case CLOSING: // We're already closing, so just let that complete. This helps make sure we allow // at least some time to acknowledge the remote before closing everything down @@ -902,53 +1165,79 @@ void NormSocket::GetSocketEvent(const NormEvent& event, NormSocketEvent& socketE // shouldn't happen break; } + break; } default: break; } + //fprintf(stderr, "NormSocket::GetSocketEvent() returning NormSocket event type:%d session:%p\n", socketEvent.type, event.session); } // end NormSocket::GetSocketEvent() +void NormSocket::SetTrace(bool state) +{ + if (NORM_SESSION_INVALID != norm_session) + NormSetMessageTrace(norm_session, state); + if (NORM_SESSION_INVALID != mcast_session) + NormSetMessageTrace(mcast_session, state); +} // end NormSocket::SetTrace() /////////////////////////////////////////////////////////////////////////////////// // NormSocket API implementation - -// TBD - provide options for binding to a specific local address, interface, etc -NormSocketHandle NormListen(NormInstanceHandle instance, UINT16 serverPort, const char* groupAddr) -{ - // TBD - check results - NormSocket* normSocket = new NormSocket(); - normSocket->Listen(instance, serverPort, groupAddr); - return (NormSocketHandle)normSocket; -} // end NormListen() -NormSocketHandle NormAccept(NormSocketHandle serverSocket, NormNodeHandle client) -{ - // TBD - VALIDATE PARAMETERS AND ERROR CHECK ALL THE API CALLS MADE HERE !!!!! - NormSocket* s = (NormSocket*)serverSocket; - return (NormSocketHandle)(s->Accept(client)); -} // end NormAccept() - - -// TBD - provide options for binding to a specific local address, interface, etc -NormSocketHandle NormConnect(NormInstanceHandle instance, const char* serverAddr, UINT16 serverPort, const char* groupAddr, NormNodeId clientId) +NormSocketHandle NormOpen(NormInstanceHandle instance) { NormSocket* normSocket = new NormSocket(); if (NULL == normSocket) { - perror("NormConnect() new NormSocket() error"); - return NULL; + perror("NormOpen() new NormSocket() error"); + return NORM_SOCKET_INVALID; } - if (normSocket->Connect(instance, serverAddr, serverPort, groupAddr, clientId)) + else if (normSocket->Open(instance)) { - return normSocket; + return (NormSocketHandle)normSocket; } else { + perror("NormOpen() error"); delete normSocket; - return NULL; + return NORM_SOCKET_INVALID; } +} // end NormOpen() + +// TBD - provide options for binding to a specific local address, interface, etc +bool NormListen(NormSocketHandle normSocket, UINT16 serverPort, const char* groupAddr, const char* serverAddr) +{ + // TBD - make sure normSocket is valid + NormSocket* s = (NormSocket*)normSocket; + return s->Listen(serverPort, groupAddr, serverAddr); +} // end NormListen() + + +NormSocketHandle NormAccept(NormSocketHandle serverSocket, NormNodeHandle client, NormInstanceHandle instance) +{ + // TBD - if another instance handle is provided use that instead + // TBD - VALIDATE PARAMETERS AND ERROR CHECK ALL THE API CALLS MADE HERE !!!!! + NormSocket* s = (NormSocket*)serverSocket; + NormInstanceHandle serverInstance = s->GetInstance(); + NormSuspendInstance(serverInstance); + NormSocketHandle clientSocket = s->Accept(client, instance); + NormResumeInstance(serverInstance); + return clientSocket; +} // end NormAccept() + + +// TBD - provide options for binding to a specific local address, interface, etc +bool NormConnect(NormSocketHandle normSocket, const char* serverAddr, UINT16 serverPort, UINT16 localPort, const char* groupAddr, NormNodeId clientId) +{ + // TBD - make sure normSocket is valid + NormSocket* s = (NormSocket*)normSocket; + NormInstanceHandle instance = s->GetInstance(); + NormSuspendInstance(instance); + bool result = s->Connect(serverAddr, serverPort, localPort, groupAddr, clientId); + NormResumeInstance(instance); + return result; } // end NormConnect() @@ -957,13 +1246,20 @@ ssize_t NormWrite(NormSocketHandle normSocket, const void *buf, size_t nbyte) // TBD - we could make write() and read() optionally blocking or non-blocking // by using GetSocketEvent() as appropriate (incl. returning error conditions, etc) NormSocket* s = (NormSocket*)normSocket; - return (ssize_t)s->Write((const char*)buf, nbyte); + NormInstanceHandle instance = s->GetInstance(); + NormSuspendInstance(instance); + ssize_t result = (ssize_t)s->Write((const char*)buf, nbyte); + NormResumeInstance(instance); + return result; } // end NormWrite() int NormFlush(NormSocketHandle normSocket) { NormSocket* s = (NormSocket*)normSocket; + NormInstanceHandle instance = s->GetInstance(); + NormSuspendInstance(instance); s->Flush(); + NormResumeInstance(instance); return 0; } // end NormFlush() @@ -972,27 +1268,51 @@ ssize_t NormRead(NormSocketHandle normSocket, void *buf, size_t nbyte) // TBD - we could make write() and read() optionally blocking or non-blocking // by using GetSocketEvent() as appropriate (incl. returning error conditions, etc) NormSocket* s = (NormSocket*)normSocket; + NormInstanceHandle instance = s->GetInstance(); + NormSuspendInstance(instance); // TBD - make sure s->rx_stream is valid unsigned int numBytes = nbyte; + ssize_t result; if (s->Read((char*)buf, numBytes)) - return numBytes; + result = numBytes; else - return -1; // broken stream error (TBD - enumerate socket error values) + result = -1; // broken stream error (TBD - enumerate socket error values) + NormResumeInstance(instance); + return result; } // end NormWrite() - void NormShutdown(NormSocketHandle normSocket) { NormSocket* s = (NormSocket*)normSocket; + NormInstanceHandle instance = s->GetInstance(); + NormSuspendInstance(instance); s->Shutdown(); + NormResumeInstance(instance); } // end NormShutdown() void NormClose(NormSocketHandle normSocket) { NormSocket* s = (NormSocket*)normSocket; + NormInstanceHandle instance = s->GetInstance(); + NormSuspendInstance(instance); s->Close(); + NormResumeInstance(instance); + delete s; } // end NormClose() +void NormSetSocketUserData(NormSocketHandle normSocket, const void* userData) +{ + if (NORM_SOCKET_INVALID != normSocket) + ((NormSocket*)normSocket)->SetUserData(userData); +} // end NormSetSocketUserData() + +const void* NormGetSocketUserData(NormSocketHandle normSocket) +{ + NormSocket* s = (NormSocket*)normSocket; + return s->GetUserData(); +} // end NormGetSocketUserData() + + // This gets and translates low level NORM API events to NormSocket events // given the "normSocket" state bool NormGetSocketEvent(NormInstanceHandle instance, NormSocketEvent* socketEvent, bool waitForEvent) @@ -1001,6 +1321,7 @@ bool NormGetSocketEvent(NormInstanceHandle instance, NormSocketEvent* socketEven NormEvent event; if (NormGetNextEvent(instance, &event, waitForEvent)) { + NormSuspendInstance(instance); NormSocket* normSocket = NULL; if (NORM_SESSION_INVALID != event.session) normSocket = (NormSocket*)NormGetUserData(event.session); @@ -1014,6 +1335,7 @@ bool NormGetSocketEvent(NormInstanceHandle instance, NormSocketEvent* socketEven { normSocket->GetSocketEvent(event, *socketEvent); } + NormResumeInstance(instance); return true; } else @@ -1024,14 +1346,27 @@ bool NormGetSocketEvent(NormInstanceHandle instance, NormSocketEvent* socketEven // Other helper functions -NormSessionHandle NormGetSession(NormSocketHandle normSocket) +void NormGetPeerName(NormSocketHandle normSocket, char* addr, unsigned int* addrLen, UINT16* port) +{ + NormSocket* s = (NormSocket*)normSocket; + s->GetPeerName(addr, addrLen, port); +} // end NormGetPeerName() + +NormSessionHandle NormGetSocketSession(NormSocketHandle normSocket) { NormSocket* s = (NormSocket*)normSocket; return s->GetSession(); -} // end NormGetSession() +} // end NormGetSocketSession() -NormSessionHandle NormGetMulticastSession(NormSocketHandle normSocket) +NormSessionHandle NormGetSocketMulticastSession(NormSocketHandle normSocket) { NormSocket* s = (NormSocket*)normSocket; return s->GetMulticastSession(); -} // end NormGetSession() +} // end NormGetSocketMulticastSession() + + +void NormSetSocketTrace(NormSocketHandle normSocket, bool enable) +{ + NormSocket* s = (NormSocket*)normSocket; + s->SetTrace(enable); +} diff --git a/examples/normSocket.h b/examples/normSocket.h index a4e3ebb..d62de1f 100644 --- a/examples/normSocket.h +++ b/examples/normSocket.h @@ -19,7 +19,7 @@ // // 0) Make "NormInstanceHandle" optional for Listen/Connect/Accept // (for eventual blocking usage, it wouldn't be needed) -// 1) Support asymmetric server->multicast, client->unicast model +// 1) Support asymmetric server->multicast, client->unicast model (DONE) // 2) Provide non-blocking option for read/write calls // 3) leverage ICMP (e.g., port unreachable) feedback to detect connect() failure (DONE) // 4) provide means for connect() to time out??? (DONE) @@ -41,6 +41,12 @@ // f) Buffer size (sender, stream, receiver) // g) cache bounds / limits + +#ifdef WIN32 +#include // for SSIZE_T +typedef SSIZE_T ssize_t; +#endif // WIN32 + #include "normApi.h" typedef const void* NormSocketHandle; @@ -55,42 +61,55 @@ extern const double NORM_DEFAULT_CONNECT_TIMEOUT; // Main NormSocket API Functions -NormSocketHandle NormOpen(NormInstanceHandle instance = NORM_INSTANCE_INVALID); +NormSocketHandle NormOpen(NormInstanceHandle instance); -NormSocketHandle NormListen(NormInstanceHandle instance, UINT16 serverPort, const char* groupAddr = NULL); +bool NormListen(NormSocketHandle normSocket, + UINT16 serverPort, + const char* groupAddr = NULL, + const char* serverAddr = NULL); -NormSocketHandle NormConnect(NormInstanceHandle instance, - const char* serverAddr, - UINT16 serverPort, - const char* groupAddr = NULL, - NormNodeId clientId = NORM_NODE_NONE); +bool NormConnect(NormSocketHandle normSocket, + const char* serverAddr, + UINT16 serverPort, + UINT16 localPort = 0, + const char* groupAddr = NULL, + NormNodeId clientId = NORM_NODE_ANY); -NormSocketHandle NormAccept(NormSocketHandle serverSocket, NormNodeHandle clientNode, NormInstanceHandle instance = NORM_INSTANCE_INVALID); +NormSocketHandle NormAccept(NormSocketHandle serverSocket, + NormNodeHandle clientNode, + NormInstanceHandle instance = NORM_INSTANCE_INVALID); void NormShutdown(NormSocketHandle normSocket); void NormClose(NormSocketHandle normSocket); -ssize_t NormRead(NormSocketHandle normSocket, void *buf, size_t nbyte); +ssize_t NormRead(NormSocketHandle normSocket, void* buf, size_t nbyte); -ssize_t NormWrite(NormSocketHandle normSocket, const void *buf, size_t nbyte); +ssize_t NormWrite(NormSocketHandle normSocket, const void* buf, size_t nbyte); int NormFlush(NormSocketHandle normSocket); -// Helper API functions +// NormSocket helper functions -NormSessionHandle NormGetSession(NormSocketHandle normSocket); -NormSessionHandle NormGetMulticastSession(NormSocketHandle normSocket); +void NormSetSocketUserData(NormSocketHandle normSocket, const void* userData); +const void* NormGetSocketUserData(NormSocketHandle normSocket); + +NormInstanceHandle NormGetSocketInstance(NormSocketHandle normSocket); +NormSessionHandle NormGetSocketSession(NormSocketHandle normSocket); +NormSessionHandle NormGetSocketMulticastSession(NormSocketHandle normSocket); +void NormGetPeerName(NormSocketHandle normSocket, char* addr, unsigned int* addrLen, UINT16* port); + +void NormSetSocketTrace(NormSocketHandle normSocket, bool enable); typedef enum NormSocketEventType { - NORM_SOCKET_NONE = 0, + NORM_SOCKET_NONE = 0, // applications should generally ignore these NORM_SOCKET_ACCEPT, // only issued for listening server sockets NORM_SOCKET_CONNECT, // notification confirming connection to server NORM_SOCKET_READ, // edge-triggered notification that socket is ready for reading NORM_SOCKET_WRITE, // edge-triggered notification that socket is ready for writing NORM_SOCKET_CLOSING, // indicates remote endpoint is closing socket (only read data at this point) - NORM_SOCKET_CLOSED // indicates socket is now closed (invalid for further operations) + NORM_SOCKET_CLOSE // indicates socket is now closed (invalid for further operations) } NormSocketEventType; typedef struct @@ -116,5 +135,4 @@ typedef struct bool NormGetSocketEvent(NormInstanceHandle normInstance, NormSocketEvent* event, bool waitForEvent = true); - #endif // !_NORM_SOCKET diff --git a/examples/normSocketNotes.txt b/examples/normSocketNotes.txt new file mode 100644 index 0000000..9a600ad --- /dev/null +++ b/examples/normSocketNotes.txt @@ -0,0 +1,155 @@ +NormSocket API Extension Notes + +The NormSocket API (currently defined in examples/normSocket.h) is an in-development extension to the +base (low-level) NORM API (defined in include/normApi.h) that provides a more familiar (socket-like), +easier-to-use API for some specific NORM use patterns. The use patterns supported here are those of a +client-server paradigm where "clients" have individual, reliable connections to the server and the +"server" can have either a one-to-many multicast connection or individual unicast connections to the set +of clients. The paradigm follows a connection-oriented socket model where the server "listens" for +incoming connections initiated by clients. The low-level NORM API is used to establish multiple NORM +sessions as needed to support the necessary data exhange relationships. Since this is a +connection-oriented model, explicit ACK-based flow control used for both client-to-server and +server-to-client(s) data transfers. This is a more controlled model of NORM use as compared to the +looser "connection-free" model of operation NORM use has typically observed. + +The NormSocket API code makes careful use of operating system socket-binding to enable port reuse and +simpler session establishment where clients can "connect" to the server listening on a well-known port +as compared to typical base NORM API use where participants can exchange data on a a priori known common +port number (and group address if multicast). However, the clients do need to know the unicast address +of the server and the paradigm is somewhat that of Single-Source Multicast (SSM). Client application +can, of course, using multiple NormSocket instances connect to multiple multicast (or unicast) servers.. +THere +A server application can use the NormListen() call to establish a session that listens for incoming +connection requests, Client applications can use the NormConnect() call to initiate a connection to a +listening server. In response, the server application can use NormAccept() to establish connections with +the clients. + +At the moment a NormGetSocketEvent() (analogous to the NormGetNextEvent() call in the base NORM API) can +be used to provide cues to changes in NormSocket state (notification of incoming connections, connection +establishment, read/write readiness, etc.). A descriptor is provided for asynchronous I/O notification +of pending events. At the moment, the default behavior is a single NormInstance, and hence a single +descriptor, can be used for multiple NormSockets, but it is possible to call NormCreateInstance() and +associate a separate NormInstance (and hence separate descriptor) with each NormSocket that is +listending, connected or accepted if desired. + +As the NormSocket API extension matures, more examples and documentation will be provided as well as +incorporating it (possibly directly) into the "normApi.h" header file and NORM library. At the moment, +"examples/normClient.cpp" and "examples/normServer.cpp" illustrate (and test) basic usage of the API. +The "normServer" can be run as a unicast server using the syntax: + +./normServer listen + +or a multicast server using the syntax: + +./normServer listen / + +Corresponding "normClient" instances can connect to unicast server using the syntax: + +./normClient connect / + +or to a multicast server using the syntax: + +./normClient connect /, + +There are some additional command-line options shared by these examples such as "trace" to enable NORM's +message trace output and "debug " to invoke different levels of debug output. Note that an +option to specify the multicast interface is planned but not yet implemented in these examples. +Similarly, an option to set a specific unicast server binding will also be provided. + +TODO (not in priority order): + +1) When client initiates connection and server-listener receives NORM_CMD(CC), add code + to directly "inject" the NORM_CMD(CC) into the newly created server-side client + session such that NORM_ACK is immediately sent. This will give client a more immediate + RTT measurement, etc. + +2) Add options to set transmit rate, congestion control, etc options via NormSocketHandle + (current code works at default tx rate of 64 kbps) + +3) The server-listener session should be modified so NORM_DATA messages are ignored. Also, + NormSetServerListener() could also set the session a silent receiver, too? + +4) Add support for additional transport modes in addition to current byte stream: +` 1) Message stream + 2) NORM_OBJECT_DATA (w/ data copy option?) + 3) NORM_OBJECT_FILE + +5) Add option to specify multicast "interface" and use in examples. + +6) Enable setting a specific unicast server binding address. + + +=========================================================================== +MISC. DEVELOPMENT NOTES + +// Notional program outline +// (optional mcastAddr iff asymm multicast/unicast (i.e. SSM)) + +normSocket = NormListen(port, [mcastAddr]) + +_or_ + +normSocket = NormConnect(addr, port, [mcastAddr]) + +while NormGetSocketEvent(normSocket) +{ + + switch (event.GetType()) + NORM_ACCEPT: + clientSocket = NormAccept(normSocket, client, [instance]); // rx-only for asymm mcast case + + NORM_CONNECTED: + // in response to NormConnect when "server" is detected (NEW_REMOTE_SENDER) + + NORM_TX_READY: + upon QUEUE_VACANCY, etc ... socket _might_ be write() ready + + NORM_RX_READY: + + upon RX_OBJECT_NEW, RX_OBJECT_UPDATE ... socket _might_ be read() ready + + NORM_CLOSING: ??? + upon stream end received from remote sender + + NORM_CLOSED: ??? + upon stream end _and_ ack to our stream end +} + + + + *** Server listen on a port (optional mcast address): + +- Create NormSession that receive-only on given port (if it's asymm mcast, then sender enabled, too) + +- For each client (detected via NEW_REMOTE_SENDER yielding a NormSocket::NORM_ACCEPT event) + Create tightly binded session for rx (and tx if unicast) ... this will fail if binding already exists + + +*** Client connect + +- Create single socket (common ephemeral tx/rx port) NormSession for server destination addr/port. +- Call NormStartSender() to initiate probing of server +- NORM_CONNECTED upon NEW_REMOTE_SENDER (from server) ??? + + +QUESTIONS: + +1) How does a server reject a connection request? (for now, just ignore) + +2) How is a "connection" gracefully terminated? (NormShutdown() vs. NormClose()?) + + +NormClose(graceful = false) ... + a) stop sender/receiver, destroy everything + +NormShutdown(graceful = true) ... + a) close stream (gracefully, w/ ack request) + b) upon ack receipt (or timeout) _and_ rx stream end, destroy everything + + +Upon receipt of rx stream end, close tx stream gracefully ... + + initiator -> stream end; upon ack set tx_stream = NULL and wait for rx_stream end + + receptor -> upon rx_stream end, initiate tx_stream shutdown and wait for ack + diff --git a/examples/normStreamer.cpp b/examples/normStreamer.cpp new file mode 100644 index 0000000..bcc23ae --- /dev/null +++ b/examples/normStreamer.cpp @@ -0,0 +1,2178 @@ + +#include "normApi.h" +#include "protoSocket.h" + +#include // for printf(), etc +#include // for srand() +#include // for strrchr(), memset(), etc +#include // for gettimeofday() +#include // for htons() +#include // for, well, fnctl() +#include // obvious child +#include // embarrassingly obvious +#include // Memory Lock. +#include // Adjust scheduler (linux) +#include // for setpriority() stuff +#ifdef LINUX +#include +#endif // LINUX + +const unsigned int LOOP_MAX = 100; + +// Setting SHOOT_FIRST to non-zero means that an ACK request +// will be used to advance the acking "watermark" point +// with each message fully written to the transmit stream. +// The alternative "ack later" behavior waits to send a new +// ACK request until any pending flow control ACK requeset +// has completed. This latter approach favors throughput +// over timeliness of message delivery. I.e., lower data +// rate applications that are concerned with low-latency message +// delivery can potentially benefit from the "shoot first" +// behavior while very high throughput applications that want +// to "keep the pipe full as possible" can benefit from the +// "ack later" behavior. The difference between these behaviors, +// since ACK requests are cued for all messages when flow +// control is _not_ pending, is somewhat subtle and developers +// may want to assess both behaviors for their application. +// Additionally, limiting ACK request to flow control only is +// another possible approach as well as dynamically updating +// something like the "tx_stream_buffer_count" with each +// message ACK request initiated could be possible. The caution +// with the SHOOT_FIRST type strategies and high throughput is +// the application may end up "chasing" the ACK request until +// flow control buffer limits are reached and end up with +// "dead air" time. There are always tradeoffs! + +#define SHOOT_FIRST 0 + +class NormStreamer +{ + public: + NormStreamer(); + ~NormStreamer(); + + // some day build these directly into NORM API + enum CCMode {NORM_FIXED, NORM_CC, NORM_CCE, NORM_CCL}; + + enum + { + MSG_HEADER_SIZE = 2, // Big Endian message length header size + MSG_SIZE_MAX = 65535 // (including length header) + }; + + void SetOutputFile(FILE* filePtr) + { + output_file = filePtr; + output_fd = fileno(filePtr); + } + + void SetLoopback(bool state) + { + loopback = state; + if (NORM_SESSION_INVALID != norm_session) + NormSetMulticastLoopback(norm_session, state); + } + + bool EnableUdpRelay(const char* relayAddr, unsigned short relayPort); + bool EnableUdpListener(unsigned short thePort, const char* groupAddr, const char * interfaceName); + bool UdpListenerEnabled() const + {return input_socket.IsOpen();} + bool UdpRelayEnabled() const + {return output_socket.IsOpen();} + + int GetInputDescriptor() const + {return (input_socket.IsOpen() ? input_socket.GetHandle() : fileno(input_file));} + int GetOutputDescriptor() const + {return (output_socket.IsOpen() ? output_socket.GetHandle() : fileno(output_file));} + + bool OpenNormSession(NormInstanceHandle instance, + const char* addr, + unsigned short port, + NormNodeId nodeId); + void CloseNormSession(); + + void SetNormCongestionControl(CCMode ccMode); + void SetFlushMode(NormFlushMode flushMode) + {flush_mode = flushMode;} + void SetNormTxRate(double bitsPerSecond) + { + assert(NORM_SESSION_INVALID != norm_session); + NormSetTxRate(norm_session, bitsPerSecond); + } + void SetNormMulticastInterface(const char* ifaceName) + { + assert(NORM_SESSION_INVALID != norm_session); + NormSetMulticastInterface(norm_session, ifaceName); + } + void SetNormMessageTrace(bool state) + { + assert(NORM_SESSION_INVALID != norm_session); + NormSetMessageTrace(norm_session, state); + } + void AddAckingNode(NormNodeId ackId) + { + assert(NORM_SESSION_INVALID != norm_session); + NormAddAckingNode(norm_session, ackId); + norm_acking = true; // invoke ack-based flow control + } + + + + bool Start(bool sender, bool receiver); + void Stop() + {is_running = false;} + bool IsRunning() const + {return is_running;} + void HandleNormEvent(const NormEvent& event); + + // Sender methods + int GetInputFile() const + {return input_fd;} + void SetInputReady() + {input_ready = true;} + bool InputReady() const + {return input_ready;} + bool InputNeeded() const + {return input_needed;} + void ReadInput(); + void ReadInputSocket(); + bool TxPending() const + {return (!input_needed && (input_index < input_msg_length));} + bool TxReady() const + {return (tx_ready && (!norm_acking || (tx_stream_buffer_count < tx_stream_buffer_max)));} + void SendData(); + unsigned int WriteToStream(const char* buffer, unsigned int numBytes); + void FlushStream(bool eom, NormFlushMode flushMode); + + // Receiver methods + bool RxNeeded() const + {return rx_needed;} + bool RxReady() const + {return rx_ready;} + void RecvData(); + int GetOutputFile() const + {return output_fd;} + void SetOutputReady() + {output_ready = true;} + bool OutputReady() const + {return output_ready;} + bool OutputPending() const + {return (!rx_needed && (output_index < output_msg_length));} + + + void SetOutputBucketRate(double bitsPerSecond) + { + output_bucket_rate = bitsPerSecond / 8.0; // convert to bytes per second + output_bucket_interval = 1.0 / output_bucket_rate; + } + + void SetOutputBucketDepth(unsigned int numBytes) + {output_bucket_depth = numBytes;} + unsigned int GetOutputBucketDepth() const + {return output_bucket_depth;} + + double GetOutputBucketTimeout() const + { + if (0 != output_bucket_depth) + { + if (OutputPending()) + { + unsigned int pendingBytes = output_msg_length - output_index; + if (pendingBytes > output_bucket_count) + { + return ((double)(pendingBytes - output_bucket_count)) * output_bucket_interval; + } + else + { + return 0.0; + } + } + else + { + return -1.0; + } + } + else + { + return 0.0; + } + } + + double GetOutputBucketFillTime() const + { + return (output_bucket_count < output_bucket_depth) ? + ((double)(output_bucket_depth - output_bucket_count)) * output_bucket_interval : + 0.0; + } + + bool OutputBucketReady() const + { + if (0 != output_bucket_depth) + { + unsigned int pendingBytes = output_msg_length - output_index; + return (output_bucket_count >= pendingBytes); + } + else + { + return true; + } + } + void CreditOutputBucket(double interval) + { + if (0 != output_bucket_depth) + { + output_bucket_count += interval * output_bucket_rate; + if (output_bucket_count > output_bucket_depth) + output_bucket_count = output_bucket_depth; + } + } + + + void WriteOutput(); + void WriteOutputSocket(); + + void OmitHeader(bool state) + {omit_header = state;} + + unsigned long GetInputByteCount() const + {return input_byte_count;} + unsigned long GetTxByteCount() const + {return tx_byte_count;} + + // These can only be called post-OpenNormSession() + void SetSilentReceiver(bool state) + {NormSetSilentReceiver(norm_session, true);} + void SetTxLoss(double txloss) + {NormSetTxLoss(norm_session, txloss);} + // Set the scheduler for running the app and norm threads. + static bool BoostPriority(); + + void SetSegmentSize(unsigned short segmentSize) + {segment_size = segmentSize;} + void SetBlockSize(unsigned short blockSize) + {block_size = blockSize;} + void SetNumParity(unsigned short numParity) + {num_parity = numParity;} + void SetAutoParity(unsigned short autoParity) + {auto_parity = autoParity;} + + void SetStreamBufferSize(unsigned int value) + {stream_buffer_size = value;} + void SetTxSocketBufferSize(unsigned int value) + {tx_socket_buffer_size = value;} + void SetRxSocketBufferSize(unsigned int value) + {rx_socket_buffer_size = value;} + + void SetInputSocketBufferSize(unsigned int value) + {input_socket_buffer_size = value;} + void SetOutputSocketBufferSize(unsigned int value) + {output_socket_buffer_size = value;} + + // Check that sequence numbers increase by one each time. + // Assumes that sequence number is 8- or 4-byte network-order first 8 bytes of buffer. + void CheckSequenceNumber(const char* buffer, const char* source); + void CheckSequenceNumber64(const char* buffer, const char* source); + void CheckSequenceNumber32(const char* buffer, const char* source); + void SetCheckSequence(unsigned int value) // 64 or 32 + {check_sequence = value;} + + private: + NormSessionHandle norm_session; + bool is_multicast; + bool loopback; + bool is_running; + + // State variables for reading input messages for transmission + ProtoSocket input_socket; // optional UDP socket to "listen" + FILE* input_file; + int input_fd; // stdin by default + bool input_ready; + bool input_needed; + char input_buffer[MSG_SIZE_MAX]; + unsigned int input_msg_length; + unsigned int input_index; + + NormObjectHandle tx_stream; + bool tx_ready; + unsigned int tx_stream_buffer_max; + unsigned int tx_stream_buffer_threshold; // flow control threshold + unsigned int tx_stream_buffer_count; + unsigned int tx_stream_bytes_remain; + bool tx_watermark_pending; + bool norm_acking; + bool tx_ack_pending; + NormFlushMode flush_mode; // TBD - allow for "none", "passive", "active" options + + // Receive stream and state variables for writing received messages to output + NormObjectHandle rx_stream; + bool rx_ready; + bool rx_needed; + bool msg_sync; + double output_bucket_rate; // bytes per second + double output_bucket_interval; // seconds per byte + unsigned int output_bucket_depth; // bytes + unsigned int output_bucket_count; // bytes + ProtoSocket output_socket; // optional UDP socket for recv msg output + ProtoAddress relay_addr; // dest addr for recv msg relay + FILE* output_file; + int output_fd; // stdout by default + bool output_ready; + char output_buffer[MSG_SIZE_MAX]; + unsigned int output_msg_length; + unsigned int output_index; + + + // These are some options mainly for testing purposes + bool omit_header; // if "true", receive message length header is _not_ written to output + //bool rx_silent; + //double tx_loss; + unsigned long input_byte_count; + unsigned long tx_byte_count; + + unsigned short segment_size; + unsigned short block_size; + unsigned short num_parity; + unsigned short auto_parity; + + unsigned long stream_buffer_size; + unsigned int tx_socket_buffer_size; + unsigned int rx_socket_buffer_size; + unsigned int input_socket_buffer_size; + unsigned int output_socket_buffer_size; + + unsigned int check_sequence; + uint64_t sequence_prev; +}; // end class NormStreamer + +NormStreamer::NormStreamer() + : norm_session(NORM_SESSION_INVALID), is_multicast(false), loopback(false), is_running(false), + input_socket(ProtoSocket::UDP), input_file(stdin), input_fd(fileno(stdin)), input_ready(true), + input_needed(false), input_msg_length(0), input_index(0), + tx_stream (NORM_OBJECT_INVALID), tx_ready(true), + tx_stream_buffer_max(0), tx_stream_buffer_count(0), tx_stream_bytes_remain(0), + tx_watermark_pending(false), norm_acking(false), tx_ack_pending(false), flush_mode(NORM_FLUSH_ACTIVE), + rx_stream(NORM_OBJECT_INVALID), rx_ready(false), rx_needed(false), msg_sync(false), + output_bucket_rate(0.0), output_bucket_interval(0.0), output_bucket_depth(0), output_bucket_count(0), + output_socket(ProtoSocket::UDP), output_file(stdout), output_fd(fileno(stdout)), output_ready(true), + output_msg_length(0), output_index(0), + omit_header(false), input_byte_count(0), tx_byte_count(0), + segment_size(1398), block_size(64), num_parity(0), auto_parity(0), + stream_buffer_size(2*1024*1024), + tx_socket_buffer_size(0), rx_socket_buffer_size(0), + input_socket_buffer_size(0), output_socket_buffer_size(0), + check_sequence(0), sequence_prev(0) +{ +} + +NormStreamer::~NormStreamer() +{ +} + +bool NormStreamer::BoostPriority() +{ +#ifdef LINUX + pid_t this_process = getpid() ; + int policy = SCHED_FIFO ; + int max_priority = sched_get_priority_max(policy) ; + struct sched_param schedule_parameters ; + memset((void*)&schedule_parameters, 0, sizeof(schedule_parameters)) ; + schedule_parameters.sched_priority = max_priority ; + int status = sched_setscheduler(this_process, policy, &schedule_parameters) ; + if (0 != status) + { + fprintf(stderr, "%s:=>sched_setscheduler failed (%d), %s\n", __PRETTY_FUNCTION__, errno, strerror(errno) ) ; + return false ; + } + else + { + fprintf(stderr, "%s:=>sched_setscheduler set priority to %d for process %u \n", __PRETTY_FUNCTION__, max_priority, this_process ) ; + } +#else + // (TBD) Do something differently if "pthread sched param"? + if (0 != setpriority(PRIO_PROCESS, getpid(), -20)) + { + PLOG(PL_ERROR, "NormStreamer::BoostPriority() error: setpriority() error: %s\n", GetErrorString()); + return false; + } +#endif // if/else LINUX + return true; +} + +#ifndef ntohll +//Convert net-order to host-order. +uint64_t ntohll(uint64_t value) +{ + static const int betest = 1 ; + union MyUnion + { + uint64_t i64; + uint32_t i32[2]; + }; + + uint64_t rval = value; + bool host_is_little_endian = ( 1 == (int)(*(char*)&betest) ) ; + if ( host_is_little_endian ) + { + + MyUnion u; + u.i64 = value; + uint32_t temp = u.i32[0]; + u.i32[0] = ntohl(u.i32[1]); + u.i32[1] = ntohl(temp); + rval = u.i64; + } + return rval ; +} +#endif // !nothll + +void NormStreamer::CheckSequenceNumber64(const char* buffer, const char* source) +{ + uint64_t temp; + memcpy((void*)&temp, (void*)buffer, sizeof(temp)); + uint64_t sequence = ntohll(temp); + if (0 != sequence_prev) + { + int64_t delta = (int64_t)(sequence - sequence_prev); + if (1 != delta) + { + fprintf(stderr, "normStreamer: %s dropped %lu packets seq:%lu seq_prev:%lu\n", + source, (unsigned long)delta, (unsigned long)sequence, + (unsigned long)sequence_prev); + } + } + sequence_prev = sequence; +} // end NormStreamer::CheckSequenceNumber64() + +void NormStreamer::CheckSequenceNumber32(const char* buffer, const char* source) +{ + uint32_t temp; + memcpy((void*)&temp, (void*)buffer, sizeof(temp)); + uint32_t sequence = ntohll(temp); + if (0 != sequence_prev) + { + int32_t delta = (int32_t)(sequence - sequence_prev); + if (1 != delta) + { + fprintf(stderr, "normStreamer: %s dropped %lu packets seq:%lu seq_prev:%lu\n", + source, (unsigned long)delta, (unsigned long)sequence, + (unsigned long)sequence_prev); + } + } + sequence_prev = sequence; +} // end NormStreamer::CheckSequenceNumber32() + +void NormStreamer::CheckSequenceNumber(const char* buffer, const char* source) +{ + switch (check_sequence) + { + case 32: + CheckSequenceNumber32(buffer, source); + break; + case 64: + CheckSequenceNumber64(buffer, source) ; + break; + default: + break; + } +} // end NormStreamer::CheckSequenceNumber() + +bool NormStreamer::EnableUdpRelay(const char* relayAddr, unsigned short relayPort) +{ + if (!output_socket.Open()) + { + fprintf(stderr, "normStreamer error: unable to open 'relay' socket\n"); + return false ; + } + if (!output_socket.SetTxBufferSize(output_socket_buffer_size)) + { + fprintf(stderr, "normStreamer warning: unable to set desired 'relay' socket buffer size (retrieved value:%u)\n", + output_socket.GetTxBufferSize()); + } + if (!relay_addr.ResolveFromString(relayAddr)) + { + fprintf(stderr, "normStreamer error: invalid relay address\n"); + output_socket.Close(); + return false; + } + relay_addr.SetPort(relayPort); // TBD - validate port number?? + return true; +} // end bool EnableUdpRelay() + +bool NormStreamer::EnableUdpListener(unsigned short thePort, const char* groupAddr, const char * interfaceName) +{ + if (!input_socket.Open(thePort)) + { + fprintf(stderr, "normStreamer error: unable to open 'listen' socket on port %hu\n", thePort); + return false; + } + if (!input_socket.SetRxBufferSize(input_socket_buffer_size)) + { + fprintf(stderr, "normStreamer error: unable to set desired 'listen' socket buffer size\n"); + return false; + } + if (NULL != groupAddr) + { + ProtoAddress addr; + if (!addr.ResolveFromString(groupAddr) || (!addr.IsMulticast())) + { + fprintf(stderr, "normStreamer error: invalid 'listen' group address\n"); + input_socket.Close(); + return false ; + } + if (!input_socket.JoinGroup(addr, interfaceName)) + { + fprintf(stderr, "normStreamer error: unable to join 'listen' group address\n"); + input_socket.Close(); + return false; + } + } + return true; +} // end NormStreamer::EnableUdpListener() + +bool NormStreamer::OpenNormSession(NormInstanceHandle instance, const char* addr, unsigned short port, NormNodeId nodeId) +{ + if (NormIsUnicastAddress(addr)) + is_multicast = false; + else + is_multicast = true; + norm_session = NormCreateSession(instance, addr, port, nodeId); + if (NORM_SESSION_INVALID == norm_session) + { + fprintf(stderr, "normStreamer error: unable to create NORM session\n"); + return false; + } + if (is_multicast) + { + NormSetRxPortReuse(norm_session, true); + if (loopback) + NormSetMulticastLoopback(norm_session, true); + } + + // Set some default parameters (maybe we should put parameter setting in Start()) + NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_STREAM); + + NormSetDefaultUnicastNack(norm_session, true); + + NormSetTxRobustFactor(norm_session, 20); + + return true; +} // end NormStreamer::OpenNormSession() + +void NormStreamer::CloseNormSession() +{ + if (NORM_SESSION_INVALID == norm_session) return; + NormDestroySession(norm_session); + norm_session = NORM_SESSION_INVALID; +} // end NormStreamer::CloseNormSession() + +void NormStreamer::SetNormCongestionControl(CCMode ccMode) +{ + assert(NORM_SESSION_INVALID != norm_session); + switch (ccMode) + { + case NORM_CC: // default TCP-friendly congestion control + NormSetEcnSupport(norm_session, false, false, false); + break; + case NORM_CCE: // "wireless-ready" ECN-only congestion control + NormSetEcnSupport(norm_session, true, true); + break; + case NORM_CCL: // "loss tolerant", non-ECN congestion control + NormSetEcnSupport(norm_session, false, false, true); + break; + case NORM_FIXED: // "fixed" constant data rate + NormSetEcnSupport(norm_session, false, false, false); + break; + } + if (NORM_FIXED != ccMode) + NormSetCongestionControl(norm_session, true); + else + NormSetCongestionControl(norm_session, false); +} // end NormStreamer::SetNormCongestionControl() + +bool NormStreamer::Start(bool sender, bool receiver) +{ + // Note the session NORM buffer size is set the same s stream_buffer_size + unsigned int bufferSize = stream_buffer_size; + if (receiver) + { + if (!NormPreallocateRemoteSender(norm_session, bufferSize, segment_size, block_size, num_parity, stream_buffer_size)) + fprintf(stderr, "normStreamer warning: unable to preallocate remote sender\n"); + fprintf(stderr, "normStreamer: receiver ready.\n"); + if (!NormStartReceiver(norm_session, bufferSize)) + { + fprintf(stderr, "normStreamer error: unable to start NORM receiver\n"); + return false; + } + if (0 != mlockall(MCL_CURRENT | MCL_FUTURE)) + fprintf(stderr, "normStreamer error: failed to lock memory for receiver.\n"); + if (0 != rx_socket_buffer_size) + NormSetRxSocketBuffer(norm_session, rx_socket_buffer_size); + rx_needed = true; + rx_ready = false; + } + if (sender) + { + NormSetGrttEstimate(norm_session, 0.001); + //NormSetGrttMax(norm_session, 0.100); + NormSetBackoffFactor(norm_session, 0); + if (norm_acking) + { + // ack-based flow control enabled on command-line, + // so disable timer-based flow control + NormSetFlowControl(norm_session, 0.0); + } + // Pick a random instance id for now + struct timeval currentTime; + gettimeofday(¤tTime, NULL); + srand(currentTime.tv_usec); // seed random number generator + NormSessionId instanceId = (NormSessionId)rand(); + if (!NormStartSender(norm_session, instanceId, bufferSize, segment_size, block_size, num_parity)) + { + fprintf(stderr, "normStreamer error: unable to start NORM sender\n"); + if (receiver) NormStopReceiver(norm_session); + return false; + } + if (auto_parity > 0) + NormSetAutoParity(norm_session, auto_parity < num_parity ? auto_parity : num_parity); + if (0 != tx_socket_buffer_size) + NormSetTxSocketBuffer(norm_session, tx_socket_buffer_size); + if (NORM_OBJECT_INVALID == (tx_stream = NormStreamOpen(norm_session, stream_buffer_size))) + { + fprintf(stderr, "normStreamer error: unable to open NORM tx stream\n"); + NormStopSender(norm_session); + if (receiver) NormStopReceiver(norm_session); + return false; + } + else + { + if (0 != mlockall(MCL_CURRENT|MCL_FUTURE)) + fprintf(stderr, "normStreamer warning: failed to lock memory for sender.\n"); + } + tx_stream_buffer_max = NormGetStreamBufferSegmentCount(bufferSize, segment_size, block_size); + tx_stream_buffer_max -= block_size; // a little safety margin (perhaps not necessary) + tx_stream_buffer_threshold = tx_stream_buffer_max / 8; + tx_stream_buffer_count = 0; + tx_stream_bytes_remain = 0; + tx_watermark_pending = false; + tx_ack_pending = false; + tx_ready = true; + input_index = input_msg_length = 0; + input_needed = true; + input_ready = true; + } + is_running = true; + return true; +} // end NormStreamer::Start(); + +void NormStreamer::ReadInputSocket() +{ + unsigned int loopCount = 0; + NormSuspendInstance(NormGetInstance(norm_session)); + while (input_needed && input_ready && (loopCount < LOOP_MAX)) + { + loopCount++; + unsigned int numBytes = MSG_SIZE_MAX - MSG_HEADER_SIZE; + ProtoAddress srcAddr; + if (input_socket.RecvFrom(input_buffer+MSG_HEADER_SIZE, numBytes, srcAddr)) + { + if (0 == numBytes) + { + input_ready = false; + break; + } + input_index = 0; + input_msg_length = numBytes + MSG_HEADER_SIZE; + input_byte_count += input_msg_length; + unsigned short msgSize = input_msg_length;; + msgSize = htons(msgSize); + memcpy(input_buffer, &msgSize, MSG_HEADER_SIZE); + input_needed = false; + if (TxReady()) SendData(); + } + else + { + // TBD - handle error? + input_ready = false; + } + } + NormResumeInstance(NormGetInstance(norm_session)); +} // end NormStreamer::ReadInputSocket() + +void NormStreamer::ReadInput() +{ + if (UdpListenerEnabled()) return ReadInputSocket(); + // The loop count makes sure we don't spend too much time here + // before going back to the main loop to handle NORM events, etc + unsigned int loopCount = 0; + NormSuspendInstance(NormGetInstance(norm_session)); + while (input_needed && input_ready && (loopCount < LOOP_MAX)) + { + loopCount++; + //if (100 == loopCount) + // fprintf(stderr, "normStreamer ReadInput() loop count max reached\n"); + unsigned int numBytes; + if (input_index < MSG_HEADER_SIZE) + { + // Reading message length header for next message to send + numBytes = MSG_HEADER_SIZE - input_index; + } + else + { + // Reading message body + assert(input_index < input_msg_length); + numBytes = input_msg_length - input_index; + } + ssize_t result = read(input_fd, input_buffer + input_index, numBytes); + if (result > 0) + { + input_index += result; + input_byte_count += result; + if (MSG_HEADER_SIZE == input_index) + { + // We have now read the message size header + // TBD - support other message header formats? + // (for now, assume 2-byte message length header) + uint16_t msgSize ; + memcpy(&msgSize, input_buffer, MSG_HEADER_SIZE); + msgSize = ntohs(msgSize); + input_msg_length = msgSize; + } + else if (input_index == input_msg_length) + { + // Message input complete + input_index = 0; // reset index for transmission phase + input_needed = false; + if (TxReady()) SendData(); + } + else + { + // Still need more input + // (wait for next input notification to read more) + input_ready = false; + } + } + else if (0 == result) + { + // end-of-file reached, TBD - trigger final flushing and wrap-up + fprintf(stderr, "normStreamer: input end-of-file detected ...\n"); + NormStreamClose(tx_stream, true); + if (norm_acking) + { + NormSetWatermark(norm_session, tx_stream, true); + tx_ack_pending = false; + } + input_needed = false; + } + else + { + switch (errno) + { + case EINTR: + continue; // interrupted, try again + case EAGAIN: + // input starved, wait for next notification + input_ready = false; + break; + default: + // TBD - handle this better + perror("normStreamer error reading input"); + break; + } + break; + } + } // end while (input_needed && input_ready) + NormResumeInstance(NormGetInstance(norm_session)); +} // end NormStreamer::ReadInput() + +void NormStreamer::SendData() +{ + while (TxReady() && !input_needed) + { + // Note WriteToStream() or FlushStream() will set "tx_ready" to + // false upon flow control thus negating TxReady() status + assert(input_index < input_msg_length); + assert(input_msg_length); + if ((0 != check_sequence) && (0 == input_index)) + CheckSequenceNumber(input_buffer+MSG_HEADER_SIZE, __func__); + input_index += WriteToStream(input_buffer + input_index, input_msg_length - input_index); + if (input_index == input_msg_length) + { + // Complete message was sent, so set eom and optionally flush + if (NORM_FLUSH_NONE != flush_mode) + FlushStream(true, flush_mode); + else + NormStreamMarkEom(tx_stream); + input_index = input_msg_length = 0; + input_needed = true; + } + else + { + //fprintf(stderr, "SendData() impeded by flow control\n"); + } + } // end while (TxReady() && !input_needed) +} // end NormStreamer::SendData() + +unsigned int NormStreamer::WriteToStream(const char* buffer, unsigned int numBytes) +{ + unsigned int bytesWritten; + if (norm_acking) + { + // This method uses NormStreamWrite(), but limits writes by explicit ACK-based flow control status + if (tx_stream_buffer_count < tx_stream_buffer_max) + { + // 1) How many buffer bytes are available? + unsigned int bytesAvailable = segment_size * (tx_stream_buffer_max - tx_stream_buffer_count); + bytesAvailable -= tx_stream_bytes_remain; // unflushed segment portion + if (bytesAvailable < numBytes) numBytes = bytesAvailable; + assert(numBytes); + // 2) Write to the stream + bytesWritten = NormStreamWrite(tx_stream, buffer, numBytes); + tx_byte_count += bytesWritten; + // 3) Update "tx_stream_buffer_count" accordingly + unsigned int totalBytes = bytesWritten + tx_stream_bytes_remain; + unsigned int numSegments = totalBytes / segment_size; + tx_stream_bytes_remain = totalBytes % segment_size; + tx_stream_buffer_count += numSegments; + + //assert(bytesWritten == numBytes); // this could fail if timer-based flow control is left enabled + // 3) Check if we need to issue a watermark ACK request? + if (!tx_watermark_pending && (tx_stream_buffer_count >= tx_stream_buffer_threshold)) + { + // Initiate flow control ACK request + //fprintf(stderr, "write-initiated flow control ACK REQUEST\n"); + NormSetWatermark(norm_session, tx_stream); + tx_watermark_pending = true; + tx_ack_pending = false; + } + } + else + { + fprintf(stderr, "normStreamer: sender flow control limited\n"); + return 0; + } + } + else + { + bytesWritten = NormStreamWrite(tx_stream, buffer, numBytes); + tx_byte_count += bytesWritten; + } + if (bytesWritten != numBytes) //NormStreamWrite() was (at least partially) blocked + { + //fprintf(stderr, "NormStreamWrite() blocked by flow control ...\n"); + tx_ready = false; + } + return bytesWritten; +} // end NormStreamer::WriteToStream() + +void NormStreamer::FlushStream(bool eom, NormFlushMode flushMode) +{ + if (norm_acking) + { + bool setWatermark = false; + if (0 != tx_stream_bytes_remain) + { + // The flush will force the runt segment out, so we increment our buffer usage count + // (and initiate flow control watermark ack request if buffer mid-point threshold exceeded + tx_stream_buffer_count++; + tx_stream_bytes_remain = 0; + if (!tx_watermark_pending && (tx_stream_buffer_count >= tx_stream_buffer_threshold)) + { + setWatermark = true; + tx_watermark_pending = true; + //fprintf(stderr, "flush-initiated flow control ACK REQUEST\n"); + } + } + // The check for "tx_watermark_pending" here prevents a new watermark + // ack request from being set until the pending flow control ack is + // received. This favors avoiding dead air time over saving "chattiness" + if (setWatermark) + { + // Flush passive since watermark will invoke active request + // (TBD - do non-acking nodes NACK to watermark when not ack target?) + NormStreamFlush(tx_stream, eom, NORM_FLUSH_PASSIVE); + } + else if (tx_watermark_pending) + { + // Pre-existing pending flow control watermark ack request +#if SHOOT_FIRST + // Go ahead and set a fresh watermark + // TBD - not sure this mode works properly ... may need to + // keep track of unacknowledged byte count and decrement accordingly + // when ack arrives + NormStreamFlush(tx_stream, eom, NORM_FLUSH_PASSIVE); + setWatermark = true; +#else // ACK_LATER + // Wait until flow control ACK is received before issuing another ACK request + NormStreamFlush(tx_stream, eom, flushMode); + tx_ack_pending = true; // will call NormSetWatermark() upon flow control ack completion +#endif + } + else + { + // Since we're acking, use active ack request in lieu of active flush + NormStreamFlush(tx_stream, eom, NORM_FLUSH_PASSIVE); + setWatermark = true; + } + if (setWatermark) + { + NormSetWatermark(norm_session, tx_stream, true); + tx_ack_pending = false; + } + } + else + { + NormStreamFlush(tx_stream, eom, flushMode); + } +} // end NormStreamer::FlushStream() + +void NormStreamer::RecvData() +{ + // The loop count makes sure we don't spend too much time here + // before going back to the main loop to handle NORM events, etc + unsigned int loopCount = 0; + // Reads data from rx_stream to available output_buffer + NormSuspendInstance(NormGetInstance(norm_session)); + while (rx_needed && rx_ready && (loopCount < LOOP_MAX)) + { + loopCount++; + //if (100 == loopCount) + // fprintf(stderr, "normStreamer RecvData() loop count max reached.\n"); + // Make sure we have msg_sync (TBD - skip this for byte streaming) + if (!msg_sync) + { + msg_sync = NormStreamSeekMsgStart(rx_stream); + if (!msg_sync) + { + rx_ready = false; + break; // wait for next NORM_RX_OBJECT_UPDATED to re-sync + } + } + unsigned int bytesWanted; + if (output_index < MSG_HEADER_SIZE) + { + // Receiving message header + bytesWanted = MSG_HEADER_SIZE - output_index; + } + else + { + // Receiving message body + assert(output_index < output_msg_length); + bytesWanted = output_msg_length - output_index; + } + unsigned bytesRead = bytesWanted; + if (!NormStreamRead(rx_stream, output_buffer + output_index, &bytesRead)) + { + // Stream broken (should _not_ happen if norm_acking flow control) + //fprintf(stderr, "normStreamer error: BROKEN stream detected, re-syncing ...\n"); + msg_sync = false; + output_index = output_msg_length = 0; + continue; + } + output_index += bytesRead; + /*if (0 == bytesRead) + { + rx_ready = false; + } + else*/ if (bytesRead != bytesWanted) + { + //continue; + rx_ready = false; // didn't get all we need + } + else if (MSG_HEADER_SIZE == output_index) + { + // We have now read the message size header + // TBD - support other message header formats? + // (for now, assume 2-byte message length header) + uint16_t msgSize ; + memcpy(&msgSize, output_buffer, MSG_HEADER_SIZE); + output_msg_length = ntohs(msgSize); + } + else if (output_index == output_msg_length) + { + // Received full message + rx_needed = false; + output_index = 0; // reset for writing to output + if (output_ready && OutputBucketReady()) + WriteOutput(); + } + } + NormResumeInstance(NormGetInstance(norm_session)); + +} // end NormStreamer::RecvData() + +void NormStreamer::WriteOutputSocket() +{ + if (output_ready && !rx_needed) + { + assert(output_index < output_msg_length); + unsigned int payloadSize = output_msg_length - MSG_HEADER_SIZE; + unsigned int numBytes = payloadSize; + if ((0 != check_sequence)) + CheckSequenceNumber(output_buffer+MSG_HEADER_SIZE, __func__); + if (output_socket.SendTo(output_buffer+MSG_HEADER_SIZE, numBytes, relay_addr)) + { + if (numBytes != payloadSize) + { + // sendto() was blocked + output_ready = false; + return; + } + if (0 != output_bucket_depth) + { + // Debit output token bucket since it's active + ASSERT(output_bucket_count >= payloadSize); + output_bucket_count -= payloadSize; + } + rx_needed = true; + output_index = output_msg_length = 0; + } + else + { + output_ready = false; + } + } +} // end NormStreamer::WriteOutputSocket() + +void NormStreamer::WriteOutput() +{ + if (UdpRelayEnabled()) + { + WriteOutputSocket(); + return; + } + while (output_ready && !rx_needed) + { + assert(output_index < output_msg_length); + if ((0 != check_sequence) && (0 == output_index)) + CheckSequenceNumber(output_buffer+MSG_HEADER_SIZE,__func__); + ssize_t result = write(output_fd, output_buffer + output_index, output_msg_length - output_index); + if (result >= 0) + { + if (0 != output_bucket_depth) + { + // Debit output token bucket since it's active + if (result > output_bucket_count) + TRACE("result:%d output_bucket_count:%u\n", (int)result, output_bucket_count); + ASSERT(output_bucket_count >= result); + output_bucket_count -= result; + } + output_index += result; + if (output_index == output_msg_length) + { + // Complete message written + rx_needed = true; + output_index = output_msg_length = 0; + if ((NORM_OBJECT_INVALID == tx_stream) && (NORM_OBJECT_INVALID == rx_stream)) + Stop(); // receive stream was terminated by sender + } + else + { + output_ready = false; + } + } + else + { + switch (errno) + { + case EINTR: + perror("normStreamer output EINTR"); + continue; // interupted, try again + case EAGAIN: + // output blocked, wait for next notification + //perror("normStreamer output blocked"); + output_ready = false; + break; + default: + perror("normStreamer error writing output"); + break; + } + break; + } + } +} // end NormStreamer::WriteOutput() + +void NormStreamer::HandleNormEvent(const NormEvent& event) +{ + switch (event.type) + { + case NORM_TX_QUEUE_EMPTY: + //TRACE("normStreamer: flow control empty ...\n"); + tx_ready = true; + break; + case NORM_TX_QUEUE_VACANCY: + //TRACE("normStreamer: flow control relieved ...\n"); + tx_ready = true; + break; + + case NORM_GRTT_UPDATED: + //fprintf(stderr, "new GRTT = %lf\n", NormGetGrttEstimate(norm_session)); + break; + + case NORM_TX_WATERMARK_COMPLETED: + if (NORM_ACK_SUCCESS == NormGetAckingStatus(norm_session)) + { + //fprintf(stderr, "WATERMARK COMPLETED\n"); + if (tx_watermark_pending) + { + // Flow control ack request was pending. + tx_watermark_pending = false; + tx_stream_buffer_count -= tx_stream_buffer_threshold; + //fprintf(stderr, "flow control ACK completed\n"); + if (tx_ack_pending) + { + NormSetWatermark(norm_session, tx_stream, true); + tx_ack_pending = false; + } + } + } + else + { + // TBD - we could see who didn't ACK and possibly remove them + // from our acking list. For now, we are infinitely + // persistent by always resetting the watermark ack request + // For example, an application could make a decision at this + // point, depending upon some count of ACK request failures + // to choose to remove a previously included receiver. + fprintf(stderr, "flow control watermark reset\n"); + if (tx_ack_pending) + { + // May as well advance the ack request point + NormSetWatermark(norm_session, tx_stream, true); + tx_ack_pending = false; + } + else + { + NormResetWatermark(norm_session); + } + } + break; + + case NORM_TX_OBJECT_PURGED: + // tx_stream graceful close completed + NormStopSender(norm_session); + tx_stream = NORM_OBJECT_INVALID; + if (NORM_OBJECT_INVALID == rx_stream) Stop(); + break; + + case NORM_REMOTE_SENDER_INACTIVE: + //fprintf(stderr, "REMOTE SENDER INACTIVE node: %u\n", NormNodeGetId(event.sender)); + //NormNodeDelete(event.sender); + break; + + case NORM_RX_OBJECT_NEW: + if ((NORM_OBJECT_INVALID == rx_stream) && + (NORM_OBJECT_STREAM == NormObjectGetType(event.object))) + { + rx_stream = event.object; + rx_ready = true; + // By setting initial "msg_sync" to true, we can detect when + // stream beginning was missed (for NORM_SYNC_STREAM only) + msg_sync = false; + rx_needed = true; + output_index = output_msg_length = 0; + } + else + { + fprintf(stderr, "normStreamer warning: NORM_RX_OBJECT_NEW while already receiving?!\n"); + } + + case NORM_RX_OBJECT_UPDATED: + rx_ready = true; + break; + + case NORM_RX_OBJECT_ABORTED: + //fprintf(stderr, "NORM_RX_OBJECT_ABORTED\n");// %hu\n", NormObjectGetTransportId(event.object)); + rx_stream = NORM_OBJECT_INVALID; + rx_needed = false; + rx_ready = false; + break; + + case NORM_RX_OBJECT_COMPLETED: + // Rx stream has closed + // TBD - set state variables so any pending output is + // written out and things shutdown if not sender, too + fprintf(stderr, "normStreamer: rx_stream completed.\n"); + // if rx_needed is true, all output has been written + if (rx_needed && (NORM_OBJECT_INVALID == tx_stream)) + { + NormNodeHandle sender = NormObjectGetSender(rx_stream); + // Wait a couple of GRTT's to ACK sender + double exitTime = 20.0 * NormNodeGetGrtt(sender); + if (exitTime < 1.0) exitTime = 1.0; + fprintf(stderr, "normStreamer reception completed, exiting in %f seconds ...\n", (float)exitTime); + sleep(exitTime); // TBD - use our user-defined NormSession timeout instead? (retaining rx_stream) + if (rx_needed && (NORM_OBJECT_INVALID == tx_stream)) + Stop(); + } + rx_stream = NORM_OBJECT_INVALID; + rx_ready = false; + rx_needed = false; + break; + + default: + break; + } + //NormReleasePreviousEvent(NormGetInstance(norm_session)); + +} // end NormStreamer::HandleNormEvent() + +void Usage() +{ + fprintf(stderr, "Usage: normStreamer id {send|recv} [addr [/]][interface ][loopback]\n" + " [cc|cce|ccl|rate ][ack [,,...][flush {none|passive|active}]\n" + " [listen [/]][linterface ]\n" + " [relay /][limit [/]][output ]\n" + " [boost][debug ][trace][log ]\n" + " [segment ][block ][parity ][auto ]\n" + " [insockbuffer ][outsockbuffer ]\n" + " [txsockbuffer ][rxsockbuffer ]\n" + " [streambuffer ]\n" + " [check64 | check32]\n"); + //" [omit][silent][txloss ]\n"); +} // end Usage() + +void PrintHelp() +{ + fprintf(stderr, "\nHelp for normStreamer:\n\n") ; + fprintf(stderr, + "The 'normStreamer' application sends messages from STDIN (or a listening UDP socket) to one or more\n" + "receiving nodes using the NORM protocol. Received messages are output to STDOUT (or relayed to\n" + "to a UDP destination address/port). Key command line options are:\n\n" + " id -- Specifies the node id for the local NORM instance (required)\n" + " send | recv -- Specifies whether this node will be a sender and/or receiver (must choose at least one)\n" + " addr [/] -- specifies the network address over which to send/receive NORM protocol\n" + " interface -- Specifies the name of the network interface on which to conduct NORM protocol\n" + " (e.g., 'eth0')\n" + " loopback -- Enables 'loopback' sessions on the same host machine. Required for multicast loopback.\n" + " rate -- sets fixed sender rate (and receiver token bucket rate if 'limit' option is used)\n" + " [cc|cce|ccl] -- Enables optional NORM congestion control mode (overrides 'rate')\n" + " ack [] -- Instructs sender to request positive acknowledgement from listed receiver nodes\n" + " flush [] -- Choose 'none', 'passive', or 'active' message stream flushing mode. If 'none',\n" + " NORM_DATA packets will always be packed with message content up to the full\n" + " segment size. If 'passive', short NORM_DATA packets will be sent to transmit\n" + " any messages as soon as possible. If 'active', NORM stream will be flushed\n" + " on a per-message basis as with 'passive' mode, but positive acknowledgment will\n" + " _also_ be requested if a list of acking receiver node ids has beeen provided.\n" + " listen [/] -- Specifies the port and optional multicast address which the sender uses to listen\n" + " for UDP packets to transmit to the receiver(s) via the NORM protocol\n" + " linterface -- Specifies the name of the network interface on which to listen for UDP packet\n" + " payloads to send to the receiver(s) via NORM protocol\n" + " relay / -- Specifies the address/port for which to relay (as UDP datagrams) received messages\n" + " limit [/] -- Token bucket rate/depth for optional receiver output limiter (smooths bursty output\n" + " upon NORM loss recovery). When UDP 'relay' is used, this option is useful to avoid\n" + " overly bursty UDP output. The is in units of bits/second and the is\n" + " in units of bytes. If not specified here, the value set by 'rate' command is used\n" + " as the token bucket rate.\n" + " check64 | check32 -- Enables checking that packet sequence numbers in the first 4/8 bytes of received\n" + " packets increment properly (optional)\n" + " insockbuffer -- Specifies the size of the 'listen' UDP socket buffer (optional).\n" + " outsockbuffer -- Specifies the size of the 'relay' UDP socket buffer (optional).\n" + " txsockbuffer -- Specifies the size of the NORM/UDP transmit socket buffer (optional).\n" + " rxsockbuffer -- Specifies the size of the NORM/UDP receive socket buffer (optional).\n" + " streambuffer -- Specifies the size of the NORM stream buffer (optional).\n\n"); + Usage(); + +} // end PrintHelp() + +int main(int argc, char* argv[]) +{ + // REQUIRED parameters initiailization + NormNodeId nodeId = NORM_NODE_NONE; + bool send = false; + bool recv = false; + + char sessionAddr[64]; + strcpy(sessionAddr, "224.1.2.3"); + unsigned int sessionPort = 6003; + + char listenAddr[64]; // UDP :listen" multicast addr + listenAddr[0] = '\0'; + unsigned int listenPort = 0; // UDP "listen" port for UDP "listen" + const char* listenIface = NULL; // UDP "listen" interface + + char relayAddr[64]; + relayAddr[0] = '\0'; + unsigned int relayPort = 0; + + double txRate = 0.0; // used for non-default NORM_FIXED ccMode + NormStreamer::CCMode ccMode = NormStreamer::NORM_CC; + const char* mcastIface = NULL; + NormNodeId ackingNodeList[256]; + unsigned int ackingNodeCount = 0; + + bool loopback = false; + int debugLevel = 0; + bool trace = false; + const char* logFile = NULL; + bool omitHeaderOnOutput = false; + bool silentReceiver = false; + double txloss = 0.0; + bool boostPriority = false; + unsigned int checkSequence = 0; // can set to 64 or 32 + // TBD - set these defaults to reasonable values or just use NormStreamer constructor defaults + unsigned long inputSocketBufferSize = 256*1024*1024; + unsigned long outputSocketBufferSize = 256*1024*1024; + unsigned long txSocketBufferSize = 6*1024*1024; + unsigned long rxSocketBufferSize = 6*1024*1024; + unsigned long streamBufferSize = 256*1024*1024; + + // Instantiate a NormStreamer and set default params + NormStreamer normStreamer; + normStreamer.SetFlushMode(NORM_FLUSH_NONE); + + // Parse command-line + int i = 1; + while (i < argc) + { + const char* cmd = argv[i++]; + size_t len = strlen(cmd); + if (0 == strncmp(cmd, "help", len)) + { + PrintHelp() ; + exit(0); + } + else if (0 == strncmp(cmd, "send", len)) + { + send = true; + } + else if (0 == strncmp(cmd, "recv", len)) + { + recv = true; + } + else if (0 == strncmp(cmd, "loopback", len)) + { + loopback = true; + } + else if (0 == strncmp(cmd, "addr", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'addr[/port]' value!\n"); + Usage(); + return -1; + } + const char* addrPtr = argv[i++]; + const char* portPtr = strchr(addrPtr, '/'); + if (NULL == portPtr) + { + strncpy(sessionAddr, addrPtr, 63); + sessionAddr[63] = '\0'; + } + else + { + size_t addrLen = portPtr - addrPtr; + if (addrLen > 63) addrLen = 63; // should issue error message + strncpy(sessionAddr, addrPtr, addrLen); + sessionAddr[addrLen] = '\0'; + portPtr++; + sessionPort = atoi(portPtr); + } + } + else if (0 == strncmp(cmd, "listen", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing '[mcastAddr/]port]' value!\n"); + Usage(); + return -1; + } + const char* addrPtr = argv[i++]; + const char* portPtr = strchr(addrPtr, '/'); + if (NULL != portPtr) + { + size_t addrLen = portPtr - addrPtr; + if (addrLen > 63) addrLen = 63; // should issue error message + strncpy(listenAddr, addrPtr, addrLen); + listenAddr[addrLen] = '\0'; + portPtr++; + listenPort = atoi(portPtr); + } + else + { + // no address, just port + listenPort = atoi(addrPtr); + addrPtr = NULL; + } + } + else if (0 == strncmp(cmd, "relay", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing relay 'dstAddr/port' value!\n"); + Usage(); + return -1; + } + const char* addrPtr = argv[i++]; + const char* portPtr = strchr(addrPtr, '/'); + if (NULL == portPtr) + { + fprintf(stderr, "normStreamer error: missing relay 'port' value!\n"); + Usage(); + return -1; + } + if (NULL != portPtr) + { + size_t addrLen = portPtr - addrPtr; + if (addrLen > 63) addrLen = 63; // should issue error message + strncpy(relayAddr, addrPtr, addrLen); + relayAddr[addrLen] = '\0'; + portPtr++; + relayPort = atoi(portPtr); + } + + } + else if (0 == strncmp(cmd, "output", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing output 'device' name!\n"); + Usage(); + return -1; + } + FILE* outfile = fopen(argv[i++], "w+"); + if (NULL == outfile) + { + fprintf(stderr, "normStreamer output device fopen() error: %s\n", GetErrorString()); + Usage(); + return -1; + } + normStreamer.SetOutputFile(outfile); + } + else if (0 == strncmp(cmd, "id", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'id' value!\n"); + Usage(); + return -1; + } + nodeId = atoi(argv[i++]); + } + else if (0 == strncmp(cmd, "ack", len)) + { + // comma-delimited acking node id list + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'id' value!\n"); + Usage(); + return -1; + } + const char* alist = argv[i++]; + while ((NULL != alist) && (*alist != '\0')) + { + // TBD - Do we need to skip leading white space? + int id; + if (1 != sscanf(alist, "%d", &id)) + { + fprintf(stderr, "normStreamer error: invalid acking node list!\n"); + Usage(); + return -1; + } + ackingNodeList[ackingNodeCount] = NormNodeId(id); + ackingNodeCount++; + alist = strchr(alist, ','); + if (NULL != alist) alist++; // point past comma + } + } + else if (0 == strncmp(cmd, "flush", len)) + { + // "none", "passive", or "active" + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'flush' !\n"); + Usage(); + return -1; + } + const char* mode = argv[i++]; + if (0 == strcmp(mode, "none")) + { + normStreamer.SetFlushMode(NORM_FLUSH_NONE); + } + else if (0 == strcmp(mode, "passive")) + { + normStreamer.SetFlushMode(NORM_FLUSH_PASSIVE); + } + else if (0 == strcmp(mode, "active")) + { + normStreamer.SetFlushMode(NORM_FLUSH_ACTIVE); + } + else + { + fprintf(stderr, "normMsgr error: invalid 'flush' mode \"%s\"\n", mode); + return -1; + } + } + else if (0 == strncmp(cmd, "rate", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'rate' value!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lf", &txRate)) + { + fprintf(stderr, "normStreamer error: invalid transmit rate!\n"); + Usage(); + return -1; + } + // set fixed-rate operation + ccMode = NormStreamer::NORM_FIXED; + normStreamer.SetOutputBucketRate(txRate); + } + else if (0 == strcmp(cmd, "cc")) + { + ccMode = NormStreamer::NORM_CC; + } + else if (0 == strcmp(cmd, "cce")) + { + ccMode = NormStreamer::NORM_CCE; + } + else if (0 == strcmp(cmd, "ccl")) + { + ccMode = NormStreamer::NORM_CCL; + } + else if (0 == strncmp(cmd, "interface", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'interface' !\n"); + Usage(); + return -1; + } + mcastIface = argv[i++]; + } + else if (0 == strncmp(cmd, "linterface", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'linterface' !\n"); + Usage(); + return -1; + } + listenIface = argv[i++]; + } + else if (0 == strncmp(cmd, "insockbuffer", len)) + { + unsigned long value = 0 ; + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'insockbuffer' size!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'insockbuffer' size\n"); + Usage(); + return -1; + } + inputSocketBufferSize = value; + } + else if (0 == strncmp(cmd, "outsockbuffer", len)) + { + unsigned long value = 0 ; + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'outsockbuffer' size!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'outsockbuffer' size!\n"); + Usage(); + return -1; + } + outputSocketBufferSize = value; + } + else if (0 == strncmp(cmd, "limit", len)) + { + // format: limit [/] with 'rate' in bps and 'size' in bytes + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'limit' size!\n"); + Usage(); + return -1; + } + const char* ratePtr = argv[i++]; + const char* sizePtr = strchr(ratePtr, '/'); + unsigned int rateLen = 0; + if (NULL != sizePtr) + rateLen = sizePtr++ - ratePtr; + else + sizePtr = ratePtr; + if (0 != rateLen) + { + if (rateLen > 63) + { + fprintf(stderr, "normStreamer error: out-of-bounds 'limit' rate\n"); + Usage(); + return -1; + } + char rateText[64]; + strncpy(rateText, ratePtr, rateLen); + rateText[rateLen] = '\0'; + double value; + if (1 != sscanf(rateText, "%lf", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'limit' rate\n"); + Usage(); + return -1; + } + normStreamer.SetOutputBucketRate(value); + } + unsigned long value; + if (1 != sscanf(sizePtr, "%lu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'limit' size\n"); + Usage(); + return -1; + } + normStreamer.SetOutputBucketDepth(value); + } + else if (0 == strncmp(cmd, "txsockbuffer", len)) + { + unsigned long value = 0 ; + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'txsockbuffer' size!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'txsockbuffer' size!\n"); + Usage(); + return -1; + } + txSocketBufferSize = value; + } + else if (0 == strncmp(cmd, "rxsockbuffer", len)) + { + unsigned long value = 0 ; + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'rxsockbuffer' size!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'rxsockbuffer' size!\n"); + Usage(); + return -1; + } + rxSocketBufferSize = value; + } + else if (0 == strncmp(cmd, "segment", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'segment' size!\n"); + Usage(); + return -1; + } + unsigned short value; + if (1 != sscanf(argv[i++], "%hu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'segment' size!\n"); + Usage(); + return -1; + } + normStreamer.SetSegmentSize(value); + } + else if (0 == strncmp(cmd, "block", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'block' size!\n"); + Usage(); + return -1; + } + unsigned short value; + if (1 != sscanf(argv[i++], "%hu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'block' size!\n"); + Usage(); + return -1; + } + normStreamer.SetBlockSize(value); + } + else if (0 == strncmp(cmd, "parity", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'parity' count!\n"); + Usage(); + return -1; + } + unsigned short value; + if (1 != sscanf(argv[i++], "%hu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'parity' count!\n"); + Usage(); + return -1; + } + normStreamer.SetNumParity(value); + } + else if (0 == strncmp(cmd, "auto", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'auto' parity count!\n"); + Usage(); + return -1; + } + unsigned short value; + if (1 != sscanf(argv[i++], "%hu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'auto' parity count!\n"); + Usage(); + return -1; + } + normStreamer.SetAutoParity(value); + } + else if (0 == strncmp(cmd, "streambuffer", len)) + { + unsigned long value = 0 ; + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'streambuffer' size!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lu", &value)) + { + fprintf(stderr, "normStreamer error: invalid 'streambuffer' size!\n"); + Usage(); + return -1; + } + streamBufferSize = value; + } + else if ( 0 == strncmp(cmd,"chkseq", len) ) + { + checkSequence = 64; // same as "check64" for "historical" reasons + } + else if ( 0 == strncmp(cmd,"check64", len) ) + { + checkSequence = 64; + } + else if ( 0 == strncmp(cmd,"check32", len) ) + { + checkSequence = 32; + } + else if (0 == strncmp(cmd, "omit", len)) + { + omitHeaderOnOutput = true; + } + else if (0 == strncmp(cmd, "silent", len)) + { + silentReceiver = true; + } + else if (0 == strncmp(cmd, "boost", len)) + { + boostPriority = true; + } + else if (0 == strncmp(cmd, "txloss", len)) + { + if (1 != sscanf(argv[i++], "%lf", &txloss)) + { + fprintf(stderr, "normStreamer error: invalid 'txloss' value!\n"); + Usage(); + return -1; + } + } + else if (0 == strncmp(cmd, "debug", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'debug' !\n"); + Usage(); + return -1; + } + debugLevel = atoi(argv[i++]); + } + else if (0 == strncmp(cmd, "trace", len)) + { + trace = true; + } + else if (0 == strncmp(cmd, "log", len)) + { + if (i >= argc) + { + fprintf(stderr, "normStreamer error: missing 'log' !\n"); + Usage(); + return -1; + } + logFile = argv[i++]; + } + else if (0 == strncmp(cmd, "help", len)) + { + Usage(); + return 0; + } + else + { + fprintf(stderr, "normStreamer error: invalid command \"%s\"!\n", cmd); + Usage(); + return -1; + } + } + + if (!send && !recv) + { + fprintf(stderr, "normStreamer error: not configured to send or recv!\n"); + Usage(); + return -1; + } + if (NORM_NODE_NONE == nodeId) + { + fprintf(stderr, "normStreamer error: no local 'id' provided!\n"); + Usage(); + return -1; + } + + if (boostPriority) + { + if (!normStreamer.BoostPriority()) + { + fprintf(stderr, "normStreamer error: setting scheduler/ priority boost failed (requires 'sudo').\n"); + return -1; + } + } + + + + if (0 != listenPort) + { + normStreamer.SetInputSocketBufferSize(inputSocketBufferSize); + if (!normStreamer.EnableUdpListener(listenPort, listenAddr, listenIface)) + { + fprintf(stderr, "normStreamer error: Failed to enable UDP listener\n") ; + return -1; + } + } + if (0 != relayPort) + { + // TBD - check addr/port validity? + normStreamer.SetOutputSocketBufferSize(outputSocketBufferSize); + if (! normStreamer.EnableUdpRelay(relayAddr, relayPort)) + { + fprintf(stderr, "normStreamer error: Failed to open UDP relay socket\n") ; + return -1; + } + } + + // TBD - should provide more error checking of NORM API calls + NormInstanceHandle normInstance = NormCreateInstance(boostPriority); + NormSetDebugLevel(debugLevel); + if ((NULL != logFile) && !NormOpenDebugLog(normInstance, logFile)) + { + perror("normStreamer error: unable to open log file"); + Usage(); + return -1; + } + normStreamer.SetCheckSequence(checkSequence); + + + normStreamer.SetTxSocketBufferSize(txSocketBufferSize); + normStreamer.SetRxSocketBufferSize(rxSocketBufferSize); + normStreamer.SetStreamBufferSize(streamBufferSize); + + normStreamer.SetLoopback(loopback); + + if (omitHeaderOnOutput) normStreamer.OmitHeader(true); + + if (!normStreamer.OpenNormSession(normInstance, sessionAddr, sessionPort, (NormNodeId)nodeId)) + { + fprintf(stderr, "normStreamer error: unable to open NORM session\n"); + NormDestroyInstance(normInstance); + return -1; + } + + if (silentReceiver) normStreamer.SetSilentReceiver(true); + if (txloss > 0.0) normStreamer.SetTxLoss(txloss); + + for (unsigned int i = 0; i < ackingNodeCount; i++) + normStreamer.AddAckingNode(ackingNodeList[i]); + + normStreamer.SetNormCongestionControl(ccMode); + if (NormStreamer::NORM_FIXED == ccMode) + normStreamer.SetNormTxRate(txRate); + if (NULL != mcastIface) + normStreamer.SetNormMulticastInterface(mcastIface); + + if (trace) normStreamer.SetNormMessageTrace(true); + + // TBD - set NORM session parameters + normStreamer.Start(send, recv); + + // TBD - add WIN32 support using win32InputHandler code + // and MsgWaitForMultipleObjectsEx() instead of select() + + int normfd = NormGetDescriptor(normInstance); + // Get input/output descriptors and set to non-blocking i/o + int inputfd = normStreamer.GetInputDescriptor(); + int outputfd = normStreamer.GetOutputDescriptor(); + if (-1 == fcntl(inputfd, F_SETFL, fcntl(inputfd, F_GETFL, 0) | O_NONBLOCK)) + perror("normStreamer: fcntl(inputfd, O_NONBLOCK) error"); + //if (!normStreamer.UdpRelayEnabled()) + if (-1 == fcntl(outputfd, F_SETFL, fcntl(outputfd, F_GETFL, 0) | O_NONBLOCK)) + perror("normStreamer: fcntl(outputfd, O_NONBLOCK) error"); + fd_set fdsetInput, fdsetOutput; + FD_ZERO(&fdsetInput); + FD_ZERO(&fdsetOutput); + +#ifdef LINUX + // We user timerfd on Linux for more precise timeouts + int timerfd = timerfd_create(CLOCK_MONOTONIC, 0); + if (timerfd < 0) + { + perror("normStreamer: timerfd_create() error"); + return -1; + } +#endif // LINUX + struct timeval lastTime; + gettimeofday(&lastTime, NULL); + struct timeval bucketTime = lastTime; + while (normStreamer.IsRunning()) + { + int maxfd = -1; + int fdMask = 0; + bool waitOnNorm = false; + double timeoutInterval = -1.0; + if (send) + { + if (normStreamer.InputNeeded()) + { + if (normStreamer.InputReady()) + { + FD_CLR(inputfd, &fdsetInput); + timeoutInterval = 0.0; + } + else + { + FD_SET(inputfd, &fdsetInput); + if (inputfd > maxfd) maxfd = inputfd; + fdMask |= 0x01; + } + } + else + { + FD_CLR(inputfd, &fdsetInput); + } + if (normStreamer.TxPending()) + { + if (normStreamer.TxReady()) + timeoutInterval = 0.0; + else + waitOnNorm = true; + } + } + if (recv) + { + if (normStreamer.RxNeeded()) + { + if (normStreamer.RxReady()) + timeoutInterval = 0.0; + else + waitOnNorm = true; + } + if (normStreamer.OutputPending()) + { + if (normStreamer.OutputReady()) + { + FD_CLR(outputfd, &fdsetOutput); + if (timeoutInterval < 0.0) + timeoutInterval = normStreamer.GetOutputBucketTimeout(); + } + else + { + FD_SET(outputfd, &fdsetOutput); + if (outputfd > maxfd) maxfd = outputfd; + fdMask |= 0x02; + } + } + else + { + FD_CLR(outputfd, &fdsetOutput); + } + } + if (waitOnNorm) + { + // we need to wait until NORM is tx_ready or rx_ready + FD_SET(normfd, &fdsetInput); + if (normfd > maxfd) maxfd = normfd; + fdMask |= 0x04; + } + else + { + FD_CLR(normfd, &fdsetInput); + } + // Set timeout for select() ... TBD - it may be a slight + // performance enhancement to skip the select() call when + // the timeout needed is zero??? + struct timeval timeout; + struct timeval* timeoutPtr = &timeout; +#ifdef LINUX + if (timeoutInterval > 0.0) + { + // On Linux, we use the timerfd with our select() call to get + // more precise timeouts than select() does alone on Linux + struct timespec timeoutSpec; + timeoutSpec.tv_sec = (unsigned int)timeoutInterval; + timeoutSpec.tv_nsec = 1.0e+09*(timeoutInterval - (double)timeoutSpec.tv_sec); + struct itimerspec timerSpec; + timerSpec.it_interval.tv_sec = timerSpec.it_interval.tv_nsec = 0; + timerSpec.it_value = timeoutSpec; + if (0 == timerfd_settime(timerfd, 0, &timerSpec, 0)) + { + timeoutPtr = NULL; + FD_SET(timerfd, &fdsetInput); + if (outputfd > maxfd) maxfd = timerfd; + fdMask |= 0x08; + } + else + { + FD_CLR(timerfd, &fdsetInput); + timeout.tv_sec = (unsigned int)timeoutInterval; + timeout.tv_usec = 1.0e+06*(timeoutInterval - (double)timeout.tv_sec); + perror("normStreamer: timerfd_settime() error"); + } + } + else + { + // No precision timing needed + FD_CLR(timerfd, &fdsetInput); + if (timeoutInterval < 0.0) + { + // We wait one second maximum for debugging purposes + timeout.tv_sec = 1; + timeout.tv_usec = 0; + } + else // if (0.0 == timeoutInterval) + { + timeout.tv_sec = timeout.tv_usec = 0; + } + } +#else // non-LINUX + if (timeoutInterval > 0.0) + { + timeout.tv_sec = (unsigned int)timeoutInterval; + timeout.tv_usec = 1.0e+06*(timeoutInterval - (double)timeout.tv_sec); + } + else if (timeoutInterval < 0.0) + { + // We wait one second maximum for debugging purposes + timeout.tv_sec = 1; + timeout.tv_usec = 0; + } + else // if (0.0 == timeoutInterval) + { + timeout.tv_sec = timeout.tv_usec = 0; + } +#endif // if/else LINUX + int result = select(maxfd+1, &fdsetInput, &fdsetOutput, NULL, timeoutPtr); + switch (result) + { + case -1: + switch (errno) + { + case EINTR: + case EAGAIN: + continue; + default: + perror("normStreamer select() error"); + // TBD - stop NormStreamer + break; + } + break; + case 0: + // timeout + break; + default: + if (FD_ISSET(inputfd, &fdsetInput)) + normStreamer.SetInputReady(); + if (FD_ISSET(outputfd, &fdsetOutput)) + normStreamer.SetOutputReady(); +#ifdef LINUX + if (FD_ISSET(timerfd, &fdsetInput)) + { + // clear the timerfd status by reading from it + uint64_t expirations = 0; + if (read(timerfd, &expirations, sizeof(expirations)) < 0) + perror("normStreamer read(timerfd) error"); + } +#endif // LINUX + break; + } + // We always clear out/handle pending NORM API events + // (to keep event queue from building up) + NormEvent event; + while (NormGetNextEvent(normInstance, &event, false)) + normStreamer.HandleNormEvent(event); + + struct timeval thisTime; + gettimeofday(&thisTime, NULL); + + if (0 != normStreamer.GetOutputBucketDepth()) + { + // Credit output token bucket for time that has passed + double interval = (double)(thisTime.tv_sec - bucketTime.tv_sec); + if (thisTime.tv_usec > bucketTime.tv_usec) + interval += 1.0e-06 * (thisTime.tv_usec - bucketTime.tv_usec); + else + interval -= 1.0e-06 * (bucketTime.tv_usec - thisTime.tv_usec); + normStreamer.CreditOutputBucket(interval); + bucketTime = thisTime; + } + + // for debugging to see if anything gets "stuck" + if ((thisTime.tv_sec - lastTime.tv_sec) >= 1) + { + if (send) + fprintf(stderr, "normStreamer: inputNeeded:%d inputReady:%d txPending:%d txReady:%d inputCount:%lu txCount:%lu fdMask:%d\n", + normStreamer.InputNeeded(), normStreamer.InputReady(), normStreamer.TxPending(), normStreamer.TxReady(), + normStreamer.GetInputByteCount(), normStreamer.GetTxByteCount(), fdMask); + if (recv) + fprintf(stderr, "normStreamer: rxNeeded:%d rxReady:%d outputPending:%d outputReady:%d fdMask:%d\n", + normStreamer.RxNeeded(), normStreamer.RxReady(), normStreamer.OutputPending(), normStreamer.OutputReady(), fdMask); + + lastTime = thisTime; + } + + // As a result of input/output ready or NORM notification events: + // 1) Recv from rx_stream if needed and ready + if (normStreamer.RxNeeded() && normStreamer.RxReady()) + normStreamer.RecvData(); + // 2) Write any pending data to output if output is ready + if (normStreamer.OutputPending() && normStreamer.OutputReady()) + { + if (normStreamer.OutputBucketReady()) + normStreamer.WriteOutput(); + } + // 3) Read from input if needed and ready + if (normStreamer.InputNeeded() && normStreamer.InputReady()) + normStreamer.ReadInput(); + // 4) Send any pending tx message + if (normStreamer.TxPending() && normStreamer.TxReady()) + normStreamer.SendData(); + + } // end while(normStreamer.IsRunning() + +#ifdef LINUX + close(timerfd); +#endif // LINUX + fflush(stderr); + + close(normStreamer.GetOutputDescriptor()); // TBD - do this in the destructor? + + NormDestroyInstance(normInstance); + + fprintf(stderr, "normStreamer exiting ...\n"); + + return 0; + +} // end main() + diff --git a/examples/win32InputHandler.cpp b/examples/win32InputHandler.cpp new file mode 100644 index 0000000..e91774f --- /dev/null +++ b/examples/win32InputHandler.cpp @@ -0,0 +1,243 @@ + + +#include +#include // for perror() +#include // for memset() +#include // for _beginthreadex(), _endthreadex() + +// This class creates a child thread to read STDIN into a buffer. +// An event is set that can be monitored by the parent to know +// when input is available. Mutexes are used to regulate the +// thread's reading (i.e., flow control). +class Win32InputHandler +{ + public: + Win32InputHandler(); + ~Win32InputHandler(); + + bool Open(int buflen = 4096); + void Close(); + + // The event handle returned here can be used in + // a call like WaitForSignalObject() to get notified + // when there is data to be read. + HANDLE GetEventHandle() const + {return input_event;} + + // This does not block and the input_event + // handle is a cue for when to call it + // Returns number of bytes copied into "buffer" + // (-1 is returned upon input error (input closed)) + int ReadData(char* buffer, int numBytes); + + private: + static unsigned int __stdcall DoThreadOpen(void* param); + unsigned int Run(); + + HANDLE input_handle; + HANDLE input_event; + HANDLE thread_handle; + bool input_ready; + char* input_buffer; + int input_buflen; + int input_index; + int input_outdex; + CONDITION_VARIABLE buffer_vacancy; + CRITICAL_SECTION buffer_lock; + bool is_running; + int input_length; + bool input_error; + +}; // end class Win32InputHandler + +Win32InputHandler::Win32InputHandler() + : input_handle(NULL), input_event(NULL), thread_handle(NULL), + input_ready(false), input_buffer(NULL), input_buflen(0), + input_index(0), input_outdex(0), is_running(false), + input_length(0), input_error(false) +{ + +} + +Win32InputHandler::~Win32InputHandler() +{ + if (is_running) Close(); +} + +bool Win32InputHandler::Open(int buflen) +{ + if (is_running) Close(); + if (NULL != input_buffer) + { + input_buffer = NULL; + delete[] input_buffer; + } + input_buflen = input_index = input_outdex = input_length = 0; + input_error = false; + if (NULL != input_event) CloseHandle(input_event); + + // Get the STDIN handle whatever it may be + input_handle = GetStdHandle(STD_INPUT_HANDLE); + /*if (FILE_TYPE_CHAR == GetFileType(input_handle)) + { + // The STDIN is a console + DWORD consoleMode; + bool result = GetConsoleMode(input_handle, &consoleMode); + consoleMode &= ~(ENABLE_LINE_INPUT | ENABLE_MOUSE_INPUT | ENABLE_WINDOW_INPUT); + SetConsoleMode(input_handle, consoleMode); + }*/ + + // Create an event handle, non-signaled initially + input_event = CreateEvent(NULL, TRUE, FALSE, NULL); + if (NULL == input_event) + { + perror("Win32InputHandler::Open() CreateEvent() error"); + return false; + } + if (NULL == (input_buffer = new char[buflen])) + { + perror("Win32InputHandler::Open() new input_buffer error"); + CloseHandle(input_event); + input_event = NULL; + return false; + } + input_buflen = buflen; + InitializeConditionVariable(&buffer_vacancy); + InitializeCriticalSection(&buffer_lock); + is_running = true; + if (0 == (thread_handle = (HANDLE)_beginthreadex(NULL, 0, DoThreadOpen, this, 0, NULL))) + { + perror("Win32InputHandler::Open() _beginthreadex() error"); + DeleteCriticalSection(&buffer_lock); + delete[] input_buffer; + input_buffer = NULL; + CloseHandle(input_event); + input_event = NULL; + return false; + } + return true; +} // end Win32InputHandler::Open() + +void Win32InputHandler::Close() +{ + EnterCriticalSection(&buffer_lock); + is_running = false; + WakeConditionVariable(&buffer_vacancy); // in case it was full + LeaveCriticalSection(&buffer_lock); + WaitForSingleObject(thread_handle, INFINITE); + DeleteCriticalSection(&buffer_lock); + delete[] input_buffer; + input_buffer = NULL; + CloseHandle(input_event); + input_event = NULL; +} // end Win32InputHandler::Close() + +unsigned int __stdcall Win32InputHandler::DoThreadOpen(void* param) +{ + Win32InputHandler* inputHandler = reinterpret_cast(param); + unsigned int exitStatus = inputHandler->Run(); + _endthreadex(exitStatus); + return exitStatus; +} // end Win32InputHandler::DoThreadOpen() + +unsigned int Win32InputHandler::Run() +{ + // This loop is for the child thread to read input one byte at a time + // (Since consoles/anonymous pipes don't support async i/o) + // The input bytes are copied to a circularly-managed "input_buffer" + // that can be accessed via the "ReadData()" method by the parent + while (is_running) + { + // Read input one byte at a time and buffer + DWORD dwRead = 0; + BOOL result = ReadFile(input_handle, input_buffer+input_index, 1, &dwRead, NULL); + EnterCriticalSection(&buffer_lock); + if (result) + { + if (0 != dwRead) + { + input_length++; + input_index++; + if (input_index >= input_buflen) + input_index = 0; + } + else + { + LeaveCriticalSection(&buffer_lock); + continue; + } + } + else + { + // What do we do on error?! + input_error = true; + is_running = false; + } + // Signals the parent of data ready (or error) + if (!input_ready) + { + SetEvent(input_event); + input_ready = true; + } + // If the buffer has been filled, this blocks this child + // loop until buffer space is available or it is stopped. + while ((input_index == input_outdex) && (0 != input_length) && is_running) + SleepConditionVariableCS(&buffer_vacancy, &buffer_lock, INFINITE); + LeaveCriticalSection(&buffer_lock); + } // end while(is_running) + return (input_error ? 1 : 0); +} // end Win32InputHandler::Run() + +// Called by parent to gobble up data (non-blocking read) +// Returns number of bytes read or -1 on error (e.g., STDIN pipe closed) +int Win32InputHandler::ReadData(char* buffer, int numBytes) +{ + EnterCriticalSection(&buffer_lock); + if ((0 != input_length) && (0 != numBytes)) + { + if (numBytes > input_length) + numBytes = input_length; + input_length -= numBytes; + if (0 == input_length) + { + // On error condition, the event is left triggered + // after the input_buffer is emptied so parent gets + // all data buffered before the error (e.g., eof) + // occurred. + if (input_ready && !input_error) + { + input_ready = false; + ResetEvent(input_event); + } + } + // Copy "numBytes" of available data to caller's "buffer" + // (two-step process when input_buffer wraps) + unsigned int offset = input_outdex; + input_outdex += numBytes; + if (input_outdex >= input_buflen) + { + input_outdex -= input_buflen; + unsigned int count = numBytes - input_outdex; + memcpy(buffer, input_buffer+offset, count); + if (0 != input_outdex) + memcpy(buffer+count, input_buffer, input_outdex); + } + else + { + memcpy(buffer, input_buffer + offset, numBytes); + } + } + else + { + // Doing the error check + if (input_error && (0 != numBytes)) + numBytes = -1; // to indicate an error has occurred and child is dead + else + numBytes = 0; + } + // This wakes up the child if it is sleeping because the buffer was full + if (numBytes > 0) + WakeConditionVariable(&buffer_vacancy); + LeaveCriticalSection(&buffer_lock); + return numBytes; +} // end Win32InputHandler::ReadData() diff --git a/examples/wintest.cpp b/examples/wintest.cpp new file mode 100644 index 0000000..ef55f3c --- /dev/null +++ b/examples/wintest.cpp @@ -0,0 +1,57 @@ + + +#include "win32InputHandler.cpp" // for class Win32InputHandler + +#include +#include +#include + +#define BUFSIZE 4096 + +int main(int argc, char** argv) +{ + HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE); + + if (argc > 1) + { + // Generate random output + const char* text = "This is some text to output\n"; + int textlen = strlen(text); + + while (true) + { + int count = rand() % 100; + while (count > 0) + { + printf(text); + DWORD dwWritten; + BOOL fSuccess = WriteFile(hStdout, text, textlen, &dwWritten, NULL); + count -= textlen; + } + + int delay = (rand() % 1000); + Sleep(1000); + + } + + return 0; + } + + Win32InputHandler inputHandler; + + inputHandler.Start(); + + while (true) + { + char buffer[BUFSIZE]; + WaitForSingleObject(inputHandler.GetEventHandle(), INFINITE); + int numBytes = inputHandler.ReadData(buffer, 1024); + if (numBytes < 0) break; + + DWORD dwWritten; + BOOL fSuccess = WriteFile(hStdout, buffer, numBytes, &dwWritten, NULL); + } + inputHandler.Stop(); + + return 0; +} // end main() diff --git a/include/normApi.h b/include/normApi.h index 6e8b702..e0ecce2 100644 --- a/include/normApi.h +++ b/include/normApi.h @@ -59,8 +59,8 @@ extern "C" { #endif /* __cplusplus */ #define NORM_VERSION_MAJOR 1 -#define NORM_VERSION_MINOR 0 -#define NORM_VERSION_PATCH 0 +#define NORM_VERSION_MINOR 5 +#define NORM_VERSION_PATCH 7 /** NORM API Types */ typedef const void* NormInstanceHandle; @@ -187,7 +187,7 @@ typedef enum NormEventType NORM_GRTT_UPDATED, NORM_CC_ACTIVE, NORM_CC_INACTIVE, - NORM_ACKING_NODE_NEW, // whe NormSetAutoAcking xxx + NORM_ACKING_NODE_NEW, // whe NormSetAutoAcking NORM_SEND_ERROR, // ICMP error (e.g. destination unreachable) NORM_USER_TIMEOUT // issues when timeout set by NormSetUserTimer() expires } NormEventType; @@ -201,6 +201,11 @@ typedef struct } NormEvent; +// For setting custom NORM_OBJECT_DATA alloc/free functions +typedef char* (*NormAllocFunctionHandle)(size_t); +typedef void (*NormFreeFunctionHandle)(char*); + + /** NORM API General Initialization and Operation Functions */ NORM_API_LINKAGE @@ -208,6 +213,9 @@ int NormGetVersion(int* major DEFAULT((int*)0), int* minor DEFAULT((int*)0), int* patch DEFAULT((int*)0)); + + + NORM_API_LINKAGE NormInstanceHandle NormCreateInstance(bool priorityBoost DEFAULT(false)); @@ -256,6 +264,11 @@ const NormDescriptor NORM_DESCRIPTOR_INVALID; NORM_API_LINKAGE NormDescriptor NormGetDescriptor(NormInstanceHandle instanceHandle); +NORM_API_LINKAGE +void NormSetAllocationFunctions(NormInstanceHandle instance, + NormAllocFunctionHandle allocFunc, + NormFreeFunctionHandle freeFunc); + /** NORM Session Creation and Control Functions */ NORM_API_LINKAGE @@ -288,6 +301,11 @@ void NormCancelUserTimer(NormSessionHandle sessionHandle); NORM_API_LINKAGE NormNodeId NormGetLocalNodeId(NormSessionHandle sessionHandle); +NORM_API_LINKAGE +bool NormGetAddress(NormSessionHandle sessionHandle, + char* addrBuffer, + unsigned int* bufferLen, + UINT16* port DEFAULT((UINT16*)0)); NORM_API_LINKAGE UINT16 NormGetRxPort(NormSessionHandle sessionHandle); @@ -315,6 +333,13 @@ bool NormChangeDestination(NormSessionHandle sessionHandle, UINT16 sessionPort, bool connectToSessionAddress DEFAULT(false)); +NORM_API_LINKAGE +void NormSetServerListener(NormSessionHandle sessionHandle, bool state); + +NORM_API_LINKAGE +bool NormTransferSender(NormSessionHandle sessionHandle, NormNodeHandle sender); + + NORM_API_LINKAGE void NormSetRxPortReuse(NormSessionHandle sessionHandle, bool enableReuse, @@ -410,7 +435,8 @@ bool NormStartSender(NormSessionHandle sessionHandle, UINT32 bufferSpace, UINT16 segmentSize, UINT16 numData, - UINT16 numParity); + UINT16 numParity, + UINT8 fecId DEFAULT(0)); NORM_API_LINKAGE void NormStopSender(NormSessionHandle sessionHandle); @@ -583,6 +609,9 @@ bool NormSendCommand(NormSessionHandle sessionHandle, NORM_API_LINKAGE void NormCancelCommand(NormSessionHandle sessionHandle); + +NORM_API_LINKAGE +void NormSetSynStatus(NormSessionHandle sessionHandle, bool state); /* NORM Receiver Functions */ @@ -646,6 +675,14 @@ NORM_API_LINKAGE void NormNodeSetRxRobustFactor(NormNodeHandle remoteSender, int robustFactor); +NORM_API_LINKAGE +bool NormPreallocateRemoteSender(NormSessionHandle sessionHandle, + unsigned long bufferSize, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + unsigned int streamBufferSize DEFAULT(0)); + NORM_API_LINKAGE bool NormStreamRead(NormObjectHandle streamHandle, char* buffer, @@ -706,6 +743,12 @@ const char* NormDataAccessData(NormObjectHandle objectHandle); NORM_API_LINKAGE char* NormDataDetachData(NormObjectHandle objectHandle); +NORM_API_LINKAGE +char* NormAlloc(size_t numBytes); + +NORM_API_LINKAGE +void NormFree(char* dataPtr); + NORM_API_LINKAGE NormNodeHandle NormObjectGetSender(NormObjectHandle objectHandle); diff --git a/include/normMessage.h b/include/normMessage.h index 057df7b..c1bf7be 100644 --- a/include/normMessage.h +++ b/include/normMessage.h @@ -134,6 +134,8 @@ class NormObjectSize // Operators bool operator==(const NormObjectSize& b) const {return (b.size == size);} + bool operator!=(const NormObjectSize& b) const + {return (b.size != size);} NormObjectSize operator+(const NormObjectSize& b) const { NormObjectSize result(size); @@ -201,9 +203,9 @@ class NormObjectId } bool operator<=(const NormObjectId& id) const - {return ((value == id.value) || (*this=(const NormObjectId& id) const - {return ((value == id.value) || (*this>id));} + {return ((value == id.value) || (*this > id));} bool operator==(const NormObjectId& id) const {return (value == id.value);} bool operator!=(const NormObjectId& id) const @@ -212,6 +214,8 @@ class NormObjectId void operator-=(UINT16 delta) {value -= delta;} + const char* GetValuePtr() const + {return (const char*)(&value);} NormObjectId& operator++(int) {value++; return *this;} NormObjectId& operator--(int) {value--; return *this;} @@ -224,33 +228,93 @@ class NormBlockId { public: NormBlockId() {}; - NormBlockId(UINT32 id) {value = id;} - NormBlockId(const NormObjectId& id) {value = (UINT32)id;} - operator UINT32() const {return value;} + NormBlockId(UINT32 id) : value(id) {} + + UINT32 GetValue() const + {return value;} + + const char* GetValuePtr() const + {return ((const char*)&value);} + bool operator==(const NormBlockId& id) const - {return (value == (UINT32)id);} + {return (value == id.value);} bool operator!=(const NormBlockId& id) const - {return (value != (UINT32)id);} + {return (value != id.value);} + // These static helper methods provide a "mask" parameter to allow + // for different bit-length NormBlockId values (depends upon FEC encoding scheme). + // The "mask" is in the eye-of-the-beholder, i.e., the given NormObject + // or NormBlockBuffer that is manipulating block id values for protocol purposes + // provide the FEC block "mask" to these methods. + // We may reconsider adding the "mask" as a NormBlockId member variable, but since + // NormBlockId usage is so ubiquitous, it could be added overhead consider within + // the context of sender, object, etc a common fec_block_mask is used. - //INT32 operator-(const NormBlockId& id) const - // {return ((INT32)value - id.value);} - - bool operator<(const NormBlockId& id) const + // Compute difference of (a - b). If a non-zero bit "mask" is given, the difference + // is for the masked word size (e.g., mask = 0x00ffffff is a 24-bit integer). + static INT32 Difference(const NormBlockId& a, const NormBlockId& b, UINT32 mask) { - UINT32 diff = value - id.value; - return ((diff > 0x80000000) || ((0x80000000 == diff) && (value > id.value))); + if (mask) + { + UINT32 sign = (mask ^ (mask >> 1)); + UINT32 result = a.value - b.value; + if (0 == (result & sign)) + return (INT32)(result & mask); + else if ((result != sign) || (a.value < b.value)) + return (INT32)(result | ~mask); + else + return (INT32)(result & mask); + } + else + { + return ((INT32)(a.value - b.value)); + } } - bool operator>(const NormBlockId& id) const + // Compare two block ids. If a non-zero bit "mask" is given, the comparison + // is a "sliding window" (signed) over the bit space. Otherwise, it is + // simply an unsigned value comparison. + // Returns -1, 0, +1 for (a < b), (a == b), and (a > b), respectively + static int Compare(const NormBlockId& a, const NormBlockId& b, UINT32 mask) { - UINT32 diff = id.value - value; - return ((diff > 0x80000000) || ((0x80000000 == diff) && (id.value > value))); + if (mask) + { + // "Sliding window" comparison + INT32 delta = Difference(a, b, mask); + if (delta < 0) + return -1; + else if (0 == delta) + return 0; + else // if delta > 0 + return 1; + } + else if (a.value < b.value) + { + return -1; + } + else if (a.value == b.value) + { + return 0; + } + else // if (a > b) + { + return 1; + } } + void Increment(UINT32 i, UINT32 mask) + { + value = value + i; + if (mask) value &= mask; + } - NormBlockId& operator++(int ) {value++; return *this;} - + void Decrement(UINT32 i, UINT32 mask) + { + if (mask && (value < i)) + value = (mask - (i - value) + 1); + else + value -= i; + } private: UINT32 value; }; // end class NormBlockId @@ -308,7 +372,7 @@ class NormHeaderExtension }; // end class NormHeaderExtension -// This class is some we use to set/get +// This class is what we use to set/get // FEC Payload Id content. The FEC Payload // Id format is dependent upon the "fec_id" (FEC Type) // and, in some cases, its field size ("m") parameter @@ -523,7 +587,10 @@ class NormMsg { buffer[SOURCE_ID_OFFSET] = htonl(sourceId); } + // For messages to be sent, "addr" is destination void SetDestination(const ProtoAddress& dst) {addr = dst;} + // For message received, "addr" is source + void SetSource(const ProtoAddress& src) {addr = src;} void AttachExtension(NormHeaderExtension& extension) { @@ -545,6 +612,8 @@ class NormMsg {return (Type)(((UINT8*)buffer)[TYPE_OFFSET] & 0x0f);} UINT16 GetHeaderLength() {return ((UINT8*)buffer)[HDR_LEN_OFFSET] << 2;} + UINT16 GetBaseHeaderLength() + {return header_length_base;} UINT16 GetSequence() const { return (ntohs((((UINT16*)buffer)[SEQUENCE_OFFSET]))); @@ -555,9 +624,11 @@ class NormMsg } const ProtoAddress& GetDestination() const {return addr;} const ProtoAddress& GetSource() const {return addr;} - const char* GetBuffer() {return ((char*)buffer);} + const char* GetBuffer() const {return ((char*)buffer);} UINT16 GetLength() const {return length;} + void Display() const; // hex output to log + // To retrieve any attached header extensions bool HasExtensions() const {return (header_length > header_length_base);} bool GetNextExtension(NormHeaderExtension& ext) const @@ -566,7 +637,7 @@ class NormMsg UINT16 nextOffset = (UINT16)(currentBuffer ? (currentBuffer - buffer + (ext.GetLength()/4)) : (header_length_base/4)); - bool result = (nextOffset < (header_length/4)); + bool result = HasExtensions() ? (nextOffset < (header_length/4)) : false; ext.AttachBuffer(result ? (buffer+nextOffset) : (UINT32*)NULL); return result; } @@ -627,7 +698,8 @@ class NormObjectMsg : public NormMsg FLAG_INFO = 0x04, FLAG_UNRELIABLE = 0x08, FLAG_FILE = 0x10, - FLAG_STREAM = 0x20 + FLAG_STREAM = 0x20, + FLAG_SYN = 0x40 //FLAG_MSG_START = 0x40 deprecated }; UINT16 GetInstanceId() const @@ -733,6 +805,51 @@ class NormFtiExtension2 : public NormHeaderExtension }; }; // end class NormFtiExtension2 +// Helper class for containing key FTI params +class NormFtiData +{ + public: + NormFtiData() + : object_size(NormObjectSize(0)), segment_size(0), + num_data(0), num_parity(0), fec_m(0), instance_id(0) {} + ~NormFtiData() {} + + void SetObjectSize(const NormObjectSize& objectSize) + {object_size = objectSize;} + void SetSegmentSize(UINT16 segmentSize) + {segment_size = segmentSize;} + void SetFecMaxBlockLen(UINT16 numData) + {num_data = numData;} + void SetFecNumParity(UINT16 numParity) + {num_parity = numParity;} + void SetFecFieldSize(UINT8 fecM) + {fec_m = fecM;} + void SetFecInstanceId(UINT16 instanceId) + {instance_id = instanceId;} + + const NormObjectSize& GetObjectSize() const + {return object_size;} + UINT16 GetSegmentSize() const + {return segment_size;} + UINT16 GetFecMaxBlockLen() const + {return num_data;} + UINT16 GetFecNumParity() const + {return num_parity;} + UINT8 GetFecFieldSize() const + {return fec_m;} + UINT16 GetFecInstanceId() const + {return instance_id;} + + private: + NormObjectSize object_size; + UINT16 segment_size; + UINT16 num_data; + UINT16 num_parity; + UINT8 fec_m; + UINT16 instance_id; + +}; // end class NormFtiData + // This FEC Object Transmission Information assumes "fec_id" == 5 (RFC 5510) // (this is the fully-defined 8-bit Reed-Solomon codec) @@ -1282,6 +1399,13 @@ class NormCmdCCMsg : public NormCmdMsg } bool GetCCNode(NormNodeId nodeId, UINT8& flags, UINT8& rtt, UINT16& rate) const; + // This function uses the "reserved" field of the NORM_CMD(CC) message + // and is not strictly compliant with RFC 5740 when invoked. + enum {FLAG_SYN = 0x01}; + bool SynIsSet() const + {return (0 != (FLAG_SYN & ((UINT8*)buffer)[RESERVED_OFFSET]));} + void SetSyn() + {((UINT8*)buffer)[RESERVED_OFFSET] = FLAG_SYN;} class Iterator; friend class Iterator; @@ -1389,7 +1513,9 @@ class NormRepairRequest void SetFlag(NormRepairRequest::Flag theFlag) {flags |= theFlag;} void ClearFlag(NormRepairRequest::Flag theFlag) - {flags &= ~theFlag;} + {flags &= ~theFlag;} + void SetFlags(int theFlags) + {flags = theFlags;} // Returns length (each repair item requires 8 bytes of space) bool AppendRepairItem(UINT8 fecId, @@ -1425,7 +1551,13 @@ class NormRepairRequest {return form;} bool FlagIsSet(NormRepairRequest::Flag theFlag) const {return (0 != (theFlag & flags));} - UINT16 GetLength() const {return length;} + int GetFlags() const + {return flags;} + UINT16 GetLength() const + {return (ITEM_LIST_OFFSET + length);} + + UINT32* GetBuffer() const + {return buffer;} // Outputs textual representation of RepairRequest content void Log(UINT8 fecId, UINT8 fecM) const; @@ -1436,10 +1568,10 @@ class NormRepairRequest // Checks for matching fecId and assumes constant 'm' ?!?! Iterator(const NormRepairRequest& theRequest, UINT8 fecId, UINT8 fecM); void Reset() {offset = 0;} - bool NextRepairItem(NormObjectId* objectId, - NormBlockId* blockId, - UINT16* blockLen, - UINT16* symbolId); + UINT16 NextRepairItem(NormObjectId* objectId, + NormBlockId* blockId, + UINT16* blockLen, + UINT16* symbolId); private: const NormRepairRequest& request; UINT8 fec_id; @@ -1731,6 +1863,22 @@ class NormNackMsg : public NormMsg length += requestLength; return requestLength; } + + // TBD - add some safety checks to these methods + void InitFrom(NormNackMsg nack) + { + // Copy header from "nack" + memcpy(buffer, nack.buffer, nack.GetHeaderLength()); + header_length_base = nack.header_length_base; + length = header_length = nack.GetHeaderLength(); + } + void AppendRepairRequest(const NormRepairRequest request) + { + memcpy(buffer+length/4, request.GetBuffer(), request.GetLength()); + length += request.GetLength(); + } + void ResetPayload() + {length = GetHeaderLength();} // Message processing NormNodeId GetSenderId() const diff --git a/include/normNode.h b/include/normNode.h index 22876a4..320c712 100644 --- a/include/normNode.h +++ b/include/normNode.h @@ -140,6 +140,10 @@ class NormLossEstimator2 {ignore_loss = state;} void SetTolerateLoss(bool state) {tolerate_loss = state;} + + unsigned short GetLagIndex() const + {return lag_index;} + private: enum {DEPTH = 8}; @@ -174,7 +178,6 @@ class NormLossEstimator2 {init = true; Sync(theSequence);} void Sync(unsigned short theSequence) {lag_index = theSequence;} - void ChangeLagDepth(unsigned int theDepth) { theDepth = (theDepth > 20) ? 20 : theDepth; @@ -270,14 +273,13 @@ class NormCCNode : public NormNode UINT16 cc_sequence; }; // end class NormCCNode -class NormSenderNode : public NormNode +class NormSenderNode : public NormNode, public ProtoTree::Item { public: enum ObjectStatus {OBJ_INVALID, OBJ_NEW, OBJ_PENDING, OBJ_COMPLETE}; enum RepairBoundary {BLOCK_BOUNDARY, OBJECT_BOUNDARY}; - enum SyncPolicy { SYNC_CURRENT, // sync to detect transmit point, iff NORM_DATA from first FEC block unless stream @@ -291,6 +293,12 @@ class NormSenderNode : public NormNode void SetInstanceId(UINT16 instanceId) {instance_id = instanceId;} + bool PreallocateRxStream(unsigned int bufferSize, + UINT16 segmentSize , + UINT16 numData, + UINT16 numParity); + bool GetFtiData(const NormObjectMsg& msg, NormFtiData& ftiData); + // Parameters NormObject::NackingMode GetDefaultNackingMode() const {return default_nacking_mode;} @@ -317,11 +325,15 @@ class NormSenderNode : public NormNode void UpdateGrttEstimate(UINT8 grttQuantized); double GetGrttEstimate() const {return grtt_estimate;} + void ResetGrttNotification() {notify_on_grtt_update = true;} bool UpdateLossEstimate(const struct timeval& currentTime, unsigned short theSequence, bool ecnStatus = false); double LossEstimate() {return loss_estimator.LossFraction();} + unsigned short GetCurrentSequence() const + {return loss_estimator.GetLagIndex();} + double GetRttEstimate() const {return rtt_estimate;} void CheckCCFeedback(); @@ -340,7 +352,13 @@ class NormSenderNode : public NormNode UINT16 GetInstanceId() {return instance_id;} bool IsOpen() const {return is_open;} void Close(); - bool AllocateBuffers(UINT8 fecId, UINT16 fecInstanceId, UINT8 fecM, UINT16 segmentSize, UINT16 numData, UINT16 numParity); + bool AllocateBuffers(unsigned int bufferSpace, + UINT8 fecId, + UINT16 fecInstanceId, + UINT8 fecM, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity); bool BuffersAllocated() {return (0 != segment_size);} void FreeBuffers(); void Activate(bool isObjectMsg); @@ -381,6 +399,15 @@ class NormSenderNode : public NormNode void DeleteObject(NormObject* obj); + NormObject* GetNextPendingObject() + { + NormObjectId objid; + if (GetNextPending(objid)) + return rx_table.Find(objid); + else + return NULL; + } + UINT16 SegmentSize() {return segment_size;} UINT16 BlockSize() {return ndata;} UINT16 NumParity() {return nparity;} @@ -437,9 +464,9 @@ class NormSenderNode : public NormNode unsigned long BufferOverunCount() const {return segment_pool.OverunCount() + block_pool.OverrunCount();} - unsigned long CurrentStreamBufferUsage() const; - unsigned long PeakStreamBufferUsage() const; - unsigned long StreamBufferOverunCount() const; + unsigned long CurrentStreamBufferUsage(); + unsigned long PeakStreamBufferUsage(); + unsigned long StreamBufferOverunCount(); //unsigned long RecvTotal() const {return recv_total;} @@ -461,6 +488,7 @@ class NormSenderNode : public NormNode recv_goodput.Reset(); } void IncrementResyncCount() {resync_count++;} + void DecrementResyncCount() {resync_count--;} unsigned long ResyncCount() const {return resync_count;} unsigned long NackCount() const {return nack_count;} unsigned long SuppressCount() const {return suppress_count;} @@ -502,9 +530,34 @@ class NormSenderNode : public NormNode bool ReadNextCmd(char* buffer, unsigned int* buflen); + void SetAddress(const ProtoAddress& address) + { + unsigned int len = address.GetLength(); + memcpy(key_buffer, address.GetRawHostAddress(), len); + UINT16 port = htons(address.GetPort()); + memcpy(key_buffer+len, &port, 2); + key_size = (len+2) << 3; + NormNode::SetAddress(address); + } + + UINT8 GetGrttQuantized() const + {return grtt_quantized;} + UINT8 GetBackoffFactor() const + {return backoff_factor;} + UINT8 GetGroupSizeQuantized() const + {return gsize_quantized;} + UINT16 GetCCSequence() const + {return cc_sequence;} + double GetSendRate() const + {return send_rate;} private: + const char* GetKey() const + {return key_buffer;} + unsigned int GetKeysize() const + {return key_size;} + static const double DEFAULT_NOMINAL_INTERVAL; static const double ACTIVITY_INTERVAL_MIN; @@ -523,7 +576,10 @@ class NormSenderNode : public NormNode void AttachCCFeedback(NormAckMsg& ack); void HandleRepairContent(const UINT32* buffer, UINT16 bufferLen); - + void FragmentNack(NormNackMsg& superNack); + + + UINT16 instance_id; int robust_factor; SyncPolicy sync_policy; @@ -540,6 +596,7 @@ class NormSenderNode : public NormNode UINT8 fec_m; unsigned int ndata; unsigned int nparity; + NormStreamObject* preset_stream; NormObjectTable rx_table; ProtoSlidingMask rx_pending_mask; @@ -561,7 +618,7 @@ class NormSenderNode : public NormNode // Watermark acknowledgement ProtoTimer ack_timer; - bool ack_pending; + bool ack_pending; NormObjectId watermark_object_id; NormBlockId watermark_block_id; NormSegmentId watermark_segment_id; @@ -574,6 +631,7 @@ class NormSenderNode : public NormNode double gsize_estimate; UINT8 gsize_quantized; double backoff_factor; + bool notify_on_grtt_update; // for API // Remote sender congestion control state NormLossEstimator2 loss_estimator; @@ -600,6 +658,10 @@ class NormSenderNode : public NormNode CmdBuffer* cmd_buffer_tail; // newly-received commands appended here CmdBuffer* cmd_buffer_pool; // we "pool" allocated buffers for possible reuse here + // Used for NormSocket API extension to index by addr/port + char key_buffer[16+2]; // big enough for IPv6 plus port + unsigned int key_size; + // For statistics tracking Accumulator recv_total; // total recvd accumulator Accumulator recv_goodput; // goodput recvd accumulator @@ -612,7 +674,8 @@ class NormSenderNode : public NormNode }; // end class NormSenderNode - // Used for binary trees of NormNodes +// Used for binary trees of NormNodes sorted by NormNodeId +// (TBD - update to use ProtoTree instead? class NormNodeTree { friend class NormNodeTreeIterator; @@ -639,7 +702,6 @@ class NormNodeTreeIterator void Reset(NormNode* prevNode = NULL); NormNode* GetNextNode(); - private: const NormNodeTree& tree; NormNode* next; @@ -689,4 +751,30 @@ class NormNodeListIterator const NormNodeList& list; NormNode* next; }; // end class NormNodeListIterator + +// Used to track remote client sender nodes for server/listener sessions +class NormClientTree : public ProtoTreeTemplate +{ + public: + void InsertNode(NormSenderNode& sender) + { + sender.Retain(); + Insert(sender); + } + void RemoveNode(NormSenderNode& sender) + { + Remove(sender); + sender.Release(); + } + NormSenderNode* FindNodeByAddress(const ProtoAddress& addr) + { + char key[16+2]; + unsigned int len = addr.GetLength(); + memcpy(key, addr.GetRawHostAddress(), len); + UINT16 port = htons(addr.GetPort()); + memcpy(key+len, &port, 2); + return Find(key, (len+2) << 3); + } +}; // end class NormClientTree + #endif // NORM_NODE diff --git a/include/normObject.h b/include/normObject.h index b5b485d..7b366e9 100644 --- a/include/normObject.h +++ b/include/normObject.h @@ -7,7 +7,15 @@ #include +#define USE_PROTO_TREE 1 // for more better performing NormObjectTable? + +#ifdef USE_PROTO_TREE +#include "protoTree.h" + +class NormObject : public ProtoSortedTree::Item +#else class NormObject +#endif // if/else USE_PROTO_TREE { friend class NormObjectTable; @@ -44,16 +52,43 @@ class NormObject // This must be reset after each update void SetNotifyOnUpdate(bool state) - { - notify_on_update = state; - } + {notify_on_update = state;} // Object information NormObject::Type GetType() const {return type;} const NormObjectId& GetId() const {return transport_id;} const NormObjectSize& GetSize() const {return object_size;} + + void SetId(const NormObjectId transportId) + {transport_id = transportId;} + + + UINT16 GetSegmentSize() const {return segment_size;} + UINT8 GetFecId() const {return fec_id;} + UINT16 GetFecMaxBlockLen() const {return ndata;} + UINT16 GetFecNumParity() const {return nparity;} + UINT8 GetFecFieldSize() const {return fec_m;} + + // returns difference (a - b) + INT32 Difference(NormBlockId a, NormBlockId b) const + {return NormBlockId::Difference(a, b, fec_block_mask);} + // returns -1, 0, or 1 for (a < b), (a == b), or (a > b), respectively + int Compare(NormBlockId a, NormBlockId b) const + {return NormBlockId::Compare(a, b, fec_block_mask);} + void Increment(NormBlockId& b, UINT32 i = 1) const + {b.Increment(i, fec_block_mask);} + void Decrement(NormBlockId& b, UINT32 i = 1) const + {b.Decrement(i, fec_block_mask);} + bool HaveInfo() const {return (info_len > 0);} bool HasInfo() const {return (NULL != info_ptr);} + void ClearInfo() + { + if (NULL != info_ptr) delete[] info_ptr; + info_ptr = NULL; + info_len = 0; + pending_info = false; + } const char* GetInfo() const {return info_ptr;} UINT16 GetInfoLength() const {return info_len;} @@ -94,8 +129,6 @@ class NormObject } void Close(); - - virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* buffer) = 0; @@ -119,17 +152,19 @@ class NormObject NormBlockId GetFinalBlockId() const {return final_block_id;} UINT32 GetBlockSize(NormBlockId blockId) const { - return (((UINT32)blockId < large_block_count) ? large_block_size : - small_block_size); + return ((blockId.GetValue() < large_block_count) ? large_block_size : + small_block_size); } NormObjectSize GetBytesPending() const; bool IsPending(bool flush = true) const; - bool IsRepairPending() const; + bool IsRepairPending(); bool IsPendingInfo() {return pending_info;} bool PendingMaskIsSet() const {return pending_mask.IsSet();} + unsigned int GetPendingMaskSize() const + {return pending_mask.GetSize();} bool GetFirstPending(NormBlockId& blockId) const { UINT32 index = 0; @@ -139,7 +174,7 @@ class NormObject } bool GetNextPending(NormBlockId& blockId) const { - UINT32 index = (UINT32)blockId; + UINT32 index = blockId.GetValue(); bool result = pending_mask.GetNextSet(index); blockId = NormBlockId(index); return result; @@ -161,7 +196,7 @@ class NormObject bool GetNextRepair(NormBlockId& blockId) const { - UINT32 index = (UINT32)blockId; + UINT32 index = blockId.GetValue(); bool result = repair_mask.GetNextSet(index); blockId = NormBlockId(index); return result; @@ -174,7 +209,7 @@ class NormObject return result; } - bool FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) const; + bool FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId); // Methods available to sender for transmission bool NextSenderMsg(NormObjectMsg* msg); @@ -185,26 +220,17 @@ class NormObject void ClearFirstPass() {first_pass = false};*/ bool TxReset(NormBlockId firstBlock = NormBlockId(0), bool requeue = false); - bool TxResetBlocks(NormBlockId nextId, NormBlockId lastId); + bool TxResetBlocks(const NormBlockId& nextId, const NormBlockId& lastId); bool TxUpdateBlock(NormBlock* theBlock, NormSegmentId firstSegmentId, NormSegmentId lastSegmentId, - UINT16 numErasures) - { - NormBlockId blockId = theBlock->GetId(); - bool result = theBlock->TxUpdate(firstSegmentId, lastSegmentId, - GetBlockSize(blockId), nparity, - numErasures); - ASSERT(result ? pending_mask.Set(blockId) : true); - result = result ? pending_mask.Set(blockId) : false; - return result; - } + UINT16 numErasures); bool HandleInfoRequest(bool holdoff); - bool HandleBlockRequest(NormBlockId nextId, NormBlockId lastId); + bool HandleBlockRequest(const NormBlockId& nextId, const NormBlockId& lastId); NormBlock* FindBlock(NormBlockId blockId) {return block_buffer.Find(blockId);} bool ActivateRepairs(); - bool IsRepairSet(NormBlockId blockId) {return repair_mask.Test(blockId);} - bool IsPendingSet(NormBlockId blockId) {return pending_mask.Test(blockId);} + bool IsRepairSet(NormBlockId blockId) {return repair_mask.Test(blockId.GetValue());} + bool IsPendingSet(NormBlockId blockId) {return pending_mask.Test(blockId.GetValue());} bool AppendRepairAdv(NormCmdRepairAdvMsg& cmd); NormBlockId GetMaxPendingBlockId() const {return max_pending_block;} @@ -236,12 +262,12 @@ class NormObject bool ReceiverRewindCheck(NormBlockId blockId, NormSegmentId segmentId); bool IsRepairPending(bool flush); - bool AppendRepairRequest(NormNackMsg& nack, bool flush); + bool AppendRepairRequest(NormNackMsg& nack, bool flush, UINT16 payloadMax); void SetRepairInfo() {repair_info = true;} bool SetRepairs(NormBlockId first, NormBlockId last) { - return (first == last) ? repair_mask.Set(first) : - repair_mask.SetBits(first, repair_mask.Delta(last,first)+1); + return ((first == last) ? repair_mask.Set(first.GetValue()) : + repair_mask.SetBits(first.GetValue(), repair_mask.Difference(last.GetValue(),first.GetValue())+1)); } void SetLastNackTime(const ProtoTime& theTime) {last_nack_time = theTime;} @@ -258,7 +284,17 @@ class NormObject const NormObjectId& objectId); void Accept() {accepted = true;} - + +#ifdef USE_PROTO_TREE + // Proto::Tree item required overrides + const char* GetKey() const + {return transport_id.GetValuePtr();} + unsigned int GetKeysize() const + {return (8*sizeof(UINT16));} + ProtoTree::Endian GetEndian() const + {return ProtoTree::GetNativeEndian();} +#endif // USE PROTO_TREE + NormObject::Type type; class NormSession& session; class NormSenderNode* sender; // NULL value indicates local (tx) object @@ -269,6 +305,7 @@ class NormObject UINT16 segment_size; UINT8 fec_id; UINT8 fec_m; + UINT32 fec_block_mask; UINT16 ndata; UINT16 nparity; NormBlockBuffer block_buffer; @@ -298,8 +335,9 @@ class NormObject bool notify_on_update; const void* user_data; // for NORM API usage only - - NormObject* next; +#ifndef USE_PROTO_TREE + NormObject* next; +#endif }; // end class NormObject @@ -325,7 +363,7 @@ class NormFileObject : public NormObject return result; } bool PadToSize() - {return file.Pad(NormObject::GetSize().GetOffset());} + {return file.IsOpen() ? file.Pad(NormObject::GetSize().GetOffset()) : false;} virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, @@ -349,9 +387,12 @@ class NormDataObject : public NormObject { // (TBD) allow support of greater than 4GB size data objects public: + typedef void (*DataFreeFunctionHandle)(char*); + NormDataObject(class NormSession& theSession, class NormSenderNode* theSender, - const NormObjectId& objectId); + const NormObjectId& objectId, + DataFreeFunctionHandle dataFreeFunc); ~NormDataObject(); bool Open(char* dataPtr, @@ -380,13 +421,17 @@ class NormDataObject : public NormObject virtual char* RetrieveSegment(NormBlockId blockId, NormSegmentId segmentId); + + private: - NormObjectSize large_block_length; - NormObjectSize small_block_length; - char* data_ptr; - UINT32 data_max; - bool data_released; // when true, data_ptr is deleted + NormObjectSize large_block_length; + NormObjectSize small_block_length; + char* data_ptr; + UINT32 data_max; + bool data_released; // when true, data_ptr is deleted + DataFreeFunctionHandle data_free_func; + // on NormDataObject destruction }; // end class NormDataObject @@ -426,7 +471,7 @@ class NormStreamObject : public NormObject bool GetPushMode() const {return push_mode;} bool IsOldBlock(NormBlockId blockId) const - {return (!stream_buffer.IsEmpty() && (blockId < stream_buffer.RangeLo()));} + {return (!stream_buffer.IsEmpty() && (Compare(blockId, stream_buffer.RangeLo()) < 0));} bool IsClosing() {return stream_closing;} bool HasVacancy() @@ -459,6 +504,11 @@ class NormStreamObject : public NormObject } bool StreamAdvance(); + NormBlockId GetSyncId() const + {return stream_sync_id;} + NormBlockId GetNextId() const + {return stream_next_id;} + virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* buffer); @@ -479,15 +529,15 @@ class NormStreamObject : public NormObject bool LockSegments(NormBlockId blockId, NormSegmentId firstId, NormSegmentId lastId); NormBlockId StreamBufferLo() const {return stream_buffer.RangeLo();} + NormBlockId RepairWindowLo() const; void Prune(NormBlockId blockId, bool updateStatus); bool IsFlushPending() {return flush_pending;} - NormBlockId FlushBlockId() - {return (write_index.segment ? write_index.block : - (NormBlockId((UINT32)write_index.block-1)));} - NormSegmentId FlushSegmentId() - {return (write_index.segment ? (write_index.segment-1) : - (ndata-1));} + + NormBlockId FlushBlockId() const; + + NormSegmentId FlushSegmentId() const + {return (write_index.segment ? (write_index.segment-1) : (ndata-1));} NormBlockId GetNextBlockId() const {return (sender ? read_index.block : write_index.block);} @@ -529,6 +579,7 @@ class NormStreamObject : public NormObject public: NormBlockId block; NormSegmentId segment; + UINT16 offset; }; // Extra state for STREAM objects bool stream_sync; @@ -551,9 +602,10 @@ class NormStreamObject : public NormObject bool msg_start; FlushMode flush_mode; bool push_mode; - + bool stream_broken; bool stream_closing; + // For threaded API purposes UINT32 block_pool_threshold; }; // end class NormStreamObject @@ -588,6 +640,10 @@ class NormSimObject : public NormObject }; // end class NormSimObject #endif // SIMULATE +#ifdef USE_PROTO_TREE +class NormObjectTree : public ProtoSortedTreeTemplate {}; +#endif // USE_PROTO_TREE + class NormObjectTable { public: @@ -600,7 +656,6 @@ class NormObjectTable void SetRangeMax(UINT16 rangeMax); void Destroy(); - bool IsInited() const {return (NULL != table);} UINT16 GetRangeMax() const {return range_max;} bool CanInsert(NormObjectId objectId) const; bool Insert(NormObject* theObject); @@ -612,7 +667,23 @@ class NormObjectTable bool IsEmpty() const {return (0 == range);} UINT32 GetCount() const {return count;} const NormObjectSize& GetSize() const {return size;} + +#ifdef USE_PROTO_TREE + class Iterator + { + public: + Iterator(NormObjectTable& objectTable); + NormObject* GetNextObject(); + NormObject* GetPrevObject(); + void Reset(); + + private: + const NormObjectTable& table; + NormObjectTree::Iterator iterator; + NormObject* next_object; + }; +#else class Iterator { public: @@ -626,12 +697,19 @@ class NormObjectTable bool reset; NormObjectId index; }; +#endif // if/else USE_PROTO_TREE private: +#ifndef USE_PROTO_TREE NormObject* Next(NormObject* o) const {return o->next;} - +#endif + +#ifdef USE_PROTO_TREE + NormObjectTree tree; +#else NormObject** table; UINT16 hash_mask; +#endif // if/else USE_PROTO_TREE UINT16 range_max; // max range of objects that can be kept UINT16 range; // zero if "object table" is empty NormObjectId range_lo; diff --git a/include/normSegment.h b/include/normSegment.h index d88e923..cf88173 100644 --- a/include/normSegment.h +++ b/include/normSegment.h @@ -4,6 +4,9 @@ #include "normMessage.h" #include "protoBitmask.h" +#define USE_PROTO_TREE 1 // for more better performing NormBlockBuffer? + + // Norm uses preallocated (or dynamically allocated) pools of // segments (vectors) for different buffering purposes @@ -43,7 +46,11 @@ class NormSegmentPool bool overrun_flag; }; // end class NormSegmentPool +#ifdef USE_PROTO_TREE +class NormBlock : public ProtoSortedTree::Item +#else class NormBlock +#endif // if/else USE_PROTO_TREE { friend class NormBlockPool; friend class NormBlockBuffer; @@ -144,7 +151,7 @@ class NormBlock UINT8 fecId, UINT8 fecM, UINT16 numData, - UINT16 segmentSize); + UINT16 payloadMax); // Receiver routines void RxInit(NormBlockId& blockId, UINT16 ndata, UINT16 nparity) @@ -238,7 +245,7 @@ class NormBlock UINT16 numParity, NormObjectId objectId, bool pendingInfo, - UINT16 segmentSize); + UINT16 payloadMax); void SetLastNackTime(const ProtoTime& theTime) @@ -254,6 +261,15 @@ class NormBlock void EmptyToPool(NormSegmentPool& segmentPool); private: +#ifdef USE_PROTO_TREE + const char* GetKey() const + {return blk_id.GetValuePtr();} + unsigned int GetKeysize() const + {return (8*sizeof(UINT32));} + ProtoTree::Endian GetEndian() const + {return ProtoTree::GetNativeEndian();} +#endif // USE_PROTO_TREE + NormBlockId blk_id; UINT16 size; char** segment_table; @@ -267,8 +283,7 @@ class NormBlock ProtoBitmask pending_mask; ProtoBitmask repair_mask; ProtoTime last_nack_time; // for stream flow control - - NormBlock* next; + NormBlock* next; // used for NormBlockPool }; // end class NormBlock class NormBlockPool @@ -314,6 +329,10 @@ class NormBlockPool bool overrun_flag; }; // end class NormBlockPool +#ifdef USE_PROTO_TREE +class NormBlockTree : public ProtoSortedTreeTemplate {}; +#endif // USE_PROTO_TREE + class NormBlockBuffer { public: @@ -322,18 +341,34 @@ class NormBlockBuffer NormBlockBuffer(); ~NormBlockBuffer(); - bool Init(unsigned long rangeMax, unsigned long tableSize = 256); + bool Init(unsigned long rangeMax, unsigned long tableSize, UINT32 fecBlockMask); void Destroy(); bool Insert(NormBlock* theBlock); - bool Remove(const NormBlock* theBlock); + bool Remove(NormBlock* theBlock); NormBlock* Find(const NormBlockId& blockId) const; NormBlockId RangeLo() const {return range_lo;} NormBlockId RangeHi() const {return range_hi;} + NormBlockId RangeMin() const; bool IsEmpty() const {return (0 == range);} bool CanInsert(NormBlockId blockId) const; + +#ifdef USE_PROTO_TREE + class Iterator + { + public: + Iterator(NormBlockBuffer& blockBuffer); + NormBlock* GetNextBlock(); + void Reset(); + + private: + NormBlockBuffer& buffer; + NormBlockTree::Iterator iterator; + NormBlock* next_block; + }; +#else class Iterator { public: @@ -346,18 +381,29 @@ class NormBlockBuffer bool reset; NormBlockId index; }; - +#endif // if/else USE_PROTO_TREE private: + int Compare(NormBlockId a, NormBlockId b) const + {return NormBlockId::Compare(a, b, fec_block_mask);} + INT32 Difference(NormBlockId a, NormBlockId b) const + {return NormBlockId::Difference(a, b, fec_block_mask);} + void Increment(NormBlockId& b, UINT32 i = 1) const + {b.Increment(i, fec_block_mask);} + void Decrement(NormBlockId& b, UINT32 i = 1) const + {b.Decrement(i, fec_block_mask);} + +#ifdef USE_PROTO_TREE + NormBlockTree tree; +#else static NormBlock* Next(NormBlock* b) {return b->next;} - NormBlock** table; - unsigned long hash_mask; + unsigned long hash_mask; +#endif // if/else USE_PROTO_TREE unsigned long range_max; // max range of blocks that can be buffered unsigned long range; // zero if "block buffer" is empty + UINT32 fec_block_mask; NormBlockId range_lo; NormBlockId range_hi; }; // end class NormBlockBuffer - - #endif // _NORM_SEGMENT diff --git a/include/normSession.h b/include/normSession.h index 7bea248..379c4c5 100644 --- a/include/normSession.h +++ b/include/normSession.h @@ -103,11 +103,18 @@ class NormSessionMgr NormController* GetController() const {return controller;} + + void SetDataFreeFunction(NormDataObject::DataFreeFunctionHandle freeFunc) + {data_free_func = freeFunc;} + NormDataObject::DataFreeFunctionHandle GetDataFreeFunction() const + {return data_free_func;} + private: - ProtoTimerMgr& timer_mgr; - ProtoSocket::Notifier& socket_notifier; - ProtoChannel::Notifier* channel_notifier; - NormController* controller; + ProtoTimerMgr& timer_mgr; + ProtoSocket::Notifier& socket_notifier; + ProtoChannel::Notifier* channel_notifier; + NormController* controller; + NormDataObject::DataFreeFunctionHandle data_free_func; class NormSession* top_session; // top of NormSession list @@ -161,6 +168,8 @@ class NormSession }; // General methods + void SetNodeId(NormNodeId nodeId) + {local_node_id = nodeId;} const NormNodeId& LocalNodeId() const {return local_node_id;} bool Open(); void Close(); @@ -210,7 +219,7 @@ class NormSession UINT16 GetTxPort() const; bool SetRxPortReuse(bool enableReuse, - const char* rxAddress, // bind() to / + const char* rxAddress = NULL, // bind() to / const char* senderAddress = (const char*)0, // connect() to / UINT16 senderPort = 0); @@ -289,6 +298,7 @@ class NormSession // flow control is imposed double GetFlowControlDelay() const { + if (0.0 == flow_control_factor) return 0.0; double fdelay = (flow_control_factor * (SenderGrtt() * (backoff_factor + 1))); return ((fdelay > 0.020) ? fdelay : 0.020); // minimum 20 msec flow control } @@ -301,6 +311,13 @@ class NormSession bool SetTxCacheBounds(NormObjectSize sizeMax, unsigned long countMin, unsigned long countMax); + + // For NormSocket API extension support only + void SetServerListener(bool state) + {is_server_listener = state;} + bool IsServerListener() const + {return is_server_listener;} + void Notify(NormController::Event event, class NormNode* node, class NormObject* object) @@ -313,7 +330,14 @@ class NormSession NormMsg* GetMessageFromPool() {return message_pool.RemoveHead();} void ReturnMessageToPool(NormMsg* msg) {message_pool.Append(msg);} void QueueMessage(NormMsg* msg); - bool SendMessage(NormMsg& msg); + enum MessageStatus + { + MSG_SEND_FAILED, + MSG_SEND_BLOCKED, + MSG_SEND_OK + + }; + MessageStatus SendMessage(NormMsg& msg); void ActivateTimer(ProtoTimer& timer) {session_mgr.ActivateTimer(timer);} void SetUserData(const void* userData) @@ -334,7 +358,8 @@ class NormSession UINT32 bufferSpace, UINT16 segmentSize, UINT16 numData, - UINT16 numParity); + UINT16 numParity, + UINT8 fecId = 0); void StopSender(); void SetTxOnly(bool txOnly, bool connectToSessionAddress = false); bool GetTxOnly() const @@ -385,6 +410,9 @@ class NormSession bool SenderSendCmd(const char* cmdBuffer, unsigned int cmdLength, bool robust); void SenderCancelCmd(); + void SenderSetSynStatus(bool state) + {syn_status = state;} + // robust factor void SetTxRobustFactor(int value) {tx_robust_factor = value;} @@ -407,6 +435,13 @@ class NormSession void SenderSetExtraParity(UINT16 extraParity) {extra_parity = extraParity;} + INT32 Difference(NormBlockId a, NormBlockId b) const + {return NormBlockId::Difference(a, b, fec_block_mask);} + int Compare(NormBlockId a, NormBlockId b) const + {return NormBlockId::Compare(a, b, fec_block_mask);} + void Increment(NormBlockId& b, UINT32 i = 1) const + {b.Increment(i, fec_block_mask);} + // EMCON Sender (useful when there are silent receivers) // (NORM_INFO is redundantly sent) @@ -431,6 +466,8 @@ class NormSession } double SenderGrtt() const {return grtt_advertised;} + void ResetGrttNotification() + {notify_on_grtt_update = true;} void SenderSetGrtt(double grttValue) { if (IsSender()) @@ -439,7 +476,7 @@ class NormSession grttValue = (grttValue < grttMin) ? grttMin : grttValue; } grtt_quantized = NormQuantizeRtt(grttValue); - grtt_measured = grtt_advertised = NormUnquantizeRtt(grtt_quantized); + grtt_measured = grtt_advertised = NormUnquantizeRtt(grtt_quantized); } double SenderGroupSize() {return gsize_measured;} void SenderSetGroupSize(double gsize) @@ -482,13 +519,21 @@ class NormSession unsigned long RemoteSenderBufferSize() const {return remote_sender_buffer_size;} + bool InsertRemoteSender(NormSenderNode& sender); + void DeleteRemoteSender(NormSenderNode& senderNode); // Call this to do remote sender memory allocations ahead of time - bool PreallocateRemoteSender(UINT16 segmentSize, UINT16 numData, UINT16 numParity); + bool PreallocateRemoteSender(unsigned int bufferSize, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + unsigned int streamBufferSize = 0); - void ReceiverSetUnicastNacks(bool state) {unicast_nacks = state;} - bool ReceiverGetUnicastNacks() const {return unicast_nacks;} + void ReceiverSetUnicastNacks(bool state) + {unicast_nacks = state;} + bool ReceiverGetUnicastNacks() const + {return unicast_nacks;} void ReceiverSetSilent(bool state) {receiver_silent = state;} @@ -500,7 +545,7 @@ class NormSession {return rcvr_ignore_info;} // The default "rcvr_max_delay = -1" corresponds to typical - // operation where source data forpartially received FEC blocks + // operation where source data for partially received FEC blocks // are only provided to the app when buffer constraints require it. // Otherwise, the "maxDelay" corresponds to the max number // of FEC blocks the receiver waits before passing partially @@ -541,7 +586,7 @@ class NormSession // Set default "max_pending_range" of NormObjects for reception void SetRxCacheMax(UINT16 maxCount) - {rx_cache_count_max = maxCount;} + {rx_cache_count_max = (maxCount > 0x7fff) ? 0x7fff : maxCount;} UINT16 GetRxCacheMax() const {return rx_cache_count_max;} @@ -560,7 +605,8 @@ class NormSession #endif // SIMULATE void SetProbeCount(unsigned probeCount) {probe_count = probeCount;} - + bool SenderQueueSquelch(NormObjectId objectId); + private: // Only NormSessionMgr can create/delete sessions NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId); @@ -607,12 +653,14 @@ class NormSession UINT16 ccSequence); void AdjustRate(bool onResponse); void SetTxRateInternal(double txRate); // here, txRate is bytes/sec - bool SenderQueueSquelch(NormObjectId objectId); + //bool SenderQueueSquelch(NormObjectId objectId); void SenderQueueFlush(); bool SenderQueueWatermarkFlush(); bool SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd); void SenderUpdateGroupSize(); bool SenderQueueAppCmd(); + // The following method is only used for NormSocket purposes + bool SenderSendAppCmd(const char* buffer, unsigned int length, const ProtoAddress& dst); // Receiver message handling routines void ReceiverHandleObjectMessage(const struct timeval& currentTime, @@ -660,6 +708,7 @@ class NormSession double tx_rate; // bytes per second double tx_rate_min; double tx_rate_max; + unsigned int tx_residual; // for NORM_CMD(CC)/NORM_DATA "packet pairing" // Sender parameters and state @@ -685,6 +734,7 @@ class NormSession NormEncoder* encoder; UINT8 fec_id; UINT8 fec_m; + INT32 fec_block_mask; NormObjectId next_tx_object_id; unsigned int tx_cache_count_min; @@ -761,6 +811,7 @@ class NormSession char* cmd_buffer; unsigned int cmd_length; ProtoTimer cmd_timer; + bool syn_status; // Receiver parameters bool is_receiver; @@ -778,6 +829,13 @@ class NormSession NormSenderNode::SyncPolicy default_sync_policy; UINT16 rx_cache_count_max; + // For NormSocket server-listener support + bool is_server_listener; + NormClientTree client_tree; + + // API-specific state variables + bool notify_on_grtt_update; + // State for some experimental congestion control bool ecn_ignore_loss; bool cc_tolerate_loss; diff --git a/include/normVersion.h b/include/normVersion.h index 38b15ae..c558a2e 100644 --- a/include/normVersion.h +++ b/include/normVersion.h @@ -36,6 +36,6 @@ #ifndef _NORM_VERSION #define _NORM_VERSION -#define VERSION "1.5r6" +#define VERSION "1.5.7" #endif // _NORM_VERSION diff --git a/makefiles/Makefile.common b/makefiles/Makefile.common index 782f17b..e1bdf27 100644 --- a/makefiles/Makefile.common +++ b/makefiles/Makefile.common @@ -178,6 +178,15 @@ normMsgr: $(MSGR_OBJ) libnorm.a $(LIBPROTO) mkdir -p ../bin cp $@ ../bin/$@ +# (normStreamer) message sender/receiver +STREAMER_SRC = $(EXAMPLE)/normStreamer.cpp +STREAMER_OBJ = $(STREAMER_SRC:.cpp=.o) + +normStreamer: $(STREAMER_OBJ) libnorm.a $(LIBPROTO) + $(CC) $(CFLAGS) -o $@ $(STREAMER_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS) + mkdir -p ../bin + cp $@ ../bin/$@ + # (pcap2norm) - parses pcap (e.g. tcpdump) file and prints NORM trace PCAP_SRC = $(COMMON)/pcap2norm.cpp PCAP_OBJ = $(PCAP_SRC:.cpp=.o) diff --git a/makefiles/Makefile.linux b/makefiles/Makefile.linux index 8c4f946..9f1e80c 100644 --- a/makefiles/Makefile.linux +++ b/makefiles/Makefile.linux @@ -35,15 +35,18 @@ SYSTEM_LIBS = -ldl # (We export these for other Makefiles as needed) # -SYSTEM_HAVES = -DLINUX -DECN_SUPPORT -DHAVE_IPV6 -DHAVE_GETLOGIN -D_FILE_OFFSET_BITS=64 -DHAVE_LOCKF \ --DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT +SYSTEM_HAVES = -DLINUX -DECN_SUPPORT -DHAVE_IPV6 -DHAVE_GETLOGIN -D_FILE_OFFSET_BITS=64 -DHAVE_LOCKF \ +-DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT -DNO_SCM_RIGHTS -DHAVE_SCHED -DUNIX \ +-DUSE_SELECT -DUSE_TIMERFD -DUSE_EVENTFD -DHAVE_PSELECT + + SYSTEM_SRC = ../protolib/src/linux/linuxCap.cpp SYSTEM = linux export CC = g++ -export SYSTEM_CFLAGS = -Wall -Wcast-align -pedantic -fPIC +export SYSTEM_CFLAGS = -Wall -Wcast-align -pedantic -fPIC -Wno-variadic-macros export SYSTEM_SOFLAGS = -shared -Wl,-soname,libnorm.so.1 SYSTEM_SOEXT = so export RANLIB = ranlib diff --git a/makefiles/android/AndroidManifest.xml b/makefiles/android/AndroidManifest.xml index d208dda..a4746b7 100644 --- a/makefiles/android/AndroidManifest.xml +++ b/makefiles/android/AndroidManifest.xml @@ -3,6 +3,6 @@ package="mil.navy.nrl.norm" android:versionCode="1" android:versionName="1.0"> - + diff --git a/makefiles/android/jni/Application.mk b/makefiles/android/jni/Application.mk index 1737164..43cd7a9 100644 --- a/makefiles/android/jni/Application.mk +++ b/makefiles/android/jni/Application.mk @@ -1,3 +1,3 @@ APP_ABI := all -APP_PLATFORM := android-9 +APP_PLATFORM := android-14 NDK_MODULE_PATH := ../..:../../protolib/ diff --git a/makefiles/android/project.properties b/makefiles/android/project.properties index 5ca7d62..6905bdf 100644 --- a/makefiles/android/project.properties +++ b/makefiles/android/project.properties @@ -9,4 +9,4 @@ android.library=true # Project target. -target=android-14 +target=android-18 diff --git a/makefiles/win32/NormDll.vcxproj b/makefiles/win32/NormDll.vcxproj index fa07e73..485a4db 100755 --- a/makefiles/win32/NormDll.vcxproj +++ b/makefiles/win32/NormDll.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -13,16 +13,17 @@ {182006F3-188F-466E-89FE-8421C0478691} Win32Proj + 10.0.16299.0 DynamicLibrary - v120 + v141 MultiByte DynamicLibrary - v120 + v141 MultiByte diff --git a/makefiles/win32/NormLib.vcxproj b/makefiles/win32/NormLib.vcxproj index 432612f..35e176e 100755 --- a/makefiles/win32/NormLib.vcxproj +++ b/makefiles/win32/NormLib.vcxproj @@ -12,19 +12,22 @@ {D7B0023C-8798-4918-8DA0-05C9054D70B9} + 10.0.16299.0 StaticLibrary - v120_xp + v141 false MultiByte + false StaticLibrary - v120_xp + v141 false MultiByte + false diff --git a/makefiles/win32/norm/norm.vcxproj b/makefiles/win32/norm/norm.vcxproj index a4772f4..9c8be51 100755 --- a/makefiles/win32/norm/norm.vcxproj +++ b/makefiles/win32/norm/norm.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -12,17 +12,18 @@ {6E1308A6-D40F-489E-A4F1-40D859380D64} + 10.0.16299.0 Application - v120 + v141 false MultiByte Application - v120 + v141 false MultiByte @@ -64,7 +65,7 @@ /vmg %(AdditionalOptions) Disabled ..\..\..\include;..\..\..\protolib\include;%(AdditionalIncludeDirectories) - _DEBUG;PROTO_DEBUG;HAVE_IPV6;HAVE_ASSERT;WIN32;_CRT_SECURE_NO_WARNINGS;_WINDOWS;%(PreprocessorDefinitions) + _CONSOLE,_DEBUG;PROTO_DEBUG;HAVE_IPV6;HAVE_ASSERT;WIN32;_CRT_SECURE_NO_WARNINGS;_WINDOWS;%(PreprocessorDefinitions) EnableFastChecks MultiThreadedDebugDLL @@ -107,7 +108,7 @@ MaxSpeed OnlyExplicitInline ..\..\..\include;..\..\..\protolib\include;%(AdditionalIncludeDirectories) - NDEBUG;PROTO_DEBUG;HAVE_IPV6;HAVE_ASSERT;WIN32;_WINDOWS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) + _CONSOLE,NDEBUG;PROTO_DEBUG;HAVE_IPV6;HAVE_ASSERT;WIN32;_WINDOWS;_CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) true MultiThreadedDLL true @@ -139,7 +140,7 @@ - + diff --git a/makefiles/win32/normTest.vcxproj b/makefiles/win32/normTest.vcxproj index 873b847..dfc2002 100755 --- a/makefiles/win32/normTest.vcxproj +++ b/makefiles/win32/normTest.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -13,17 +13,18 @@ {32CF83E0-D2E0-4E43-8F7B-72BD0AB64647} normTest + 10.0.16299.0 Application - v120 + v141 false MultiByte Application - v120 + v141 false MultiByte diff --git a/makefiles/win32/npc.vcxproj b/makefiles/win32/npc.vcxproj index 8198a7d..0381218 100755 --- a/makefiles/win32/npc.vcxproj +++ b/makefiles/win32/npc.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -13,17 +13,18 @@ {7BF4525B-9185-4296-A2BF-FFE77BB66EAF} normTest + 10.0.16299.0 Application - v120 + v141 false MultiByte Application - v120 + v141 false MultiByte diff --git a/src/common/normApi.cpp b/src/common/normApi.cpp index 84e7b4d..ff510aa 100644 --- a/src/common/normApi.cpp +++ b/src/common/normApi.cpp @@ -68,6 +68,13 @@ class NormInstance : public NormController bool GetNextEvent(NormEvent* theEvent); bool SetCacheDirectory(const char* cachePath); + void SetAllocationFunctions(NormAllocFunctionHandle allocFunc, + NormFreeFunctionHandle freeFunc) + { + data_alloc_func = allocFunc; + session_mgr.SetDataFreeFunction(freeFunc); + } + void ReleasePreviousEvent(); bool NotifyQueueIsEmpty() const @@ -158,14 +165,15 @@ class NormInstance : public NormController ProtoDispatcher dispatcher; bool priority_boost; - NormSessionMgr session_mgr; + NormSessionMgr session_mgr; + NormAllocFunctionHandle data_alloc_func; private: void ResetNotificationEvent() { #ifdef WIN32 if (0 == ResetEvent(notify_event)) - PLOG(PL_ERROR, "NormInstance::GetNextEvent() ResetEvent error: %s\n", GetErrorString()); + PLOG(PL_ERROR, "NormInstance::ResetNotificationEvent() ResetEvent error: %s\n", GetErrorString()); #else char byte[32]; while (read(notify_fd[0], byte, 32) > 0); // TBD - error check @@ -190,7 +198,7 @@ class NormInstance : public NormController NormInstance::Notification::Queue::Queue() : head(NULL), tail(NULL) { - + } @@ -212,13 +220,14 @@ NormInstance::NormInstance() session_mgr(static_cast(dispatcher), static_cast(dispatcher), static_cast(&dispatcher)), - previous_notification(NULL), rx_cache_path(NULL) + data_alloc_func(NULL), previous_notification(NULL), rx_cache_path(NULL) { #ifdef WIN32 notify_event = NULL; #else notify_fd[0] = notify_fd[1] = -1; #endif // if/else WIN32/UNIX + dispatcher.SetUserData(&session_mgr); // for debugging session_mgr.SetController(static_cast(this)); } @@ -374,7 +383,7 @@ void NormInstance::Notify(NormController::Event event, { NormDataObject* dataObj = static_cast(object); unsigned int dataLen = (unsigned int)(object->GetSize().GetOffset()); - char* dataPtr = new char[dataLen]; + char* dataPtr = (NULL != data_alloc_func) ? data_alloc_func(dataLen) : new char[dataLen]; if (NULL == dataPtr) { PLOG(PL_FATAL, "NormInstance::Notify(RX_OBJECT_NEW) new dataPtr error: %s\n", @@ -474,7 +483,7 @@ void NormInstance::PurgeObjectNotifications(NormObjectHandle objectHandle) // TBD - check if event queue is emptied and reset event/fd } // end NormInstance::PurgeObjectNotifications() -// Purge any notifications associated with a specific object +// Purge any notifications associated with a specific remote sender node void NormInstance::PurgeNodeNotifications(NormNodeHandle nodeHandle) { if (NORM_NODE_INVALID == nodeHandle) return; @@ -507,7 +516,7 @@ void NormInstance::PurgeNodeNotifications(NormNodeHandle nodeHandle) // "Release" any previously-retained object or node handle if (NORM_OBJECT_INVALID != previous_notification->event.object) ((NormObject*)(previous_notification->event.object))->Release(); - else if (NORM_NODE_INVALID != previous_notification->event.sender) + else ((NormNode*)(previous_notification->event.sender))->Release(); notify_pool.Append(previous_notification); previous_notification = NULL; @@ -649,8 +658,6 @@ bool NormInstance::GetNextEvent(NormEvent* theEvent) return (NULL != n); } // end NormInstance::GetNextEvent() - - bool NormInstance::WaitForEvent() { if (!dispatcher.IsThreaded()) @@ -924,6 +931,16 @@ NormDescriptor NormGetDescriptor(NormInstanceHandle instanceHandle) } // end NormGetDescriptor() +NORM_API_LINKAGE +void NormSetAllocationFunctions(NormInstanceHandle instanceHandle, + NormAllocFunctionHandle allocFunc, + NormFreeFunctionHandle freeFunc) +{ + NormInstance* instance = (NormInstance*)instanceHandle; + instance->SetAllocationFunctions(allocFunc, freeFunc); +} // end NormSetAllocationFunctions() + + // if "waitForEvent" is false, this is a non-blocking call // (TBD) add a timeout option to this? @@ -986,7 +1003,7 @@ NormSessionHandle NormCreateSession(NormInstanceHandle instanceHandle, NormSession* session = instance->session_mgr.NewSession(sessionAddr, sessionPort, localNodeId); instance->dispatcher.ResumeThread(); - if (session) + if (NULL != session) return ((NormSessionHandle)session); } return NORM_SESSION_INVALID; @@ -1067,7 +1084,7 @@ NORM_API_LINKAGE void NormCancelUserTimer(NormSessionHandle sessionHandle) { NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); - if (instance) + if (NULL != instance) { if (instance->dispatcher.SuspendThread()) { @@ -1090,6 +1107,38 @@ NormNodeId NormGetLocalNodeId(NormSessionHandle sessionHandle) return NORM_NODE_NONE; } // end NormGetLocalNodeId() +NORM_API_LINKAGE +bool NormGetAddress(NormSessionHandle sessionHandle, + char* addrBuffer, + unsigned int* bufferLen, + UINT16* port) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (NULL != instance) + { + if (instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + const ProtoAddress& sessionAddr = session->Address(); + unsigned int addrLen = sessionAddr.GetLength(); + if (addrBuffer && bufferLen && (addrLen <= *bufferLen)) + { + memcpy(addrBuffer, sessionAddr.GetRawHostAddress(), addrLen); + result = true; + } + else if (NULL == addrBuffer) + { + result = true; // just a query for addrLen and/or port + } + if (bufferLen) *bufferLen = addrLen; + if (port) *port = sessionAddr.GetPort(); + instance->dispatcher.ResumeThread(); + } + } + return result; +} // end NormGetAddress() + NORM_API_LINKAGE UINT16 NormGetRxPort(NormSessionHandle sessionHandle) { @@ -1146,22 +1195,44 @@ void NormSetTxOnly(NormSessionHandle sessionHandle, } } // end NormSetTxOnly() +// These functions are used internally by the NormSocket API extensiont + NORM_API_LINKAGE -bool NormChangeDestination(NormSessionHandle sessionHandle, - const char* sessionAddress, - UINT16 sessionPort, - bool connectToSessionAddress) +void NormSetId(NormSessionHandle sessionHandle, NormNodeId normId) { - // First, see if we can make a valid ProtoAddress - ProtoAddress dest; - if (!dest.ResolveFromString(sessionAddress)) - return false; - dest.SetPort(sessionPort); NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); if (instance && instance->dispatcher.SuspendThread()) { NormSession* session = (NormSession*)sessionHandle; - if (session) + if (NULL != session) session->SetNodeId(normId); + instance->dispatcher.ResumeThread(); + } +} // end NormSetId() + +NORM_API_LINKAGE +bool NormChangeDestination(NormSessionHandle sessionHandle, + const char* sessionAddress, + UINT16 sessionPort, + bool connectToSessionAddress) +{ + NormSession* session = (NormSession*)sessionHandle; + if (NULL == session) return false; + // First, see if we can make a valid ProtoAddress + ProtoAddress dest; + if (NULL != sessionAddress) + { + if (!dest.ResolveFromString(sessionAddress)) + return false; + } + else + { + dest = session->Address(); + } + dest.SetPort(sessionPort); + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + if (NULL != session) { session->SetAddress(dest); if (connectToSessionAddress) @@ -1172,6 +1243,46 @@ bool NormChangeDestination(NormSessionHandle sessionHandle, return true; } // end NormChangeDestination() +NORM_API_LINKAGE +void NormSetServerListener(NormSessionHandle sessionHandle, bool state) +{ + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (NULL != instance) + { + if (instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) session->SetServerListener(state); + instance->dispatcher.ResumeThread(); + } + } +} // end NormSetServerListener() + + +NORM_API_LINKAGE +bool NormTransferSender(NormSessionHandle sessionHandle, NormNodeHandle senderHandle) +{ + bool result = false; + NormInstance* dstInstance = NormInstance::GetInstanceFromSession(sessionHandle); + if (NULL != dstInstance) + { + if (dstInstance->dispatcher.SuspendThread()) + { + NormInstance* srcInstance = NormInstance::GetInstanceFromNode(senderHandle); + if (srcInstance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + NormSenderNode* sender = (NormSenderNode*)senderHandle; + if ((NULL != session) && (NULL != sender)) + result = session->InsertRemoteSender(*sender); + srcInstance->dispatcher.ResumeThread(); + } + dstInstance->dispatcher.ResumeThread(); + } + } + return result; +} // end NormTransferSender() + NORM_API_LINKAGE void NormSetRxPortReuse(NormSessionHandle sessionHandle, bool enableReuse, @@ -1447,7 +1558,8 @@ bool NormStartSender(NormSessionHandle sessionHandle, UINT32 bufferSpace, UINT16 segmentSize, UINT16 numData, - UINT16 numParity) + UINT16 numParity, + UINT8 fecId) { bool result = false; NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); @@ -1455,7 +1567,7 @@ bool NormStartSender(NormSessionHandle sessionHandle, { NormSession* session = (NormSession*)sessionHandle; if (session) - result = session->StartSender(sessionId, bufferSpace, segmentSize, numData, numParity); + result = session->StartSender(sessionId, bufferSpace, segmentSize, numData, numParity, fecId); else result = false; instance->dispatcher.ResumeThread(); @@ -1596,18 +1708,16 @@ void NormSetGrttEstimate(NormSessionHandle sessionHandle, NORM_API_LINKAGE double NormGetGrttEstimate(NormSessionHandle sessionHandle) { - NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); - if (instance && instance->dispatcher.SuspendThread()) + if (NORM_SESSION_INVALID != sessionHandle) { NormSession* session = (NormSession*)sessionHandle; - if (session) session->SenderGrtt(); - instance->dispatcher.ResumeThread(); + session->ResetGrttNotification(); + return (session->SenderGrtt()); } - - if (NORM_SESSION_INVALID != sessionHandle) - return (((NormSession*)sessionHandle)->SenderGrtt()); else + { return -1.0; + } } // end NormGetGrttEstimate() NORM_API_LINKAGE @@ -1728,7 +1838,7 @@ NormObjectHandle NormDataEnqueue(NormSessionHandle sessionHandle, { NormObject* obj = static_cast(session->QueueTxData(dataPtr, dataLen, infoPtr, infoLen)); - if (obj) objectHandle = (NormObjectHandle)obj; + if (NULL != obj) objectHandle = (NormObjectHandle)obj; } instance->dispatcher.ResumeThread(); } @@ -1803,6 +1913,8 @@ NORM_API_LINKAGE unsigned int NormGetStreamBufferSegmentCount(unsigned int bufferBytes, UINT16 segmentSize, UINT16 blockSize) { // This same computation is performed in NormStreamObject::Open() in "normObject.cpp" + // (Note the number of stream buffer segments is always smaller than the + // blockSize*pending_mask.GetSize()) unsigned int numBlocks = bufferBytes / (blockSize * segmentSize); if (numBlocks < 2) numBlocks = 2; // NORM enforces a 2-block minimum buffer size return (numBlocks * blockSize); @@ -1813,14 +1925,13 @@ unsigned int NormStreamWrite(NormObjectHandle streamHandle, const char* buffer, unsigned int numBytes) { - // TBD - the thread suspend/resume here may _not_ be sufficient. - // we may need to do something to make sure the NORM thread - // wakes up right away (i.e. SignalThread() - // Also so the same for flush, send command, and data/file enqueue calls! - // (i.e. in any case where immediate data send action may be invoked) + // Note: Since an underlying issue with ProtoDispatcher::SignalThread() had been resolved, + // using ProtoDispatcher::SuspendThread() should be sufficient since the underlying + // protolib time scheduling, etc. code actually invokes SignalThread() on an + // as-needed basis. Thus, using SuspendThread() (lighter weight) should suffice unsigned int result = 0; NormInstance* instance = NormInstance::GetInstanceFromObject(streamHandle); - if (instance && instance->dispatcher.SuspendThread()) + if ((NULL != instance) && instance->dispatcher.SuspendThread()) { NormStreamObject* stream = static_cast((NormObject*)streamHandle); @@ -1836,7 +1947,7 @@ void NormStreamFlush(NormObjectHandle streamHandle, NormFlushMode flushMode) { NormInstance* instance = NormInstance::GetInstanceFromObject(streamHandle); - if (instance && instance->dispatcher.SuspendThread()) + if ((NULL != instance) && instance->dispatcher.SuspendThread()) { NormStreamObject* stream = static_cast((NormObject*)streamHandle); @@ -1908,7 +2019,7 @@ bool NormSetWatermark(NormSessionHandle sessionHandle, NormObject* obj = (NormObject*)objectHandle; if (session && obj) { - // Purge any existing to NORM_TX_WATERMARK_COMPLETED notification to be safe + // Purge any existing NORM_TX_WATERMARK_COMPLETED notifications to be safe instance->PurgeNotifications(sessionHandle, NORM_TX_WATERMARK_COMPLETED); // (segmentId doesn't matter for non-stream) if (obj->IsStream()) @@ -1941,6 +2052,8 @@ bool NormResetWatermark(NormSessionHandle sessionHandle) NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); if (instance && instance->dispatcher.SuspendThread()) { + // Purge any existing NORM_TX_WATERMARK_COMPLETED notifications to be safe + instance->PurgeNotifications(sessionHandle, NORM_TX_WATERMARK_COMPLETED); NormSession* session = (NormSession*)sessionHandle; session->SenderResetWatermark(); instance->dispatcher.ResumeThread(); @@ -1997,12 +2110,17 @@ NORM_API_LINKAGE NormNodeHandle NormGetAckingNodeHandle(NormSessionHandle sessionHandle, NormNodeId nodeId) { - if (NORM_SESSION_INVALID != sessionHandle) + if (NORM_SESSION_INVALID != sessionHandle) { - NormSession* session = (NormSession*)sessionHandle; - NormAckingNode* acker = session->SenderFindAckingNode(nodeId); - if (NULL != acker) - return ((NormNodeHandle)static_cast(acker)); + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + NormAckingNode* acker = session->SenderFindAckingNode(nodeId); + instance->dispatcher.ResumeThread(); + if (NULL != acker) + return ((NormNodeHandle)static_cast(acker)); + } } return NORM_NODE_INVALID; } // end NormGetAckingNodeHandle() @@ -2104,6 +2222,19 @@ void NormCancelCommand(NormSessionHandle sessionHandle) } } // end NormCancelCommand() +NORM_API_LINKAGE +void NormSetSynStatus(NormSessionHandle sessionHandle, bool state) +{ + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + session->SenderSetSynStatus(state); + instance->dispatcher.ResumeThread(); + } +} // end NormSetSynStatus() + + /** NORM Receiver Functions */ NORM_API_LINKAGE @@ -2138,8 +2269,6 @@ NORM_API_LINKAGE void NormSetRxCacheLimit(NormSessionHandle sessionHandle, unsigned short countMax) { - // We limit this to 1/4 of the 16-bit NormObjectId space - if (countMax > 16384) countMax = 16384; NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); if (instance && instance->dispatcher.SuspendThread()) { @@ -2290,6 +2419,25 @@ void NormNodeSetRxRobustFactor(NormNodeHandle nodeHandle, } } // end NormNodeSetRxRobustFactor() +NORM_API_LINKAGE +bool NormPreallocateRemoteSender(NormSessionHandle sessionHandle, + unsigned long bufferSize, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + unsigned int streamBufferSize) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + result = session->PreallocateRemoteSender(bufferSize, segmentSize, numData, numParity, streamBufferSize); + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormPreallocateRemoteSender() + NORM_API_LINKAGE bool NormStreamRead(NormObjectHandle streamHandle, char* buffer, @@ -2545,6 +2693,19 @@ char* NormDataDetachData(NormObjectHandle dataHandle) return ptr; } // end NormDataDetachData() +NORM_API_LINKAGE +char* NormAlloc(size_t numBytes) +{ + return new char[numBytes]; +} // end NormAlloc() + +NORM_API_LINKAGE +void NormFree(char* dataPtr) +{ + if (NULL != dataPtr) + delete[] dataPtr; +} // end NormFree() + NORM_API_LINKAGE NormNodeHandle NormObjectGetSender(NormObjectHandle objectHandle) { @@ -2576,7 +2737,7 @@ bool NormNodeGetAddress(NormNodeHandle nodeHandle, //NormInstance* instance = NormInstance::GetInstanceFromNode(nodeHandle); //if (instance && instance->dispatcher.SuspendThread()) { - NormNode* node = (NormSenderNode*)nodeHandle; + NormNode* node = (NormNode*)nodeHandle; const ProtoAddress& nodeAddr = node->GetAddress(); unsigned int addrLen = nodeAddr.GetLength(); if (addrBuffer && bufferLen && (addrLen <= *bufferLen)) @@ -2623,6 +2784,7 @@ double NormNodeGetGrtt(NormNodeHandle nodeHandle) if ((NULL != node) && (NormNode::SENDER == node->GetType())) { NormSenderNode* sender = static_cast(node); + sender->ResetGrttNotification(); return sender->GetGrttEstimate(); } else diff --git a/src/common/normApp.cpp b/src/common/normApp.cpp index f053f6b..3a9c527 100644 --- a/src/common/normApp.cpp +++ b/src/common/normApp.cpp @@ -1,5 +1,10 @@ -// normApp.cpp - Command-line NORM application +// normApp.cpp - Command-line NORM "demo" application + +// Note this code does _not_ use the NORM API. It is kept as a demonstrator +// and test tool and serves as a performance comparison reference for testing +// the threaded NORM API code. So this should _not_ be used as a NORM +// programming example!!!! #include "protokit.h" #include "normSession.h" @@ -51,7 +56,6 @@ class NormApp : public NormController, public ProtoApp {object_id = objectId;} NormObjectId GetObjectId() const {return object_id;} - private: // ProtoTree::Item overrides @@ -62,9 +66,7 @@ class NormApp : public NormController, public ProtoApp char file_path[PATH_MAX]; NormObjectId object_id; - }; - - + }; // end class NormApp::FileCacheItem void ShowHelp(); void OnInputReady(); @@ -145,14 +147,17 @@ class NormApp : public NormController, public ProtoApp UINT16 extra_parity; double backoff_factor; double grtt_estimate; // initial grtt estimate + NormSession::ProbingMode grtt_probing_mode; double group_size; unsigned long tx_buffer_size; // bytes - unsigned int tx_sock_buffer_size; + unsigned int tx_sock_buffer_size; unsigned long tx_cache_min; unsigned long tx_cache_max; NormObjectSize tx_cache_size; NormFileList tx_file_list; + char tx_file_name[PATH_MAX]; + bool tx_file_info; ProtoTree tx_file_cache; bool tx_one_shot; bool tx_ack_shot; @@ -188,6 +193,7 @@ class NormApp : public NormController, public ProtoApp bool rx_persistent; bool process_aborted_files; bool preallocate_sender; + NormSenderNode::RepairBoundary repair_boundary; // Debug parameters bool tracing; @@ -222,17 +228,16 @@ NormApp::NormApp() cc_enable(false), ecn_mode(ECN_OFF), tolerate_loss(false), node_id(NORM_NODE_ANY), segment_size(1024), ndata(32), nparity(16), auto_parity(0), extra_parity(0), backoff_factor(NormSession::DEFAULT_BACKOFF_FACTOR), grtt_estimate(NormSession::DEFAULT_GRTT_ESTIMATE), - group_size(NormSession::DEFAULT_GSIZE_ESTIMATE), + grtt_probing_mode(NormSession::PROBE_ACTIVE), group_size(NormSession::DEFAULT_GSIZE_ESTIMATE), tx_buffer_size(1024*1024), tx_sock_buffer_size(0), tx_cache_min(8), tx_cache_max(256), tx_cache_size((UINT32)20*1024*1024), - tx_one_shot(false), tx_ack_shot(false), tx_file_queued(false), + tx_file_info(true), tx_one_shot(false), tx_ack_shot(false), tx_file_queued(false), tx_robust_factor(NormSession::DEFAULT_ROBUST_FACTOR), tx_object_interval(0.0), tx_repeat_count(0), tx_repeat_interval(2.0), tx_repeat_clear(true), tx_requeue(0), tx_requeue_count(0), acking_node_list(NULL), acking_flushes(false), watermark_pending(false), rx_buffer_size(1024*1024), rx_sock_buffer_size(0), rx_cache_path(NULL), post_processor(NULL), unicast_nacks(false), silent_receiver(false), low_delay(false), realtime(false), rx_robust_factor(NormSession::DEFAULT_ROBUST_FACTOR), rx_persistent(true), process_aborted_files(false), - preallocate_sender(false), tracing(false), tx_loss(0.0), rx_loss(0.0) + preallocate_sender(false), repair_boundary(NormSenderNode::BLOCK_BOUNDARY), tracing(false), tx_loss(0.0), rx_loss(0.0) { - control_pipe.SetListener(this, &NormApp::OnControlEvent); control_pipe.SetNotifier(&GetSocketNotifier()); @@ -243,6 +248,8 @@ NormApp::NormApp() interval_timer.SetInterval(0.0); interval_timer.SetRepeat(0); + tx_file_name[0] = '\0'; + struct timeval currentTime; ProtoSystemTime(currentTime); srand(currentTime.tv_usec); @@ -307,10 +314,12 @@ const char* const NormApp::cmd_list[] = "+minput", // send message stream input "+moutput", // recv message stream output "+sendfile", // file/directory list to transmit + "+info", // 'on' | 'off' to enable/disable file info (name) transmission (default = 'on') "+interval", // delay time (sec) between files (0.0 sec default) "+repeatcount", // How many times to repeat the file/directory list tx "+rinterval", // Interval (sec) between file/directory list repeats "+requeue", // how many times files are retransmitted w/ same objId + "+boundary", // 'block' or 'file' to set NORM_REPAIR_BOUNDARY (default is 'block') "-oneshot", // Transmit file(s), exiting upon TX_FLUSH_COMPLETED "-ackshot", // Transmit file(s), exiting upon TX_WATERMARK_COMPLETED "-updatesOnly", // only send updated files on repeat transmission @@ -325,6 +334,7 @@ const char* const NormApp::cmd_list[] = "+extra", // Number of extra FEC packets sent in response to repair requests "+backoff", // Backoff factor to use "+grtt", // Set sender's initial GRTT estimate + "+probe", // {'active', 'passive' | 'none'} Set sender;s GRTT probing mode 'active' is default) "+gsize", // Set sender's group size estimate "+txbuffer", // Size of sender's buffer "+txsockbuffer", // tx socket buffer size @@ -377,10 +387,12 @@ void NormApp::ShowHelp() " +minput, // sender message stream input\n" " +moutput, // receiver message stream output\n" " +sendfile, // file/directory list to transmit\n" + " +info, // 'on' | 'off' to enable/disable file info (name) transmission (default = 'on')\n" " +interval, // delay time (sec) between files (0.0 sec default)\n" " +repeatcount, // How many times to repeat the file/directory list tx\n" " +rinterval, // Interval (sec) between file/directory list repeats\n" " +requeue, // how many times files are retransmitted w/ same objId\n" + " +boundary // 'block' or 'file' to set NORM_REPAIR_BOUNDARY (default is 'block')\n" " -oneshot, // Exit upon sender TX_FLUSH_COMPLETED event (sender exits after transmission)\n" " -ackshot, // Exit upon sender TX_WATERMARK_COMPLETED event (sender exits after transmission)\n" " -updatesOnly, // only send updated files on repeat transmission\n" @@ -395,6 +407,7 @@ void NormApp::ShowHelp() " +extra, // Number of extra FEC packets sent in response to repair requests\n" " +backoff, // Backoff factor to use\n" " +grtt, // Set sender's initial GRTT estimate\n" + " +probe, // {'active', 'passive' | 'none'} Set sender;s GRTT probing mode 'active' is default)\n" " +gsize, // Set sender's group size estimate\n" " +txbuffer, // Size of sender's buffer\n" " +txcachebounds,// limits on sender tx object caching\n" @@ -428,7 +441,7 @@ void NormApp::OnControlEvent(ProtoSocket& /*theSocket*/, ProtoSocket::Event theE if (control_pipe.Recv(buffer, len)) { buffer[len] = '\0'; - PLOG(PL_ERROR, "norm: received command \"%s\"\n", buffer); + PLOG(PL_DEBUG, "norm: received command \"%s\"\n", buffer); char* cmd = buffer; char* val = NULL; for (unsigned int i = 0; i < len; i++) @@ -631,6 +644,8 @@ bool NormApp::OnCommand(const char* cmd, const char* val) } bool result = session ? session->SetLoopback(loopTemp) : true; loopback = result ? loopTemp : loopback; + if (loopback && (NULL != session) && session->Address().IsMulticast()) + session->SetRxPortReuse(true); if (!result) { PLOG(PL_FATAL, "NormApp::OnCommand(loopback) error setting socket loopback!\n"); @@ -828,17 +843,25 @@ bool NormApp::OnCommand(const char* cmd, const char* val) // (This will allow the "send" command to be invoked at run-time via the ProtoPipe // remote control interface). } - else if (!strncmp("interval", cmd, len)) + else if (!strncmp("info", cmd, len)) { - if (1 != sscanf(val, "%lf", &tx_object_interval)) - tx_object_interval = -1.0; - if (tx_object_interval < 0.0) + if (!strcmp("on", val)) + tx_file_info = true; + else if (!strcmp("off", val)) + tx_file_info = false; + else { - PLOG(PL_FATAL, "NormApp::OnCommand(interval) Invalid tx object interval: %s\n", - val); - tx_object_interval = 0.0; + PLOG(PL_FATAL, "NormApp::OnCommand(info) invalid argument!\n"); return false; } + } + else if (!strncmp("interval", cmd, len)) + { + if (1 != sscanf(val, "%lf", &tx_object_interval)) + { + PLOG(PL_FATAL, "NormApp::OnCommand(interval) Invalid tx object interval: %s\n", val); + return false; + } } else if (!strncmp("repeatcount", cmd, len)) { @@ -860,11 +883,27 @@ bool NormApp::OnCommand(const char* cmd, const char* val) { tx_requeue = tx_requeue_count = atoi(val); } + else if (!strncmp("boundary", cmd, len)) + { + if (0 == strcmp("block", val)) + { + repair_boundary = NormSenderNode::BLOCK_BOUNDARY; + } + else if (0 == strcmp("file", val)) + { + repair_boundary = NormSenderNode::OBJECT_BOUNDARY; + } + else + { + PLOG(PL_FATAL, "NormApp::OnCommand(boundary) error: invalid repair boundary \"%s\"\n", val); + return false; + } + if (session) session->ReceiverSetDefaultRepairBoundary(repair_boundary); + } else if (!strncmp("oneshot", cmd, len)) { tx_one_shot = true; - } - + } else if (!strncmp("ackshot", cmd, len)) { tx_ack_shot = true; @@ -998,6 +1037,22 @@ bool NormApp::OnCommand(const char* cmd, const char* val) grtt_estimate = grttEstimate; if (session) session->SenderSetGrtt(grttEstimate); } + else if (!strncmp("probe", cmd, len)) + { + size_t valLen = strlen(val); + if (!strncmp("none", val, valLen)) + grtt_probing_mode = NormSession::PROBE_NONE; + else if (!strncmp("passive", val, valLen)) + grtt_probing_mode = NormSession::PROBE_PASSIVE; + else if (!strncmp("active", val, valLen)) + grtt_probing_mode = NormSession::PROBE_ACTIVE; + else + { + PLOG(PL_FATAL, "NormApp::OnCommand(probe) invalid grtt probing mode!\n"); + return false; + } + if (NULL != session) session->SetGrttProbingMode(grtt_probing_mode); + } else if (!strncmp("gsize", cmd, len)) { double groupSize = atof(val); @@ -1028,7 +1083,11 @@ bool NormApp::OnCommand(const char* cmd, const char* val) tx_cache_min = countMin; tx_cache_max = countMax; tx_cache_size = sizeMax; - if (session) session->SetTxCacheBounds(tx_cache_size, tx_cache_min, tx_cache_max); + if (session) + { + session->SetTxCacheBounds(tx_cache_size, tx_cache_min, tx_cache_max); + session->SetRxCacheMax(tx_cache_max); + } } else if (!strncmp("txrobustfactor", cmd, len)) { @@ -1393,7 +1452,7 @@ void NormApp::OnInputReady() { if (feof(input)) { - PLOG(PL_FATAL, "norm: input end-of-file.\n"); + PLOG(PL_ALWAYS, "norm: input end-of-file.\n"); if (input_active) { #ifdef WIN32 @@ -1461,9 +1520,9 @@ void NormApp::OnInputReady() if (input_active) { #ifdef WIN32 - ASSERT(0); // no Win32 support for stream i/o yet + ASSERT(0); // no Win32 support for stream i/o yet #else - dispatcher.RemoveGenericInput(fileno(input)); + dispatcher.RemoveGenericInput(fileno(input)); #endif // if/else WIN32/UNIX input_active = false; } @@ -1516,7 +1575,7 @@ void NormApp::Notify(NormController::Event event, if ((msg_test || input) && (object == tx_stream)) OnInputReady(); } - else + else if (tx_object_interval >= 0.0) { // Can queue a new object for transmission if (interval_timer.IsActive()) interval_timer.Deactivate(); @@ -1535,9 +1594,18 @@ void NormApp::Notify(NormController::Event event, PLOG(PL_DEBUG, "NormApp::Notify(TX_FLUSH_COMPLETED) ...\n"); if (tx_one_shot) { - PLOG(PL_FATAL, "norm: transmit flushing completed, exiting.\n"); + PLOG(PL_ALWAYS, "norm: transmit flushing completed, exiting.\n"); Stop(); } + else if (tx_object_interval < 0.0) + { + // Can queue a new object for transmission + if (interval_timer.IsActive()) interval_timer.Deactivate(); + if (interval_timer.GetInterval() > 0.0) + ActivateTimer(interval_timer); + else + OnIntervalTimeout(interval_timer); + } break; case TX_WATERMARK_COMPLETED: @@ -1547,7 +1615,7 @@ void NormApp::Notify(NormController::Event event, watermark_pending = false; // enable new watermark to be set if (tx_ack_shot) { - PLOG(PL_ERROR, "norm: transmit flushing completed, exiting.\n"); + PLOG(PL_ALWAYS, "norm: transmit flushing completed, exiting.\n"); Stop(); } break; @@ -1954,12 +2022,20 @@ void NormApp::Notify(NormController::Event event, } // end NormApp::Notify() -bool NormApp::OnIntervalTimeout(ProtoTimer& /*theTimer*/) +bool NormApp::OnIntervalTimeout(ProtoTimer& theTimer) { char fileName[PATH_MAX]; - if (tx_file_list.GetNextFile(fileName)) + if (('\0' != tx_file_name[0]) || tx_file_list.GetNextFile(fileName)) { tx_repeat_clear = true; + + // The "tx_file_name" is used to cache the path of the current file for enqueue + // re-attempt upon possible flow control NormSession::QueueTxFile() failure + if ('\0' == tx_file_name[0]) + strcpy(tx_file_name, fileName); + else + strcpy(fileName, tx_file_name); // used cached tx_file_name for enqueue re-attempt + // 1) Build up the full path name of the file char pathName[PATH_MAX]; tx_file_list.GetCurrentBasePath(pathName); @@ -1978,9 +2054,6 @@ bool NormApp::OnIntervalTimeout(ProtoTimer& /*theTimer*/) if (PROTO_PATH_DELIMITER == fileNameInfo[i]) fileNameInfo[i] = '/'; } - char temp[PATH_MAX]; - strncpy(temp, fileNameInfo, len); - temp[len] = '\0'; FileCacheItem* fileCacheItem = NULL; NormFileObject* obj = NULL; @@ -1994,22 +2067,44 @@ bool NormApp::OnIntervalTimeout(ProtoTimer& /*theTimer*/) bool requeueSuccess = false; if (NULL != obj) requeueSuccess = session->RequeueTxObject(obj); if (!requeueSuccess) + { PLOG(PL_ERROR, "norm warning: requeue attempt exceeded configured tx cache bounds!\n"); + // TBD - remove file from tx_file_cache??? + } + // RequeueTxObject() is not subject to flow control since the object is already buffered + tx_file_name[0] = '\0'; // reset so next file will be fetched } } if (NULL == obj) { - // It's a new object (or requeue failure) - if (NULL == (obj = session->QueueTxFile(fileName, fileNameInfo, (UINT16)len))) + // This makes sure it's not an empty file + // (TBD - provide for empty files to be sent) + if (0 == NormFile::GetSize(fileName)) { - PLOG(PL_ERROR, "NormApp::OnIntervalTimeout() Error queuing tx file: %s\n", - fileName); - // Wait a second, then try the next file in the list + PLOG(PL_WARN, "norm warning: empty file \"%s\" will not be transmitted\n"); + tx_file_name[0] = '\0'; // reset so next file will be fetched + return OnIntervalTimeout(theTimer); + } + // It's a new object + const char* infoPtr = tx_file_info ? fileNameInfo : NULL; + UINT16 infoLen = tx_file_info ? (UINT16)len : 0; + if (NULL == (obj = session->QueueTxFile(fileName, infoPtr, infoLen))) + { + PLOG(PL_DEBUG, "NormApp::OnIntervalTimeout() error queuing tx file: %s\n", fileName); + // The code currently assumes that QueueTxFile() failed because of + // flow control and so we simply return and wait for a TX_QUEUE_EMPTY notification + // to trigger a re-attempt to enqueue the file name in "tx_file_name" if (interval_timer.IsActive()) interval_timer.Deactivate(); - interval_timer.SetInterval(1.0); - ActivateTimer(interval_timer); + // Old behavior was to wait a second and go to next file + // (commented out here) + //interval_timer.SetInterval(1.0); + //ActivateTimer(interval_timer); return false; } + else + { + tx_file_name[0] = '\0'; // reset so next file will be fetched + } if (0 != tx_requeue) { if (NULL == fileCacheItem) @@ -2094,7 +2189,7 @@ bool NormApp::OnIntervalTimeout(ProtoTimer& /*theTimer*/) else { tx_repeat_clear = true; - PLOG(PL_FATAL, "norm: End of tx file list reached.\n"); + PLOG(PL_ALWAYS, "norm: End of tx file list reached.\n"); if (tx_ack_shot && !watermark_pending && tx_file_queued) { session->SenderSetWatermark(tx_last_object_id, @@ -2169,18 +2264,26 @@ bool NormApp::OnStartup(int argc, const char*const* argv) session->SetRxLoss(rx_loss); session->SetTTL(ttl); session->SetLoopback(loopback); + if (loopback && session->Address().IsMulticast()) + session->SetRxPortReuse(true); if (NULL != interface_name) session->SetMulticastInterface(interface_name); session->SetEcnSupport(ECN_OFF != ecn_mode, ECN_ONLY == ecn_mode, tolerate_loss); + // We turn off flow control when no NACKing is happening + // (TBD - make flow control a command-line option) + if (silent_receiver) + session->SetFlowControl(0.0); + if (msg_test || input || !tx_file_list.IsEmpty()) { NormObjectId baseId = (unsigned short)(rand() * (65535.0/ (double)RAND_MAX)); session->SenderSetBaseObjectId(baseId); session->SetCongestionControl(cc_enable); session->SetBackoffFactor(backoff_factor); - //session->SenderSetGrtt(grtt_estimate); + session->SenderSetGrtt(grtt_estimate); + session->SetGrttProbingMode(grtt_probing_mode); session->SenderSetGroupSize(group_size); session->SetTxRobustFactor(tx_robust_factor); session->SetTxCacheBounds(tx_cache_size, tx_cache_min, tx_cache_max); @@ -2226,11 +2329,12 @@ bool NormApp::OnStartup(int argc, const char*const* argv) if (output || rx_cache_path) { // StartReceiver(bufferMax (per-sender)) + session->SetRxCacheMax(tx_cache_max); session->ReceiverSetUnicastNacks(unicast_nacks); session->ReceiverSetSilent(silent_receiver); if (preallocate_sender) { - if (!session->PreallocateRemoteSender(segment_size, ndata, nparity)) + if (!session->PreallocateRemoteSender(rx_buffer_size, segment_size, ndata, nparity)) { PLOG(PL_FATAL, "NormApp::OnStartup() remote sender preallocation error!\n"); session_mgr.Destroy(); @@ -2244,6 +2348,7 @@ bool NormApp::OnStartup(int argc, const char*const* argv) if (realtime) session->RcvrSetRealtime(true); session->SetRxRobustFactor(rx_robust_factor); + session->ReceiverSetDefaultRepairBoundary(repair_boundary); if (!session->StartReceiver(rx_buffer_size)) { PLOG(PL_FATAL, "NormApp::OnStartup() start receiver error!\n"); diff --git a/src/common/normFile.cpp b/src/common/normFile.cpp index af943c9..2ba3acf 100644 --- a/src/common/normFile.cpp +++ b/src/common/normFile.cpp @@ -188,8 +188,7 @@ void NormFile::Unlock() flock(fd, LOCK_UN); #else #ifdef HAVE_LOCKF - int ret = lockf(fd, F_ULOCK, 0); - ret = 0; // done to avoid compiler warning (should optimize out) + lockf(fd, F_ULOCK, 0); #endif // HAVE_LOCKF #endif // if/elseHAVE_FLOCK fchmod(fd, 0640); diff --git a/src/common/normMessage.cpp b/src/common/normMessage.cpp index ee775c1..1a474b2 100644 --- a/src/common/normMessage.cpp +++ b/src/common/normMessage.cpp @@ -7,7 +7,6 @@ NormHeaderExtension::NormHeaderExtension() } - NormMsg::NormMsg() : length(8), header_length(8), header_length_base(8) { @@ -92,6 +91,13 @@ bool NormMsg::InitFromBuffer(UINT16 msgLength) } } // end NormMsg::InitFromBuffer() +void NormMsg::Display() const +{ + const unsigned char* ptr = (const unsigned char*)buffer; + for (unsigned int i = 0; i < length; i++) + PLOG(PL_ALWAYS, "%02x", *ptr++); +} // end NormMsg::Display() + bool NormCmdCCMsg::GetCCNode(NormNodeId nodeId, UINT8& flags, UINT8& rtt, @@ -150,10 +156,10 @@ bool NormRepairRequest::AppendRepairItem(UINT8 fecId, { if (RANGES == form) PLOG(PL_TRACE, "NormRepairRequest::AppendRepairItem-Range(fecId>%d obj>%hu blk>%lu seg>%hu) ...\n", - fecId, (UINT16)objectId, (UINT32)blockId, (UINT16)symbolId); + fecId, (UINT16)objectId, (unsigned long)blockId.GetValue(), (UINT16)symbolId); else PLOG(PL_TRACE, "NormRepairRequest::AppendRepairItem(fecId>%d obj>%hu blk>%lu seg>%hu) ...\n", - fecId, (UINT16)objectId, (UINT32)blockId, (UINT16)symbolId); + fecId, (UINT16)objectId, (unsigned long)blockId.GetValue(), (UINT16)symbolId); ASSERT(NormPayloadId::IsValid(fecId)); UINT16 itemLength = RepairItemLength(fecId); @@ -164,7 +170,7 @@ bool NormRepairRequest::AppendRepairItem(UINT8 fecId, ((UINT8*)ptr)[RESERVED_OFFSET] = 0; // 1 byte ((UINT16*)ptr)[OBJ_ID_OFFSET] = htons((UINT16)objectId); // 2 bytes NormPayloadId payloadId(fecId, fecM, ptr + 1); - payloadId.SetFecPayloadId(blockId, symbolId, blockLen); + payloadId.SetFecPayloadId(blockId.GetValue(), symbolId, blockLen); length += itemLength; return true; } @@ -186,8 +192,8 @@ bool NormRepairRequest::AppendRepairRange(UINT8 fecId, UINT16 endSymbolId) { PLOG(PL_TRACE, "NormRepairRequest::AppendRepairRange(%hu:%lu:%hu->%hu:%lu:%hu) ...\n", - (UINT16)startObjectId, (UINT32)startBlockId, (UINT16)startSymbolId, - (UINT16)endObjectId, (UINT32)endBlockId, (UINT16)endSymbolId); + (UINT16)startObjectId, (unsigned long)startBlockId.GetValue(), (UINT16)startSymbolId, + (UINT16)endObjectId, (unsigned long)endBlockId.GetValue(), (UINT16)endSymbolId); // Note a "range" is two repair items UINT16 rangeLength = RepairRangeLength(fecId); if (buffer_len >= (ITEM_LIST_OFFSET+length+rangeLength)) @@ -198,14 +204,14 @@ bool NormRepairRequest::AppendRepairRange(UINT8 fecId, ((UINT8*)ptr)[RESERVED_OFFSET] = 0; ((UINT16*)ptr)[OBJ_ID_OFFSET] = htons((UINT16)startObjectId); NormPayloadId startId(fecId, fecM, ptr + 1); - startId.SetFecPayloadId(startBlockId, startSymbolId, startBlockLen); + startId.SetFecPayloadId(startBlockId.GetValue(), startSymbolId, startBlockLen); // range end ptr += (rangeLength/8); // advance ptr to second part of repair range ((UINT8*)ptr)[FEC_ID_OFFSET] = fecId; ((UINT8*)ptr)[RESERVED_OFFSET] = 0; ((UINT16*)ptr)[OBJ_ID_OFFSET] = htons((UINT16)endObjectId); NormPayloadId endId(fecId, fecM, ptr + 1); - endId.SetFecPayloadId(endBlockId, endSymbolId, endBlockLen); + endId.SetFecPayloadId(endBlockId.GetValue(), endSymbolId, endBlockLen); length += rangeLength; return true; } @@ -230,7 +236,7 @@ bool NormRepairRequest::AppendErasureCount(UINT8 fecId, ((UINT8*)ptr)[RESERVED_OFFSET] = 0; ((UINT16*)ptr)[OBJ_ID_OFFSET] = htons((UINT16)objectId); NormPayloadId erasureId(fecId, fecM, ptr + 1); - erasureId.SetFecPayloadId(blockId, erasureCount, blockLen); + erasureId.SetFecPayloadId(blockId.GetValue(), erasureCount, blockLen); length += itemLength; return true; } @@ -319,10 +325,10 @@ NormRepairRequest::Iterator::Iterator(const NormRepairRequest& theRequest, UINT8 } // For erasure requests, symbolId is loaded with erasureCount -bool NormRepairRequest::Iterator::NextRepairItem(NormObjectId* objectId, - NormBlockId* blockId, - UINT16* blockLen, - UINT16* symbolId) +UINT16 NormRepairRequest::Iterator::NextRepairItem(NormObjectId* objectId, + NormBlockId* blockId, + UINT16* blockLen, + UINT16* symbolId) { UINT8 fecId; // for check UINT16 itemLength = request.RetrieveRepairItem(fec_m, offset, &fecId, objectId, blockId, blockLen, symbolId); @@ -334,12 +340,8 @@ bool NormRepairRequest::Iterator::NextRepairItem(NormObjectId* objectId, return false; } offset += itemLength; - return true; - } - else - { - return false; } + return itemLength; } // end NormRepairRequest::Iterator::NextRepairItem() @@ -366,18 +368,18 @@ void NormRepairRequest::Log(UINT8 fecId, UINT8 fecM) const while (iterator.NextRepairItem(&objId, &blkId, &blkLen, &segId)) { if (FlagIsSet(SEGMENT)) - PLOG(PL_ALWAYS, "RepairItem> %hu:%lu:%hu", (UINT16)objId, (UINT32)blkId, (UINT16)segId); + PLOG(PL_ALWAYS, "RepairItem> %hu:%lu:%hu", (UINT16)objId, (unsigned long)blkId.GetValue(), (UINT16)segId); else if (FlagIsSet(BLOCK)) - PLOG(PL_ALWAYS, "RepairItem> %hu:%lu", (UINT16)objId, (UINT32)blkId, (UINT16)segId); + PLOG(PL_ALWAYS, "RepairItem> %hu:%lu", (UINT16)objId, (unsigned long)blkId.GetValue(), (UINT16)segId); else PLOG(PL_ALWAYS, "RepairItem> %hu", (UINT16)objId); if (form == RANGES) { iterator.NextRepairItem(&objId, &blkId, &blkLen, &segId); if (FlagIsSet(SEGMENT)) - PLOG(PL_ALWAYS, " -> %hu:%lu:%hu", (UINT16)objId, (UINT32)blkId, (UINT16)segId); + PLOG(PL_ALWAYS, " -> %hu:%lu:%hu", (UINT16)objId, (unsigned long)blkId.GetValue(), (UINT16)segId); else if (FlagIsSet(BLOCK)) - PLOG(PL_ALWAYS, " -> %hu:%lu", (UINT16)objId, (UINT32)blkId, (UINT16)segId); + PLOG(PL_ALWAYS, " -> %hu:%lu", (UINT16)objId, (unsigned long)blkId.GetValue(), (UINT16)segId); else PLOG(PL_ALWAYS, " -> %hu", (UINT16)objId); } diff --git a/src/common/normNode.cpp b/src/common/normNode.cpp index daad585..505471f 100644 --- a/src/common/normNode.cpp +++ b/src/common/normNode.cpp @@ -68,9 +68,9 @@ const double NormSenderNode::ACTIVITY_INTERVAL_MIN = 1.0; // 1 second min activ NormSenderNode::NormSenderNode(class NormSession& theSession, NormNodeId nodeId) : NormNode(SENDER, theSession, nodeId), instance_id(0), robust_factor(session.GetRxRobustFactor()), synchronized(false), sync_id(0), - is_open(false), segment_size(0), fec_m(0), ndata(0), nparity(0), + is_open(false), segment_size(0), fec_m(0), ndata(0), nparity(0), preset_stream(NULL), repair_boundary(BLOCK_BOUNDARY), decoder(NULL), erasure_loc(NULL), - retrieval_loc(NULL), retrieval_pool(NULL), ack_pending(false), + retrieval_loc(NULL), retrieval_pool(NULL), ack_pending(false), notify_on_grtt_update(true), cc_sequence(0), cc_enable(false), cc_feedback_needed(false), cc_rate(0.0), rtt_confirmed(false), is_clr(false), is_plr(false), slow_start(true), send_rate(0.0), recv_rate(0.0), recv_rate_prev(0.0), @@ -84,7 +84,7 @@ NormSenderNode::NormSenderNode(class NormSession& theSession, NormNodeId nodeId) unicast_nacks = session.ReceiverGetUnicastNacks(); max_pending_range = session.GetRxCacheMax(); - + repair_timer.SetListener(this, &NormSenderNode::OnRepairTimeout); repair_timer.SetInterval(0.0); repair_timer.SetRepeat(1); @@ -152,7 +152,7 @@ bool NormSenderNode::Open(UINT16 instanceId) } is_open = true; synchronized = false; - resync_count = 0; // reset resync_count + //resync_count = 0; // reset resync_count return true; } // end NormSenderNode::Open() @@ -178,24 +178,21 @@ void NormSenderNode::Close() cmd_buffer_pool = buf->GetNext(); delete buf; } - rx_repair_mask.Destroy(); rx_pending_mask.Destroy(); rx_table.Destroy(); - if (NULL != decoder) { - delete decoder; - } synchronized = false; is_open = false; } // end NormSenderNode::Close() -bool NormSenderNode::AllocateBuffers(UINT8 fecId, - UINT16 fecInstanceId, - UINT8 fecM, - UINT16 segmentSize, - UINT16 numData, - UINT16 numParity) +bool NormSenderNode::AllocateBuffers(unsigned int bufferSpace, + UINT8 fecId, + UINT16 fecInstanceId, + UINT8 fecM, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity) { ASSERT(IsOpen()); // Calculate how much memory each buffered block will require @@ -203,8 +200,6 @@ bool NormSenderNode::AllocateBuffers(UINT8 fecId, unsigned long maskSize = blockSize >> 3; if (0 != (blockSize & 0x07)) maskSize++; unsigned long blockStateSpace = sizeof(NormBlock) + blockSize * sizeof(char*) + 2*maskSize; - unsigned long bufferSpace = session.RemoteSenderBufferSize(); - // The "bufferFactor" weight determines the ratio of segment buffers (blockSegmentSpace) to // allocated NormBlock (blockStateSpace). // If "bufferFactor = 1.0", this is equivalent to the old scheme, where every allocated @@ -426,7 +421,7 @@ void NormSenderNode::FreeBuffers() segment_size = ndata = nparity = 0; } // end NormSenderNode::FreeBuffers() -unsigned long NormSenderNode::CurrentStreamBufferUsage() const +unsigned long NormSenderNode::CurrentStreamBufferUsage() { unsigned long usage = 0; NormObjectTable::Iterator it(rx_table); @@ -439,7 +434,7 @@ unsigned long NormSenderNode::CurrentStreamBufferUsage() const return usage; } // end NormSenderNode::CurrentStreamBufferUsage() -unsigned long NormSenderNode::PeakStreamBufferUsage() const +unsigned long NormSenderNode::PeakStreamBufferUsage() { unsigned long usage = 0; NormObjectTable::Iterator it(rx_table); @@ -452,7 +447,7 @@ unsigned long NormSenderNode::PeakStreamBufferUsage() const return usage; } // end NormSenderNode::PeakStreamBufferUsage() -unsigned long NormSenderNode::StreamBufferOverunCount() const +unsigned long NormSenderNode::StreamBufferOverunCount() { unsigned long count = 0; NormObjectTable::Iterator it(rx_table); @@ -471,13 +466,17 @@ bool NormSenderNode::ReadNextCmd(char* buffer, unsigned int* buflen) if (NULL == buflen) return false; // (TBD) indicate error type if (NULL != cmd_buffer_head) { - if ((NULL == buffer) || - (*buflen < cmd_buffer_head->GetContentLength())) + if (NULL == buffer) { // User is just querying for content length size. *buflen = cmd_buffer_head->GetContentLength(); - return false; + return true; } + else if (*buflen < cmd_buffer_head->GetContentLength()) + { + *buflen = cmd_buffer_head->GetContentLength(); + return false; + } else { // a) remove cmd from cmd_buffer queue @@ -519,7 +518,7 @@ void NormSenderNode::UpdateGrttEstimate(UINT8 grttQuantized) grtt_quantized = grttQuantized; grtt_estimate = NormUnquantizeRtt(grttQuantized); PLOG(PL_DEBUG, "NormSenderNode::UpdateGrttEstimate() node>%lu sender>%lu new grtt: %lf sec\n", - LocalNodeId(), GetId(), grtt_estimate); + (unsigned long)LocalNodeId(), (unsigned long)GetId(), grtt_estimate); // activity timer depends upon sender's grtt estimate // (TBD) do a proper rescaling here instead? double activityInterval = 2*session.GetTxRobustFactor()*grtt_estimate; @@ -527,7 +526,11 @@ void NormSenderNode::UpdateGrttEstimate(UINT8 grttQuantized) activity_timer.SetInterval(activityInterval); if (activity_timer.IsActive()) activity_timer.Reschedule(); // (TBD) Scale/reschedule repair_timer and/or cc_timer??? - session.Notify(NormController::GRTT_UPDATED, this, (NormObject*)NULL); + if (notify_on_grtt_update) + { + notify_on_grtt_update = false; + session.Notify(NormController::GRTT_UPDATED, this, (NormObject*)NULL); + } } // end NormSenderNode::UpdateGrttEstimate() @@ -542,7 +545,7 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, gsize_quantized = gsizeQuantized; gsize_estimate = NormUnquantizeGroupSize(gsizeQuantized); PLOG(PL_DEBUG, "NormSenderNode::HandleCommand() node>%lu sender>%lu new group size:%lf\n", - LocalNodeId(), GetId(), gsize_estimate); + (unsigned long)LocalNodeId(), (unsigned long)GetId(), gsize_estimate); } backoff_factor = (double)cmd.GetBackoffFactor(); @@ -574,6 +577,8 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, if ((NULL != obj) && (NormObject::STREAM == obj->GetType())) { NormBlockId blockId = squelch.GetFecBlockId(fec_m); + //NormStreamObject* stream = (NormStreamObject*)obj; + //TRACE("Prune(%u) 8 (read_index = %u.%hu)...\n", (UINT32)blockId, (UINT32)stream->GetNextBlockId(), stream->GetNextSegmentId()); static_cast(obj)->Prune(blockId, true); } // 3) Discard any invalidated objects (those listed in the squelch) @@ -676,15 +681,15 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, r = MAX(r, 0.5); r = (r - 0.5) / 0.4; } - //DMSG(0, "NormSenderNode::HandleCommand(CC) node>%lu bias:%lf recv_rate:%lf send_rate:%lf " - // "grtt:%lf gsize:%lf\n", - // LocalNodeId(), r, 8.0e-03*recv_rate, 8.0e-03*send_rate, + //DMSG(0, "NormSenderNode::HandleCommand(CC) node>%lu bias:%lf " + // "recv_rate:%lf send_rate:%lf grtt:%lf gsize:%lf\n", + // (unsigned long)LocalNodeId(), r, 8.0e-03*recv_rate, 8.0e-03*send_rate, // backoffTime = 0.25 * r * maxBackoff + 0.75 * backoffTime; cc_timer.SetInterval(backoffTime); PLOG(PL_TRACE, "NormSenderNode::HandleCommand() node>%lu begin CC back-off: %lf sec)...\n", - LocalNodeId(), backoffTime); + (unsigned long)LocalNodeId(), backoffTime); session.ActivateTimer(cc_timer); break; } // end if (CC_RATE == ext.GetType()) @@ -756,7 +761,8 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, watermark_segment_id = symbolId; if (!ack_timer.IsActive()) { - double ackBackoff = session.Address().IsMulticast() ? UniformRand(grtt_estimate) : 0.0; + double ackBackoff = (session.Address().IsMulticast() && (backoff_factor > 0.0)) ? + UniformRand(grtt_estimate) : 0.0; ack_timer.SetInterval(ackBackoff); ack_pending = true; session.ActivateTimer(ack_timer); @@ -793,13 +799,11 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, repairAdv.GetRepairContentLength()); } break; - } - - + } case NormCmdMsg::APPLICATION: { PLOG(PL_TRACE, "NormSenderNode::HandleCommand(APPLICATION) node>%lu recvd app-defined cmd...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); const NormCmdAppMsg& appCmd = static_cast(cmd); // 1) Buffer the received command either using a buffer structure // cmd_buffer pool or allocating a new one as needed. @@ -811,7 +815,7 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, if (NULL == buf) { PLOG(PL_ERROR, "NormSenderNode::HandleCommand(APPLICATION) node>%lu NewCmdCBuffer() error: %s\n", - GetErrorString()); + (unsigned long)LocalNodeId(), GetErrorString()); } else { @@ -836,7 +840,8 @@ void NormSenderNode::HandleCommand(const struct timeval& currentTime, else { PLOG(PL_ERROR, "NormSenderNode::HandleCommand(APPLICATION) node>%lu error: " - "cmd content greater than sender's segment_size?!\n"); + "cmd content greater than sender's segment_size?!\n", + (unsigned long)LocalNodeId()); buf->Append(cmd_buffer_pool); cmd_buffer_pool = buf; } @@ -997,7 +1002,7 @@ void NormSenderNode::HandleRepairContent(const UINT32* buffer, UINT16 bufferLen) &lastBlockLen, &lastSegmentId)) { PLOG(PL_ERROR, "NormSenderNode::HandleRepairContent() node>%lu recvd incomplete RANGE request!\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); continue; // (TBD) break/return instead??? } // (TBD) test for valid range form/level @@ -1114,7 +1119,9 @@ void NormSenderNode::DeleteObject(NormObject* obj) NormBlock* NormSenderNode::GetFreeBlock(NormObjectId objectId, NormBlockId blockId) { NormBlock* b = block_pool.Get(); - if (!b) + //TRACE("enter NormSenderNode::GetFreeBlock() empty:%d\n", (NULL == b)); + + if (NULL == b) { if (session.ReceiverIsSilent() || session.RcvrIsRealtime()) { @@ -1200,6 +1207,129 @@ char* NormSenderNode::GetFreeSegment(NormObjectId objectId, NormBlockId blockId) return result; } // end NormSenderNode::GetFreeSegment() +bool NormSenderNode::PreallocateRxStream(unsigned int bufferSize, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity) +{ + if (NULL!= preset_stream) delete preset_stream; + if (NULL == (preset_stream = new NormStreamObject(session, this, 0))) + { + PLOG(PL_ERROR, "NormSenderNode::PreallocateRxStream() new NormStreamObject error: %s\n", + GetErrorString()); + return false; + } + UINT8 fecId; + UINT8 fecM = 8; + if ((numData + numParity) > 255) + { + fecId = 2; + fecM = 16; + } + else + { + fecId = 5; + } + UINT32 blockSize = segmentSize * numData; + UINT32 numBlocks = bufferSize / blockSize; + // Buffering requires at least 2 blocks + numBlocks = MAX(2, numBlocks); + // Recompute "bufferSize" to match any adjustments + bufferSize = numBlocks * blockSize; + if (!preset_stream->RxOpen(NormObjectSize(bufferSize), + true, + segmentSize, + fecId, + fecM, + numData, + numParity)) + { + PLOG(PL_ERROR, "NormSenderNode::PreallocateRxStream() error: RxOpen() failure\n"); + delete preset_stream; + preset_stream = NULL; + return false; + } + if (!preset_stream->Accept(bufferSize, true)) + { + PLOG(PL_ERROR, "NormSenderNode::PreallocateRxStream() error: Accept() failure\n"); + delete preset_stream; + preset_stream = NULL; + return false; + } + return true; +} // end NormSenderNode::PreallocateRxStream() + + +bool NormSenderNode::GetFtiData(const NormObjectMsg& msg, NormFtiData& ftiData) +{ + UINT8 fecId = msg.GetFecId(); + switch (fecId) + { + case 2: + { + NormFtiExtension2 fti; + while (msg.GetNextExtension(fti)) + { + if (NormHeaderExtension::FTI == fti.GetType()) + { + ASSERT(1 == fti.GetFecGroupSize()); // TBD - allow for different groupings + ftiData.SetFecInstanceId(0); + ftiData.SetFecFieldSize(fti.GetFecFieldSize()); + ftiData.SetSegmentSize(fti.GetSegmentSize()); + ftiData.SetFecMaxBlockLen(fti.GetFecMaxBlockLen()); + ftiData.SetFecNumParity(fti.GetFecNumParity()); + ftiData.SetObjectSize(fti.GetObjectSize()); + return true; + } + } + break; + } + case 5: + { + NormFtiExtension5 fti; + while (msg.GetNextExtension(fti)) + { + if (NormHeaderExtension::FTI == fti.GetType()) + { + ftiData.SetFecInstanceId(0); + ftiData.SetFecFieldSize(8); + ftiData.SetSegmentSize(fti.GetSegmentSize()); + ftiData.SetFecMaxBlockLen(fti.GetFecMaxBlockLen()); + ftiData.SetFecNumParity(fti.GetFecNumParity()); + ftiData.SetObjectSize(fti.GetObjectSize()); + return true; + } + } + break; + } + case 129: + { + NormFtiExtension129 fti; + while (msg.GetNextExtension(fti)) + { + if (NormHeaderExtension::FTI == fti.GetType()) + { + ftiData.SetFecInstanceId(fti.GetFecInstanceId()); + ftiData.SetFecFieldSize(8); + ftiData.SetSegmentSize(fti.GetSegmentSize()); + ftiData.SetFecMaxBlockLen(fti.GetFecMaxBlockLen()); + ftiData.SetFecNumParity(fti.GetFecNumParity()); + ftiData.SetObjectSize(fti.GetObjectSize()); + return true; + } + } + break; + } + default: + PLOG(PL_ERROR, "NormSenderNode::GetFtiData() node>%lu sender>%lu unknown fec_id type:%d\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId(), (int)fecId); + break; + } // end switch (fecId) + PLOG(PL_ERROR, "NormSenderNode::GetFtiData() node>%lu sender>%lu unknown fec_id type:%d\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId(), (int)fecId); + return false; +} // end NormSenderNode::GetFtiData() + void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) { UINT8 grttQuantized = msg.GetGrtt(); @@ -1210,7 +1340,7 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) gsize_quantized = gsizeQuantized; gsize_estimate = NormUnquantizeGroupSize(gsizeQuantized); PLOG(PL_DEBUG, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu new group size: %lf\n", - LocalNodeId(), GetId(), gsize_estimate); + (unsigned long)LocalNodeId(), (unsigned long)GetId(), gsize_estimate); } backoff_factor = (double)msg.GetBackoffFactor(); @@ -1226,16 +1356,47 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) // 2) When the FEC parameters have changed (TBD) // bool allocateBuffers = true; + bool gotFTI = false; + NormFtiData ftiData; + + if (BuffersAllocated()) { - if (fec_id == fecId) - allocateBuffers = false; // TBD - validate fec params? + // Validate that allocated buffers match object FEC params + if (fecId == fec_id) + { + if (GetFtiData(msg, ftiData)) + { + gotFTI = true; + if ((ftiData.GetSegmentSize() != segment_size) || + (ftiData.GetFecFieldSize() != fec_m) || + (ftiData.GetFecMaxBlockLen() != ndata) || + (ftiData.GetFecNumParity() != nparity)) + { + FreeBuffers(); // force reallocation because fec params changed + } + else + { + allocateBuffers = false; // FEC params match + } + } + else + { + // TBD - handle case where only NORM_INFO carries FTI Info to reduce overhead + PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu - no FTI provided!\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); + return; // (TBD) notify app of error ?? + } + } else + { FreeBuffers(); // force reallocation because fec id changed + } } NormBlockId blockId; NormSegmentId segmentId; + if (NormMsg::INFO == msgType) { fec_id = fecId; @@ -1258,86 +1419,30 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) if (allocateBuffers) { PLOG(PL_DEBUG, "NormSenderNode::HandleObjectMessage() node>%lu allocating sender>%lu buffers ...\n", - LocalNodeId(), GetId()); + (unsigned long)LocalNodeId(), (unsigned long)GetId()); // Currently,, our implementation requires the FEC Object Transmission Information // to properly allocate resources for FEC buffering and decoding - UINT16 fecInstanceId = 0; - UINT8 fecM = 8; - UINT16 segmentSize = 0; - UINT16 fecMaxBlockLen = 0; - UINT16 fecNumParity = 0; - bool noFTI = true; - switch (fecId) + // So, get the FEC Transport Information (FTI) from header extension + // TBD - allow for application preset FTI + if (!gotFTI) { - case 2: + if (GetFtiData(msg, ftiData)) { - NormFtiExtension2 fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - ASSERT(1 == fti.GetFecGroupSize()); // TBD - allow for different groupings - fecM = fti.GetFecFieldSize(); - segmentSize = fti.GetSegmentSize(); - fecMaxBlockLen = fti.GetFecMaxBlockLen(); - fecNumParity = fti.GetFecNumParity(); - noFTI = false; - break; - } - } - break; + gotFTI = true; } - case 5: + else { - NormFtiExtension5 fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - segmentSize = fti.GetSegmentSize(); - fecMaxBlockLen = fti.GetFecMaxBlockLen(); - fecNumParity = fti.GetFecNumParity(); - noFTI = false; - break; - } - } - break; + PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu - no FTI provided!\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); + // (TBD) notify app of error ?? + return; } - case 129: - { - NormFtiExtension129 fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - fecInstanceId = fti.GetFecInstanceId(); - segmentSize = fti.GetSegmentSize(); - fecMaxBlockLen = fti.GetFecMaxBlockLen(); - fecNumParity = fti.GetFecNumParity(); - noFTI = false; - break; - } - } - break; - } - default: - PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu unknown fec_id type:%d\n", - LocalNodeId(), GetId(), fecId); - return; - - } // end switch (fecId) - - if (noFTI) - { - PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu - no FTI provided!\n", - LocalNodeId(), GetId()); - // (TBD) notify app of error ?? - return; } - if (!AllocateBuffers(fecId, fecInstanceId, fecM, segmentSize, fecMaxBlockLen, fecNumParity)) + if (!AllocateBuffers(session.RemoteSenderBufferSize(), fecId, ftiData.GetFecInstanceId(), ftiData.GetFecFieldSize(), + ftiData.GetSegmentSize(), ftiData.GetFecMaxBlockLen(), ftiData.GetFecNumParity())) { PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu buffer allocation error\n", - LocalNodeId(), GetId()); + (unsigned long)LocalNodeId(), (unsigned long)GetId()); // (TBD) notify app of error ?? return; } @@ -1378,7 +1483,7 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) return; } } - + bool presetStream = false; NormObject* obj = NULL; switch (status) { @@ -1389,10 +1494,53 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) { if (msg.FlagIsSet(NormObjectMsg::FLAG_STREAM)) { - if (!(obj = new NormStreamObject(session, this, objectId))) + if (NULL != preset_stream) { - PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() new NORM_OBJECT_STREAM error: %s\n", - GetErrorString()); + obj = static_cast(preset_stream); + // Validate FTI params + if (!gotFTI) + { + // need to get FTI data + if (GetFtiData(msg, ftiData)) + { + gotFTI = true; + } + else + { + PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu - no FTI provided!\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); + // (TBD) notify app of error ?? + return; + } + } + if ((obj->GetSize() != ftiData.GetObjectSize()) || + (obj->GetFecId() != fecId) || + (obj->GetSegmentSize() != ftiData.GetSegmentSize()) || + (obj->GetFecMaxBlockLen() != ftiData.GetFecMaxBlockLen()) || + (obj->GetFecNumParity() != ftiData.GetFecNumParity()) || + (obj->GetFecFieldSize() != ftiData.GetFecFieldSize())) + { + PLOG(PL_WARN, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu warning: " + "FTI does not match preset_stream!\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); + obj = NULL; + } + else + { + // Init preset_stream objectId and INFO status + obj->SetId(objectId); + if (!msg.FlagIsSet(NormObjectMsg::FLAG_INFO)) + obj->ClearInfo(); + presetStream = true; + } + } + if (NULL == obj) + { + if (NULL == (obj = new NormStreamObject(session, this, objectId))) + { + PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() new NORM_OBJECT_STREAM error: %s\n", + GetErrorString()); + } } } else if (msg.FlagIsSet(NormObjectMsg::FLAG_FILE)) @@ -1409,7 +1557,7 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) } else { - if (!(obj = new NormDataObject(session, this, objectId))) + if (!(obj = new NormDataObject(session, this, objectId, session.GetSessionMgr().GetDataFreeFunction()))) { PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() new NORM_OBJECT_DATA error: %s\n", GetErrorString()); @@ -1424,119 +1572,75 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) ASSERT(rx_table.CanInsert(objectId)); ASSERT(rx_pending_mask.Test(objectId)); rx_table.Insert(obj); - UINT8 fecId = msg.GetFecId(); - UINT8 fecM = 8; - UINT16 segmentSize = 0; - UINT16 fecMaxBlockLen = 0; - UINT16 fecNumParity = 0; - NormObjectSize objectSize; - bool noFTI = true; - switch (fecId) - { - case 2: - { - NormFtiExtension2 fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - ASSERT(1 == fti.GetFecGroupSize()); // TBD - allow for different groupings - fecM = fti.GetFecFieldSize(); - segmentSize = fti.GetSegmentSize(); - fecMaxBlockLen = fti.GetFecMaxBlockLen(); - fecNumParity = fti.GetFecNumParity(); - objectSize = fti.GetObjectSize(); - noFTI = false; - break; - } - } - break; - } - case 5: - { - NormFtiExtension5 fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - segmentSize = fti.GetSegmentSize(); - fecMaxBlockLen = fti.GetFecMaxBlockLen(); - fecNumParity = fti.GetFecNumParity(); - objectSize = fti.GetObjectSize(); - noFTI = false; - break; - } - } - break; - } - case 129: - { - NormFtiExtension129 fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - segmentSize = fti.GetSegmentSize(); - fecMaxBlockLen = fti.GetFecMaxBlockLen(); - fecNumParity = fti.GetFecNumParity(); - objectSize = fti.GetObjectSize(); - noFTI = false; - break; - } - } - break; - } - default: - segmentSize = fecMaxBlockLen = fecNumParity = 0; - PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu unknown fec_id type:%d\n", - LocalNodeId(), GetId(), fecId); - return; - - } // end switch (fecId) - - if (noFTI) + // Pull out FTI parameters from header extension if we didn't get it above + if (!gotFTI && !GetFtiData(msg, ftiData)) { PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu " - "new obj>%hu - no FTI provided!\n", LocalNodeId(), GetId(), (UINT16)objectId); - DeleteObject(obj); + "new obj>%hu - no FTI provided!\n", (unsigned long)LocalNodeId(), + (unsigned long)GetId(), (UINT16)objectId); + if (!presetStream) DeleteObject(obj); obj = NULL; } - else if (obj->RxOpen(objectSize, - msg.FlagIsSet(NormObjectMsg::FLAG_INFO), - segmentSize, - fecId, - fecM, - fecMaxBlockLen, - fecNumParity)) + else { - session.Notify(NormController::RX_OBJECT_NEW, this, obj); - if (obj->Accepted()) + if (presetStream || + obj->RxOpen(ftiData.GetObjectSize(), + msg.FlagIsSet(NormObjectMsg::FLAG_INFO), + ftiData.GetSegmentSize(), + fecId, + ftiData.GetFecFieldSize(), + ftiData.GetFecMaxBlockLen(), + ftiData.GetFecNumParity())) { - if (obj->IsStream()) + session.Notify(NormController::RX_OBJECT_NEW, this, obj); + if (obj->Accepted()) { - // This initial "StreamUpdateStatus()" syncs the stream according to our sync policy - NormStreamObject* stream = static_cast(obj); - if (SYNC_CURRENT != sync_policy) - stream->StreamResync(NormBlockId(0)); + if (obj->IsStream()) + { + if (presetStream) preset_stream = NULL; // we're using it up + // This initial "StreamUpdateStatus()" syncs the stream according to our sync policy + NormStreamObject* stream = static_cast(obj); + if (SYNC_CURRENT == sync_policy) + { + // Just "sync" to first received blockId + stream->StreamUpdateStatus(blockId); + } + else + { + // This forces the sender to do a maximum "rewind" + // If the resultant "syncId" is close to zero, assume + // we are "in-range" of sender initial (block zero) stream start + NormBlockId syncId = blockId; + stream->Decrement(syncId, stream->GetPendingMaskSize() - 1); + if ((stream->Compare(blockId, NormBlockId(0)) >= 0) && + (stream->Compare(syncId, NormBlockId(0)) <= 0)) + { + // Assume we are "in-range" of sender initial stream startup + syncId = NormBlockId(0); + } + stream->StreamUpdateStatus(syncId); + } + } + PLOG(PL_DETAIL, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu new obj>%hu\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId(), (UINT16)objectId); + } + else + { + PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() object not accepted\n"); + if (presetStream) + rx_table.Remove(obj); else - stream->StreamUpdateStatus(blockId); - } - PLOG(PL_DETAIL, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu new obj>%hu\n", - LocalNodeId(), GetId(), (UINT16)objectId); + DeleteObject(obj); + obj = NULL; + } } - else + else { - PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() object not accepted\n"); + PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() error opening object\n"); DeleteObject(obj); - obj = NULL; + obj = NULL; } } - else - { - PLOG(PL_ERROR, "NormSenderNode::HandleObjectMessage() error opening object\n"); - DeleteObject(obj); - obj = NULL; - } } break; } @@ -1551,7 +1655,6 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) if (NULL != obj) { obj->HandleObjectMessage(msg, msgType, blockId, segmentId); - bool objIsPending = obj->IsPending(); // Silent receivers may be configured to allow obj completion w/out INFO @@ -1608,7 +1711,7 @@ bool NormSenderNode::SyncTest(const NormObjectMsg& msg) const bool result = msg.FlagIsSet(NormObjectMsg::FLAG_STREAM); // Allow sync on INFO or block zero DATA message result = result || (NormMsg::INFO == msg.GetType()) ? - true : (0 == ((const NormDataMsg&)msg).GetFecBlockId(fec_m)); + true : (NormBlockId(0) == ((const NormDataMsg&)msg).GetFecBlockId(fec_m)); // Never sync on repair messages result = result && !msg.FlagIsSet(NormObjectMsg::FLAG_REPAIR); return result; @@ -1666,25 +1769,29 @@ void NormSenderNode::Sync(NormObjectId objectId) GetLastPending(lastPending); if ((objectId > lastPending) || ((next_id - objectId) > max_pending_range)) { + bool incrementResyncCount = objectId <= lastPending; // may just be a squelch trim NormObject* obj; while ((obj = rx_table.Find(rx_table.RangeLo()))) { + incrementResyncCount = true; AbortObject(obj); } rx_pending_mask.Clear(); - resync_count++; + if (incrementResyncCount) IncrementResyncCount(); } else if (objectId > firstPending) { + bool incrementResyncCount = false; // may just be a squelch trim NormObject* obj; while ((obj = rx_table.Find(rx_table.RangeLo())) && (obj->GetId() < objectId)) { AbortObject(obj); + incrementResyncCount = true; // more than just a trim } unsigned long numBits = (UINT16)(objectId - firstPending); rx_pending_mask.UnsetBits(firstPending, numBits); - resync_count++; + if (incrementResyncCount) IncrementResyncCount(); } } if ((next_id < objectId) || ((next_id - objectId) > max_pending_range)) @@ -1726,11 +1833,11 @@ NormSenderNode::ObjectStatus NormSenderNode::UpdateSyncStatus(const NormObjectId // (TBD) We may want to control resync policy options // or revert to fresh sync if sync is totally lost, // otherwise SQUELCH process will get things in order - PLOG(PL_INFO, "NormSenderNode::UpdateSyncStatus() node>%lu resync to sender>%lu obj>%hu...\n", - LocalNodeId(), GetId(), (UINT16)objectId); + PLOG(PL_DEBUG, "NormSenderNode::UpdateSyncStatus() node>%lu resync to sender>%lu obj>%hu...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId(), (UINT16)objectId); NormObjectId syncId = objectId; - /* + // This code avoids grosser resyncs (if uncommented) ... // However, attempts at finer-grained resync // (i.e. preserving some partially-received objects) @@ -1740,7 +1847,7 @@ NormSenderNode::ObjectStatus NormSenderNode::UpdateSyncStatus(const NormObjectId // and experiment! if (rx_pending_mask.IsSet()) { - NormObjectId lastPending(65535); + NormObjectId lastPending;//(65535); GetLastPending(lastPending); if (syncId > lastPending) { @@ -1751,11 +1858,8 @@ NormSenderNode::ObjectStatus NormSenderNode::UpdateSyncStatus(const NormObjectId } } } - */ Sync(syncId); - //resync_count++; (this is now incremented in NormNode::Sync()) - status = OBJ_NEW; - break; + return UpdateSyncStatus(objectId); } case OBJ_NEW: SetPending(objectId); @@ -1798,8 +1902,9 @@ NormSenderNode::ObjectStatus NormSenderNode::GetObjectStatus(const NormObjectId& // this is too aggressive a resync rule? if ((sync_id - objectId) > 2*max_pending_range) { - PLOG(PL_WARN, "NormSenderNode::GetObjectStatus() INVALID object>%hu sync_id>%hu\n", - (UINT16)objectId, (UINT16)sync_id); + // This can happen with NORM_SYNC_ALL sync policy + PLOG(PL_DEBUG, "NormSenderNode::GetObjectStatus() INVALID object>%hu sync_id>%hu\n", + (UINT16)objectId, (UINT16)sync_id); return OBJ_INVALID; } else @@ -1833,8 +1938,8 @@ NormSenderNode::ObjectStatus NormSenderNode::GetObjectStatus(const NormObjectId& { NormObjectId fp; GetFirstPending(fp); - PLOG(PL_WARN, "NormSenderNode::GetObjectStatus() INVALID object>%hu firstPending>%hu\n", - (UINT16)objectId, (UINT16)fp); + PLOG(PL_DEBUG, "NormSenderNode::GetObjectStatus() INVALID object>%hu firstPending>%hu\n", + (UINT16)objectId, (UINT16)fp); return OBJ_INVALID; } } @@ -1843,8 +1948,8 @@ NormSenderNode::ObjectStatus NormSenderNode::GetObjectStatus(const NormObjectId& NormObjectId delta = objectId - next_id + 1; if (delta > NormObjectId((UINT16)rx_pending_mask.GetSize())) { - PLOG(PL_WARN, "NormSenderNode::GetObjectStatus() INVALID object>%hu next_id>%hu\n", - (UINT16)objectId, (UINT16)next_id); + PLOG(PL_DEBUG, "NormSenderNode::GetObjectStatus() INVALID object>%hu next_id>%hu\n", + (UINT16)objectId, (UINT16)next_id); return OBJ_INVALID; } @@ -1943,7 +2048,7 @@ void NormSenderNode::RepairCheck(NormObject::CheckLevel checkLevel, ExponentialRand(grtt_estimate*backoff_factor, gsize_estimate) : 0.0; PLOG(PL_DEBUG, "NormSenderNode::RepairCheck() node>%lu begin NACK backoff: %lf sec)...\n", - LocalNodeId(), backoffInterval); + (unsigned long)LocalNodeId(), backoffInterval); // Here, we clear NormSenderNode repair_mask // that is used for NACK suppression. // (object/block repair_masks are cleared as needed in NormObject::ReceiverRepairCheck @@ -1979,7 +2084,7 @@ void NormSenderNode::RepairCheck(NormObject::CheckLevel checkLevel, { repair_timer.Deactivate(); PLOG(PL_DEBUG, "NormSenderNode::RepairCheck() node>%lu sender rewind detected, ending NACK holdoff ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); // Immediately do a repair check to see if rewind was sufficient // TBD - will we get too much unnecessary NACKing with out-of-order packet delivery ??? RepairCheck(checkLevel, objectId, blockId, segmentId); @@ -1995,14 +2100,14 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) switch(repair_timer.GetRepeatCount()) { case 0: // hold-off time complete - PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu end NACK hold-off ...\n", - LocalNodeId()); + PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sender>%lu end NACK hold-off ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); break; case 1: // back-off timeout complete { - PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu end NACK back-off ...\n", - LocalNodeId()); + PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sender>%lu end NACK back-off ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); // 1) Were we suppressed? NormObjectId nextId; if (GetFirstPending(nextId)) @@ -2028,15 +2133,19 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) if (repairPending) { // We weren't completely suppressed, so build NACK - NormNackMsg* nack = (NormNackMsg*)session.GetMessageFromPool(); - if (!nack) + NormNackMsg* nack = static_cast(session.GetMessageFromPool()); + if (NULL == nack) { PLOG(PL_WARN, "NormSenderNode::OnRepairTimeout() node>%lu Warning! " - "message pool empty ...\n", LocalNodeId()); + "message pool empty ...\n", (unsigned long)LocalNodeId()); repair_timer.Deactivate(); return false; } nack->Init(); + UINT16 payloadMax = 4*segment_size; + // If we sync'd to non-DATA, we don't yet know the sender segment_size + if (0 == payloadMax) + payloadMax = 4*NormNackMsg::DEFAULT_LENGTH_MAX; bool nackAppended = false; if (cc_enable) @@ -2068,6 +2177,7 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) else { double nominalSize = (nominal_packet_size > segment_size) ? nominal_packet_size : segment_size; + if (0 == nominalSize) nominalSize = 512; // TBD - what should this really be double ccRate = NormSession::CalculateRate(nominalSize, rtt_estimate, ccLoss); @@ -2086,8 +2196,8 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) ext.SetCCRate(NormQuantizeRate(ccRate)); } PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sending NACK rate:%lf kbps (rtt:%lf loss:%lf s:%hu) slow_start:%d\n", - LocalNodeId(), 8.0e-03*NormUnquantizeRate(ext.GetCCRate()), - rtt_estimate, ccLoss, (UINT16)nominal_packet_size, slow_start); + (unsigned long)LocalNodeId(), 8.0e-03*NormUnquantizeRate(ext.GetCCRate()), + rtt_estimate, ccLoss, (UINT16)nominal_packet_size, slow_start); ext.SetCCSequence(cc_sequence); // Cancel potential pending NORM_ACK(CC) since we are NACKing if (cc_timer.IsActive()) @@ -2111,13 +2221,13 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) { bool appendRequest = false; NormObject* obj = iterating ? rx_table.Find(nextId) : NULL; + UINT16 diff = nextId - prevId; if (obj) appendRequest = true; - else if (iterating && ((nextId - prevId) == consecutiveCount)) + else if (iterating && (diff == consecutiveCount)) consecutiveCount++; // consecutive series of missing objs starts/continues else appendRequest = true; // consecutive series broken or finished - if (appendRequest) { NormRepairRequest::Form nextForm; @@ -2133,7 +2243,7 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) default: nextForm = NormRepairRequest::RANGES; break; - } + } if (prevForm != nextForm) { if ((NormRepairRequest::INVALID != prevForm) && @@ -2148,10 +2258,7 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) } if (NormRepairRequest::INVALID != nextForm) { - if (0 != segment_size) - nack->AttachRepairRequest(req, segment_size); // (TBD) error check - else - nack->AttachRepairRequest(req, NormNackMsg::DEFAULT_LENGTH_MAX); + nack->AttachRepairRequest(req, payloadMax); // (TBD) error check req.SetForm(nextForm); req.ResetFlags(); // Set flags for missing objects according to @@ -2193,7 +2300,7 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) } prevForm = NormRepairRequest::INVALID; bool flush = (nextId != max_pending_object); - nackAppended |= obj->AppendRepairRequest(*nack, flush); + nackAppended |= obj->AppendRepairRequest(*nack, flush, payloadMax); } consecutiveCount = 0; } @@ -2233,8 +2340,16 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) ASSERT(nack->GetRepairContentLength() > 0); if (!session.ReceiverIsSilent()) { - session.SendMessage(*nack); - nack_count++; + UINT16 singleNackSize = segment_size ? segment_size : NormNackMsg::DEFAULT_LENGTH_MAX; + if (nack->GetRepairContentLength() <= singleNackSize) + { + session.SendMessage(*nack); + nack_count++; + } + else + { + FragmentNack(*nack); + } } session.ReturnMessageToPool(nack); } @@ -2244,8 +2359,8 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) // perhaps because of our "nacking mode" // even though there were pending objects // TBD - should we avoid NACK hold-off when this happens? - PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu zero content nack ...\n", - LocalNodeId()); + PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sender>%lu zero content nack ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); session.ReturnMessageToPool(nack); } } @@ -2254,8 +2369,8 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) if (!session.ReceiverIsSilent()) { suppress_count++; - PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu NACK SUPPRESSED ...\n", - LocalNodeId()); + PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sender>%lu NACK SUPPRESSED ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); } } // end if/else(repairPending) // BACKOFF related code @@ -2277,16 +2392,17 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) holdoffInterval += grtt_estimate; } } - //holdoffInterval = (backoff_factor > 0.0) ? holdoffInterval : 1.0*grtt_estimate; + // Uncommenting the line below treats ((0 == nparity) && 0.0 == backoff_factor) + // as a special case (assumes zero sender aggregateInterval) + holdoffInterval = ((0 != nparity) || (backoff_factor > 0.0)) ? holdoffInterval : grtt_estimate; repair_timer.SetInterval(holdoffInterval); - PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu begin NACK hold-off: %lf sec ...\n", - LocalNodeId(), holdoffInterval); - + PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sender>%lu begin NACK hold-off: %lf sec ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId(), holdoffInterval); } else { - PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu nothing pending ...\n", - LocalNodeId()); + PLOG(PL_DEBUG, "NormSenderNode::OnRepairTimeout() node>%lu sender>%lu nothing pending ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); // (TBD) cancel hold-off timeout ??? } // end if/else (repair_mask.IsSet()) } @@ -2299,6 +2415,104 @@ bool NormSenderNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) return true; } // end NormSenderNode::OnRepairTimeout() +void NormSenderNode::FragmentNack(NormNackMsg& superNack) +{ + // Parse a "super" NACK and refactor it into a series of smaller + // NACK messages as needed (per "segment_size" constraint) + // and send them. + NormNackMsg* nack = (NormNackMsg*)session.GetMessageFromPool(); + if (!nack) + { + PLOG(PL_WARN, "NormSenderNode::FragmentNack() node>%lu Warning! " + "message pool empty ...\n", (unsigned long)LocalNodeId()); + return; + } + nack->InitFrom(superNack); + // GRTT response is deferred until transmit time + if (unicast_nacks) + nack->SetDestination(GetAddress()); + else + nack->SetDestination(session.Address()); + + UINT16 payloadLength = 0; + NormRepairRequest superReq; + UINT16 requestOffset = 0; + UINT16 requestLength = 0; + while (0 != (requestLength = superNack.UnpackRepairRequest(superReq, requestOffset))) + { + const UINT16 REQ_HDR_LEN = 4; // TBD - get from normMessage.h instead + requestOffset += requestLength; + if ((payloadLength + requestLength) <= segment_size) + { + // Copy whole request over + nack->AppendRepairRequest(superReq); + payloadLength += requestLength; + } + else if ((payloadLength + REQ_HDR_LEN) < segment_size) + { + // Duplicate request and add individual repair items + NormRepairRequest::Form requestForm = superReq.GetForm(); + NormRepairRequest req; + nack->AttachRepairRequest(req, segment_size); + req.SetForm(requestForm); + req.SetFlags(superReq.GetFlags()); + payloadLength += REQ_HDR_LEN; + + NormRepairRequest::Iterator iterator(superReq, fec_id, fec_m); + NormObjectId objectId, lastObjectId; + NormBlockId blockId, lastBlockId; + UINT16 blockLen, lastBlockLen; + NormSegmentId segmentId, lastSegmentId; + UINT16 itemLength; + while (0 != (itemLength = iterator.NextRepairItem(&objectId, &blockId, &blockLen, &segmentId))) + { + if (NormRepairRequest::RANGES == requestForm) + { + itemLength += iterator.NextRepairItem(&lastObjectId, &lastBlockId, + &lastBlockLen, &lastSegmentId); + } + if ((payloadLength + itemLength) > segment_size) + { + // We have filled the NACK, so pack, send, and reset request + nack->PackRepairRequest(req); + session.SendMessage(*nack); + nack_count++; + nack->ResetPayload(); + nack->AttachRepairRequest(req, segment_size); + payloadLength = REQ_HDR_LEN; + } + if (NormRepairRequest::RANGES == requestForm) + { + req.AppendRepairRange(fec_id, fec_m, objectId, blockId, blockLen, segmentId, + lastObjectId, lastBlockId, lastBlockLen, lastSegmentId); + } + else + { + req.AppendRepairItem(fec_id, fec_m, objectId, blockId, blockLen, segmentId); + } + payloadLength += itemLength; + } + nack->PackRepairRequest(req); + ASSERT(nack->GetRepairContentLength() == payloadLength); + } + else + { + session.SendMessage(*nack); + nack_count++; + nack->ResetPayload(); + payloadLength = 0; + } + } + if (0 != payloadLength) + { + ASSERT(nack->GetRepairContentLength() == payloadLength); + session.SendMessage(*nack); + nack_count++; + } + session.ReturnMessageToPool(nack); + +} // end NormSenderNode::FragmentNack() + void NormSenderNode::UpdateRecvRate(const struct timeval& currentTime, unsigned short msgSize) { if (prev_update_time.tv_sec || prev_update_time.tv_usec) @@ -2400,14 +2614,14 @@ bool NormSenderNode::OnActivityTimeout(ProtoTimer& /*theTimer*/) { // Remote sender completely inactive? PLOG(PL_INFO, "NormSenderNode::OnActivityTimeout() node>%lu sender>%lu gone inactive?\n", - LocalNodeId(), GetId()); + (unsigned long)LocalNodeId(), (unsigned long)GetId()); //FreeBuffers(); This now needs to be done by the app as of norm version 1.4b3 session.Notify(NormController::REMOTE_SENDER_INACTIVE, this, NULL); } else { PLOG(PL_INFO, "NormSenderNode::OnActivityTimeout() node>%lu for sender>%lu\n", - LocalNodeId(), GetId()); + (unsigned long)LocalNodeId(), (unsigned long)GetId()); struct timeval currentTime; ::ProtoSystemTime(currentTime); UpdateRecvRate(currentTime, 0); @@ -2569,9 +2783,12 @@ void NormSenderNode::AttachCCFeedback(NormAckMsg& ack) #endif // LIMIT_CC_RATE ext.SetCCRate(NormQuantizeRate(ccRate)); } - PLOG(PL_DEBUG, "NormSenderNode::AttachCCFeedback() node>%lu sending ACK rate:%lf kbps (rtt:%lf loss:%lf s:%lf recvRate:%lf) slow_start:%d\n", - LocalNodeId(), 8.0e-03*NormUnquantizeRate(ext.GetCCRate()) , - rtt_estimate, ccLoss, nominal_packet_size, 8.0e-03*recv_rate, slow_start); + PLOG(PL_DEBUG, "NormSenderNode::AttachCCFeedback() node>%lu sender>%lu sending ACK rate:%lf kbps " + "(rtt:%lf loss:%lf s:%lf recvRate:%lf) slow_start:%d\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId(), + 8.0e-03*NormUnquantizeRate(ext.GetCCRate()) , + rtt_estimate, ccLoss, nominal_packet_size, + 8.0e-03*recv_rate, slow_start); ext.SetCCSequence(cc_sequence); } // end NormSenderNode::AttachCCFeedback() @@ -2580,7 +2797,7 @@ bool NormSenderNode::OnCCTimeout(ProtoTimer& /*theTimer*/) // Build and send NORM_ACK(CC) if (ack_pending && (1 == cc_timer.GetRepeatCount())) { - // Send ACK flush right away (CC feedback is included + // Send ACK flush right away (CC feedback is included) if (ack_timer.IsActive()) ack_timer.Deactivate(); if (cc_timer.IsActive()) cc_timer.Deactivate(); // will be reactivated if needed OnAckTimeout(ack_timer); @@ -2598,8 +2815,8 @@ bool NormSenderNode::OnCCTimeout(ProtoTimer& /*theTimer*/) NormAckMsg* ack = (NormAckMsg*)session.GetMessageFromPool(); if (!ack) { - PLOG(PL_WARN, "NormSenderNode::OnCCTimeout() node>%lu warning: " - "message pool empty ...\n", LocalNodeId()); + PLOG(PL_WARN, "NormSenderNode::OnCCTimeout() node>%lu sender>%lu warning: message pool empty ...\n", + (unsigned long)LocalNodeId(), (unsigned long)GetId()); if (cc_timer.IsActive()) cc_timer.Deactivate(); return false; } @@ -2611,6 +2828,7 @@ bool NormSenderNode::OnCCTimeout(ProtoTimer& /*theTimer*/) AttachCCFeedback(*ack); // cc feedback extension + // TBD - we need to provide a multicast_acks option if (unicast_nacks) ack->SetDestination(GetAddress()); else @@ -2671,7 +2889,7 @@ bool NormSenderNode::OnAckTimeout(ProtoTimer& /*theTimer*/) blockLen = ndata; else blockLen = watermark_segment_id; - ack->SetFecPayloadId(fec_id, watermark_block_id, watermark_segment_id, blockLen, fec_m); + ack->SetFecPayloadId(fec_id, watermark_block_id.GetValue(), watermark_segment_id, blockLen, fec_m); if (unicast_nacks) ack->SetDestination(GetAddress()); @@ -2793,7 +3011,6 @@ void NormNodeTree::AttachNode(NormNode *node) void NormNodeTree::DetachNode(NormNode* node) { ASSERT(NULL != node); - node->Release(); NormNode* x; NormNode* y; if (!node->left || !node->right) @@ -2845,7 +3062,8 @@ void NormNodeTree::DetachNode(NormNode* node) } if ((y->left = node->left)) y->left->parent = y; if ((y->right = node->right)) y->right->parent = y; - } + } + node->Release(); } // end NormNodeTree::DetachNode() @@ -3217,8 +3435,6 @@ bool NormLossEstimator2::Update(const struct timeval& currentTime, double windowScale = ignore_loss ? 2.0 : 1.0; if (deltaTime > windowScale*event_window) { - //if (CONFIRMING == seeking_loss_event) - // TRACE("Clearing unconfirmed loss event ...\n"); seeking_loss_event = SEEKING; } } diff --git a/src/common/normObject.cpp b/src/common/normObject.cpp index c3c8e4b..416c5dd 100644 --- a/src/common/normObject.cpp +++ b/src/common/normObject.cpp @@ -16,7 +16,10 @@ NormObject::NormObject(NormObject::Type theType, current_block_id(0), next_segment_id(0), max_pending_block(0), max_pending_segment(0), info_ptr(NULL), info_len(0), first_pass(true), accepted(false), notify_on_update(true), - user_data(NULL), next(NULL) + user_data(NULL) +#ifndef USE_PROTO_TREE + , next(NULL) +#endif { if (theSender) { @@ -32,7 +35,7 @@ NormObject::NormObject(NormObject::Type theType, NormObject::~NormObject() { Close(); - if (info_ptr) + if (NULL != info_ptr) { delete[] info_ptr; info_ptr = NULL; @@ -47,10 +50,8 @@ void NormObject::Retain() void NormObject::Release() { - //if ((NULL != sender) && (reference_count == 1)) // TRACE("NormObject final release node>%lu object>%hu\n", (unsigned long)sender->GetId(), (unsigned short)transport_id); - if (sender) sender->Release(); if (reference_count) { @@ -105,7 +106,7 @@ bool NormObject::Open(const NormObjectSize& objectSize, { if (infoPtr) { - if (info_ptr) delete []info_ptr; + if (NULL != info_ptr) delete[] info_ptr; if (infoLen > segmentSize) { PLOG(PL_FATAL, "NormObject::Open() info too big error\n"); @@ -128,10 +129,26 @@ bool NormObject::Open(const NormObjectSize& objectSize, // (Note NormObjectSize divide operator always rounds _upwards_) NormObjectSize numSegments = objectSize / NormObjectSize(segmentSize); NormObjectSize numBlocks = numSegments / NormObjectSize(numData); - ASSERT(0 == numBlocks.MSB()); - - if (!block_buffer.Init(numBlocks.LSB())) + // TBD - make sure numBlocks is in range of fec_block_mask + fec_block_mask = NormPayloadId::GetFecBlockMask(fecId, fecM); + + UINT32 blockIdMax = fec_block_mask; + if (IsStream()) blockIdMax = (blockIdMax / 2) - 1; + if ((numBlocks.MSB() > 0) || (numBlocks.LSB() > blockIdMax)) + { + PLOG(PL_FATAL, "NormObject::Open() error: object size exceeds FEC blocking and segmentation parameter capability\n"); + Close(); + return false; + } + + // No NormBlockId wrapping for NORM_DATA or NORM_FILE objects + // (so we zero fec_block_mask for "unsigned" value behaviors + // in NormBlockId and ProtoBitmask classes + + if (!IsStream()) fec_block_mask = 0; + + if (!block_buffer.Init(numBlocks.LSB(), 256, fec_block_mask)) { PLOG(PL_FATAL, "NormObject::Open() init block_buffer error\n"); Close(); @@ -141,17 +158,16 @@ bool NormObject::Open(const NormObjectSize& objectSize, // Init pending_mask (everything pending) // Note the pending_mask and repair_mask both need to be inited with a // "rangeMask" according to their fecId (e.g. 24-bit mask for some)!!! - UINT32 rangeMask = NormPayloadId::GetFecBlockMask(fecId, fecM); - if (!pending_mask.Init(numBlocks.LSB(), rangeMask)) + if (!pending_mask.Init(numBlocks.LSB(), fec_block_mask)) { PLOG(PL_FATAL, "NormObject::Open() init pending_mask (%lu) error: %s\n", - numBlocks.LSB(), GetErrorString()); + (unsigned long)numBlocks.LSB(), GetErrorString()); Close(); return false; } // Init repair_mask - if (!repair_mask.Init(numBlocks.LSB(), rangeMask)) + if (!repair_mask.Init(numBlocks.LSB(), fec_block_mask)) { PLOG(PL_FATAL, "NormObject::Open() init pending_mask error\n"); Close(); @@ -254,13 +270,13 @@ NormObjectSize NormObject::GetBytesPending() const if (block) pendingBytes += block->GetBytesPending(GetBlockSize(nextId), segment_size, final_block_id, final_segment_size); - else if ((UINT32)nextId < large_block_count) + else if (nextId.GetValue() < large_block_count) pendingBytes += largeBlockBytes; else if (nextId == final_block_id) pendingBytes += lastBlockBytes; else pendingBytes += smallBlockBytes; - nextId++; + Increment(nextId); } while (GetNextPending(nextId)); return pendingBytes; } @@ -296,29 +312,68 @@ bool NormObject::HandleInfoRequest(bool holdoff) return increasedRepair; } // end NormObject::HandleInfoRequest() -bool NormObject::HandleBlockRequest(NormBlockId nextId, NormBlockId lastId) +/* +bool NormObject::HandleBlockRequest(const NormBlockId& firstId, const NormBlockId& lastId) { PLOG(PL_TRACE, "NormObject::HandleBlockRequest() node>%lu blk>%lu -> blk>%lu\n", - LocalNodeId(), (UINT32)nextId, (UINT32)lastId); + (unsigned long)LocalNodeId(), + (unsigned long)firstId.GetValue(), + (unsigned long)lastId.GetValue()); bool increasedRepair = false; - lastId++; - while (nextId != lastId) + NormBlockId nextId = firstId; + while (Compare(nextId, lastId) <= 0) { - if (!repair_mask.Test(nextId)) + if (!repair_mask.Test(nextId.GetValue())) { // (TBD) these tests can probably go away if everything else is done right - if (!pending_mask.CanSet(nextId)) + if (!pending_mask.CanSet(nextId.GetValue())) PLOG(PL_ERROR, "NormObject::HandleBlockRequest() pending_mask.CanSet(%lu) error\n", - (UINT32)nextId); - if (!repair_mask.Set(nextId)) + (unsigned long)nextId.GetValue()); + if (!repair_mask.Set(nextId.GetValue())) PLOG(PL_ERROR, "NormObject::HandleBlockRequest() repair_mask.Set(%lu) error\n", - (UINT32)nextId); + (unsigned long)nextId.GetValue()); increasedRepair = true; } - nextId++; + Increment(nextId); } return increasedRepair; } // end NormObject::HandleBlockRequest(); +*/ + +bool NormObject::HandleBlockRequest(const NormBlockId& firstId, const NormBlockId& lastId) +{ + PLOG(PL_TRACE, "NormObject::HandleBlockRequest() node>%lu blk>%lu -> blk>%lu\n", + (unsigned long)LocalNodeId(), + (unsigned long)firstId.GetValue(), + (unsigned long)lastId.GetValue()); + bool blockInRange = true; + NormBlockId nextId = firstId; + while (Compare(nextId, lastId) <= 0) + { + if (!repair_mask.Test(nextId.GetValue())) + { + // (TBD) these tests can probably go away if everything else is done right + if (!pending_mask.CanSet(nextId.GetValue())) + { + PLOG(PL_ERROR, "NormObject::HandleBlockRequest() pending_mask.CanSet(%lu) error\n", + (unsigned long)nextId.GetValue()); + blockInRange = false; + if (IsStream()) + static_cast(this)->UnlockBlock(nextId); + } + else if (!repair_mask.Set(nextId.GetValue())) + { + PLOG(PL_ERROR, "NormObject::HandleBlockRequest() repair_mask.Set(%lu) error\n", + (unsigned long)nextId.GetValue()); + blockInRange = false; + if (IsStream()) + static_cast(this)->UnlockBlock(nextId); + } + } + Increment(nextId); + } + return blockInRange; +} // end NormObject::HandleBlockRequest(); bool NormObject::TxReset(NormBlockId firstBlock, bool requeue) @@ -330,20 +385,22 @@ bool NormObject::TxReset(NormBlockId firstBlock, bool requeue) pending_info = true; } repair_info = false; - repair_mask.Reset((UINT32)firstBlock); + repair_mask.Reset(firstBlock.GetValue()); repair_mask.Xor(pending_mask); if (repair_mask.IsSet()) { increasedRepair = true; - pending_mask.Reset((UINT32)firstBlock); + pending_mask.Reset(firstBlock.GetValue()); } repair_mask.Clear(); + // TBD - Use "firstBlock" as starting point for the iterator NormBlockBuffer::Iterator iterator(block_buffer); NormBlock* block; while ((block = iterator.GetNextBlock())) { NormBlockId blockId = block->GetId(); - if (blockId >= firstBlock) + //if (blockId >= firstBlock) + if (Compare(blockId, firstBlock) >= 0) { increasedRepair |= block->TxReset(GetBlockSize(blockId), nparity, @@ -360,26 +417,49 @@ bool NormObject::TxReset(NormBlockId firstBlock, bool requeue) return increasedRepair; } // end NormObject::TxReset() -bool NormObject::TxResetBlocks(NormBlockId nextId, NormBlockId lastId) +bool NormObject::TxResetBlocks(const NormBlockId& firstId, const NormBlockId& lastId) { bool increasedRepair = false; UINT16 autoParity = session.SenderAutoParity(); - lastId++; - while (nextId != lastId) + NormBlockId nextId = firstId; + while (Compare(nextId, lastId) <= 0) { - if (!pending_mask.Test(nextId)) + if (!pending_mask.Test(nextId.GetValue())) { - pending_mask.Set(nextId); + pending_mask.Set(nextId.GetValue()); increasedRepair = true; } NormBlock* block = block_buffer.Find(nextId); - if (block) - increasedRepair |= block->TxReset(GetBlockSize(block->GetId()), nparity, autoParity, segment_size); - nextId++; + if (NULL != block) + increasedRepair |= block->TxReset(GetBlockSize(nextId), nparity, autoParity, segment_size); + Increment(nextId); } return increasedRepair; } // end NormObject::TxResetBlocks() +bool NormObject::TxUpdateBlock(NormBlock* theBlock, + NormSegmentId firstSegmentId, + NormSegmentId lastSegmentId, + UINT16 numErasures) +{ + NormBlockId blockId = theBlock->GetId(); + if (pending_mask.CanSet(blockId.GetValue())) + { + if (theBlock->TxUpdate(firstSegmentId, lastSegmentId, + GetBlockSize(blockId), nparity, + numErasures)) + { + pending_mask.Set(blockId.GetValue()); + return true; + } + } + else if (IsStream()) + { + static_cast(this)->UnlockBlock(blockId); + } + return false; +} // end NormObject::TxUpdateBlock() + bool NormObject::ActivateRepairs() { bool repairsActivated = false; @@ -397,42 +477,43 @@ bool NormObject::ActivateRepairs() NormBlockId lastId; GetLastRepair(lastId); // for debug output only PLOG(PL_DEBUG, "NormObject::ActivateRepairs() node>%lu obj>%hu activating blk>%lu->%lu block repairs ...\n", - LocalNodeId(), (UINT16)transport_id, (UINT32)nextId, (UINT32)lastId); + (unsigned long)LocalNodeId(), + (UINT16)transport_id, + (unsigned long)nextId.GetValue(), + (unsigned long)lastId.GetValue()); //repairsActivated = true; UINT16 autoParity = session.SenderAutoParity(); do { - NormBlock* block = block_buffer.Find(nextId); - if (block) block->TxReset(GetBlockSize(nextId), nparity, autoParity, segment_size); - // (TBD) This check can be eventually eliminated if everything else is done right - if (!pending_mask.Set(nextId)) - { - PLOG(PL_ERROR, "NormObject::ActivateRepairs() pending_mask.Set(%lu) error 1!\n", (UINT32)nextId); - if (block) block->ClearPending(); - if (IsStream()) - static_cast(this)->UnlockBlock(nextId); - } - else + if (pending_mask.CanSet(nextId.GetValue())) { + NormBlock* block = block_buffer.Find(nextId); + if (NULL != block) block->TxReset(GetBlockSize(nextId), nparity, autoParity, segment_size); + pending_mask.Set(nextId.GetValue()); repairsActivated = true; } - nextId++; + else if (IsStream()) + { + static_cast(this)->UnlockBlock(nextId); + } + repair_mask.Unset(nextId.GetValue()); + Increment(nextId); } while (GetNextRepair(nextId)); - repair_mask.Clear(); + ASSERT(!repair_mask.IsSet()); } // Activate partial block (segment) repairs + // TBD - Can we make this more efficient for larger block_buffer sizes + // (e.g., have a separate list (or bitmask) of repair pending blocks?) NormBlockBuffer::Iterator iterator(block_buffer); NormBlock* block; - while ((block = iterator.GetNextBlock())) { if (block->ActivateRepairs(nparity)) { PLOG(PL_TRACE, "NormObject::ActivateRepairs() node>%lu obj>%hu activated blk>%lu segment repairs ...\n", - LocalNodeId(), (UINT16)transport_id, (UINT32)block->GetId()); - if (!pending_mask.Set(block->GetId())) + (unsigned long)LocalNodeId(), (UINT16)transport_id, (unsigned long)block->GetId().GetValue()); + if (!pending_mask.Set(block->GetId().GetValue())) { - PLOG(PL_ERROR, "NormObject::ActivateRepairs() pending_mask.Set(%lu) error 2!\n", (UINT32)block->GetId()); block->ClearPending(); if (IsStream()) static_cast(this)->UnlockBlock(block->GetId()); @@ -456,7 +537,7 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) GetLastRepair(endId); if (block_buffer.IsEmpty()) { - if (repair_mask.IsSet()) endId++; + if (repair_mask.IsSet()) Increment(endId); } else { @@ -464,17 +545,16 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) NormBlockId hi = block_buffer.RangeHi(); if (repair_mask.IsSet()) { - nextId = (lo < nextId) ? lo : nextId; - endId = (hi > endId) ? hi : endId; + nextId = (Compare(lo, nextId) < 0) ? lo : nextId; + endId = (Compare(hi, endId) > 0) ? hi : endId; } else { nextId = lo; endId = hi; } - endId++; + Increment(endId); } - // Instantiate a repair request for BLOCK level repairs NormRepairRequest req; bool requestAppended = false; @@ -484,18 +564,20 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) // Iterate through the range of blocks possibly pending repair NormBlockId firstId; UINT32 blockCount = 0; - while (nextId < endId) + //while (nextId < endId) + while (Compare(nextId, endId) < 0) { NormBlockId currentId = nextId; - nextId++; - bool repairEntireBlock = repair_mask.Test(currentId); + Increment(nextId); + bool repairEntireBlock = repair_mask.Test(currentId.GetValue()); if (repairEntireBlock) { if (!blockCount) firstId = currentId; blockCount++; } // Check for break in continuity or end - if (blockCount && (!repairEntireBlock || (nextId >= endId))) + //if (blockCount && (!repairEntireBlock || (nextId >= endId))) + if (blockCount && (!repairEntireBlock || (Compare(nextId, endId) >= 0))) { NormRepairRequest::Form form; switch (blockCount) @@ -560,8 +642,10 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) } prevForm = NormRepairRequest::INVALID; } - block->AppendRepairAdv(cmd, transport_id, repair_info, fec_id, fec_m, GetBlockSize(currentId), segment_size); // (TBD) error check - requestAppended = true; + if (block->AppendRepairAdv(cmd, transport_id, repair_info, fec_id, fec_m, GetBlockSize(currentId), segment_size)) + requestAppended = true; + else + return requestAppended; } } } // end while(nextId < endId) @@ -586,11 +670,11 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) return requestAppended; } } - return true; + return requestAppended; } // end NormObject::AppendRepairAdv() // This is used by sender (only) for watermark check -bool NormObject::FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) const +bool NormObject::FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) { //ASSERT(NULL == sender); if (repair_info) @@ -599,13 +683,16 @@ bool NormObject::FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) segmentId = 0; return true; } + // TBD - Can we make this more efficient for large block_buffer sizes + // (e.g., by maintaining a separate bitmask for blocks needing partial repair) NormBlockBuffer::Iterator iterator(block_buffer); NormBlock* block; while ((block = iterator.GetNextBlock())) if (block->IsRepairPending()) break; if (GetFirstRepair(blockId)) { - if (!block || (blockId <= block->GetId())) + //if (!block || (blockId <= block->GetId())) + if ((NULL == block) || (Compare(blockId, block->GetId()) <= 0)) { segmentId = 0; return true; @@ -624,11 +711,13 @@ bool NormObject::FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) // Called by sender only -bool NormObject::IsRepairPending() const +bool NormObject::IsRepairPending() { //ASSERT(NULL == sender); if (repair_info) return true; if (repair_mask.IsSet()) return true; + // TBD - Can we make this more efficient for large block_buffer sizes + // (e.g., by maintaining a separate bitmask for blocks needing partial repair) NormBlockBuffer::Iterator iterator(block_buffer); NormBlock* block; while ((block = iterator.GetNextBlock())) @@ -653,11 +742,13 @@ bool NormObject::IsPending(bool flush) const NormBlockId firstId; if (GetFirstPending(firstId)) { - if (firstId < max_pending_block) + //if (firstId < max_pending_block) + if (Compare(firstId, max_pending_block) < 0) { return true; } - else if (firstId > max_pending_block) + //else if (firstId > max_pending_block) + else if (Compare(firstId, max_pending_block) > 0) { return false; } @@ -703,13 +794,13 @@ bool NormObject::PassiveRepairCheck(NormBlockId blockId, NormBlockId firstPendingBlock; if (GetFirstPending(firstPendingBlock)) { - if (firstPendingBlock < blockId) + //if (firstPendingBlock < blockId) + if (Compare(firstPendingBlock, blockId) < 0) { return true; } else if (firstPendingBlock == blockId) { - NormBlock* block = block_buffer.Find(firstPendingBlock); if (NULL != block) { @@ -773,7 +864,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // (TBD) set "thruObject" if info-only obj break; case TO_BLOCK: - if (blockId >= max_pending_block) + //if (blockId >= max_pending_block) + if (Compare(blockId, max_pending_block) >= 0) { max_pending_block = blockId; max_pending_segment = 0; @@ -782,7 +874,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, { if (!holdoffPhase) { - if (blockId < current_block_id) + //if (blockId < current_block_id) + if (Compare(blockId, current_block_id) < 0) { current_block_id = blockId; next_segment_id = 0; @@ -803,7 +896,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, NormBlockId firstPending; if (GetFirstPending(firstPending)) { - if (firstPending < blockId) + //if (firstPending < blockId) + if (Compare(firstPending, blockId) < 0) { startRepairTimer = true; // Cache pending blockId range for TO_BLOCK @@ -818,7 +912,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // (TBD) set "thruObject" if info-only obj break; case THRU_SEGMENT: - if (blockId > max_pending_block) + //if (blockId > max_pending_block) + if (Compare(blockId, max_pending_block) > 0) { max_pending_block = blockId; max_pending_segment = segmentId + 1; @@ -838,7 +933,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, { if (!holdoffPhase) { - if (blockId < current_block_id) + //if (blockId < current_block_id) + if (Compare(blockId, current_block_id) < 0) { current_block_id = blockId; next_segment_id = segmentId + 1; @@ -858,14 +954,15 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // Cache pending blockId range for THRU_SEGMENT GetFirstPending(nextBlockId); endBlockId = blockId; - endBlockId++; + Increment(endBlockId); } else { NormBlockId firstPending; if (GetFirstPending(firstPending)) { - if (firstPending < blockId) + //if (firstPending < blockId) + if (Compare(firstPending, blockId) < 0) { startRepairTimer = true; } @@ -890,7 +987,7 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // Cache pending blockId range for THRU_SEGMENT nextBlockId = firstPending; endBlockId = blockId; - endBlockId++; + Increment(endBlockId); } } } @@ -899,7 +996,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, } break; case THRU_BLOCK: - if (blockId > max_pending_block) + //if (blockId > max_pending_block) + if (Compare(blockId, max_pending_block) > 0) { max_pending_block = blockId; max_pending_segment = GetBlockSize(blockId); @@ -910,7 +1008,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, { if (!holdoffPhase) { - if (blockId < current_block_id) + //if (blockId < current_block_id) + if (Compare(blockId, current_block_id) < 0) { current_block_id = blockId; next_segment_id = GetBlockSize(blockId); @@ -925,20 +1024,21 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // Cache pending blockId range for THRU_BLOCK GetFirstPending(nextBlockId); endBlockId = blockId; - endBlockId++; + Increment(endBlockId); } else { NormBlockId firstPending; if (GetFirstPending(firstPending)) { - if (firstPending <= blockId) + //if (firstPending <= blockId) + if (Compare(firstPending, blockId) <= 0) { startRepairTimer = true; // Cache pending blockId range for THRU_BLOCK nextBlockId = firstPending; endBlockId = blockId; - endBlockId++; + Increment(endBlockId); } } } @@ -949,7 +1049,8 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, case THRU_OBJECT: if (!IsStream()) max_pending_block = final_block_id; - else if (blockId > max_pending_block) + //else if (blockId > max_pending_block) + else if (Compare(blockId, max_pending_block) > 0) max_pending_block = blockId; max_pending_segment = GetBlockSize(max_pending_block); thruObject = true; @@ -961,7 +1062,7 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // Cache pending blockId range for THRU_OBJECT GetFirstPending(nextBlockId); GetLastPending(endBlockId); - endBlockId++; + Increment(endBlockId); } if (IsStream()) current_block_id = max_pending_block; @@ -987,11 +1088,12 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, // here at the start of a fresh NACK cycle) repair_info = false; // info repair mask if (repair_mask.IsSet()) repair_mask.Clear(); - while (nextBlockId < endBlockId) + //while (nextBlockId < endBlockId) + while (Compare(nextBlockId, endBlockId) < 0) { NormBlock* block = block_buffer.Find(nextBlockId); if (NULL != block) block->ClearRepairs(); - nextBlockId++; + Increment(nextBlockId); if (!GetNextPending(nextBlockId)) break; } return true; @@ -1005,12 +1107,13 @@ bool NormObject::ReceiverRepairCheck(CheckLevel level, bool NormObject::ReceiverRewindCheck(NormBlockId blockId, NormSegmentId segmentId) { - if (blockId > current_block_id) + //if (blockId > current_block_id) + if (Compare(blockId, current_block_id) > 0) return false; - else if (blockId < current_block_id) - return true; - else // (blockId == current_block_id) + else if (blockId == current_block_id) return ((segmentId+1) < next_segment_id); + else // if (blockId < current_block_id) + return true; } // end NormObject::ReceiverRewindCheck() // Note this partially clears "repair_mask" state @@ -1021,7 +1124,7 @@ bool NormObject::ReceiverRewindCheck(NormBlockId blockId, // up through the "current_block_id" bool NormObject::IsRepairPending(bool flush) { - // Only NORM receiver code sshould call this + // Only NORM receiver code should call this if (pending_info && !repair_info) return true; // Calculate repair_mask = pending_mask - repair_mask repair_mask.XCopy(pending_mask); @@ -1030,13 +1133,15 @@ bool NormObject::IsRepairPending(bool flush) { do { - if (!flush && (nextId > current_block_id)) break; + //if (!flush && (nextId > current_block_id)) break; + if (!flush && (Compare(nextId, current_block_id) > 0)) break; NormBlock* block = block_buffer.Find(nextId); if (block) { bool isPending; UINT16 numData = GetBlockSize(nextId); - if (flush || (nextId < current_block_id)) + //if (flush || (nextId < current_block_id)) + if (flush || (Compare(nextId, current_block_id) < 0)) { isPending = block->IsRepairPending(numData, nparity); } @@ -1053,14 +1158,15 @@ bool NormObject::IsRepairPending(bool flush) { return true; // We need the whole thing } - nextId++; + Increment(nextId); } while (GetNextRepair(nextId)); } return false; } // end NormObject::IsRepairPending() bool NormObject::AppendRepairRequest(NormNackMsg& nack, - bool flush) + bool flush, + UINT16 payloadMax) { // If !flush, we request only up _to_ max_pending_block::max_pending_segment. NormRepairRequest req; @@ -1070,7 +1176,8 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, NormBlockId nextId; bool iterating = GetFirstPending(nextId); NormBlockId prevId = nextId; - iterating = iterating && (flush || (nextId <= max_pending_block)); + //iterating = iterating && (flush || (nextId <= max_pending_block)); + iterating = iterating && (flush || (Compare(nextId, max_pending_block) <= 0)); UINT32 consecutiveCount = 0; while (iterating || (0 != consecutiveCount)) { @@ -1080,14 +1187,15 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, if (PL_TRACE <= GetDebugLevel()) { PLOG(PL_TRACE, "NormObject::AppendRepairRequest() node>%lu obj>%hu, blk>%lu->%lu ", - LocalNodeId(), (UINT16)GetId(), (UINT32)nextId, (UINT32)lastId); - PLOG(PL_ALWAYS, "(maxPending = %lu)\n", (UINT32)max_pending_block); + (unsigned long)LocalNodeId(), (UINT16)GetId(), + (unsigned long)nextId.GetValue(), (unsigned long)lastId.GetValue()); + PLOG(PL_ALWAYS, "(maxPending = %lu)\n", (unsigned long)max_pending_block.GetValue()); } bool appendRequest = false; NormBlock* block = iterating ? block_buffer.Find(nextId) : NULL; - if (block) + if (NULL != block) appendRequest = true; - else if (iterating && ((UINT32)(nextId - prevId) == consecutiveCount)) + else if (iterating && ((UINT32)Difference(nextId, prevId) == consecutiveCount)) consecutiveCount++; else appendRequest = true; @@ -1121,7 +1229,7 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, } if (NormRepairRequest::INVALID != nextForm) { - nack.AttachRepairRequest(req, segment_size); + nack.AttachRepairRequest(req, payloadMax); req.SetForm(nextForm); req.ResetFlags(); if (NACK_NORMAL == nacking_mode) @@ -1139,16 +1247,18 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, req.AppendRepairItem(fec_id, fec_m, transport_id, prevId, GetBlockSize(prevId), 0); // (TBD) error check if (2 == consecutiveCount) { - prevId++; + Increment(prevId); req.AppendRepairItem(fec_id, fec_m, transport_id, prevId, GetBlockSize(prevId), 0); // (TBD) error check } break; case NormRepairRequest::RANGES: { - NormBlockId lastId = prevId+consecutiveCount-1; + // lastId = prevId + consecutiveCount - 1 + NormBlockId endId = prevId; + Increment(endId, consecutiveCount - 1); req.AppendRepairRange(fec_id, fec_m, transport_id, prevId, GetBlockSize(prevId), 0, - transport_id, lastId, GetBlockSize(lastId), 0); // (TBD) error check + transport_id, endId, GetBlockSize(endId), 0); // (TBD) error check break; } default: @@ -1180,23 +1290,35 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, PLOG(PL_WARN, "NormObject::AppendRepairRequest() warning: full NACK msg\n"); return requestAppended; } + requestAppended = true; } - if (flush || (nextId != max_pending_block)) + bool blockRequestAppended; + if (flush || (nextId != max_pending_block)) { - block->AppendRepairRequest(nack, fec_id, fec_m, numData, nparity, transport_id, - pending_info, segment_size); // (TBD) error check + blockRequestAppended = + block->AppendRepairRequest(nack, fec_id, fec_m, numData, nparity, + transport_id, pending_info, payloadMax); } else { if (max_pending_segment < numData) - block->AppendRepairRequest(nack, fec_id, fec_m, max_pending_segment, 0, transport_id, - pending_info, segment_size); // (TBD) error check + { + blockRequestAppended = + block->AppendRepairRequest(nack, fec_id, fec_m, max_pending_segment, 0, + transport_id, pending_info, payloadMax); + } else - block->AppendRepairRequest(nack, fec_id, fec_m, numData, nparity, transport_id, - pending_info, segment_size); // (TBD) error check + { + blockRequestAppended = + block->AppendRepairRequest(nack, fec_id, fec_m, numData, nparity, + transport_id, pending_info, payloadMax); + } } - requestAppended = true; - prevForm = NormRepairRequest::INVALID; + if (blockRequestAppended) + requestAppended = true; + else // Must have filled NACK message + return requestAppended; + prevForm = NormRepairRequest::INVALID; } consecutiveCount = 0; } @@ -1210,9 +1332,10 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, } prevId = nextId; } // end if (appendRequest) - nextId++; + Increment(nextId); iterating = GetNextPending(nextId); - iterating = iterating && (flush || (nextId <= max_pending_block)); + //iterating = iterating && (flush || (nextId <= max_pending_block)); + iterating = iterating && (flush || (Compare(nextId, max_pending_block) <= 0)); } // end while (iterating || (0 != consecutiveCount)) // This conditional makes sure any outstanding requests constructed @@ -1230,7 +1353,7 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, if (!requestAppended && pending_info && (NACK_NONE != nacking_mode)) { // INFO_ONLY repair request - nack.AttachRepairRequest(req, segment_size); + nack.AttachRepairRequest(req, payloadMax); req.SetForm(NormRepairRequest::ITEMS); req.ResetFlags(); req.SetFlag(NormRepairRequest::INFO); @@ -1261,8 +1384,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, { info_len = segment_size; PLOG(PL_WARN, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " - "Warning! info too long.\n", LocalNodeId(), sender->GetId(), - (UINT16)transport_id); + "Warning! info too long.\n", (unsigned long)LocalNodeId(), + (unsigned long)sender->GetId(), (UINT16)transport_id); } memcpy(info_ptr, infoMsg.GetInfo(), info_len); pending_info = false; @@ -1273,7 +1396,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, // (TBD) Verify info hasn't changed? PLOG(PL_DEBUG, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " "received duplicate info ...\n", - LocalNodeId(), sender->GetId(), (UINT16)transport_id); + (unsigned long)LocalNodeId(), (unsigned long)sender->GetId(), + (UINT16)transport_id); } } else // NORM_MSG_DATA @@ -1289,7 +1413,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, if (!stream->StreamUpdateStatus(blockId)) { PLOG(PL_WARN, "NormObject::HandleObjectMessage() node:%lu sender:%lu obj>%hu blk>%lu " - "broken stream ...\n", LocalNodeId(), sender->GetId(), (UINT16)transport_id, (UINT32)blockId); + "broken stream ...\n", (unsigned long)LocalNodeId(), (unsigned long)sender->GetId(), + (UINT16)transport_id, (unsigned long)blockId.GetValue()); sender->IncrementResyncCount(); while (!stream->StreamUpdateStatus(blockId)) { @@ -1303,19 +1428,19 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, block_buffer.Remove(block); sender->PutFreeBlock(block); } - pending_mask.Unset(firstId); + pending_mask.Unset(firstId.GetValue()); } else { // If we try to resync to too far back, we end up chasing our tail here - // do we just sync to the current block under these circumstances + // so we just sync to the current block under these circumstances stream->StreamResync(blockId); // - pending_mask.GetSize()/2); break; } } } } - if (pending_mask.Test(blockId)) + if (pending_mask.Test(blockId.GetValue())) { NormBlock* block = block_buffer.Find(blockId); if (!block) @@ -1323,8 +1448,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, if (!(block = sender->GetFreeBlock(transport_id, blockId))) { //DMSG(2, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " - // "Warning! no free blocks ...\n", LocalNodeId(), sender->GetId(), - // (UINT16)transport_id); + // "Warning! no free blocks ...\n", (unsigned long)LocalNodeId(), (unsigned long) + // sender->GetId(), (UINT16)transport_id); return; } block->RxInit(blockId, numData, nparity); @@ -1336,8 +1461,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, if (segmentLength > segment_size) { PLOG(PL_ERROR, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " - "Error! segment too large ...\n", LocalNodeId(), sender->GetId(), - (UINT16)transport_id); + "Error! segment too large ...\n", (unsigned long)LocalNodeId(), + (unsigned long)sender->GetId(),(UINT16)transport_id); return; } UINT16 payloadLength = data.GetPayloadLength(); @@ -1365,8 +1490,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, else { //DMSG(2, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " - // "Warning! no free segments ...\n", LocalNodeId(), sender->GetId(), - // (UINT16)transport_id); + // "Warning! no free segments ...\n", (unsigned long)LocalNodeId(), + // (unsigned long)sender->GetId(), (UINT16)transport_id); if (!isSourceSymbol) return; } block->UnsetPending(segmentId); @@ -1401,8 +1526,9 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, // Decode (if pending_mask.FirstSet() < numData) // and write any decoded data segments to object PLOG(PL_DETAIL, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu blk>%lu " - "completed block ...\n", LocalNodeId(), sender->GetId(), - (UINT16)transport_id, (UINT32)block->GetId()); + "completed block ...\n", (unsigned long)LocalNodeId(), + (unsigned long)sender->GetId(), (UINT16)transport_id, + (unsigned long)block->GetId().GetValue()); UINT16 erasureCount = 0; UINT16 nextErasure = 0; UINT16 retrievalCount = 0; @@ -1487,10 +1613,10 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, for (UINT16 i = 0; i < retrievalCount; i++) block->DetachSegment(sender->GetRetrievalLoc(i)); // OK, we're done with this block - pending_mask.Unset(blockId); + pending_mask.Unset(blockId.GetValue()); block_buffer.Remove(block); sender->PutFreeBlock(block); - } + } // if erasureCount <= parityCount (i.e., block complete) // Notify application of new data available // (TBD) this could be improved for stream objects // so it's not called unnecessarily @@ -1505,21 +1631,23 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, } else { - PLOG(PL_TRACE, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " - "received duplicate segment ...\n", LocalNodeId(), - sender->GetId(), (UINT16)transport_id); + PLOG(PL_DEBUG, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " + "received duplicate segment blk>%lu segment>%hu...\n", (unsigned long)LocalNodeId(), + (unsigned long)sender->GetId(), (UINT16)transport_id,(unsigned long)blockId.GetValue(), + segmentId); } } else { - PLOG(PL_TRACE, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " - "received duplicate block message ...\n", LocalNodeId(), - sender->GetId(), (UINT16)transport_id); + PLOG(PL_DEBUG, "NormObject::HandleObjectMessage() node>%lu sender>%lu obj>%hu " + "received duplicate block message blk>%lu ...\n", (unsigned long)LocalNodeId(), + (unsigned long)sender->GetId(), (UINT16)transport_id, (unsigned long)blockId.GetValue()); } // end if/else pending_mask.Test(blockId) } // end if/else (NORM_MSG_INFO) + } // end NormObject::HandleObjectMessage() -// Returns source symbol segments to pool for first block with such resources +// Returns source symbol segments to pool for ordinally _first_ block with such resources bool NormObject::ReclaimSourceSegments(NormSegmentPool& segmentPool) { NormBlockBuffer::Iterator iterator(block_buffer); @@ -1543,7 +1671,7 @@ bool NormObject::ReclaimSourceSegments(NormSegmentPool& segmentPool) } // end NormObject::ReclaimSourceSegments() -// Steals non-pending block (oldest first) for _sender_ resource management +// Steals non-pending block (ordinally _first_) for _sender_ resource management // (optionally excludes block indicated by blockId) NormBlock* NormObject::StealNonPendingBlock(bool excludeBlock, NormBlockId excludeId) { @@ -1559,8 +1687,8 @@ NormBlock* NormObject::StealNonPendingBlock(bool excludeBlock, NormBlockId exclu { NormBlockId bid = block->GetId(); if (block->IsTransmitPending() || - pending_mask.Test(bid) || - repair_mask.Test(bid) || + pending_mask.Test(bid.GetValue()) || + repair_mask.Test(bid.GetValue()) || (excludeBlock && (excludeId == bid))) { continue; @@ -1623,11 +1751,8 @@ NormBlock* NormObject::StealOldestBlock(bool excludeBlock, NormBlockId excludeId } } // end NormObject::StealOldestBlock() - - bool NormObject::NextSenderMsg(NormObjectMsg* msg) -{ - +{ // Init() the message if (pending_info) { @@ -1700,7 +1825,6 @@ bool NormObject::NextSenderMsg(NormObjectMsg* msg) ASSERT(0); return false; } - if (pending_info) { // (TBD) set REPAIR_FLAG for retransmitted info @@ -1709,209 +1833,231 @@ bool NormObject::NextSenderMsg(NormObjectMsg* msg) pending_info = false; return true; } + // This block gets the next pending block/segment + // (The loop handles NORM_OBJECT_STREAM advancement + // without the prior approach that used recursion) + NormDataMsg* data = static_cast(msg); + NormBlock* block = NULL; NormBlockId blockId; - if (!GetFirstPending(blockId)) + UINT16 numData = 0; + NormSegmentId segmentId; + bool squelchQueued = false; + while (NULL == block) { - // Attempt to advance stream (probably had been repair-delayed) - if (IsStream()) + if (!GetFirstPending(blockId)) { - if (static_cast(this)->StreamAdvance()) + // Attempt to advance stream (probably had been repair-delayed) + if (IsStream()) { - return NextSenderMsg(msg); + if (static_cast(this)->StreamAdvance()) + { + continue; // return NextSenderMsg(msg); + } + else + { + //ASSERT(IsRepairPending()); + return false; + } } else { - //ASSERT(IsRepairPending()); + PLOG(PL_FATAL, "NormObject::NextSenderMsg() pending object w/ no pending blocks?!\n"); return false; } } - else + numData = GetBlockSize(blockId); + block = block_buffer.Find(blockId); + if (!block) { - PLOG(PL_FATAL, "NormObject::NextSenderMsg() pending object w/ no pending blocks?!\n"); - return false; + if (NULL == (block = session.SenderGetFreeBlock(transport_id, blockId))) + { + PLOG(PL_INFO, "NormObject::NextSenderMsg() node>%lu warning: sender resource " + "constrained (no free blocks).\n", (unsigned long)LocalNodeId()); + return false; + } + // Load block with zero initialized parity segments + UINT16 totalBlockLen = numData + nparity; + for (UINT16 i = numData; i < totalBlockLen; i++) + { + char* s = session.SenderGetFreeSegment(transport_id, blockId); + if (s) + { + UINT16 payloadMax = segment_size + NormDataMsg::GetStreamPayloadHeaderLength(); +#ifdef SIMULATE + payloadMax = MIN(payloadMax, SIM_PAYLOAD_MAX); +#endif // SIMULATE + memset(s, 0, payloadMax); // extra byte for msg flags + block->AttachSegment(i, s); + } + else + { + PLOG(PL_INFO, "NormObject::NextSenderMsg() node>%lu warning: sender resource " + "constrained (no free segments).\n", (unsigned long)LocalNodeId()); + session.SenderPutFreeBlock(block); + return false; + } + } + block->TxInit(blockId, numData, session.SenderAutoParity()); + //if (blockId < max_pending_block) + if (Compare(blockId, max_pending_block) < 0) + block->SetFlag(NormBlock::IN_REPAIR); + while (!block_buffer.Insert(block)) + { + //ASSERT(STREAM == type); + //if (blockId > block_buffer.RangeLo()) + if (Compare(blockId, block_buffer.RangeLo()) > 0) + { + NormBlock* lowBlock = block_buffer.Find(block_buffer.RangeLo()); + NormBlockId lowBlockId = lowBlock->GetId(); + bool push = static_cast(this)->GetPushMode(); + if (!push && (lowBlock->IsRepairPending() || IsRepairSet(lowBlockId))) + { + // Pending repairs delaying stream advance + PLOG(PL_DEBUG, "NormObject::NextSenderMsg() node>%lu pending repairs delaying stream progress\n", + (unsigned long)LocalNodeId()); + session.SenderPutFreeBlock(block); + return false; + } + else + { + // Prune old non-pending block (or even pending if "push" enabled stream) + block_buffer.Remove(lowBlock); + repair_mask.Unset(lowBlockId.GetValue()); // just in case + pending_mask.Unset(lowBlockId.GetValue()); + if (IsStream()) // always true + static_cast(this)->UnlockBlock(lowBlockId); + session.SenderPutFreeBlock(lowBlock); + continue; + } + } + else if (IsStream()) + { + PLOG(PL_WARN, "NormObject::NextSenderMsg() node>%lu Warning! can't repair old stream block\n", + (unsigned long)LocalNodeId()); + if (!squelchQueued) + { + session.SenderQueueSquelch(transport_id); + squelchQueued = true; + } + session.SenderPutFreeBlock(block); + repair_mask.Unset(blockId.GetValue()); // just in case + pending_mask.Unset(blockId.GetValue()); + // Unlock (set to non-pending status) the corresponding stream_buffer block + static_cast(this)->UnlockBlock(blockId); + block = NULL; + break; //return NextSenderMsg(msg); + } + else + { + PLOG(PL_FATAL, "NormObject::NextSenderMsg() invalid non-stream state!\n"); + ASSERT(0); + return false; + } + } // end while (!block_buffer.Insert()) + if (NULL == block) continue; + } // end if (!block) + if (!block->GetFirstPending(segmentId)) + { + PLOG(PL_ERROR, "NormObject::NextSenderMsg() warning: found pending block %lu with nothing pending!?\n", + (unsigned long)blockId.GetValue()); + pending_mask.Unset(blockId.GetValue()); + block = NULL; + continue; //return NextSenderMsg(msg); } - } - - NormDataMsg* data = (NormDataMsg*)msg; - UINT16 numData = GetBlockSize(blockId); - NormBlock* block = block_buffer.Find(blockId); - if (!block) - { - if (NULL == (block = session.SenderGetFreeBlock(transport_id, blockId))) - { - PLOG(PL_INFO, "NormObject::NextSenderMsg() node>%lu warning: sender resource " - "constrained (no free blocks).\n", LocalNodeId()); - return false; - } - // Load block with zero initialized parity segments - UINT16 totalBlockLen = numData + nparity; - for (UINT16 i = numData; i < totalBlockLen; i++) - { - char* s = session.SenderGetFreeSegment(transport_id, blockId); - if (s) + // Try to read segment + if (segmentId < numData) + { + // Try to read data segment (Note "ReadSegment" copies in offset/length info also) + char* buffer = data->AccessPayload(); + UINT16 payloadLength = ReadSegment(blockId, segmentId, buffer); + if (0 == payloadLength) { + // (TBD) deal with read error + //(for streams, it currently means the stream is pending, but app hasn't yet written data) + if (!IsStream()) + { + PLOG(PL_FATAL, "NormObject::NextSenderMsg() ReadSegment() error\n"); + return false; + } + else if (static_cast(this)->IsOldBlock(blockId)) + { + PLOG(PL_ERROR, "NormObject::NextSenderMsg() node>%lu Warning! can't repair old stream segment\n", + (unsigned long)LocalNodeId()); + block->UnsetPending(segmentId); + if (!block->IsPending()) + { + // End of old block reached + block->ResetParityCount(nparity); + pending_mask.Unset(blockId.GetValue()); + // for EMCON sending, mark NORM_INFO for re-transmission, if applicable + if (session.SndrEmcon() && HaveInfo()) + pending_info = true; + } + block = NULL; + continue; //return NextSenderMsg(msg); + } + else + { + // App hasn't written data for this block yet + return false; + } + } + data->SetPayloadLength(payloadLength); + + // Perform incremental FEC encoding as needed + if ((block->ParityReadiness() == segmentId) && (0 != nparity)) + // (TBD) && ((incrementalParity == true) || (auto_parity != 0)) + { + // (TBD) for non-stream objects, catch alternate "last block/segment len" + // ZERO pad any "runt" segments before encoding UINT16 payloadMax = segment_size + NormDataMsg::GetStreamPayloadHeaderLength(); #ifdef SIMULATE payloadMax = MIN(payloadMax, SIM_PAYLOAD_MAX); #endif // SIMULATE - memset(s, 0, payloadMax); // extra byte for msg flags - block->AttachSegment(i, s); - } - else - { - PLOG(PL_INFO, "NormObject::NextSenderMsg() node>%lu warning: sender resource " - "constrained (no free segments).\n", LocalNodeId()); - session.SenderPutFreeBlock(block); - return false; - } - } - block->TxInit(blockId, numData, session.SenderAutoParity()); - if (blockId < max_pending_block) block->SetFlag(NormBlock::IN_REPAIR); - while (!block_buffer.Insert(block)) - { - //ASSERT(STREAM == type); - if (blockId > block_buffer.RangeLo()) - { - NormBlock* lowBlock = block_buffer.Find(block_buffer.RangeLo()); - NormBlockId lowBlockId = lowBlock->GetId(); - bool push = static_cast(this)->GetPushMode(); - if (!push && (lowBlock->IsRepairPending() || IsRepairSet(lowBlockId))) - { - // Pending repairs delaying stream advance - PLOG(PL_DEBUG, "NormObject::NextSenderMsg() node>%lu pending repairs delaying stream progress\n", LocalNodeId()); - session.SenderPutFreeBlock(block); - return false; - } - else - { - // Prune old non-pending block (or even pending if "push" enabled stream) - block_buffer.Remove(lowBlock); - repair_mask.Unset(lowBlockId); // just in case - pending_mask.Unset(lowBlockId); - if (IsStream()) // always true - static_cast(this)->UnlockBlock(lowBlockId); - session.SenderPutFreeBlock(lowBlock); - continue; - } - } - else if (IsStream()) - { - PLOG(PL_WARN, "NormObject::NextSenderMsg() node>%lu Warning! can't repair old stream block\n", LocalNodeId()); - session.SenderPutFreeBlock(block); - repair_mask.Unset(blockId); // just in case - pending_mask.Unset(blockId); - // Unlock (set to non-pending status) the corresponding stream_buffer block - static_cast(this)->UnlockBlock(blockId); - return NextSenderMsg(msg); - } - else - { - PLOG(PL_FATAL, "NormObject::NextSenderMsg() invalid non-stream state!\n"); - ASSERT(0); - return false; - } - } - } // end if (!block) - NormSegmentId segmentId = 0; - if (!block->GetFirstPending(segmentId)) - { - PLOG(PL_ERROR, "NormObject::NextSenderMsg() nothing pending!?\n"); - fprintf(stderr, "NormObject::NextSenderMsg() nothing pending!?\n"); - ASSERT(0); - return false; - } - // Try to read segment - if (segmentId < numData) - { - // Try to read data segment (Note "ReadSegment" copies in offset/length info also) - char* buffer = data->AccessPayload(); - UINT16 payloadLength = ReadSegment(blockId, segmentId, buffer); - if (0 == payloadLength) - { - // (TBD) deal with read error - //(for streams, it currently means the stream is pending, but app hasn't yet written data) - if (!IsStream()) - { - PLOG(PL_FATAL, "NormObject::NextSenderMsg() ReadSegment() error\n"); - return false; - } - else if (static_cast(this)->IsOldBlock(blockId)) - { - PLOG(PL_ERROR, "NormObject::NextSenderMsg() node>%lu Warning! can't repair old stream segment\n", LocalNodeId()); - block->UnsetPending(segmentId); - if (!block->IsPending()) - { - // End of old block reached - block->ResetParityCount(nparity); - pending_mask.Unset(blockId); - // for EMCON sending, mark NORM_INFO for re-transmission, if applicable - if (session.SndrEmcon() && HaveInfo()) - pending_info = true; - } - return NextSenderMsg(msg); - } - else - { - // App hasn't written data for this block yet - return false; + if (payloadLength < payloadMax) + memset(buffer+payloadLength, 0, payloadMax-payloadLength); + // (TBD) the encode routine could update the block's parity readiness + block->UpdateSegSizeMax(payloadLength); + session.SenderEncode(segmentId, data->AccessPayload(), block->SegmentList(numData)); + block->IncreaseParityReadiness(); } } - data->SetPayloadLength(payloadLength); - - // Perform incremental FEC encoding as needed - if ((block->ParityReadiness() == segmentId) && (0 != nparity)) - // (TBD) && ((incrementalParity == true) || (auto_parity != 0)) - { - // (TBD) for non-stream objects, catch alternate "last block/segment len" - // ZERO pad any "runt" segments before encoding - UINT16 payloadMax = segment_size + NormDataMsg::GetStreamPayloadHeaderLength(); + else + { + if (!block->ParityReady(numData)) + { + ASSERT(0 == block->ParityReadiness()); + CalculateBlockParity(block); + } + char* segment = block->GetSegment(segmentId); + ASSERT(NULL != segment); + // We only need to send FEC content to cover the biggest segment + // sent for the block. #ifdef SIMULATE - payloadMax = MIN(payloadMax, SIM_PAYLOAD_MAX); -#endif // SIMULATE - if (payloadLength < payloadMax) - memset(buffer+payloadLength, 0, payloadMax-payloadLength); - // (TBD) the encode routine could update the block's parity readiness - block->UpdateSegSizeMax(payloadLength); - session.SenderEncode(segmentId, data->AccessPayload(), block->SegmentList(numData)); - block->IncreaseParityReadiness(); - } - } - else - { - if (!block->ParityReady(numData)) - { - ASSERT(0 == block->ParityReadiness()); - CalculateBlockParity(block); - } - char* segment = block->GetSegment(segmentId); - ASSERT(NULL != segment); - // We only need to send FEC content to cover the biggest segment - // sent for the block. -#ifdef SIMULATE - UINT16 payloadMax = MIN(block->GetSegSizeMax(), SIM_PAYLOAD_MAX); - data->SetPayload(segment, payloadMax); - data->SetPayloadLength(block->GetSegSizeMax()); // correct the msg length + UINT16 payloadMax = MIN(block->GetSegSizeMax(), SIM_PAYLOAD_MAX); + data->SetPayload(segment, payloadMax); + data->SetPayloadLength(block->GetSegSizeMax()); // correct the msg length #else - data->SetPayload(segment, block->GetSegSizeMax()); + data->SetPayload(segment, block->GetSegSizeMax()); #endif // if/else SIMULATE - } + } + } // end while (NULL == block) block->UnsetPending(segmentId); //if (block->InRepair()) // data->SetFlag(NormObjectMsg::FLAG_REPAIR); - - data->SetFecPayloadId(fec_id, blockId, segmentId, numData, fec_m); + data->SetFecPayloadId(fec_id, blockId.GetValue(), segmentId, numData, fec_m); if (!block->IsPending()) { // End of block reached block->ResetParityCount(nparity); - pending_mask.Unset(blockId); + pending_mask.Unset(blockId.GetValue()); // for EMCON sending, mark NORM_INFO for re-transmission, if applicable if (session.SndrEmcon() && HaveInfo()) pending_info = true; // Advance sender use of "max_pending_block" so we always // know when a block should be flagged as IN_REPAIR if (blockId == max_pending_block) - max_pending_block++; + Increment(max_pending_block); } // We update the object/block flow control timestamp to @@ -1950,19 +2096,19 @@ bool NormStreamObject::StreamAdvance() // (TBD) should we make sure !pending_mask.IsSet()??? NormBlockId nextBlockId = stream_next_id; // Make sure we won't prevent any pending repairs - if (repair_mask.CanSet(nextBlockId)) + if (repair_mask.CanSet(nextBlockId.GetValue())) { - if (block_buffer.CanInsert(nextBlockId)) + if (block_buffer.CanInsert(nextBlockId.GetValue())) { - if (pending_mask.Set(nextBlockId)) + if (pending_mask.Set(nextBlockId.GetValue())) { - stream_next_id++; + Increment(stream_next_id); return true; } else { PLOG(PL_ERROR, "NormStreamObject::StreamAdvance() error: node>%lu couldn't set set stream pending mask (1)\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } } else @@ -1973,32 +2119,57 @@ bool NormStreamObject::StreamAdvance() { // ??? (TBD) Should this block be returned to the pool right now??? // especially for "push" enabled streams - if (pending_mask.Set(nextBlockId)) + if (pending_mask.Set(nextBlockId.GetValue())) { - stream_next_id++; + Increment(stream_next_id); return true; } else { PLOG(PL_ERROR, "NormStreamObject::StreamAdvance() error: node>%lu couldn't set stream pending mask (2)\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } } else { - PLOG(PL_WARN, "NormStreamObject::StreamAdvance() warning: node>%lu pending segment repairs (blk>%lu) " - "delaying stream advance ...\n", LocalNodeId(), (UINT32)block->GetId()); + PLOG(PL_DEBUG, "NormStreamObject::StreamAdvance() warning: node>%lu pending segment repairs (blk>%lu) " + "delaying stream advance ...\n", (unsigned long)LocalNodeId(), + (unsigned long)block->GetId().GetValue()); } } } else { PLOG(PL_WARN, "NormStreamObject::StreamAdvance() warning: node>%lu pending block repair delaying stream advance ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } return false; } // end NormStreamObject::StreamAdvance() +NormBlockId NormStreamObject::RepairWindowLo() const +{ + NormBlockId blockId(0); + if (!stream_buffer.IsEmpty()) + blockId = StreamBufferLo(); + if (!block_buffer.IsEmpty()) + { + NormBlockId rangeMin = block_buffer.RangeMin(); + if (Compare(rangeMin, blockId) > 0) + blockId = rangeMin; + } + return blockId; + + /* + // TBD - use block_buffer.range_hi information to compute + // (i.e. range_hi - range_max + while (!block_buffer.CanInsert(blockId)) + { + Increment(blockId); + } + return blockId; + */ +} // end NormStreamObject::RepairWindowLo() + bool NormObject::CalculateBlockParity(NormBlock* block) { if (0 == nparity) return true; @@ -2063,7 +2234,8 @@ NormBlock* NormObject::SenderRecoverBlock(NormBlockId blockId) if (!block_buffer.Insert(block)) { session.SenderPutFreeBlock(block); - PLOG(PL_DEBUG, "NormObject::SenderRecoverBlock() node>%lu couldn't buffer recovered block\n"); + PLOG(PL_DEBUG, "NormObject::SenderRecoverBlock() node>%lu couldn't buffer recovered block\n", + (unsigned long)LocalNodeId()); return NULL; } return block; @@ -2077,7 +2249,7 @@ NormBlock* NormObject::SenderRecoverBlock(NormBlockId blockId) else { //DMSG(2, "NormObject::SenderRecoverBlock() node>%lu Warning! sender resource " - // "constrained (no free blocks).\n", LocalNodeId()); + // "constrained (no free blocks).\n", (unsigned long)LocalNodeId()); return (NormBlock*)NULL; } } // end NormObject::SenderRecoverBlock() @@ -2218,14 +2390,14 @@ bool NormFileObject::WriteSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset; NormObjectSize segmentSize = NormObjectSize(segment_size); - if (((UINT32)blockId) < large_block_count) + if (blockId.GetValue() < large_block_count) { - segmentOffset = (large_block_length*(UINT32)blockId) + (segmentSize*segmentId); + segmentOffset = (large_block_length*blockId.GetValue()) + (segmentSize*segmentId); } else { segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this - UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + UINT32 smallBlockIndex = blockId.GetValue() - large_block_count; segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } @@ -2243,14 +2415,11 @@ UINT16 NormFileObject::ReadSegment(NormBlockId blockId, NormSegmentId segmentId, char* buffer) { - - - // Determine segment length from blockId::segmentId size_t len; if (blockId == final_block_id) { - if (segmentId == (GetBlockSize(blockId)-1)) + if (segmentId == (GetBlockSize(blockId) - 1)) len = final_segment_size; else len = segment_size; @@ -2263,14 +2432,14 @@ UINT16 NormFileObject::ReadSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset; NormObjectSize segmentSize = NormObjectSize(segment_size); - if ((UINT32)blockId < large_block_count) + if (blockId.GetValue() < large_block_count) { - segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + segmentOffset = large_block_length*blockId.GetValue() + segmentSize*segmentId; } else { segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this - UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + UINT32 smallBlockIndex = blockId.GetValue() - large_block_count; segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } @@ -2325,22 +2494,28 @@ char* NormFileObject::RetrieveSegment(NormBlockId blockId, // NormDataObject::NormDataObject(class NormSession& theSession, class NormSenderNode* theSender, - const NormObjectId& objectId) + const NormObjectId& objectId, + DataFreeFunctionHandle dataFreeFunc) : NormObject(DATA, theSession, theSender, objectId), large_block_length(0), small_block_length(0), - data_ptr(NULL), data_max(0), data_released(false) + data_ptr(NULL), data_max(0), data_released(false), + data_free_func(dataFreeFunc) { } NormDataObject::~NormDataObject() { + //TRACE("NormDataObject destructor ...\n"); Close(); if (data_released) { if (NULL != data_ptr) { - delete[] data_ptr; + if (NULL != data_free_func) + data_free_func(data_ptr); + else + delete[] data_ptr; data_ptr = NULL; } data_released = false; @@ -2356,7 +2531,10 @@ bool NormDataObject::Open(char* dataPtr, { if (data_released && (NULL != data_ptr)) { - delete[] data_ptr; + if (NULL != data_free_func) + data_free_func(data_ptr); + else + delete[] data_ptr; data_ptr = NULL; data_released = false; } @@ -2432,14 +2610,14 @@ bool NormDataObject::WriteSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset; NormObjectSize segmentSize = NormObjectSize(segment_size); - if ((UINT32)blockId < large_block_count) + if (blockId.GetValue() < large_block_count) { - segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + segmentOffset = large_block_length*blockId.GetValue() + segmentSize*segmentId; } else { segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this - UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + UINT32 smallBlockIndex = blockId.GetValue() - large_block_count; segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } @@ -2479,14 +2657,14 @@ UINT16 NormDataObject::ReadSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset; NormObjectSize segmentSize = NormObjectSize(segment_size); - if ((UINT32)blockId < large_block_count) + if (blockId.GetValue() < large_block_count) { - segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + segmentOffset = large_block_length*blockId.GetValue() + segmentSize*segmentId; } else { segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this - UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + UINT32 smallBlockIndex = blockId.GetValue() - large_block_count; segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } @@ -2524,14 +2702,14 @@ char* NormDataObject::RetrieveSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset; NormObjectSize segmentSize = NormObjectSize(segment_size); - if ((UINT32)blockId < large_block_count) + if (blockId.GetValue() < large_block_count) { - segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + segmentOffset = large_block_length*blockId.GetValue() + segmentSize*segmentId; } else { segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this - UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + UINT32 smallBlockIndex = blockId.GetValue() - large_block_count; segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } @@ -2570,7 +2748,7 @@ NormStreamObject::NormStreamObject(class NormSession& theSession, read_init(true), read_ready(false), flush_pending(false), msg_start(true), flush_mode(FLUSH_NONE), push_mode(false), - stream_closing(false), + stream_broken(false), stream_closing(false), block_pool_threshold(0) { } @@ -2591,21 +2769,35 @@ NormStreamObject::~NormStreamObject() block_pool.Destroy(); } +NormBlockId NormStreamObject::FlushBlockId() const +{ + if (0 != write_index.segment) + { + return write_index.block; + } + else + { + // blockId = write_index.block - 1 + NormBlockId blockId = write_index.block; + Decrement(blockId); + return blockId; + } +} // end NormStreamObject::FlushBlockId() + bool NormStreamObject::Open(UINT32 bufferSize, bool doubleBuffer, const char* infoPtr, UINT16 infoLen) { - if (!bufferSize) + if (0 == bufferSize) { PLOG(PL_FATAL, "NormStreamObject::Open() zero bufferSize error\n"); return false; } - UINT16 segmentSize, numData; - if (sender) + if (NULL != sender) { - // receive streams have already beeen pre-opened + // receive streams have already beeen pre-opened via RxOpen() segmentSize = segment_size; numData = ndata; } @@ -2613,12 +2805,33 @@ bool NormStreamObject::Open(UINT32 bufferSize, { segmentSize = session.SenderSegmentSize(); numData = session.SenderBlockSize(); + stream_next_id = pending_mask.GetSize(); } - UINT32 blockSize = segmentSize * numData; UINT32 numBlocks = bufferSize / blockSize; // Buffering requires at least 2 blocks numBlocks = MAX(2, numBlocks); + // Recompute "bufferSize" to match any adjustments + bufferSize = numBlocks * blockSize; + + if (NULL == sender) + { + // This is a tx stream object, open with adjusted "bufferSize" + if (!NormObject::Open(NormObjectSize((UINT32)bufferSize), + infoPtr, + infoLen, + session.SenderSegmentSize(), + session.GetSenderFecId(), + session.GetSenderFecFieldSize(), + session.SenderBlockSize(), + session.SenderNumParity())) + { + PLOG(PL_FATAL, "NormStreamObject::Open() object open error\n"); + Close(); + return false; + } + } + if (doubleBuffer) numBlocks *= 2; UINT32 numSegments = numBlocks * numData; @@ -2636,7 +2849,7 @@ bool NormStreamObject::Open(UINT32 bufferSize, return false; } - if (!stream_buffer.Init(numBlocks)) + if (!stream_buffer.Init(numBlocks, 256, fec_block_mask)) { PLOG(PL_FATAL, "NormStreamObject::Open() stream_buffer init error\n"); Close(); @@ -2646,30 +2859,11 @@ bool NormStreamObject::Open(UINT32 bufferSize, // since our objects are exclusively read _or_ write read_init = true; - read_index.block = read_index.segment = 0; + read_index.block = read_index.segment = read_index.offset = 0; write_index.block = write_index.segment = 0; tx_index.block = tx_index.segment = 0; tx_offset = write_offset = read_offset = 0; write_vacancy = true; - - if (!sender) - { - if (!NormObject::Open(NormObjectSize((UINT32)bufferSize), - infoPtr, - infoLen, - session.SenderSegmentSize(), - session.GetSenderFecId(), - session.GetSenderFecFieldSize(), - session.SenderBlockSize(), - session.SenderNumParity())) - { - PLOG(PL_FATAL, "NormStreamObject::Open() object open error\n"); - Close(); - return false; - } - stream_next_id = pending_mask.GetSize(); - } - stream_sync = false; flush_pending = false; msg_start = true; @@ -2679,6 +2873,7 @@ bool NormStreamObject::Open(UINT32 bufferSize, bool NormStreamObject::Accept(UINT32 bufferSize, bool doubleBuffer) { + if (Accepted()) return true; // was preset_stream (already pre-accepted) if (Open(bufferSize, doubleBuffer)) { NormObject::Accept(); @@ -2709,14 +2904,16 @@ bool NormStreamObject::LockBlocks(NormBlockId firstId, NormBlockId lastId, const { NormBlockId nextId = firstId; // First check that we _can_ lock/reset them all - while (nextId <= lastId) + //while (nextId <= lastId) + while (Compare(nextId, lastId) <= 0) { NormBlock* block = stream_buffer.Find(nextId); if (NULL == block) return false; - nextId++; + Increment(nextId); } nextId = firstId; - while (nextId <= lastId) + //while (nextId <= lastId) + while (Compare(nextId, lastId) <= 0) { NormBlock* block = stream_buffer.Find(nextId); if (NULL != block) @@ -2725,7 +2922,7 @@ bool NormStreamObject::LockBlocks(NormBlockId firstId, NormBlockId lastId, const block->SetPending(0, numData); // TBD - should this reset auto-parity, too? block->SetLastNackTime(currentTime); } - nextId++; + Increment(nextId); } return true; } // end NormStreamObject::LockBlocks() @@ -2756,7 +2953,8 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) { if (stream_sync) { - if (blockId < stream_sync_id) + //if (blockId < stream_sync_id) + if (Compare(blockId, stream_sync_id) < 0) { // it's an old block or stream is _very_ broken // (nothing to update) @@ -2764,7 +2962,8 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) } else { - if (blockId < stream_next_id) + //if (blockId < stream_next_id) + if (Compare(blockId, stream_next_id) < 0) { // it's pending or complete (nothing to update) return true; @@ -2773,40 +2972,43 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) { if (pending_mask.IsSet()) { - if (pending_mask.CanSet(blockId)) + if (pending_mask.CanSet(blockId.GetValue())) { - UINT32 numBits = pending_mask.Delta(blockId, stream_next_id) + 1; - pending_mask.SetBits(stream_next_id, numBits); - stream_next_id = blockId + 1; + UINT32 numBits = (UINT32)Difference(blockId, stream_next_id) + 1; + pending_mask.SetBits(stream_next_id.GetValue(), numBits); + //stream_next_id = blockId + 1; + stream_next_id = blockId; + Increment(stream_next_id); // Handle potential stream_sync_id wrap - NormBlockId delta = stream_next_id - stream_sync_id; - if (delta > NormBlockId(2*pending_mask.GetSize())) + // Note: stream_next_id is always greater than stream_sync_id + UINT32 delta = (UINT32)Difference(stream_next_id, stream_sync_id); + if (delta > 2*pending_mask.GetSize()) GetFirstPending(stream_sync_id); return true; } else { // Stream broken - NormBlockId firstPending; - GetFirstPending(firstPending); return false; } } else { - NormBlockId delta = blockId - stream_next_id + 1; - if (delta > NormBlockId(pending_mask.GetSize())) + UINT32 delta = (UINT32)Difference(blockId, stream_next_id) + 1; + if (delta > pending_mask.GetSize()) { // Stream broken return false; } else { - pending_mask.SetBits(blockId, pending_mask.GetSize()); - stream_next_id = blockId + NormBlockId(pending_mask.GetSize()); + pending_mask.SetBits(blockId.GetValue(), pending_mask.GetSize()); + // stream_next_id = block_id + pending_mask.GetSize(); + stream_next_id = blockId; + Increment(stream_next_id, pending_mask.GetSize()); // Handle potential stream_sync_id wrap - NormBlockId delta = stream_next_id - stream_sync_id; - if (delta > NormBlockId(2*pending_mask.GetSize())) + UINT32 delta = (UINT32)Difference(stream_next_id, stream_sync_id); + if (delta > (2*pending_mask.GetSize())) GetFirstPending(stream_sync_id); return true; } @@ -2826,20 +3028,26 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) } pending_mask.Clear(); - pending_mask.SetBits(blockId, pending_mask.GetSize()); + pending_mask.SetBits(blockId.GetValue(), pending_mask.GetSize()); stream_sync = true; stream_sync_id = blockId; - stream_next_id = blockId + pending_mask.GetSize(); - + //stream_next_id = blockId + pending_mask.GetSize(); + stream_next_id = blockId; + Increment(stream_next_id, pending_mask.GetSize()); if (NULL != sender) { if (read_init && (NormSenderNode::SYNC_CURRENT != sender->GetSyncPolicy())) { // This is a fresh rx stream, so init the read indices read_init = false; - read_index.block = blockId; // for initial SYNC_STREAM sync, this will be zero (stream beginning) - read_index.segment = 0; + PLOG(PL_DEBUG, "NormStreamObject::StreamUpdateStatus() syncing stream to blockId: %lu\n", + (unsigned long)blockId.GetValue()); + read_index.block = blockId; + if (0 != blockId.GetValue()) stream_broken = true; + read_index.segment = 0; + read_index.offset = 0; read_offset = 0; + sender->DecrementResyncCount(); // correction since stream sync here will falsely increment } } @@ -2879,18 +3087,23 @@ UINT16 NormStreamObject::ReadSegment(NormBlockId blockId, { // (TBD) compare blockId with stream_buffer.RangeLo() and stream_buffer.RangeHi() NormBlock* block = stream_buffer.Find(blockId); - if (!block) + if (NULL == block) { //DMSG(0, "NormStreamObject::ReadSegment() stream starved (1)\n"); - if (!stream_buffer.IsEmpty() && (blockId < stream_buffer.RangeLo())) - PLOG(PL_ERROR, "NormStreamObject::ReadSegment() error: attempted to read old block> %lu\n", (UINT32)blockId); + //if (!stream_buffer.IsEmpty() && (blockId < stream_buffer.RangeLo())) + if (!stream_buffer.IsEmpty() && (Compare(blockId, stream_buffer.RangeLo()) < 0)) + { + PLOG(PL_ERROR, "NormStreamObject::ReadSegment() error: attempted to read old block> %lu\n", + (unsigned long)blockId.GetValue()); + } return 0; } // (TBD) should we check to see if "blockId > write_index.block" ? if ((blockId == write_index.block) && (segmentId >= write_index.segment)) { //DMSG(0, "NormStreamObject::ReadSegment(blk>%lu seg>%hu) stream starved (2) (write_index>%lu:%hu)\n", - // (UINT32)blockId, (UINT16)segmentId, (UINT32)write_index.block, (UINT16)write_index.segment); + // (unsigned long)blockId.GetValue(), (UINT16)segmentId, + // (unsigned long)write_index.block.GetValue(), (UINT16)write_index.segment); return 0; } block->UnsetPending(segmentId); @@ -2903,7 +3116,8 @@ UINT16 NormStreamObject::ReadSegment(NormBlockId blockId, //INT32 offsetDelta = segmentOffset - tx_offset; //if (offsetDelta > 0) tx_offset = segmentOffset; // Update tx_index if (blockId::segmentId > tx_index) - if (blockId > tx_index.block) + //if (blockId > tx_index.block) + if (Compare(blockId, tx_index.block) > 0) { tx_index.block = blockId; tx_index.segment = segmentId; @@ -2920,8 +3134,9 @@ UINT16 NormStreamObject::ReadSegment(NormBlockId blockId, //offsetDelta = write_offset - tx_offset; //ASSERT(offsetDelta >= 0); //if ((UINT32)offsetDelta < object_size.LSB()) - ASSERT(tx_index.block <= write_index.block); - UINT32 blockDelta = write_index.block - tx_index.block; + //ASSERT(write_index.block >= tx_index.block); + ASSERT(Compare(write_index.block, tx_index.block) >= 0); + UINT32 blockDelta = (UINT32)Difference(write_index.block, tx_index.block); if (blockDelta <= (block_pool.GetTotal() >> 1)) { NormBlock* b = stream_buffer.Find(stream_buffer.RangeLo()); @@ -2982,29 +3197,32 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, read_init = false; read_index.block = blockId; read_index.segment = segmentId; + read_index.offset = 0; read_offset = segmentOffset; read_ready = true; } - - if ((blockId < read_index.block) || - ((blockId == read_index.block) && - (segmentId < read_index.segment))) + + //if ((blockId < read_index.block) || + if ((Compare(blockId, read_index.block) < 0) || + ((blockId == read_index.block) && (segmentId < read_index.segment))) { PLOG(PL_DEBUG, "NormStreamObject::WriteSegment() block/segment < read_index!?\n"); return false; - } + } + /* // if (segmentOffset < read_offset) UINT32 diff = segmentOffset - read_offset; if ((diff > 0x80000000) || ((0x80000000 == diff) && (segmentOffset > read_offset))) { - PLOG(PL_ERROR, "NormStreamObject::WriteSegment() diff:%lu segmentOffset:%lu < read_offset:%lu \n", - diff, segmentOffset, read_offset); + PLOG(PL_DEBUG, "NormStreamObject::WriteSegment() diff:%lu segmentOffset:%lu < read_offset:%lu \n", + (unsigned long)diff, (unsigned long)segmentOffset, (unsigned long)read_offset); return false; } + */ NormBlock* block = stream_buffer.Find(blockId); - if (!block) + if (NULL == block) { bool broken = false; bool dataLost = false; @@ -3012,19 +3230,35 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, { block = stream_buffer.Find(stream_buffer.RangeLo()); ASSERT(NULL != block); - if (blockId < block->GetId()) + //if (blockId < block->GetId()) + if (Compare(blockId, block->GetId()) < 0) { PLOG(PL_DEBUG, "NormStreamObject::WriteSegment() blockId too old!?\n"); return false; } while (block->IsPending()) { - broken = true; // Force read_index forward, giving app a chance to read data - read_index.block = block->GetId(); - block->GetFirstPending(read_index.segment); + NormSegmentId firstPending; + block->GetFirstPending(firstPending); + if (read_index.block != block->GetId()) + { + // The block's id > read_index, so force read_index forward + read_index.block = block->GetId(); + read_index.segment = firstPending; + read_index.offset = 0; + broken = true; + stream_broken = true; + } + if (read_index.segment < firstPending) + { + read_index.segment = firstPending; + read_index.offset = 0; + broken = true; + stream_broken = true; + } NormBlock* tempBlock = block; - UINT32 tempOffset = read_offset; + //UINT32 tempOffset = read_offset; NormStreamObject::Index tempIndex = read_index; // (TBD) uncomment the code so that only a single // UPDATED notification is posted??? @@ -3036,21 +3270,26 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, block = stream_buffer.Find(stream_buffer.RangeLo()); if (tempBlock == block) { - if ((tempOffset == read_offset) && + if (//(tempOffset == read_offset) && (tempIndex.block == read_index.block) && - (tempIndex.segment == read_index.segment)) + (tempIndex.segment == read_index.segment) && + (tempIndex.offset == read_index.offset)) { - // App didn't want any data here, purge segment + // App didn't grab data here, purge segment dataLost = true; + broken = true; + stream_broken = true; block->UnsetPending(read_index.segment++); + read_index.offset = 0; if (read_index.segment >= ndata) { - read_index.block++; + Increment(read_index.block); read_index.segment = 0; stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); block = NULL; + //TRACE("Prune(%u) 6 ...\n", (unsigned int)read_index.block.GetValue()); Prune(read_index.block, false); break; } @@ -3063,30 +3302,37 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, break; } } // end while (block->IsPending()) - if (block) + if (NULL != block) { - if (block->GetId() == read_index.block) - { - read_index.block++; - read_index.segment = 0; - Prune(read_index.block, false); - } + NormBlockId bid = block->GetId(); stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); + if (bid == read_index.block) + { + stream_broken = true; + broken = true; + Increment(read_index.block); + read_index.segment = 0; + read_index.offset = 0; + //TRACE("Prune(%lu) 7 ...\n", (unsigned long)read_index.block.GetValue()); + Prune(read_index.block, false); + } } } // end while (block_pool.IsEmpty() || !stream_buffer.CanInsert(blockId)) if (broken) { PLOG(PL_WARN, "NormStreamObject::WriteSegment() node>%lu obj>%hu blk>%lu seg>%hu broken stream ...\n", - LocalNodeId(), (UINT16)transport_id, (UINT32)blockId, (UINT16)segmentId); + (unsigned long)LocalNodeId(), (UINT16)transport_id, + (unsigned long)blockId.GetValue(), (UINT16)segmentId); if (dataLost) PLOG(PL_ERROR, "NormStreamObject::WriteSegment() broken stream data not read by app!\n"); } block = block_pool.Get(); block->SetId(blockId); block->ClearPending(); - ASSERT(blockId >= read_index.block); + //ASSERT(blockId >= read_index.block); + ASSERT(Compare(blockId, read_index.block) >= 0); bool success = stream_buffer.Insert(block); ASSERT(success); } // end if (!block) @@ -3121,12 +3367,11 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, } else if (session.RcvrIsLowDelay()) { - INT32 delta = blockId - read_index.block; + INT32 delta = (UINT32)Difference(blockId, read_index.block); if (delta > session.RcvrGetMaxDelay()) read_ready = true; } } - } return true; } // end NormStreamObject::WriteSegment() @@ -3139,10 +3384,11 @@ void NormStreamObject::Prune(NormBlockId blockId, bool updateStatus) NormBlock* block; while (NULL != (block = block_buffer.Find(block_buffer.RangeLo()))) { - if (block->GetId() < blockId) + //if (block->GetId() < blockId) + if (Compare(block->GetId(), blockId) < 0) { resync = true; - pending_mask.Unset(block->GetId()); + pending_mask.Unset(block->GetId().GetValue()); block_buffer.Remove(block); sender->PutFreeBlock(block); } @@ -3154,11 +3400,12 @@ void NormStreamObject::Prune(NormBlockId blockId, bool updateStatus) NormBlockId firstId; if (GetFirstPending(firstId)) { - if (firstId < blockId) + //if (firstId < blockId) + if (Compare(firstId, blockId) < 0) { resync = true; - UINT32 count = pending_mask.Delta(blockId, firstId); - pending_mask.UnsetBits(firstId, count); + UINT32 numBits = (UINT32)Difference(blockId, firstId); + pending_mask.UnsetBits(firstId.GetValue(), numBits); } } while (!StreamUpdateStatus(blockId)) @@ -3169,12 +3416,12 @@ void NormStreamObject::Prune(NormBlockId blockId, bool updateStatus) if (GetFirstPending(firstId)) { NormBlock* block = block_buffer.Find(firstId); - if (block) + if (NULL != block) { block_buffer.Remove(block); sender->PutFreeBlock(block); } - pending_mask.Unset(firstId); + pending_mask.Unset(firstId.GetValue()); } else { @@ -3182,7 +3429,20 @@ void NormStreamObject::Prune(NormBlockId blockId, bool updateStatus) break; } } - if (resync) sender->IncrementResyncCount(); + if (resync) + { + //if (updateStatus && (read_index.block < blockId) && notify_on_update) + //if (updateStatus && notify_on_update && (Compare(read_index.block, blockId) < 0)) + if (notify_on_update) + { + // If "updateStatus" is true, then Prune() was invoked due to SQUELCH + // This will prompt the app to read from the stream which, in turn + // will force the stream read_index forward + notify_on_update = false; + session.Notify(NormController::RX_OBJECT_UPDATED, sender, this); + } + sender->IncrementResyncCount(); + } } } // end NormStreamObject::Prune() @@ -3191,18 +3451,19 @@ void NormStreamObject::Prune(NormBlockId blockId, bool updateStatus) bool NormStreamObject::PassiveReadCheck(NormBlockId blockId, NormSegmentId segmentId) { bool result; - if (read_index.block < blockId) + //if (read_index.block < blockId) + if (Compare(read_index.block, blockId) < 0) result = true; - else if (read_index.block > blockId) - result = false; - else // read_index.block == blockId + else if (read_index.block == blockId) result = (read_index.segment <= segmentId); + else //if (read_index.block > blockId) + result = false; /*if (result) { TRACE("NormStreamObject::PassiveReadCheck() %s\n", result ? "data not yet read by app" : "OK"); TRACE(" (read_index>%lu:%hu check>%lu:%hu usage:%u)\n", - (UINT32)read_index.block, read_index.segment, - (UINT32)blockId, segmentId, GetCurrentBufferUsage()); + (unsigned long)read_index.block.GetValue(), (UINT16)read_index.segment, + (unsigned long)blockId.GetValue(), (UINT16)segmentId, GetCurrentBufferUsage()); } */ return result; @@ -3210,9 +3471,41 @@ bool NormStreamObject::PassiveReadCheck(NormBlockId blockId, NormSegmentId segme bool NormStreamObject::Read(char* buffer, unsigned int* buflen, bool seekMsgStart) { + // NOTES: ReadPrivate() always returns "true" if it reads any bytes + // It returns "false" upon a stream break. + // To properly reset "update" notification, it needs to + // be called until it passes back a read byte count of zero. + // So, upon read byte count < bytesWanted, we add an extra + // call to ReadPrivate() to do that. I.e., so the "update" + // notification is reset upon a short read count. The + // "stream_broken" state variable is used for when this + // extra ReadPrivate() call reveals the broken stream condition. + if (stream_broken && !seekMsgStart) + { + if (NULL != buflen) *buflen = 0; + stream_broken = false; + return false; + } + unsigned int bytesWanted; + if (NULL == buflen) + { + bytesWanted = 0; + buflen = &bytesWanted; + } + else + { + bytesWanted = *buflen; + } bool result = ReadPrivate(buffer, buflen, seekMsgStart); - if (!read_ready) - SetNotifyOnUpdate(true); // reset notification when stream is no longer "read_ready" + if (!read_ready) notify_on_update = true; + if (!seekMsgStart && result && (0 != *buflen) && (*buflen < bytesWanted)) + { + char dummyBuffer[8]; + unsigned int dummyCount = 8; + stream_broken = ReadPrivate(dummyBuffer, &dummyCount, false) ? false : true; + ASSERT(0 == dummyCount); + if (!read_ready) notify_on_update = true; + } return result; } // end NormStreamObject::Read() @@ -3230,12 +3523,13 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek Retain(); unsigned int bytesRead = 0; unsigned int bytesToRead = *buflen; + bool brokenStream = false; do { NormBlock* block = stream_buffer.Find(read_index.block); if (NULL == block) { - //DMSG(0, "NormStreamObject::Read() stream buffer empty (1) (sbEmpty:%d)\n", stream_buffer.IsEmpty()); + //DMSG(0, "NormStreamObject::ReadPrivate() stream buffer empty (1) (sbEmpty:%d)\n", stream_buffer.IsEmpty()); read_ready = false; *buflen = bytesRead; if (bytesRead > 0) @@ -3255,30 +3549,37 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek else if (session.RcvrIsLowDelay()) { // Has the sender moved forward to the next FEC blocks - INT32 delta = max_pending_block - read_index.block; - if (delta > session.RcvrGetMaxDelay()) - forceForward = true; + if (Compare(max_pending_block, read_index.block) >= 0) + { + INT32 delta = (INT32)Difference(max_pending_block, read_index.block); + if (delta > session.RcvrGetMaxDelay()) + forceForward = true; + } } else { NormBlockId firstPending; if (GetFirstPending(firstPending)) { - if (read_index.block < firstPending) + //if (read_index.block < firstPending) + if (Compare(read_index.block, firstPending) < 0) forceForward = true; } } if (forceForward) { - read_index.block++; + Increment(read_index.block); read_index.segment = 0; + read_index.offset = 0; + //TRACE("Prune(%u) 1 ...\n", (unsigned int)read_index.block.GetValue()); + if (!seekMsgStart) brokenStream = true; Prune(read_index.block, false); continue; } else { Release(); - return seekMsgStart ? false : true; + return (seekMsgStart || brokenStream) ? false : true; } } } @@ -3289,7 +3590,7 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek if (NULL == segment) { //DMSG(0, "NormStreamObject::ReadPrivate(%lu:%hu) stream buffer empty (read_offset>%lu) (2)\n", - // (UINT32)read_index.block, read_index.segment, read_offset); + // (unsigned long)read_index.block.GetValue(), read_index.segment, (unsigned long)read_offset); read_ready = false; *buflen = bytesRead; if (bytesRead > 0) @@ -3308,43 +3609,58 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek } else if (session.RcvrIsLowDelay()) { - // Has the sender moved forward to the next FEC blocks - INT32 delta = max_pending_block - read_index.block; - if (delta > session.RcvrGetMaxDelay()) - forceForward = true; + // Has the sender moved forward to the next FEC block + if (Compare(max_pending_block, read_index.block) >= 0) + { + INT32 delta = (UINT32)Difference(max_pending_block, read_index.block); + if (delta > session.RcvrGetMaxDelay()) + forceForward = true; + } } else { NormBlockId firstPending; if (GetFirstPending(firstPending)) { - if (read_index.block < firstPending) + //if (read_index.block < firstPending) + if (Compare(read_index.block, firstPending) < 0) forceForward = true; } } if (forceForward) { - // Force read_index forward and try again. + // Force read_index forward and try again if seeking msg start if (++read_index.segment >= ndata) { stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); - read_index.block++; + Increment(read_index.block); read_index.segment = 0; + read_index.offset = 0; + //TRACE("Prune(%u) 2 ...\n", (unsigned int)read_index.block.GetValue()); Prune(read_index.block, false); // prevents repair requests for data we - } // no longer care about - continue; + } // no longer care about (i.e, prior to this block) + if (!seekMsgStart) brokenStream = true; + continue; // attempt to find next valid read_index beyond break } else { Release(); - return seekMsgStart ? false : true; + return (seekMsgStart || brokenStream) ? false : true; } } } // end if (NULL = segment) ASSERT(NULL != segment); read_ready = true; + if (brokenStream) + { + // We have found the next valid read_index and block/segment but stream + // continuity was broken so report immediately by returning false + Release(); + return false; + } + UINT16 length = NormDataMsg::ReadStreamPayloadLength(segment); if (0 == length) { @@ -3362,8 +3678,9 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); - read_index.block++; + Increment(read_index.block); read_index.segment = 0; + read_index.offset = 0; //Prune(read_index.block, false); } continue; @@ -3373,76 +3690,86 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek else if (length > segment_size) { // This segment is an invalid segment because its length is too long - Release(); + read_ready = false; if (bytesRead > 0) { // Go ahead and return data read thus so far *buflen = bytesRead; + Release(); return true; } else { // Skip this invalid segment PLOG(PL_ERROR, "NormStreamObject::ReadPrivate() node>%lu obj>%hu blk>%lu seg>%hu invalid stream segment!\n", - LocalNodeId(), (UINT16)transport_id, (UINT32)read_index.block, read_index.segment, read_offset, length); + (unsigned long)LocalNodeId(), (UINT16)transport_id, + (unsigned long)read_index.block.GetValue(), (UINT16)read_index.segment); if (++read_index.segment >= ndata) { stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); - read_index.block++; + Increment(read_index.block); read_index.segment = 0; + read_index.offset = 0; + //TRACE("Prune(%u) 3 ...\n", (unsigned int)read_index.block.GetValue()); Prune(read_index.block, false); } *buflen = 0; - read_ready = false; //DetermineReadReadiness(); + Release(); return false; } } + UINT32 segmentOffset = NormDataMsg::ReadStreamPayloadOffset(segment); - // if (read_offset < segmentOffset) + + /*// if (read_offset < segmentOffset) UINT32 diff = read_offset - segmentOffset; if ((diff > 0x80000000) || ((0x80000000 == diff) && (read_offset > segmentOffset))) { - Release(); + read_ready = false; //DetermineReadReadiness(); if (bytesRead > 0) { // Go ahead and return data read thus so far *buflen = bytesRead; + Release(); return true; } else { PLOG(PL_WARN, "NormStreamObject::ReadPrivate() node>%lu obj>%hu blk>%lu seg>%hu broken stream (read_offset:%lu segmentOffset:%lu)!\n", - (UINT32)LocalNodeId(), (UINT16)transport_id, (UINT32)read_index.block, (UINT16)read_index.segment, - read_offset, segmentOffset); + (unsigned long)LocalNodeId(), (UINT16)transport_id, + (unsigned long)read_index.block.GetValue(), (UINT16)read_index.segment, + (unsigned long)read_offset, (unsigned long)segmentOffset); read_offset = segmentOffset; *buflen = 0; - read_ready = false; //DetermineReadReadiness(); + Release(); return false; } - } + }*/ + //UINT32 index = read_offset - segmentOffset; - UINT32 index = read_offset - segmentOffset; - - if ((length > 0) && (index >= length)) + if ((length > 0) && (read_index.offset >= length)) { - Release(); + read_ready = false; //DetermineReadReadiness(); if (bytesRead > 0) { // Go ahead and return data read thus so far *buflen = bytesRead; + Release(); return true; } else { - PLOG(PL_ERROR, "NormStreamObject::ReadPrivate() node>%lu obj>%hu blk>%lu seg>%hu mangled stream! index:%hu length:%hu " - "read_offset:%lu segmentOffset:%lu\n", - LocalNodeId(), (UINT16)transport_id, (UINT32)read_index.block, read_index.segment, index, length, read_offset, segmentOffset); + PLOG(PL_ERROR, "NormStreamObject::ReadPrivate() node>%lu obj>%hu blk>%lu seg>%hu mangled stream! " + "offset:%hu length:%hu read_offset:%lu segmentOffset:%lu\n", + (unsigned long)read_index.block.GetValue(), (UINT16)read_index.segment, + read_index.offset, length, (unsigned long)read_offset, (unsigned long)segmentOffset); // Reset our read_offset ... read_offset = segmentOffset; + read_index.offset = 0; *buflen = 0; - read_ready = false; //DetermineReadReadiness(); + Release(); return false; } } @@ -3465,13 +3792,15 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek //block->DetachSegment(read_index.segment); //segment_pool.Put(segment); block->UnsetPending(read_index.segment++); + read_index.offset = 0; if (read_index.segment >= ndata) { stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); - read_index.block++; + Increment(read_index.block); read_index.segment = 0; + //TRACE("Prune(%u) 4 ...\n", (unsigned int)read_index.block.GetValue()); Prune(read_index.block, false); } continue; @@ -3479,25 +3808,25 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek else { read_offset += (msgStart - 1); - index += (msgStart - 1); + read_index.offset = (msgStart - 1); seekMsgStart = false; } } - UINT16 count = length - index; + UINT16 count = length - read_index.offset; count = MIN(count, bytesToRead); #ifdef SIMULATE - UINT16 simCount = index + count + NormDataMsg::GetStreamPayloadHeaderLength(); + UINT16 simCount = read_index.offset + count + NormDataMsg::GetStreamPayloadHeaderLength(); simCount = (simCount < SIM_PAYLOAD_MAX) ? (SIM_PAYLOAD_MAX - simCount) : 0; - memcpy(buffer+bytesRead, segment+index+NormDataMsg::GetStreamPayloadHeaderLength(), simCount); + memcpy(buffer+bytesRead, segment+read_index.offset+NormDataMsg::GetStreamPayloadHeaderLength(), simCount); #else - memcpy(buffer+bytesRead, segment+index+NormDataMsg::GetStreamPayloadHeaderLength(), count); + memcpy(buffer+bytesRead, segment+read_index.offset+NormDataMsg::GetStreamPayloadHeaderLength(), count); #endif // if/else SIMULATE - index += count; + read_index.offset += count; bytesRead += count; read_offset += count; bytesToRead -= count; - if (index >= length) + if (read_index.offset >= length) { bool streamEnded = (0 == NormDataMsg::ReadStreamPayloadLength(segment)); // NormDataMsg::StreamPayloadFlagIsSet(segment, NormDataMsg::FLAG_STREAM_END); @@ -3506,13 +3835,15 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek // block->DetachSegment(read_index.segment); // segment_pool.Put(segment); block->UnsetPending(read_index.segment++); + read_index.offset = 0; if (read_index.segment >= ndata) { stream_buffer.Remove(block); block->EmptyToPool(segment_pool); block_pool.Put(block); - read_index.block++; + Increment(read_index.block); read_index.segment = 0; + //TRACE("Prune(%u) 5 ...\n", (unsigned int)read_index.block.GetValue()); Prune(read_index.block, false); if (0 == bytesToRead) read_ready = DetermineReadReadiness(); @@ -3522,7 +3853,6 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek if (0 == bytesToRead) read_ready = (NULL != block->GetSegment(read_index.segment)); } - if (streamEnded) { PLOG(PL_DEBUG, "NormStreamObject::ReadPrivate() stream ended by sender 2\n"); @@ -3531,11 +3861,11 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek sender->DeleteObject(this); } } - } while ((bytesToRead > 0) || seekMsgStart); // end while (len > 0) + } while ((bytesToRead > 0) || seekMsgStart); *buflen = bytesRead; Release(); return true; -} // end NormStreamObject::Read() +} // end NormStreamObject::ReadPrivate() void NormStreamObject::Terminate() @@ -3553,8 +3883,8 @@ void NormStreamObject::Terminate() if (block->IsPending()) { NormBlockId blockId = block->GetId(); - pending_mask.Unset(blockId); - repair_mask.Unset(blockId); + pending_mask.Unset(blockId.GetValue()); + repair_mask.Unset(blockId.GetValue()); NormBlock* b = FindBlock(blockId); if (b) { @@ -3563,8 +3893,10 @@ void NormStreamObject::Terminate() } if (!pending_mask.IsSet()) { - pending_mask.Set(write_index.block); - stream_next_id = write_index.block + 1; + pending_mask.Set(write_index.block.GetValue()); + //stream_next_id = write_index.block + 1; + stream_next_id = write_index.block; + Increment(stream_next_id); } } stream_buffer.Remove(block); @@ -3585,8 +3917,8 @@ void NormStreamObject::Terminate() if (b->IsPending()) { NormBlockId blockId = b->GetId(); - pending_mask.Unset(blockId); - repair_mask.Unset(blockId); + pending_mask.Unset(blockId.GetValue()); + repair_mask.Unset(blockId.GetValue()); NormBlock* c = FindBlock(blockId); if (c) { @@ -3595,8 +3927,10 @@ void NormStreamObject::Terminate() } if (!pending_mask.IsSet()) { - pending_mask.Set(write_index.block); - stream_next_id = write_index.block + 1; + pending_mask.Set(write_index.block.GetValue()); + //stream_next_id = write_index.block + 1; + stream_next_id = write_index.block; + Increment(stream_next_id); } } stream_buffer.Remove(b); @@ -3620,7 +3954,7 @@ void NormStreamObject::Terminate() block->SetPending(write_index.segment); if (++write_index.segment >= ndata) { - write_index.block++; + Increment(write_index.block); write_index.segment = 0; } flush_pending = true; @@ -3628,7 +3962,7 @@ void NormStreamObject::Terminate() } // end NormStreamObject::Terminate() UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) -{ +{ UINT32 nBytes = 0; do { @@ -3636,7 +3970,8 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) { if (0 != len) { - PLOG(PL_ERROR, "NormStreamObject::Write() error: stream is closing (len:%lu eom:%d)\n", len, eom); + PLOG(PL_ERROR, "NormStreamObject::Write() error: stream is closing (len:%lu eom:%d)\n", + (unsigned long)len, eom); len = 0; } break; @@ -3647,8 +3982,9 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) //INT32 deltaOffset = write_offset - tx_offset; // (TBD) deprecate tx_offset //ASSERT(deltaOffset >= 0); //if (deltaOffset >= (INT32)object_size.LSB()) - ASSERT(write_index.block >= tx_index.block); - UINT32 deltaBlock = write_index.block - tx_index.block; + //ASSERT(write_index.block >= tx_index.block); + ASSERT(Compare(write_index.block, tx_index.block) >= 0); + UINT32 deltaBlock = (UINT32)Difference(write_index.block, tx_index.block); if (deltaBlock > (block_pool.GetTotal() >> 1)) { write_vacancy = false; @@ -3670,8 +4006,8 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) if (push_mode) { NormBlockId blockId = block->GetId(); - pending_mask.Unset(blockId); - repair_mask.Unset(blockId); + pending_mask.Unset(blockId.GetValue()); + repair_mask.Unset(blockId.GetValue()); NormBlock* b = FindBlock(blockId); if (b) { @@ -3680,8 +4016,10 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) } if (!pending_mask.IsSet()) { - pending_mask.Set(write_index.block); - stream_next_id = write_index.block + 1; + pending_mask.Set(write_index.block.GetValue()); + //stream_next_id = write_index.block + 1; + stream_next_id = write_index.block; + Increment(stream_next_id); } } else @@ -3708,7 +4046,7 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) block->ClearPending(); bool success = stream_buffer.Insert(block); ASSERT(success); - } // end if (!block) + } // end if (NULL == block) char* segment = block->GetSegment(write_index.segment); if (NULL == segment) { @@ -3722,8 +4060,8 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) if (push_mode) { NormBlockId blockId = b->GetId(); - pending_mask.Unset(blockId); - repair_mask.Unset(blockId); + pending_mask.Unset(blockId.GetValue()); + repair_mask.Unset(blockId.GetValue()); NormBlock* c = FindBlock(blockId); if (c) { @@ -3732,8 +4070,10 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) } if (!pending_mask.IsSet()) { - pending_mask.Set(write_index.block); - stream_next_id = write_index.block + 1; + pending_mask.Set(write_index.block.GetValue()); + //stream_next_id = write_index.block + 1; + stream_next_id = write_index.block; + Increment(stream_next_id); } } else @@ -3789,7 +4129,7 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) ProtoTime currentTime; currentTime.GetCurrentTime(); block->SetLastNackTime(currentTime); - write_index.block++; + Increment(write_index.block); write_index.segment = 0; } } @@ -3804,8 +4144,10 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) flush_pending = true; else if (!stream_closing) flush_pending = false; - if ((0 != nBytes) || (FLUSH_NONE != flush_mode)) - session.TouchSender(); + if ((0 != nBytes) || (FLUSH_NONE != flush_mode)) + { + session.TouchSender(); + } } else { @@ -3882,7 +4224,12 @@ char* NormSimObject::RetrieveSegment(NormBlockId blockId, // NormObjectTable::NormObjectTable() - : table((NormObject**)NULL), range_max(0), range(0), +#ifndef USE_PROTO_TREE + : table((NormObject**)NULL), +#else + : +#endif // if/else USE_PROTO_TREE + range_max(0), range(0), count(0), size(0) { } @@ -3894,7 +4241,9 @@ NormObjectTable::~NormObjectTable() bool NormObjectTable::Init(UINT16 rangeMax, UINT16 tableSize) { - if (table) Destroy(); + Destroy(); + if (!rangeMax || !tableSize) return false; +#ifndef USE_PROTO_TREE // Make sure (rangeMax > 0) and tableSize is greater than 0 and 2^n if (!rangeMax || !tableSize) return false; if (0 != (tableSize & 0x07)) tableSize = (tableSize >> 3) + 1; @@ -3905,8 +4254,10 @@ bool NormObjectTable::Init(UINT16 rangeMax, UINT16 tableSize) } memset(table, 0, tableSize*sizeof(char*)); hash_mask = tableSize - 1; +#endif // !USE_PROTO_TREE range_max = rangeMax; - range = 0; + count = range = 0; + size = NormObjectSize(0); return true; } // end NormObjectTable::Init() @@ -3936,25 +4287,31 @@ void NormObjectTable::SetRangeMax(UINT16 rangeMax) range_max = rangeMax; } // end NormObjectTable::SetRangeMax() +#ifdef USE_PROTO_TREE + +NormObject* NormObjectTable::Find(const NormObjectId& objectId) const +{ + if ((0 == range) || (objectId < range_lo) || (objectId > range_hi)) + return NULL; + else + return tree.Find(objectId.GetValuePtr(), 8*sizeof(UINT16)); +} // end NormObjectTable::Find() + void NormObjectTable::Destroy() { - if (table) + NormObject* obj; + while(NULL != (obj = Find(range_lo))) { - NormObject* obj; - while((obj = Find(range_lo))) - { - // TBD - should we issue PURGED/ABORTED notifications here??? - // (We haven't since this is destroyed only when session is terminated) - // or when a NormSenderNode is deleted - Remove(obj); - obj->Release(); - } - delete []table; - table = (NormObject**)NULL; - range = range_max = 0; - } + // TBD - should we issue PURGED/ABORTED notifications here??? + // (We haven't since this is destroyed only when session is terminated) + // or when a NormSenderNode is deleted + Remove(obj); + obj->Release(); + } + count = range = range_max = 0; } // end NormObjectTable::Destroy() +#else NormObject* NormObjectTable::Find(const NormObjectId& objectId) const { if (0 != range) @@ -3971,6 +4328,27 @@ NormObject* NormObjectTable::Find(const NormObjectId& objectId) const } } // end NormObjectTable::Find() +void NormObjectTable::Destroy() +{ + if (NULL != table) + { + NormObject* obj; + while((obj = Find(range_lo))) + { + // TBD - should we issue PURGED/ABORTED notifications here??? + // (We haven't since this is destroyed only when session is terminated) + // or when a NormSenderNode is deleted + Remove(obj); + obj->Release(); + } + delete[] table; + table = (NormObject**)NULL; + count = range = range_max = 0; + } +} // end NormObjectTable::Destroy() + +#endif // if/else USE_PROTO_TREE + bool NormObjectTable::CanInsert(NormObjectId objectId) const { if (0 != range) @@ -4009,11 +4387,12 @@ bool NormObjectTable::Insert(NormObject* theObject) range_lo = range_hi = objectId; range = 1; } - if (objectId < range_lo) + else if (objectId < range_lo) { UINT16 newRange = range_lo - objectId + range; if (newRange > range_max) return false; range_lo = objectId; + ASSERT(range_lo <= range_hi); range = newRange; } else if (objectId > range_hi) @@ -4021,8 +4400,14 @@ bool NormObjectTable::Insert(NormObject* theObject) UINT16 newRange = objectId - range_hi + range; if (newRange > range_max) return false; range_hi = objectId; + ASSERT(range_lo <= range_hi); range = newRange; } + +#ifdef USE_PROTO_TREE + ASSERT(NULL == Find(theObject->GetId())); + tree.Insert(*theObject); +#else UINT16 index = ((UINT16)objectId) & hash_mask; NormObject* prev = NULL; NormObject* entry = table[index]; @@ -4037,12 +4422,59 @@ bool NormObjectTable::Insert(NormObject* theObject) table[index] = theObject; ASSERT(((NULL != entry) ? (objectId != entry->GetId()) : true)); theObject->next = entry; +#endif // if/else USE_PROTO_TREE count++; size = size + theObject->GetSize(); theObject->Retain(); return true; } // end NormObjectTable::Insert() +#ifdef USE_PROTO_TREE +bool NormObjectTable::Remove(NormObject* theObject) +{ + ASSERT(NULL != theObject); + const NormObjectId& objectId = theObject->GetId(); + if (range) + { + if ((objectId < range_lo) || (objectId > range_hi)) return false; + if (range > 1) + { + if (objectId == range_lo) + { + const NormObject* next = static_cast(theObject->GetNext()); + if (NULL == next) next = static_cast(tree.GetHead()); + ASSERT(NULL != next); + range_lo = next->GetId(); + ASSERT(range_lo <= range_hi); + range = range_hi - range_lo + 1; + } + else if (objectId == range_hi) + { + const NormObject* prev = static_cast(theObject->GetPrev()); + if (NULL == prev) prev = static_cast(tree.GetTail()); + ASSERT(NULL != prev); + range_hi = prev->GetId(); + ASSERT(range_lo <= range_hi); + range = range_hi - range_lo + 1; + } + } + else + { + range = 0; + } + ASSERT(NULL != tree.Find(theObject->GetId().GetValuePtr(), 8*sizeof(UINT16))); + tree.Remove(*theObject); + count--; + size = size - theObject->GetSize(); + theObject->Release(); + return true; + } + else + { + return false; + } +} // end NormObjectTable::Remove() +#else bool NormObjectTable::Remove(NormObject* theObject) { ASSERT(NULL != theObject); @@ -4111,7 +4543,6 @@ bool NormObjectTable::Remove(NormObject* theObject) endex = index; entry = NULL; UINT16 offset = 0; - //printf("preving i:%lu endex:%lu lo:%lu hi:%lu\n", i, endex, (UINT16)range_lo, (UINT16) range_hi); NormObjectId prevId = range_lo; do { @@ -4120,7 +4551,6 @@ bool NormObjectTable::Remove(NormObject* theObject) if ((entry = table[i])) { NormObjectId id = (UINT16)objectId - offset; - //printf("Looking for id:%lu at index:%lu\n", (UINT16)transport_id, i); while(entry && (entry->GetId() != id)) { if ((entry->GetId() < objectId) && @@ -4151,7 +4581,73 @@ bool NormObjectTable::Remove(NormObject* theObject) return false; } } // end NormObjectTable::Remove() +#endif // if/else USE_PROTO_TREE +#ifdef USE_PROTO_TREE +NormObjectTable::Iterator::Iterator(NormObjectTable& objectTable) + : table(objectTable), iterator(objectTable.tree, false, objectTable.range_lo.GetValuePtr(), 8*sizeof(UINT16)) +{ + next_object = iterator.GetNextItem(); + ASSERT((NULL == next_object) || (objectTable.range_lo == next_object->GetId())) +} + +void NormObjectTable::Iterator::Reset() +{ + iterator.Reset(false, table.range_lo.GetValuePtr(), 8*sizeof(UINT16)); + next_object = iterator.GetNextItem(); + ASSERT(table.IsEmpty() || (NULL != next_object)); + ASSERT((NULL == next_object) || (table.range_lo == next_object->GetId())) +} + +NormObject* NormObjectTable::Iterator::GetNextObject() +{ + NormObject* nextObj = next_object; + if (NULL != nextObj) + { + next_object = iterator.GetNextItem(); + if (NULL == next_object) + { + iterator.Reset(); + next_object = iterator.GetNextItem(); + if (next_object->GetId() <= nextObj->GetId()) + next_object = NULL; + } + else if (next_object->GetId() <= nextObj->GetId()) + { + next_object = NULL; + } + } + return nextObj; +} // end NormObjectTable::Iterator::GetNextObject() + +NormObject* NormObjectTable::Iterator::GetPrevObject() +{ + if (!iterator.IsReversed()) + { + iterator.Reset(true, table.range_hi.GetValuePtr(), 8*sizeof(UINT16)); + next_object = iterator.GetPrevItem(); + } + NormObject* prevObj = next_object; + if (NULL != prevObj) + { + next_object = iterator.GetPrevItem(); + if (NULL == next_object) + { + iterator.Reset(true); + next_object = iterator.GetPrevItem(); + if (next_object->GetId() >= prevObj->GetId()) + next_object = NULL; + } + else if (next_object->GetId() >= prevObj->GetId()) + { + next_object = NULL; + } + } + return prevObj; +} // end NormObjectTable::Iterator::GetPrevObject() + + +#else NormObjectTable::Iterator::Iterator(const NormObjectTable& objectTable) : table(objectTable), reset(true) { @@ -4274,3 +4770,5 @@ NormObject* NormObjectTable::Iterator::GetPrevObject() } } } // end NormObjectTable::Iterator::GetPrevObject() + +#endif // if/else USE_PROTO_TREE diff --git a/src/common/normPrecode.cpp b/src/common/normPrecode.cpp index 5864a09..10c5ddc 100644 --- a/src/common/normPrecode.cpp +++ b/src/common/normPrecode.cpp @@ -1,7 +1,16 @@ #include "protoApp.h" #include "normFile.h" + +// Commment this #define out to use new, faster RS8 codec instead +// (TBD - provide option use 16-bit Reed Solomon for large block sizes?) +//#define USE_MDP_FEC + +#ifdef USE_MDP_FEC +#include "normEncoderMDP.h" +#else #include "normEncoderRS8.h" +#endif // if/else USE_MDP_FEC #include // for BYTE_ORDER macro #include // for atoi() @@ -48,8 +57,8 @@ class NormPrecodeApp : public ProtoApp { case 8: { - long* outPtr = (long*)&result; - long* inPtr = (long*)&offset; + UINT32* outPtr = (UINT32*)&result; + UINT32* inPtr = (UINT32*)&offset; outPtr[0] = ntohl(inPtr[1]); outPtr[1] = ntohl(inPtr[0]); break; @@ -559,7 +568,11 @@ bool NormPrecodeApp::Encode() InitInterleaver(numOutputSegments); // 1) Init our FEC encoder +#ifdef USE_MDP_FEC + NormEncoderMDP encoder; +#else NormEncoderRS8 encoder; +#endif // if/else USE_MDP_FEC if (!encoder.Init(num_data, num_parity, dataSegmentSize)) // 4 CRC bytes are _not_ encoded { PLOG(PL_FATAL, "npc: error initializing FEC encoder\n"); @@ -607,7 +620,7 @@ bool NormPrecodeApp::Encode() parityVec[i] = pvec; pvec += segment_size; } - + // 3) Build "meta_data" segment for the file // (TBD) This could be built directly into iBuffer segment zero char metaData[SEGMENT_MAX+4]; @@ -638,7 +651,7 @@ bool NormPrecodeApp::Encode() strncpy(metaData+8, ptr, segment_size - 12); // 2) Read "in_file" segments, encode, and output to "out_file" - PLOG(PL_FATAL, "npc: encoding file ... (progress: 0%%)"); + PLOG(PL_ALWAYS, "npc: encoding file ... (progress: 0%%)"); // State to track/display encoding progress NormFile::Offset progressThreshold = numOutputSegments / 100; double progressIncrement = 100.0; @@ -659,61 +672,57 @@ bool NormPrecodeApp::Encode() { NormFile::Offset interleaverOffset = ComputeInterleaverOffset(outputSegmentId, numOutputSegments); char* segment = useBuffering ? (iBuffer + (interleaverOffset % interleaverBytes)) : iBuffer; - - switch (parityReady) + if (parityReady) { - case false: + // D) Output parity segment for this block + char* pvec = parityVec[num_parity - parityCount]; + memcpy(segment, pvec, dataSegmentSize); + if (0 == --parityCount) { - // Read and encode a segment - inputSegmentId++; - if (1 == inputSegmentId) + memset(parityVec[0], 0, num_parity * segment_size); + parityReady = false; + blockId++; + } + } + else + { + // Read and encode a segment + inputSegmentId++; + if (1 == inputSegmentId) + { + // A) Segment '0' is the meta-data segment + memcpy(segment, metaData, dataSegmentSize); + } + else + { + // B) Read in data portion of next "segment" + unsigned int bytesToRead; + if (inputSegmentId != numInputSegments) { - // A) Segment '0' is the meta-data segment - memcpy(segment, metaData, dataSegmentSize); + bytesToRead = dataSegmentSize; } else { - // B) Read in data portion of next "segment" - unsigned int bytesToRead; - if (inputSegmentId != numInputSegments) - { - bytesToRead = dataSegmentSize; - } - else - { - memset(segment, 0, dataSegmentSize); - bytesToRead = lastFecSegSize; - } - if (in_file.Read(segment, bytesToRead) != bytesToRead) - { - PLOG(PL_FATAL, "\nnpc: unexpected error reading input file: %s\n", GetErrorString()); - return false; - } + memset(segment, 0, dataSegmentSize); + bytesToRead = lastFecSegSize; } - // C) Encode and check for parity readiness - encoder.Encode(outputSegmentId, segment, parityVec); - unsigned int numData = (blockId != lastBlockId) ? fecBlockSize : lastBlockSize; - if (numData == ++parityCount) + if (in_file.Read(segment, bytesToRead) != bytesToRead) { - parityCount = num_parity; - parityReady = true; + PLOG(PL_FATAL, "\nnpc: unexpected error reading input file: %s\n", GetErrorString()); + return false; } - break; } - case true: + // C) Encode and check for parity readiness + //TRACE("outputSegmentId:%lu\n", outputSegmentId); + + encoder.Encode(outputSegmentId % fecBlockSize, segment, parityVec); + unsigned int numData = (blockId != lastBlockId) ? fecBlockSize : lastBlockSize; + if (numData == ++parityCount) { - // D) Output parity segment for this block - char* pvec = parityVec[num_parity - parityCount]; - memcpy(segment, pvec, dataSegmentSize); - if (0 == --parityCount) - { - memset(parityVec[0], 0, num_parity * segment_size); - parityReady = false; - blockId++; - } - break; + parityCount = num_parity; + parityReady = true; } - } + } // E) Calculate and add CRC32 checksum to each "segment" UINT32 checksum = ComputeCRC32(segment, dataSegmentSize); checksum = htonl(checksum); @@ -760,17 +769,17 @@ bool NormPrecodeApp::Encode() if (++progressCounter >= progressThreshold) { if (progressPercent < 9) - PLOG(PL_FATAL, "\b\b\b%d%%)", progressPercent + 1); + PLOG(PL_ALWAYS, "\b\b\b%d%%)", progressPercent + 1); else if (progressPercent < 99) - PLOG(PL_FATAL, "\b\b\b\b%d%%)", progressPercent + 1); + PLOG(PL_ALWAYS, "\b\b\b\b%d%%)", progressPercent + 1); if (progressPercent < 99) progressPercent++; progressCounter = 0; } } if (progressPercent < 10) - PLOG(PL_FATAL, "\b\b\b100%%)\n"); + PLOG(PL_ALWAYS, "\b\b\b100%%)\n"); else - PLOG(PL_FATAL, "\b\b\b\b100%%)\n"); + PLOG(PL_ALWAYS, "\b\b\b\b100%%)\n"); in_file.Close(); out_file.Close(); @@ -815,7 +824,11 @@ bool NormPrecodeApp::Decode() InitInterleaver(numInputSegments); // 2) init FEC decoder +#ifdef USE_MDP_FEC + NormDecoderMDP decoder; +#else NormDecoderRS8 decoder; +#endif // if/else USE_MDP_FEC unsigned int dataSegmentSize = segment_size - 4; // leaves space for our CRC if (!decoder.Init(num_data, num_parity, dataSegmentSize)) { @@ -894,7 +907,7 @@ bool NormPrecodeApp::Decode() // Read and decode each block in "in_file" NormFile::Offset fecBlockId = 0; - NormFile::Offset outFileSize; + NormFile::Offset outFileSize = 0; NormFile::Offset lastInterleaverBlockId = numInputSegments / interleaver_size; NormFile::Offset lastInterleaverBytes = (numInputSegments % interleaver_size) * segment_size; @@ -1084,9 +1097,9 @@ bool NormPrecodeApp::Decode() if (++progressCounter >= progressThreshold) { if (progressPercent < 9) - PLOG(PL_FATAL, "\b\b\b%d%%)", progressPercent + 1); + PLOG(PL_ALWAYS, "\b\b\b%d%%)", progressPercent + 1); else if (progressPercent < 99) - PLOG(PL_FATAL, "\b\b\b\b%d%%)", progressPercent + 1); + PLOG(PL_ALWAYS, "\b\b\b\b%d%%)", progressPercent + 1); if (progressPercent < 99) progressPercent++; progressCounter = 0; } @@ -1094,9 +1107,9 @@ bool NormPrecodeApp::Decode() } // end while (inputSegmentId < numInputSegments) if (progressPercent < 10) - PLOG(PL_FATAL, "\b\b\b100%%)\n"); + PLOG(PL_ALWAYS, "\b\b\b100%%)\n"); else - PLOG(PL_FATAL, "\b\b\b\b100%%)\n"); + PLOG(PL_ALWAYS, "\b\b\b\b100%%)\n"); // Cleanup, cleanup delete[] iBuffer; diff --git a/src/common/normSegment.cpp b/src/common/normSegment.cpp index d7f8da1..0cb468c 100644 --- a/src/common/normSegment.cpp +++ b/src/common/normSegment.cpp @@ -6,7 +6,6 @@ NormSegmentPool::NormSegmentPool() { } - NormSegmentPool::~NormSegmentPool() { Destroy(); @@ -282,7 +281,6 @@ bool NormBlock::TxUpdate(NormSegmentId nextId, NormSegmentId lastId, UINT16 numData, UINT16 numParity, UINT16 erasureCount) { bool increasedRepair = false; - if (nextId < numData) { // Explicit data repair request @@ -314,6 +312,8 @@ bool NormBlock::TxUpdate(NormSegmentId nextId, NormSegmentId lastId, } else { + // TBD - double-check this ... not sure this is exactly right + // (may need to always do explicit repair here?) // Use any remaining fresh parity ... if (parity_count < parityAvailable) { @@ -342,7 +342,7 @@ bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, UINT16 numData, UINT16 numParity, UINT16 erasureCount) { PLOG(PL_TRACE, "NormBlock::HandleSegmentRequest() blk>%lu seg>%hu:%hu erasures:%hu\n", - (UINT32)blk_id, (UINT16)nextId, (UINT16)lastId, erasureCount); + (unsigned long)blk_id.GetValue(), (UINT16)nextId, (UINT16)lastId, erasureCount); bool increasedRepair = false; if (nextId < numData) { @@ -375,6 +375,8 @@ bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, } else { + // TBD - double-check this. It may not be exactly right + // (may need to alwayds do explicitr repair here) // Use any remaining fresh parity ... if (parity_count < parityAvailable) { @@ -399,15 +401,16 @@ bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, return increasedRepair; } // end NormBlock::HandleSegmentRequest() -// (TBD) this should return true is something is appending, false otherwise +// (TBD) this should return true if something is appended, false otherwise bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, NormObjectId objectId, bool repairInfo, UINT8 fecId, UINT8 fecM, UINT16 numData, - UINT16 segmentSize) + UINT16 payloadMax) { + bool requestAppended = false; NormRepairRequest req; req.SetFlag(NormRepairRequest::SEGMENT); if (repairInfo) req.SetFlag(NormRepairRequest::INFO); @@ -447,12 +450,14 @@ bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, { if (0 == cmd.PackRepairRequest(req)) { + prevForm = NormRepairRequest::INVALID; PLOG(PL_WARN, "NormBlock::AppendRepairAdv() warning: full msg\n"); break; } + requestAppended = true; } req.SetForm(form); - cmd.AttachRepairRequest(req, segmentSize); // (TBD) error check + cmd.AttachRepairRequest(req, payloadMax); // (TBD) error check prevForm = form; } switch(form) @@ -480,9 +485,11 @@ bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, { if (0 == cmd.PackRepairRequest(req)) PLOG(PL_WARN, "NormBlock::AppendRepairAdv() warning: full msg\n"); + else + requestAppended = true; } } - return true; + return requestAppended; } // end NormBlock::AppendRepairAdv() NormObjectSize NormBlock::GetBytesPending(UINT16 numData, @@ -521,8 +528,9 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, UINT16 numParity, NormObjectId objectId, bool pendingInfo, - UINT16 segmentSize) + UINT16 payloadMax) { + bool requestAppended = false; NormSegmentId nextId = 0; NormSegmentId endId; if (erasure_count > numParity) @@ -580,11 +588,13 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, { if (0 == nack.PackRepairRequest(req)) { + prevForm = NormRepairRequest::INVALID; // so we don't re-attempt pack PLOG(PL_WARN, "NormBlock::AppendRepairRequest() warning: full NACK msg\n"); break; } + requestAppended = true; } - nack.AttachRepairRequest(req, segmentSize); // (TBD) error check + nack.AttachRepairRequest(req, payloadMax); // (TBD) error check req.SetForm(form); prevForm = form; } @@ -614,8 +624,10 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, { if (0 == nack.PackRepairRequest(req)) PLOG(PL_WARN, "NormBlock::AppendRepairRequest() warning: full NACK msg\n"); + else + requestAppended = true; } - return true; + return requestAppended; } // end NormBlock::AppendRepairRequest() NormBlockPool::NormBlockPool() @@ -671,7 +683,12 @@ void NormBlockPool::Destroy() } // end NormBlockPool::Destroy() NormBlockBuffer::NormBlockBuffer() - : table((NormBlock**)NULL), range_max(0), range(0) +#ifdef USE_PROTO_TREE + : +#else + : table((NormBlock**)NULL), +#endif // if/else USE_PROTO_TREE + range_max(0), range(0), fec_block_mask(0) { } @@ -680,16 +697,17 @@ NormBlockBuffer::~NormBlockBuffer() Destroy(); } -bool NormBlockBuffer::Init(unsigned long rangeMax, unsigned long tableSize) +bool NormBlockBuffer::Init(unsigned long rangeMax, unsigned long tableSize, UINT32 fecBlockMask) { - if (table) Destroy(); + Destroy(); // Make sure tableSize is greater than 0 and 2^n if (!rangeMax || !tableSize) { PLOG(PL_FATAL, "NormBlockBuffer::Init() bad range(%lu) or tableSize(%lu)\n", - rangeMax, tableSize); + rangeMax, tableSize); return false; } +#ifndef USE_PROTO_TREE if (0 != (tableSize & 0x07)) tableSize = (tableSize >> 3) + 1; if (!(table = new NormBlock*[tableSize])) { @@ -698,14 +716,38 @@ bool NormBlockBuffer::Init(unsigned long rangeMax, unsigned long tableSize) } memset(table, 0, tableSize*sizeof(char*)); hash_mask = tableSize - 1; +#endif // !USE_PROTO_TREE range_max = rangeMax; range = 0; + fec_block_mask = fecBlockMask; return true; } // end NormBlockBuffer::Init() +#ifdef USE_PROTO_TREE +void NormBlockBuffer::Destroy() +{ + NormBlock* block; + while((block = Find(range_lo))) + { + PLOG(PL_ERROR, "NormBlockBuffer::Destroy() buffer not empty!?\n"); + Remove(block); + delete block; + } + range_max = range = 0; +} // end NormBlockBuffer::Destroy() + +NormBlock* NormBlockBuffer::Find(const NormBlockId& blockId) const +{ + if ((0 == range) || (Compare(blockId, range_lo) < 0) || (Compare(blockId, range_hi) > 0)) + return NULL; + else + return tree.Find(blockId.GetValuePtr(), 8*sizeof(UINT32)); +} // end NormBlockBuffer::Find() + +#else + void NormBlockBuffer::Destroy() { - range_max = range = 0; if (table) { NormBlock* block; @@ -717,18 +759,19 @@ void NormBlockBuffer::Destroy() } delete []table; table = (NormBlock**)NULL; - range_max = 0; - } + } + range_max = range = 0; } // end NormBlockBuffer::Destroy() NormBlock* NormBlockBuffer::Find(const NormBlockId& blockId) const { if (range) { - if ((blockId < range_lo) || (blockId > range_hi)) + //if ((blockId < range_lo) || (blockId > range_hi)) + if ((Compare(blockId, range_lo) < 0) || (Compare(blockId, range_hi) > 0)) return (NormBlock*)NULL; - NormBlock* theBlock = table[((UINT32)blockId) & hash_mask]; - while (theBlock && (blockId != theBlock->GetId())) + NormBlock* theBlock = table[(blockId.GetValue()) & hash_mask]; + while ((NULL != theBlock) && (blockId != theBlock->GetId())) theBlock = theBlock->next; return theBlock; } @@ -738,21 +781,38 @@ NormBlock* NormBlockBuffer::Find(const NormBlockId& blockId) const } } // end NormBlockBuffer::Find() +#endif // if/else USE_PROTO_TREE + +NormBlockId NormBlockBuffer::RangeMin() const +{ + if (range_max > 1) + { + NormBlockId rangeMin = range_hi; + Decrement(rangeMin, range_max - 1); + return rangeMin; + } + else + { + return range_lo; + } +} // end NormBlockBuffer::RangeMin() bool NormBlockBuffer::CanInsert(NormBlockId blockId) const { if (0 != range) { - if (blockId < range_lo) + // if (blockId < range_lo) + if (Compare(blockId, range_lo) < 0) { - if ((range_lo - blockId + range) > range_max) + if (((UINT32)Difference(range_lo, blockId) + range) > range_max) return false; else return true; } - else if (blockId > range_hi) + // else if (blockId > range_hi) + else if (Compare(blockId, range_hi) > 0) { - if ((blockId - range_hi + range) > range_max) + if (((UINT32)Difference(blockId, range_hi) + range) > range_max) return false; else return true; @@ -772,29 +832,38 @@ bool NormBlockBuffer::CanInsert(NormBlockId blockId) const bool NormBlockBuffer::Insert(NormBlock* theBlock) { const NormBlockId& blockId = theBlock->GetId(); - if (!range) + if (0 == range) { range_lo = range_hi = blockId; range = 1; } - if (blockId < range_lo) + // else if (blockId < range_lo) + else if (Compare(blockId, range_lo) < 0) { - UINT32 newRange = range_lo - blockId + range; + UINT32 newRange = (UINT32)Difference(range_lo, blockId) + range; if (newRange > range_max) return false; range_lo = blockId; range = newRange; } - else if (blockId > range_hi) + // else if (blockId > range_hi) + else if (Compare(blockId, range_hi) > 0) { - UINT32 newRange = blockId - range_hi + range; + UINT32 newRange = (UINT32)Difference(blockId, range_hi) + range; if (newRange > range_max) return false; range_hi = blockId; range = newRange; } - UINT32 index = ((UINT32)blockId) & hash_mask; + ASSERT(Compare(range_hi, range_lo) >= 0); + // else unchanged range +#ifdef USE_PROTO_TREE + ASSERT(NULL == Find(theBlock->GetId())); + tree.Insert(*theBlock); +#else + UINT32 index = blockId.GetValue() & hash_mask; NormBlock* prev = NULL; NormBlock* entry = table[index]; - while (entry && (entry->GetId() < blockId)) + // while (entry && (entry->GetId() < blockId)) + while ((NULL != entry) && (Compare(entry->GetId(), blockId) < 0)) { prev = entry; entry = entry->next; @@ -803,21 +872,66 @@ bool NormBlockBuffer::Insert(NormBlock* theBlock) prev->next = theBlock; else table[index] = theBlock; - ASSERT((entry ? (blockId != entry->GetId()) : true)); - theBlock->next = entry; +#endif // if/else USE_PROTO_TREE return true; } // end NormBlockBuffer::Insert() -bool NormBlockBuffer::Remove(const NormBlock* theBlock) +#ifdef USE_PROTO_TREE + +bool NormBlockBuffer::Remove(NormBlock* theBlock) +{ + ASSERT(NULL != theBlock); + const NormBlockId& blockId = theBlock->GetId(); + switch (range) + { + case 0: + return false; // empty NormBlockBuffer + case 1: + if (blockId != range_lo) + return false; // out-of-range + range = 0; + break; + default: + if ((Compare(blockId, range_lo) < 0) || (Compare(blockId, range_hi) > 0)) + return false; // out-of-range + if (blockId == range_lo) + { + const NormBlock* next = static_cast(theBlock->GetNext()); + if (NULL == next) next = static_cast(tree.GetHead()); + ASSERT(NULL != next); + range_lo = next->GetId(); + range = Difference(range_hi, range_lo) + 1; + } + else if (blockId == range_hi) + { + const NormBlock* prev = static_cast(theBlock->GetPrev()); + if (NULL == prev) prev = static_cast(tree.GetTail()); + ASSERT(NULL != prev); + range_hi = prev->GetId(); + range = Difference(range_hi, range_lo) + 1; + } + // else range unchanged + break; + } + ASSERT(NULL != tree.Find(theBlock->GetId().GetValuePtr(), 8*sizeof(UINT32))); + tree.Remove(*theBlock); + return true; +} // end NormBlockBuffer::Remove() + +#else + +bool NormBlockBuffer::Remove(NormBlock* theBlock) { ASSERT(NULL != theBlock); if (range) { const NormBlockId& blockId = theBlock->GetId(); - if ((blockId < range_lo) || (blockId > range_hi)) return false; - UINT32 index = ((UINT32)blockId) & hash_mask; + // if ((blockId < range_lo) || (blockId > range_hi)) + if ((Compare(blockId, range_lo) < 0) || (Compare(blockId, range_hi) > 0)) + return false; + UINT32 index = blockId.GetValue() & hash_mask; NormBlock* prev = NULL; NormBlock* entry = table[index]; while (entry && (entry->GetId() != blockId)) @@ -825,8 +939,8 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) prev = entry; entry = entry->next; } - if (!entry) return false; - if (prev) + if (NULL == entry) return false; + if (NULL != prev) prev->next = entry->next; else table[index] = entry->next; @@ -835,7 +949,7 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) { if (blockId == range_lo) { - // Find next entry for range_lo + // Find next entry for new range_lo UINT32 i = index; UINT32 endex; if (range <= hash_mask) @@ -849,29 +963,33 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) { ++i &= hash_mask; offset++; - if ((entry = table[i])) + if (NULL != (entry = table[i])) { - //NormBlockId id = (UINT32)index + offset; - NormBlockId id = (UINT32)blockId + offset; + // NormBlockId id = blockId + offset; + NormBlockId id = blockId; + Increment(id, offset); while(entry && (entry->GetId() != id)) { - if ((entry->GetId() > blockId) && - (entry->GetId() < nextId)) nextId = entry->GetId(); + // if ((entry->GetId() > blockId) && (entry->GetId() < nextId) + if ((Compare(entry->GetId(), blockId) > 0) && + (Compare(entry->GetId(), nextId) < 0)) + { + nextId = entry->GetId(); + } entry = entry->next; - } - if (entry) break; + if (NULL != entry) break; } } while (i != endex); - if (entry) + if (NULL != entry) range_lo = entry->GetId(); else range_lo = nextId; - range = range_hi - range_lo + 1; + range = (UINT32)Difference(range_hi, range_lo) + 1; } else if (blockId == range_hi) { - // Find prev entry for range_hi + // Find prev entry for new range_hi UINT32 i = index; UINT32 endex; if (range <= hash_mask) @@ -880,7 +998,6 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) endex = index; entry = NULL; UINT32 offset = 0; - //printf("preving i:%lu endex:%lu lo:%lu hi:%lu\n", i, endex, (UINT32)range_lo, (UINT32) range_hi); NormBlockId prevId = range_lo; do { @@ -888,23 +1005,27 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) offset++; if ((entry = table[i])) { - //NormBlockId id = (UINT32)index - offset; - NormBlockId id = (UINT32)blockId - offset; - //printf("Looking for id:%lu at index:%lu\n", (UINT32)id, i); + // NormBlockId id = blockId - offset; + NormBlockId id = blockId; + Decrement(id, offset); while(entry && (entry->GetId() != id)) { - if ((entry->GetId() < blockId) && - (entry->GetId() > prevId)) prevId = entry->GetId(); + // if ((entry->GetId() < blockId) && (entry->GetId() > prevId)) + if ((Compare(entry->GetId(), blockId) < 0) && + (Compare(entry->GetId(), prevId) > 0)) + { + prevId = entry->GetId(); + } entry = entry->next; } - if (entry) break; + if (NULL != entry) break; } } while (i != endex); - if (entry) + if (NULL != entry) range_hi = entry->GetId(); else range_hi = prevId; - range = range_hi - range_lo + 1; + range = (UINT32)Difference(range_hi, range_lo) + 1; } } else @@ -919,6 +1040,48 @@ bool NormBlockBuffer::Remove(const NormBlock* theBlock) } } // end NormBlockBuffer::Remove() +#endif // if/else USE_PROTO_TREE + + +#ifdef USE_PROTO_TREE +NormBlockBuffer::Iterator::Iterator(NormBlockBuffer& blockBuffer) + : buffer(blockBuffer), iterator(blockBuffer.tree, false, blockBuffer.range_lo.GetValuePtr(), 8*sizeof(UINT32)) +{ + next_block = iterator.GetNextItem(); + ASSERT((NULL == next_block) || (blockBuffer.range_lo == next_block->GetId())) +} + +void NormBlockBuffer::Iterator::Reset() +{ + iterator.Reset(false, buffer.range_lo.GetValuePtr(), 8*sizeof(UINT32)); + next_block = iterator.GetNextItem(); + ASSERT(buffer.IsEmpty() || (NULL != next_block)); + ASSERT((NULL == next_block) || (buffer.range_lo == next_block->GetId())) +} // end NormBlockBuffer::Iterator::Reset() + +NormBlock* NormBlockBuffer::Iterator::GetNextBlock() +{ + NormBlock* nextBlock = next_block; + if (NULL != nextBlock) + { + next_block = iterator.GetNextItem(); + if (NULL == next_block) + { + iterator.Reset(); + next_block = iterator.GetNextItem(); + if (buffer.Compare(next_block->GetId(), nextBlock->GetId()) <= 0) + next_block = NULL; + } + else if (buffer.Compare(next_block->GetId(), nextBlock->GetId()) <= 0) + { + next_block = NULL; + } + } + return nextBlock; +} // end NormBlockBuffer::Iterator::GetNextBlock() + +#else + NormBlockBuffer::Iterator::Iterator(const NormBlockBuffer& blockBuffer) : buffer(blockBuffer), reset(true) { @@ -941,30 +1104,38 @@ NormBlock* NormBlockBuffer::Iterator::GetNextBlock() } else { - if (buffer.range && - (index < buffer.range_hi) && - (index >= buffer.range_lo)) + // if (buffer.range && (index < buffer.range_hi) && (index >= buffer.range_lo)) + if ((0 != buffer.range) && + (buffer.Compare(index, buffer.range_hi) < 0) && + (buffer.Compare(index, buffer.range_lo) >= 0)) { // Find next entry _after_ current "index" - UINT32 i = index; + UINT32 i = index.GetValue();; UINT32 endex; - if ((UINT32)(buffer.range_hi - index) <= buffer.hash_mask) - endex = buffer.range_hi & buffer.hash_mask; + // if ((UINT32)(buffer.range_hi - index) <= buffer.hash_mask) + if ((UINT32)buffer.Difference(buffer.range_hi, index) <= buffer.hash_mask) + endex = buffer.range_hi.GetValue() & buffer.hash_mask; else - endex = index; + endex = index.GetValue(); UINT32 offset = 0; NormBlockId nextId = buffer.range_hi; do { ++i &= buffer.hash_mask; offset++; - NormBlockId id = (UINT32)index + offset; + // NormBlockId id = (UINT32)index + offset; + NormBlockId id = index; + buffer.Increment(id, offset); ASSERT(i < 256); NormBlock* entry = buffer.table[i]; while ((NULL != entry ) && (entry->GetId() != id)) { - if ((entry->GetId() > index) && (entry->GetId() < nextId)) + // if ((entry->GetId() > index) && (entry->GetId() < nextId)) + if ((buffer.Compare(entry->GetId(), index) > 0) && + (buffer.Compare(entry->GetId(), nextId) < 0)) + { nextId = entry->GetId(); + } entry = NormBlockBuffer::Next(entry); } if (entry) @@ -983,3 +1154,5 @@ NormBlock* NormBlockBuffer::Iterator::GetNextBlock() } } } // end NormBlockBuffer::Iterator::GetNextBlock() + +#endif // if/else USE_PROTO_TREE diff --git a/src/common/normSession.cpp b/src/common/normSession.cpp index e498cdf..56161ed 100644 --- a/src/common/normSession.cpp +++ b/src/common/normSession.cpp @@ -28,12 +28,23 @@ const UINT16 NormSession::DEFAULT_RX_CACHE_MAX = 256; const int NormSession::DEFAULT_ROBUST_FACTOR = 20; // default robust factor + +// This is extra stuff defined for NormSocket API extension purposes. As the NormSocket +// extension is finalized, these may be refined/relocated +enum {NORM_SOCKET_VERSION = 1}; +enum NormSocketCommand +{ + NORM_SOCKET_CMD_NULL = 0, // reserved, invalid/null command + NORM_SOCKET_CMD_REJECT, // sent by server-listener to reject invalid connection messages + NORM_SOCKET_CMD_ALIVE // TBD - for NormSocket "keep-alive" option? +}; + NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) : session_mgr(sessionMgr), notify_pending(false), tx_port(0), tx_port_reuse(false), tx_socket_actual(ProtoSocket::UDP), tx_socket(&tx_socket_actual), rx_socket(ProtoSocket::UDP), rx_cap(NULL), rx_port_reuse(false), local_node_id(localNodeId), ttl(DEFAULT_TTL), tos(0), loopback(false), mcast_loopback(false), fragmentation(false), ecn_enabled(false), - tx_rate(DEFAULT_TRANSMIT_RATE/8.0), tx_rate_min(-1.0), tx_rate_max(-1.0), + tx_rate(DEFAULT_TRANSMIT_RATE/8.0), tx_rate_min(-1.0), tx_rate_max(-1.0), tx_residual(0), backoff_factor(DEFAULT_BACKOFF_FACTOR), is_sender(false), tx_robust_factor(DEFAULT_ROBUST_FACTOR), instance_id(0), ndata(DEFAULT_NDATA), nparity(DEFAULT_NPARITY), auto_parity(0), extra_parity(0), @@ -55,12 +66,12 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) grtt_response(false), grtt_current_peak(0.0), grtt_age(0.0), probe_count(1), cc_enable(false), cc_adjust(true), cc_sequence(0), cc_slow_start(true), cc_active(false), flow_control_factor(DEFAULT_FLOW_CONTROL_FACTOR), - cmd_count(0), cmd_buffer(NULL), cmd_length(0), + cmd_count(0), cmd_buffer(NULL), cmd_length(0), syn_status(false), is_receiver(false), rx_robust_factor(DEFAULT_ROBUST_FACTOR), preset_sender(NULL), unicast_nacks(false), receiver_silent(false), rcvr_ignore_info(false), rcvr_max_delay(-1), rcvr_realtime(false), - default_repair_boundary(NormSenderNode::BLOCK_BOUNDARY), + default_repair_boundary(NormSenderNode::BLOCK_BOUNDARY), default_nacking_mode(NormObject::NACK_NORMAL), default_sync_policy(NormSenderNode::SYNC_CURRENT), - rx_cache_count_max(DEFAULT_RX_CACHE_MAX), + rx_cache_count_max(DEFAULT_RX_CACHE_MAX), is_server_listener(false), notify_on_grtt_update(true), ecn_ignore_loss(false), trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0), user_data(NULL), next(NULL) @@ -173,7 +184,7 @@ bool NormSession::Open() } else { - tx_socket = &rx_socket; + tx_socket = &rx_socket; } } if (!rx_socket.IsOpen() && (!tx_only || (&rx_socket == tx_socket))) @@ -187,7 +198,7 @@ bool NormSession::Open() rx_socket.EnableRecvDstAddr(); if (rx_port_reuse) { - // Enable port/addr reuse and bind socket to destination address + // Enable port/addr reuse and bind socket to destination address if (!rx_socket.SetReuse(true)) { PLOG(PL_FATAL, "NormSession::Open() rx_socket::SetReuse() error\n"); @@ -199,19 +210,21 @@ bool NormSession::Open() if (rx_bind_addr.IsValid()) { #ifdef WIN32 - if (!rx_bind_addr.IsMulticast()) // Win32 doesn't like to bind() mcast addr?? + if (rx_bind_addr.IsMulticast()) // Win32 doesn't like to bind() mcast addr?? + PLOG(PL_WARN, "NormSession::Open() warning: WIN32 multicast bind() issue!\n"); + else #endif bindAddr = &rx_bind_addr; } - if(!rx_socket.Bind(address.GetPort(), bindAddr)) + if(!rx_socket.Bind(address.GetPort(), bindAddr)) { PLOG(PL_FATAL, "NormSession::Open() error: rx_socket.Bind() error\n"); Close(); return false; } - if (rx_connect_addr.IsValid() && (0 != rx_connect_addr.GetPort())) + if (rx_connect_addr.IsValid() && (0 != rx_connect_addr.GetPort())) { - // For unicast, we use the "connect()" call to effectively + // For unicast, we use the "connect()" call to effectively // uniquely "bind" our rx_socket to the remote addr. // (it _may_ be the case that "tx_port" == "address.GetPort()" // for this to work?) @@ -432,6 +445,8 @@ bool NormSession::SetSSM(const char* sourceAddress) } } // end NormSession::SetSSM() +// This must be called _before_ sender or receiver is started +// (i.e., before socket(s) are opened) bool NormSession::SetRxPortReuse(bool enableReuse, const char* rxBindAddress, // bind() to / const char* senderAddress, // connect() to / @@ -633,10 +648,14 @@ void NormSession::SetTxRateInternal(double txRate) if (grttQuantizedOld != grtt_quantized) { PLOG(PL_DEBUG, "NormSession::SetTxRateInternal() node>%lu %s to new grtt to: %lf sec\n", - LocalNodeId(), + (unsigned long)LocalNodeId(), (grttQuantizedOld < grtt_quantized) ? "increased" : "decreased", grtt_advertised); - Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + if (notify_on_grtt_update) + { + notify_on_grtt_update = false; + Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + } } // wakeup grtt/cc probing if necessary if (probe_reset) @@ -699,9 +718,10 @@ bool NormSession::StartSender(UINT16 instanceId, UINT32 bufferSpace, UINT16 segmentSize, UINT16 numData, - UINT16 numParity) + UINT16 numParity, + UINT8 fecId) { - if (!IsOpen()) + if (!IsOpen()) { if (!Open()) return false; } @@ -774,7 +794,10 @@ bool NormSession::StartSender(UINT16 instanceId, StopSender(); return false; } - fec_id = 5; + if (0 != fecId) + fec_id = fecId; + else + fec_id = 5; fec_m = 8; #endif } @@ -786,6 +809,7 @@ bool NormSession::StartSender(UINT16 instanceId, StopSender(); return false; } + // TBD - Investigate if fec_id == 129 can also support 16-bit Reed Solomon fec_id = 2; fec_m = 16; } @@ -806,10 +830,15 @@ bool NormSession::StartSender(UINT16 instanceId, else { // for now use RS8 fec_id with no parity (TBD - support "compact" null FEC type) - fec_id = 5; + if (0 != fecId) + fec_id = fecId; + else + fec_id = 5; fec_m = 8; } + fec_block_mask = NormPayloadId::GetFecBlockMask(fec_id, fec_m); + // Initialize optional app-defined command state cmd_count = cmd_length = 0; if (NULL == (cmd_buffer = new char[segmentSize])) @@ -938,15 +967,21 @@ bool NormSession::StartReceiver(unsigned long bufferSize) void NormSession::StopReceiver() { // Iterate sender_tree and close/release sender nodes - NormSenderNode* senderNode = - static_cast(sender_tree.GetRoot()); - while (NULL != senderNode) + if (IsServerListener()) { - sender_tree.DetachNode(senderNode); - senderNode->Close(); - senderNode->Release(); - senderNode = + client_tree.Destroy(); + } + else + { + NormSenderNode* senderNode = static_cast(sender_tree.GetRoot()); + while (NULL != senderNode) + { + sender_tree.DetachNode(senderNode); + senderNode->Close(); + senderNode->Release(); + senderNode = static_cast(sender_tree.GetRoot()); + } } is_receiver = false; if (!is_sender) Close(); @@ -955,14 +990,19 @@ void NormSession::StopReceiver() void NormSession::DeleteRemoteSender(NormSenderNode& senderNode) { // TBD - confirm that "senderNode" is valid??? - sender_tree.DetachNode(&senderNode); + if (IsServerListener()) + client_tree.RemoveNode(senderNode); + else + sender_tree.DetachNode(&senderNode); senderNode.Close(); senderNode.Release(); } // end NormSession::DeleteRemoteSender() -bool NormSession::PreallocateRemoteSender(UINT16 segmentSize, - UINT16 numData, - UINT16 numParity) +bool NormSession::PreallocateRemoteSender(unsigned int bufferSpace, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + unsigned int streamBufferSize) { if (NULL != preset_sender) delete preset_sender; preset_sender = new NormSenderNode(*this, NORM_NODE_ANY); @@ -985,21 +1025,34 @@ bool NormSession::PreallocateRemoteSender(UINT16 segmentSize, { fecId = 5; } - - if (!preset_sender->AllocateBuffers(fecId, 0, fecM, segmentSize, numData, numParity)) + if (!preset_sender->AllocateBuffers(bufferSpace, fecId, 0, fecM, segmentSize, numData, numParity)) { PLOG(PL_ERROR, "NormSession::PreallocateRemoteSender() error: buffer allocation failure!\n"); delete preset_sender; preset_sender = NULL; return false; } + if (0 != streamBufferSize) + { + if (!preset_sender->PreallocateRxStream(streamBufferSize, segmentSize, numData, numParity)) + { + PLOG(PL_ERROR, "NormSession::PreallocateRemoteSender() error: preset_stream allocation failure!\n"); + delete preset_sender; + preset_sender = NULL; + return false; + } + } return true; } // end NormSession::PreallocateRemoteSender() void NormSession::Serve() { - // Only send new data when no other messages are queued for transmission - if (!message_queue.IsEmpty()) return; + // Only send new data when no other messages are queued for transmission + if (!message_queue.IsEmpty()) + { + ASSERT(tx_timer.IsActive()); + return; + } // Queue next sender message NormObjectId objectId; @@ -1020,6 +1073,7 @@ void NormSession::Serve() bool watermarkJustCompleted = false; if (watermark_pending && !flush_timer.IsActive()) { + PLOG(PL_DEBUG, "NormSession::Serve() watermark status check ...\n"); // Determine next message (objectId::blockId::segmentId) to be sent NormObject* nextObj; NormObjectId nextObjectId = next_tx_object_id; @@ -1060,31 +1114,49 @@ void NormSession::Serve() nextSegmentId = static_cast(nextObj)->GetNextSegmentId(); } } + PLOG(PL_DEBUG, " nextPending index>%hu:%lu:%hu\n", + (UINT16)nextObjectId, + (unsigned long)nextBlockId.GetValue(), + (UINT16)nextSegmentId); if (tx_repair_pending) { + PLOG(PL_DEBUG, " tx_repair index>%hu:%lu:%hu\n", + (UINT16)tx_repair_object_min, + (unsigned long)tx_repair_block_min.GetValue(), + (UINT16)tx_repair_segment_min); if ((tx_repair_object_min < nextObjectId) || ((tx_repair_object_min == nextObjectId) && - ((tx_repair_block_min < nextBlockId) || + //((tx_repair_block_min < nextBlockId) || + ((Compare(tx_repair_block_min, nextBlockId) < 0) || ((tx_repair_block_min == nextBlockId) && (tx_repair_segment_min < nextSegmentId))))) { nextObjectId = tx_repair_object_min; nextBlockId = tx_repair_block_min; nextSegmentId = tx_repair_segment_min; - PLOG(PL_DETAIL, "watermark>%hu:%lu:%hu check against repair index>%hu:%lu:%hu\n", - (UINT16)watermark_object_id, (UINT32)watermark_block_id, (UINT16)watermark_segment_id, - (UINT16)nextObjectId, (UINT32)nextBlockId, (UINT16)nextSegmentId); + PLOG(PL_DEBUG, " updated nextPending index>%hu:%lu:%hu\n", + (UINT16)nextObjectId, + (unsigned long)nextBlockId.GetValue(), + (UINT16)nextSegmentId); + } } // end if (tx_repair_pending) + ASSERT(nextBlockId.GetValue() <= (UINT32)0x00ffffff); + + PLOG(PL_DEBUG, " watermark>%hu:%lu:%hu check against next pending index>%hu:%lu:%hu\n", + (UINT16)watermark_object_id, (unsigned long)watermark_block_id.GetValue(), (UINT16)watermark_segment_id, + (UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), (UINT16)nextSegmentId); if ((nextObjectId > watermark_object_id) || ((nextObjectId == watermark_object_id) && - ((nextBlockId > watermark_block_id) || + //((nextBlockId > watermark_block_id) || + ((Compare(nextBlockId, watermark_block_id) > 0) || ((nextBlockId == watermark_block_id) && (nextSegmentId > watermark_segment_id))))) { + PLOG(PL_DEBUG, " calling SenderQueueWatermarkFlush() ...\n"); // The sender tx position is > watermark if (SenderQueueWatermarkFlush()) { @@ -1114,8 +1186,8 @@ void NormSession::Serve() // watermark flush just completed if (watermark_flushes) flush_count = GetTxRobustFactor(); + watermarkJustCompleted = true; } - watermarkJustCompleted = true; } } else @@ -1132,7 +1204,7 @@ void NormSession::Serve() next->ResetReqCount(GetTxRobustFactor()); } } - } // end if (watermark_pending) + } // end if (watermark_pending && !flush_timer.IsActive()) if (NULL != obj) { @@ -1199,7 +1271,7 @@ void NormSession::Serve() { PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender stream flush complete ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode*)NULL, stream); flush_count++; data_active = false; @@ -1234,14 +1306,14 @@ void NormSession::Serve() else { PLOG(PL_ERROR, "NormSession::Serve() pending non-stream obj, no message?.\n"); - ASSERT(repair_timer.IsActive()); + //ASSERT(repair_timer.IsActive()); } } } else { PLOG(PL_ERROR, "NormSession::Serve() node>%lu Warning! message_pool empty.\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } } else @@ -1254,11 +1326,12 @@ void NormSession::Serve() SenderQueueFlush(); else PLOG(PL_DETAIL, "NormSession::Serve() node>%lu NORM_CMD(FLUSH) deferred by pending repairs ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } else if (GetTxRobustFactor() == flush_count) { - PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", LocalNodeId()); + PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", + (unsigned long)LocalNodeId()); Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); @@ -1273,6 +1346,8 @@ void NormSession::SenderSetWatermark(NormObjectId objectId, NormSegmentId segmentId, bool overrideFlush) { + PLOG(PL_DEBUG, "NormSession::SenderSetWatermark() watermark>%hu:%lu:%hu\n", + (UINT16)objectId, (unsigned long)blockId.GetValue(), (UINT16)segmentId); watermark_flushes = overrideFlush; watermark_pending = true; watermark_active = false; @@ -1342,9 +1417,12 @@ void NormSession::SenderRemoveAckingNode(NormNodeId nodeId) { NormAckingNode* theNode = static_cast(acking_node_tree.FindNodeById(nodeId)); - if (theNode) + if (NULL != theNode) { acking_node_tree.DetachNode(theNode); + theNode->Release(); + // TBD - if a watermark was pending and this is the only + // non-pending acker, can we immediately issue WATERMARK_COMPLETED? acking_node_count--; } } // end NormSession::RemoveAckingNode() @@ -1447,8 +1525,7 @@ bool NormSession::SenderQueueWatermarkFlush() blockLen = ndata; else blockLen = watermark_segment_id; - - flush->SetFecPayloadId(fec_id, watermark_block_id, watermark_segment_id, blockLen, fec_m); + flush->SetFecPayloadId(fec_id, watermark_block_id.GetValue(), watermark_segment_id, blockLen, fec_m); NormNodeTreeIterator iterator(acking_node_tree); NormAckingNode* next; @@ -1495,7 +1572,7 @@ bool NormSession::SenderQueueWatermarkFlush() } else { - PLOG(PL_DETAIL, "NormSession::ServeQueueWatermarkFlush() full cmd ...\n"); + PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() full cmd ...\n"); } } if (watermark_pending) @@ -1506,13 +1583,14 @@ bool NormSession::SenderQueueWatermarkFlush() //if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) // flush_count++; QueueMessage(flush); - PLOG(PL_DETAIL, "NormSession::ServeQueueWatermarkFlush() node>%lu cmd queued ...\n", - LocalNodeId()); + PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu cmd queued ...\n", + (unsigned long)LocalNodeId()); } else if (NULL != acking_node_tree.GetRoot()) { ReturnMessageToPool(flush); - PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu watermark ack finished.\n"); + PLOG(PL_DEBUG, "NormSession::ServeQueueWatermarkFlush() node>%lu watermark ack finished.\n", + (unsigned long)LocalNodeId()); Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); return false; } @@ -1526,8 +1604,9 @@ bool NormSession::SenderQueueWatermarkFlush() else { PLOG(PL_ERROR, "NormSession::SenderQueueWatermarkRequest() node>%lu message_pool exhausted! (couldn't req)\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } + PLOG(PL_DEBUG, "NormSession::SenderQueueWatermarkFlush() starting flush timeout: %lf sec ....\n", 2*grtt_advertised); flush_timer.SetInterval(2*grtt_advertised); ActivateTimer(flush_timer); return true; @@ -1566,18 +1645,18 @@ void NormSession::SenderQueueFlush() flush->SetGroupSize(gsize_quantized); flush->SetObjectId(objectId); - flush->SetFecPayloadId(fec_id, blockId, segmentId, obj->GetBlockSize(blockId), fec_m); + flush->SetFecPayloadId(fec_id, blockId.GetValue(), segmentId, obj->GetBlockSize(blockId), fec_m); QueueMessage(flush); if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) flush_count++; PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() node>%lu, flush queued (flush_count:%u)...\n", - LocalNodeId(), flush_count); + (unsigned long)LocalNodeId(), flush_count); } else { PLOG(PL_ERROR, "NormSession::SenderQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } } else @@ -1594,20 +1673,22 @@ void NormSession::SenderQueueFlush() if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) flush_count++; PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() node>%lu squelch queued (flush_count:%u)...\n", - LocalNodeId(), flush_count); + (unsigned long)LocalNodeId(), flush_count); } else { PLOG(PL_ERROR, "NormSession::SenderQueueFlush() warning: node>%lu unable to queue squelch\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } } + PLOG(PL_DEBUG, "NormSession::SenderQueueFlush() starting flush timeout: %lf sec ....\n", 2*grtt_advertised); flush_timer.SetInterval(2*grtt_advertised); ActivateTimer(flush_timer); } // end NormSession::SenderQueueFlush() bool NormSession::OnFlushTimeout(ProtoTimer& /*theTimer*/) { + PLOG(PL_DEBUG, "NormSession::OnFlushTimeout() deactivating flush_timer ....\n"); flush_timer.Deactivate(); PromptSender(); return false; @@ -1631,13 +1712,12 @@ void NormSession::QueueMessage(NormMsg* msg) */ // (TBD) if (0.0 == tx_rate), should we just dump the // message rather than queueing it? - if (!tx_timer.IsActive() && (tx_rate > 0.0)) + if (!tx_timer.IsActive() && (tx_rate > 0.0)) { tx_timer.SetInterval(0.0); ActivateTimer(tx_timer); } - if (msg) - message_queue.Append(msg); + if (NULL != msg) message_queue.Append(msg); } // end NormSesssion::QueueMessage(NormMsg& msg) @@ -1652,7 +1732,7 @@ NormFileObject* NormSession::QueueTxFile(const char* path, return NULL; } NormFileObject* file = new NormFileObject(*this, (NormSenderNode*)NULL, next_tx_object_id); - if (!file) + if (NULL == file) { PLOG(PL_FATAL, "NormSession::QueueTxFile() new file object error: %s\n", GetErrorString()); @@ -1686,7 +1766,7 @@ NormDataObject* NormSession::QueueTxData(const char* dataPtr, PLOG(PL_FATAL, "NormSession::QueueTxData() Error: sender is closed\n"); return NULL; } - NormDataObject* obj = new NormDataObject(*this, (NormSenderNode*)NULL, next_tx_object_id); + NormDataObject* obj = new NormDataObject(*this, (NormSenderNode*)NULL, next_tx_object_id, session_mgr.GetDataFreeFunction()); if (!obj) { PLOG(PL_FATAL, "NormSession::QueueTxData() new data object error: %s\n", @@ -1799,20 +1879,20 @@ bool NormSession::QueueTxObject(NormObject* obj) // The cases when pruning is needed include: // // 1) When the cache bounds dictate: - // i.e., ((count >= count_min) && ((count > count_max) || (size > size_max))), or + // i.e., ((count > count_min) && ((count > count_max) || (size > size_max))), or // 2) When the "tx_table" state (from insert/remove history) doesn't allow // i.e., !tx_table.CanInsert(obj) unsigned long newCount = tx_table.GetCount() + 1; while (!tx_table.CanInsert(obj->GetId()) || - ((newCount >= tx_cache_count_min) && - ((newCount >= tx_cache_count_max) || + ((newCount > tx_cache_count_min) && + ((newCount > tx_cache_count_max) || ((tx_table.GetSize() + obj->GetSize()) > tx_cache_size_max)))) { // Remove oldest non-pending NormObject* oldest = tx_table.Find(tx_table.RangeLo()); if (oldest->IsRepairPending() || oldest->IsPending()) { - PLOG(PL_DEBUG, "NormSession::QueueTxObject() all held objects repair pending:%d (repair active:%d) pending:%d\n", + PLOG(PL_ALWAYS, "NormSession::QueueTxObject() all held objects repair pending:%d (repair active:%d) pending:%d\n", oldest->IsRepairPending(), repair_timer.IsActive(), oldest->IsPending()); posted_tx_queue_empty = false; return false; @@ -1827,7 +1907,7 @@ bool NormSession::QueueTxObject(NormObject* obj) } else { - PLOG(PL_DEBUG, "NormSession::QueueTxObject() asserting flow control for object delay:%lf sec\n", delay); + PLOG(PL_ALWAYS, "NormSession::QueueTxObject() asserting flow control for object delay:%lf sec\n", delay); // TBD - flow control as should allow for TX_QUEUE_VACANCY posting for session ActivateFlowControl(delay, oldest->GetId(), NormController::TX_QUEUE_EMPTY); posted_tx_queue_empty = false; @@ -1906,6 +1986,9 @@ bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, tx_cache_count_max = (countMax > countMin) ? countMax : countMin; if (tx_cache_count_max < 1) tx_cache_count_max = 1; + tx_cache_count_min &= 0x00007fff; // limited to one-half of 16-bit NormObjectId space + tx_cache_count_max &= 0x00007fff; + if (IsSender()) { // Trim/resize the tx_table and tx masks as needed @@ -1932,7 +2015,7 @@ bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, if (!result) { countMax = tx_pending_mask.GetSize(); - if (tx_repair_mask.GetSize() < (INT32)countMax) + if (tx_repair_mask.GetSize() < countMax) countMax = tx_repair_mask.GetSize(); if (tx_cache_count_max > countMax) tx_cache_count_max = countMax; @@ -2020,7 +2103,7 @@ void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket, while (true) { - if (theSocket.RecvFrom(msg.AccessBuffer(), + if (theSocket.RecvFrom(msg.AccessBuffer(), msgLength, msg.AccessAddress())) { @@ -2039,7 +2122,7 @@ void NormSession::TxSocketRecvHandler(ProtoSocket& theSocket, else { // Probably an ICMP "port unreachable" error - if (Address().IsUnicast()) + if (Address().IsUnicast()) Notify(NormController::SEND_ERROR, NULL, NULL); break; } @@ -2078,11 +2161,11 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, unsigned int msgLength = NormMsg::MAX_SIZE; while (true) { - ProtoAddress destAddr; // we get the pkt destAddr to determine unicast/multicast + ProtoAddress destAddr; // we get the pkt destAddr to determine unicast/multicast if (theSocket.RecvFrom(msg.AccessBuffer(), - msgLength, - msg.AccessAddress(), - destAddr)) + msgLength, + msg.AccessAddress(), + destAddr)) { if (0 == msgLength) break; if (msg.InitFromBuffer(msgLength)) @@ -2157,7 +2240,7 @@ void NormSession::RxSocketRecvHandler(ProtoSocket& theSocket, else { // Probably an ICMP "port unreachable" error - if (Address().IsUnicast()) + if (Address().IsUnicast()) Notify(NormController::SEND_ERROR, NULL, NULL); break; } @@ -2363,7 +2446,7 @@ void NormTrace(const struct timeval& currentTime, #endif // if/else _WIN32_WCE PLOG(PL_ALWAYS, "trace>%02d:%02d:%02d.%06lu ", (int)ct->tm_hour, (int)ct->tm_min, (int)ct->tm_sec, (unsigned int)currentTime.tv_usec); - PLOG(PL_ALWAYS, "node>%lu %s>%s ", (UINT32)localId, status, addr.GetHostString()); + PLOG(PL_ALWAYS, "node>%lu %s>%s/%hu ", (unsigned long)localId, status, addr.GetHostString(), addr.GetPort()); bool clrFlag = false; switch (msgType) @@ -2383,28 +2466,28 @@ void NormTrace(const struct timeval& currentTime, seq, //data.IsData() ? "DATA" : "PRTY", (UINT16)data.GetObjectId(), - (UINT32)data.GetFecBlockId(fecM), + (unsigned long)data.GetFecBlockId(fecM).GetValue(), (UINT16)data.GetFecSymbolId(fecM)); if (data.IsStream()) { UINT32 offset = NormDataMsg::ReadStreamPayloadOffset(data.GetPayload()); - PLOG(PL_ALWAYS, "offset>%lu ", offset); - } - /* - if (data.IsData() && data.IsStream()) - { - //if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_MSG_START)) - UINT16 msgStartOffset = NormDataMsg::ReadStreamPayloadMsgStart(data.GetPayload()); - if (0 != msgStartOffset) + PLOG(PL_ALWAYS, "offset>%lu ", (unsigned long)offset); + + /*if (data.GetFecSymbolId(fecM) < 32) { - PLOG(PL_ALWAYS, "start word>%hu ", msgStartOffset - 1); + //if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_MSG_START)) + UINT16 msgStartOffset = NormDataMsg::ReadStreamPayloadMsgStart(data.GetPayload()); + if (0 != msgStartOffset) + { + PLOG(PL_ALWAYS, "start word>%hu ", msgStartOffset - 1); + } + //if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_STREAM_END)) + if (0 == NormDataMsg::ReadStreamPayloadLength(data.GetPayload())) + PLOG(PL_ALWAYS, "(stream end) "); } - //if (NormDataMsg::StreamPayloadFlagIsSet(data.GetPayload(), NormDataMsg::FLAG_STREAM_END)) - if (0 == NormDataMsg::ReadStreamPayloadLength(data.GetPayload())) - PLOG(PL_ALWAYS, "(stream end) "); + */ } - */ break; } case NormMsg::CMD: @@ -2427,7 +2510,7 @@ void NormTrace(const struct timeval& currentTime, static_cast(msg); PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", (UINT16)squelch.GetObjectId(), - (UINT32)squelch.GetFecBlockId(fecM), + (unsigned long)squelch.GetFecBlockId(fecM).GetValue(), (UINT16)squelch.GetFecSymbolId(fecM)); break; } @@ -2437,7 +2520,7 @@ void NormTrace(const struct timeval& currentTime, static_cast(msg); PLOG(PL_ALWAYS, " obj>%hu blk>%lu seg>%hu ", (UINT16)flush.GetObjectId(), - (UINT32)flush.GetFecBlockId(fecM), + (unsigned long)flush.GetFecBlockId(fecM).GetValue(), (UINT16)flush.GetFecSymbolId(fecM)); if (0 != flush.GetAckingNodeCount()) @@ -2469,7 +2552,7 @@ void NormTrace(const struct timeval& currentTime, case NormMsg::ACK: case NormMsg::NACK: { - PLOG(PL_ALWAYS, "inst>%hu ", instId); + PLOG(PL_ALWAYS, "inst>%hu ", instId); // look for NormCCFeedback extension NormHeaderExtension ext; while (msg.GetNextExtension(ext)) @@ -2488,7 +2571,7 @@ void NormTrace(const struct timeval& currentTime, const NormAckFlushMsg& flushAck = static_cast(ack); PLOG(PL_ALWAYS, "ACK(FLUSH) obj>%hu blk>%lu seg>%hu ", (UINT16)flushAck.GetObjectId(), - (UINT32)flushAck.GetFecBlockId(fecM), + (unsigned long)flushAck.GetFecBlockId(fecM).GetValue(), (UINT16)flushAck.GetFecSymbolId(fecM)); } else if (NormAck::CC == ack.GetAckType()) @@ -2531,42 +2614,91 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt if (trace) { // Initially assume it's a message we generated (or similarly configured sender) - UINT8 fecM = fec_m; - UINT16 instId = instance_id; - NormNodeId senderId; - switch (msg.GetType()) - { - case NormMsg::ACK: - senderId = static_cast(msg).GetSenderId(); - break; - case NormMsg::NACK: - senderId = static_cast(msg).GetSenderId(); - break; - default: - senderId = msg.GetSourceId(); - break; - } - if (IsReceiver() && (senderId != LocalNodeId())) - { - // Use our receiver state to look up sender if possible - NormSenderNode* sender = static_cast(sender_tree.FindNodeById(senderId)); - if (NULL != sender) - { - fecM = sender->GetFecFieldSize(); - instId = sender->GetInstanceId(); - } - else - { - fecM = 16; // reasonable assumption - instId = 0; - } - } - + UINT8 fecM = fec_m; + UINT16 instId = instance_id; + NormNodeId senderId; + switch (msg.GetType()) + { + case NormMsg::ACK: + senderId = static_cast(msg).GetSenderId(); + break; + case NormMsg::NACK: + senderId = static_cast(msg).GetSenderId(); + break; + default: + senderId = msg.GetSourceId(); + break; + } + if (IsReceiver() && (senderId != LocalNodeId())) + { + // Use our receiver state to look up sender if possible + NormSenderNode* sender; + if (IsServerListener()) + sender = client_tree.FindNodeByAddress(msg.GetSource()); + else + sender = static_cast(sender_tree.FindNodeById(senderId)); + if (NULL != sender) + { + fecM = sender->GetFecFieldSize(); + instId = sender->GetInstanceId(); + } + else + { + fecM = 16; // reasonable assumption + instId = 0; + } + } NormTrace(currentTime, LocalNodeId(), msg, false, fecM, instId); // TBD don't assume m == 16 (i.e. for fec_id == 2) - } + } // end if (trace) NormMsg::Type msgType = msg.GetType(); + if (IsServerListener()) + { + // Only pay attention to packets with FLAG_SYN set + // (NORM_CMD(CC), NORM_INFO, or NORM_DATA messages + // (Note FLAG_SYN is not part of RFC 5740) + bool syn = false; + bool senderMsg = true; + switch (msgType) + { + case NormMsg::CMD: + { + NormCmdMsg& cmd = static_cast(msg); + if ((NormCmdMsg::CC == cmd.GetFlavor()) && + static_cast(cmd).SynIsSet()) + { + syn = true; + } + break; + } + case NormMsg::INFO: + case NormMsg::DATA: + if (static_cast(msg).FlagIsSet(NormObjectMsg::FLAG_SYN)) + { + syn = true; + } + break; + default: + senderMsg = false; + // Receiver messages are ignored by unicast server-listener, + // but multicast server needs to process ACKS/NACKS from client receivers + if (!Address().IsMulticast()) return; + } + if (senderMsg) + { + if (!syn) + { + // Send "reject" command to source + char buffer[2]; + buffer[0] = NORM_SOCKET_VERSION; + buffer[1] = NORM_SOCKET_CMD_REJECT; + SenderSendAppCmd(buffer, 2, msg.GetSource()); + return; + } + } + } + // Add newly detected nodes to acking list _before_ processing message if (IsSender() && (TRACK_NONE != acking_auto_populate)) { @@ -2590,7 +2722,7 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt NormNodeId sourceId = msg.GetSourceId(); if (NULL == acking_node_tree.FindNodeById(sourceId)) { - if (!SenderAddAckingNode(msg.GetSourceId())) + if (!SenderAddAckingNode(msg.GetSourceId(), &msg.GetSource())) PLOG(PL_ERROR, "NormSession::HandleReceiveMessage() error: unable to add acking node!\n"); NormAckingNode* acker = (NormAckingNode*)acking_node_tree.FindNodeById(sourceId); Notify(NormController::ACKING_NODE_NEW, acker, NULL); @@ -2633,7 +2765,7 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecnSt case NormMsg::REPORT: case NormMsg::INVALID: - PLOG(PL_FATAL, "NormSession::HandleReceiveMessage(NormMsg::INVALID)\n"); + PLOG(PL_ERROR, "NormSession::HandleReceiveMessage(NormMsg::INVALID)\n"); break; } } // end NormSession::HandleReceiveMessage() @@ -2645,18 +2777,23 @@ void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTim { // Do common updates for senders we already know. NormNodeId sourceId = msg.GetSourceId(); - NormSenderNode* theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId); + NormSenderNode* theSender; + if (IsServerListener()) + theSender = client_tree.FindNodeByAddress(msg.GetSource()); + else + theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId); if (theSender) { if (msg.GetInstanceId() != theSender->GetInstanceId()) { PLOG(PL_INFO, "NormSession::ReceiverHandleObjectMessage() node>%lu sender>%lu instanceId change - resyncing.\n", - LocalNodeId(), theSender->GetId()); + (unsigned long)LocalNodeId(), (unsigned long)theSender->GetId()); theSender->Close(); Notify(NormController::REMOTE_SENDER_RESET, theSender, NULL); if (!theSender->Open(msg.GetInstanceId())) { - PLOG(PL_ERROR, "NormSession::ReceiverHandleObjectMessage() node>%lu error re-opening NormSenderNode\n"); + PLOG(PL_ERROR, "NormSession::ReceiverHandleObjectMessage() node>%lu error re-opening NormSenderNode\n", + (unsigned long)LocalNodeId()); // (TBD) notify application of error return; } @@ -2669,28 +2806,36 @@ void NormSession::ReceiverHandleObjectMessage(const struct timeval& currentTim theSender = preset_sender; preset_sender = NULL; theSender->SetId(msg.GetSourceId()); - theSender->SetAddress(msg.GetSource()); theSender->SetInstanceId(msg.GetInstanceId()); - sender_tree.AttachNode(theSender); + theSender->SetAddress(msg.GetSource()); + if (IsServerListener()) + client_tree.InsertNode(*theSender); + else + sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n", - LocalNodeId(), msg.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId()); + Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else if (NULL != (theSender = new NormSenderNode(*this, msg.GetSourceId()))) { theSender->SetAddress(msg.GetSource()); - Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); if (theSender->Open(msg.GetInstanceId())) { - sender_tree.AttachNode(theSender); + if (IsServerListener()) + client_tree.InsertNode(*theSender); + else + sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n", - LocalNodeId(), msg.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)msg.GetSourceId()); } else { - PLOG(PL_FATAL, "NormSession::ReceiverHandleObjectMessage() node>%lu error opening NormSenderNode\n"); + PLOG(PL_FATAL, "NormSession::ReceiverHandleObjectMessage() node>%lu error opening NormSenderNode\n", + (unsigned long)LocalNodeId()); // (TBD) notify application of error return; - } + } + Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else { @@ -2723,18 +2868,23 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, { // Do common updates for senders we already know. NormNodeId sourceId = cmd.GetSourceId(); - NormSenderNode* theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId); + NormSenderNode* theSender; + if (IsServerListener()) + theSender = client_tree.FindNodeByAddress(cmd.GetSource()); + else + theSender = (NormSenderNode*)sender_tree.FindNodeById(sourceId); if (NULL != theSender) { if (cmd.GetInstanceId() != theSender->GetInstanceId()) { PLOG(PL_INFO, "NormSession::ReceiverHandleCommand() node>%lu sender>%lu instanceId change - resyncing.\n", - LocalNodeId(), theSender->GetId()); + (unsigned long)LocalNodeId(), theSender->GetId()); theSender->Close(); Notify(NormController::REMOTE_SENDER_RESET, theSender, NULL); if (!theSender->Open(cmd.GetInstanceId())) { - PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() node>%lu error re-opening NormSenderNode\n"); + PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() node>%lu error re-opening NormSenderNode\n", + (unsigned long)LocalNodeId()); // (TBD) notify application of error return; } @@ -2743,27 +2893,33 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, else { //DMSG(0, "NormSession::ReceiverHandleCommand() node>%lu recvd command from unknown sender ...\n", - // LocalNodeId()); + // (unsigned long)LocalNodeId()); if (NULL != preset_sender) { theSender = preset_sender; preset_sender = NULL; theSender->SetId(cmd.GetSourceId()); - theSender->SetAddress(cmd.GetSource()); theSender->SetInstanceId(cmd.GetInstanceId()); - sender_tree.AttachNode(theSender); - PLOG(PL_DEBUG, "NormSession::ReceiverHandleObjectMessage() node>%lu new remote sender:%lu ...\n", - LocalNodeId(), cmd.GetSourceId()); + theSender->SetAddress(cmd.GetSource()); + if (IsServerListener()) + client_tree.InsertNode(*theSender); + else + sender_tree.AttachNode(theSender); + PLOG(PL_DEBUG, "NormSession::ReceiverHandleCommand() node>%lu new remote sender:%lu ...\n", + (unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId()); + Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else if ((theSender = new NormSenderNode(*this, cmd.GetSourceId()))) { - Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); theSender->SetAddress(cmd.GetSource()); if (theSender->Open(cmd.GetInstanceId())) { - sender_tree.AttachNode(theSender); + if (IsServerListener()) + client_tree.InsertNode(*theSender); + else + sender_tree.AttachNode(theSender); PLOG(PL_DEBUG, "NormSession::ReceiverHandleCommand() node>%lu new remote sender:%lu ...\n", - LocalNodeId(), cmd.GetSourceId()); + (unsigned long)LocalNodeId(), (unsigned long)cmd.GetSourceId()); } else { @@ -2771,11 +2927,12 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, // (TBD) notify application of error return; } + Notify(NormController::REMOTE_SENDER_NEW, theSender, NULL); } else { - PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() new NormSenderNode error: %s\n", - GetErrorString()); + PLOG(PL_ERROR, "NormSession::ReceiverHandleCommand() new NormSenderNode node>%lu error: %s\n", + (unsigned long)LocalNodeId(), GetErrorString()); // (TBD) notify application of error return; } @@ -2799,6 +2956,42 @@ void NormSession::ReceiverHandleCommand(const struct timeval& currentTime, // and cc feedback was not provided in response otherwise } // end NormSession::ReceiverHandleCommand() +bool NormSession::InsertRemoteSender(NormSenderNode& sender) +{ + // Build a NORM_CMD(CC) message with information from + // a "sender" being inserted from another NormSession + // (supports NormSocket server operations) + if (!IsReceiver()) return false; + NormCmdCCMsg cmd; + cmd.Init(); + cmd.SetSequence(sender.GetCurrentSequence()); + cmd.SetSourceId(sender.GetId()); + cmd.SetDestination(sender.GetAddress()); + cmd.SetInstanceId(sender.GetInstanceId()); + cmd.SetGrtt(sender.GetGrttQuantized()); + cmd.SetBackoffFactor(sender.GetBackoffFactor()); + cmd.SetGroupSize(sender.GetGroupSizeQuantized()); + cmd.SetCCSequence(sender.GetCCSequence()); + // Adjust send time for any current hold time + // since it will be "rehandled" + struct timeval adjustedSendTime; + struct timeval currentTime; + ::ProtoSystemTime(currentTime); + sender.CalculateGrttResponse(currentTime, adjustedSendTime); + cmd.SetSendTime(adjustedSendTime); + + // Insert NORM-CC header extension, if applicable + // (Note we set the extension "rate" _after_ AdjustRate() done below) + NormCCRateExtension ext; + cmd.AttachExtension(ext); + ext.SetSendRate(NormQuantizeRate(sender.GetSendRate())); + + HandleReceiveMessage(cmd, false); + + return true; // TBD - confirm the node was added + +} // end NormSession::InsertRemoteSender() + double NormSession::CalculateRtt(const struct timeval& currentTime, const struct timeval& grttResponse) @@ -2858,9 +3051,14 @@ void NormSession::SenderUpdateGrttEstimate(double receiverRtt) grtt_current_peak = grtt_measured; if (grttQuantizedOld != grtt_quantized) { + if (notify_on_grtt_update) + { + notify_on_grtt_update = false; + Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); + } Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); PLOG(PL_DEBUG, "NormSession::SenderUpdateGrttEstimate() node>%lu increased to new grtt>%lf sec\n", - LocalNodeId(), grtt_advertised); + (unsigned long)LocalNodeId(), grtt_advertised); } } else if (receiverRtt > grtt_current_peak) @@ -2895,8 +3093,8 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, { PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() cc feedback recvd at time %lu.%lf ccRate:%9.3lf ccRtt:%lf ccLoss:%lf ccFlags:%02x\n", - (unsigned long)currentTime.tv_sec, ((double)currentTime.tv_usec)*1.0e-06, - ccRate*8.0/1000.0, ccRtt, ccLoss, ccFlags); + (unsigned long)currentTime.tv_sec, ((double)currentTime.tv_usec)*1.0e-06, + ccRate*8.0/1000.0, ccRtt, ccLoss, ccFlags); // Keep track of current suppressing feedback // (non-CLR, lowest rate, unconfirmed RTT) if (0 == (ccFlags & NormCC::CLR)) @@ -2947,7 +3145,8 @@ void NormSession::SenderHandleCCFeedback(struct timeval currentTime, } PLOG(PL_DEBUG, "NormSession::SenderHandleCCFeedback() node>%lu rate>%lf (rtt>%lf loss>%lf slow_start>%d limit>%d)\n", - nodeId, ccRate * 8.0 / 1000.0, ccRtt, ccLoss, (0 != (ccFlags & NormCC::START)), (0 != (ccFlags & NormCC::LIMIT))); + (unsigned long)nodeId, ccRate * 8.0 / 1000.0, ccRtt, ccLoss, (0 != (ccFlags & NormCC::START)), + (0 != (ccFlags & NormCC::LIMIT))); // Keep the active CLR (if there is one) at the head of the list NormNodeListIterator iterator(cc_node_list); @@ -3121,6 +3320,9 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons struct timeval grttResponse; ack.GetGrttResponse(grttResponse); double receiverRtt = CalculateRtt(currentTime, grttResponse); + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu sender received ACK from node>%lu rtt>%lf\n", + (unsigned long)LocalNodeId(), (unsigned long)ack.GetSourceId(), receiverRtt); + if (receiverRtt >= 0.0) SenderUpdateGrttEstimate(receiverRtt); // Look for NORM-CC Feedback header extension @@ -3191,7 +3393,8 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons } if (!watermark_pending) { - PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu watermark ack finished.\n"); + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu watermark ack finished.\n", + (unsigned long)LocalNodeId()); Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); } */ @@ -3221,29 +3424,27 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons // (TBD) Handle other acknowledgement types default: - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() node>%lu received " - "unsupported ack type:%d\n", LocalNodeId(), ack.GetAckType()); + PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() node>%lu received unsupported ack type:%d\n", + (unsigned long)LocalNodeId(), ack.GetAckType()); break; } } // end SenderHandleAckMessage() void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, NormNackMsg& nack) { - - if (GetDebugLevel() >= PL_DEBUG) - { - PLOG(PL_ALWAYS, "node>%lu sender (tactive:%d) received NACK message with content:\n", - LocalNodeId(), repair_timer.IsActive()); - LogRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength(), fec_id, fec_m); - PLOG(PL_ALWAYS, "\n"); - } - - // (TBD) maintain average of "numErasures" for SEGMENT repair requests - // to use as input to a future automatic "auto parity" adjustor??? - // Update GRTT estimate struct timeval grttResponse; nack.GetGrttResponse(grttResponse); double receiverRtt = CalculateRtt(currentTime, grttResponse); + if (GetDebugLevel() >= PL_DEBUG) + { + PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu sender received NACK message from node>%lu rtt>%lf (tactive>%d) with content:\n", + (unsigned long)LocalNodeId(), (unsigned long)nack.GetSourceId(), receiverRtt, repair_timer.IsActive()); + LogRepairContent(nack.GetRepairContent(), nack.GetRepairContentLength(), fec_id, fec_m); + PLOG(PL_ALWAYS, "\n"); + } + // (TBD) maintain average of "numErasures" for SEGMENT repair requests + // to use as input to a future automatic "auto parity" adjustor??? + // Update GRTT estimate if (receiverRtt >= 0.0) SenderUpdateGrttEstimate(receiverRtt); // Look for NORM-CC Feedback header extension @@ -3293,7 +3494,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor } else if (obj->GetFirstPending(txBlockIndex)) { - txBlockIndex++; + Increment(txBlockIndex); } else { @@ -3309,7 +3510,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor bool holdoff = (repair_timer.IsActive() && !repair_timer.GetRepeatCount()); enum NormRequestLevel {SEGMENT, BLOCK, INFO, OBJECT}; - while ((requestLength = nack.UnpackRepairRequest(req, requestOffset))) + while (0 != (requestLength = nack.UnpackRepairRequest(req, requestOffset))) { NormRepairRequest::Form requestForm = req.GetForm(); requestOffset += requestLength; @@ -3333,7 +3534,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor else { PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu recvd repair request w/ invalid repair level\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); continue; } @@ -3351,7 +3552,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor &lastBlockLen, &lastSegmentId)) { PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu recvd incomplete RANGE request!\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); continue; // (TBD) break/return instead??? } // (TBD) test for valid range form/level @@ -3374,7 +3575,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor if (!(object = tx_table.Find(nextObjectId))) { PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu recvd repair request " - "for unknown object ...\n", LocalNodeId()); + "for unknown object ...\n", (unsigned long)LocalNodeId()); if (!squelchQueued) { SenderQueueSquelch(nextObjectId); @@ -3474,7 +3675,9 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor break; case BLOCK: PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(BLOCK) obj>%hu blks>%lu:%lu\n", - (UINT16)nextObjectId, (UINT32)nextBlockId, (UINT32)lastBlockId); + (UINT16)nextObjectId, + (unsigned long)nextBlockId.GetValue(), + (unsigned long)lastBlockId.GetValue()); inRange = false; // BLOCK requests are processed in one pass // (TBD) if entire object is TxReset(), continue if (object->IsStream()) @@ -3488,9 +3691,11 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor // Only lock blocks for which we're going to accept the repair request if (nextObjectId == txObjectIndex) { - if (lastBlockId < txBlockIndex) + //if (lastBlockId < txBlockIndex) + if (Compare(lastBlockId, txBlockIndex) < 0) attemptLock = false; - else if (nextBlockId < txBlockIndex) + //else if (nextBlockId < txBlockIndex) + else if (Compare(nextBlockId, txBlockIndex) < 0) firstLockId = txBlockIndex; } else if (nextObjectId < txObjectIndex) @@ -3508,7 +3713,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor if (!((NormStreamObject*)object)->LockBlocks(firstLockId, lastBlockId, currentTime)) { PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu LockBlocks() failure\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); if (!squelchQueued) { SenderQueueSquelch(nextObjectId); @@ -3526,9 +3731,11 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor { if (nextObjectId == txObjectIndex) { - if (nextBlockId >= txBlockIndex) + //if (nextBlockId >= txBlockIndex) + if (Compare(nextBlockId, txBlockIndex) >= 0) object->TxResetBlocks(nextBlockId, lastBlockId); - else if (lastBlockId >= txBlockIndex) + //else if (lastBlockId >= txBlockIndex) + else if (Compare(lastBlockId, txBlockIndex) >= 0) object->TxResetBlocks(txBlockIndex, lastBlockId); } else if (nextObjectId > txObjectIndex) @@ -3554,7 +3761,8 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor } else if (nextObjectId == tx_repair_object_min) { - if (nextBlockId <= tx_repair_block_min) + //if (nextBlockId <= tx_repair_block_min) + if (Compare(nextBlockId, tx_repair_block_min) <= 0) { tx_repair_block_min = nextBlockId; tx_repair_segment_min = 0; @@ -3568,14 +3776,21 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor tx_repair_block_min = nextBlockId; tx_repair_segment_min = 0; } - object->HandleBlockRequest(nextBlockId, lastBlockId); + if (!object->HandleBlockRequest(nextBlockId, lastBlockId)) + { + if (!squelchQueued) + { + SenderQueueSquelch(nextObjectId); + squelchQueued = true; + } + } startTimer = true; } break; case SEGMENT: PLOG(PL_DETAIL, "NormSession::SenderHandleNackMessage(SEGMENT) obj>%hu blk>%lu segs>%hu:%hu\n", - (UINT16)nextObjectId, (UINT32)nextBlockId, - (UINT32)nextSegmentId, (UINT32)lastSegmentId); + (UINT16)nextObjectId, (unsigned long)nextBlockId.GetValue(), + (UINT16)nextSegmentId, (UINT16)lastSegmentId); inRange = false; // SEGMENT repairs are also handled in one pass if (nextBlockId != prevBlockId) freshBlock = true; if (freshBlock) @@ -3590,7 +3805,8 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor { // Entire block already tx pending, don't worry about individual segments PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu " - "recvd SEGMENT repair request for pending block.\n"); + "recvd SEGMENT repair request for pending block.\n", + (unsigned long)LocalNodeId()); continue; } else @@ -3602,7 +3818,8 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor { PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu " "recvd repair request for old stream block(%lu) ...\n", - LocalNodeId(), (UINT32)nextBlockId); + (unsigned long)LocalNodeId(), + (unsigned long)nextBlockId.GetValue()); if (!squelchQueued) { SenderQueueSquelch(nextObjectId); @@ -3613,7 +3830,8 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor { // Resource constrained, move on to next repair request PLOG(PL_INFO, "NormSession::SenderHandleNackMessage() node>%lu " - "Warning - sender is resource constrained ...\n"); + "Warning - sender is resource constrained ...\n", + (unsigned long)LocalNodeId()); } continue; } @@ -3641,9 +3859,11 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor { if (nextObjectId == txObjectIndex) { - if (nextBlockId < txBlockIndex) + //if (nextBlockId < txBlockIndex) + if (Compare(nextBlockId, txBlockIndex) < 0) { - if (1 == (txBlockIndex - nextBlockId)) + //if (1 == (txBlockIndex - nextBlockId)) + if (1 == (UINT32)Difference(txBlockIndex, nextBlockId)) { // We're currently sending this block if (block->IsPending()) @@ -3670,13 +3890,13 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor { attemptLock = false; // NACK arrived too late } - } + } // end if (holdoff) if (attemptLock) { if (!((NormStreamObject*)object)->LockSegments(nextBlockId, firstLockId, lastLockId)) { PLOG(PL_ERROR, "NormSession::SenderHandleNackMessage() node>%lu " - "LockSegments() failure\n", LocalNodeId()); + "LockSegments() failure\n", (unsigned long)LocalNodeId()); if (!squelchQueued) { SenderQueueSquelch(nextObjectId); @@ -3708,11 +3928,13 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor } else if (nextObjectId == txObjectIndex) { - if (nextBlockId >= txBlockIndex) + //if (nextBlockId >= txBlockIndex) + if (Compare(nextBlockId, txBlockIndex) >= 0) { object->TxUpdateBlock(block, nextSegmentId, lastSegmentId, numErasures); } - else if (1 == (txBlockIndex - nextBlockId)) + //else if (1 == (txBlockIndex - nextBlockId)) + else if (1 == (UINT32)Difference(txBlockIndex, nextBlockId)) { NormSegmentId firstPending = 0; if (block->GetFirstPending(firstPending)) @@ -3732,7 +3954,7 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor } } } - else + else // !holdoff { // Update our minimum tx repair index as needed ASSERT(nextBlockId == block->GetId()); @@ -3747,7 +3969,8 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor } else if (nextObjectId == tx_repair_object_min) { - if (nextBlockId < tx_repair_block_min) + //if (nextBlockId < tx_repair_block_min) + if (Compare(nextBlockId, tx_repair_block_min) < 0) { tx_repair_block_min = nextBlockId; tx_repair_segment_min = (nextSegmentId < nextBlockSize) ? @@ -3788,8 +4011,9 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor // BACKOFF related code double aggregateInterval = address.IsMulticast() ? grtt_advertised * (backoff_factor + 1.0) : 0.0; - // backoff == 0.0 is a special case - //aggregateInterval = (backoff_factor > 0.0) ? aggregateInterval : 0.0; + // Uncommenting the line below treats ((0 == ndata) && 0.0 == backoff_factor) + // as a special case (sets zero sender aggregateInterval) + aggregateInterval = ((0 != nparity) || (backoff_factor > 0.0)) ? aggregateInterval : 0.0; // TBD - why did we do this thing here to limit the min aggregateInterval??? // (I think to allow "11th hour NACKs to be incorporated .. so this should be @@ -3801,7 +4025,8 @@ void NormSession::SenderHandleNackMessage(const struct timeval& currentTime, Nor } repair_timer.SetInterval(aggregateInterval); PLOG(PL_DEBUG, "NormSession::SenderHandleNackMessage() node>%lu starting sender " - "NACK aggregation timer (%lf sec)...\n", LocalNodeId(), aggregateInterval); + "NACK aggregation timer (%lf sec)...\n", + (unsigned long)LocalNodeId(), aggregateInterval); ActivateTimer(repair_timer); } } // end NormSession::SenderHandleNackMessage() @@ -3814,10 +4039,10 @@ void NormSession::ReceiverHandleAckMessage(const NormAckMsg& ack) { theSender->HandleAckMessage(ack); } - else + else if (ack.GetSenderId() != LocalNodeId()) { - PLOG(PL_DEBUG, "NormSession::ReceiverHandleAckMessage() node>%lu heard ACK for unknown sender>%lu.\n", - LocalNodeId(), ack.GetSenderId()); + PLOG(PL_DEBUG, "NormSession::ReceiverHandleAckMessage() node>%lu heard ACK for unknown sender>%lu\n", + (unsigned long)LocalNodeId(), (unsigned long)ack.GetSenderId(), IsServerListener()); } } // end NormSession::ReceiverHandleAckMessage() @@ -3828,10 +4053,10 @@ void NormSession::ReceiverHandleNackMessage(const NormNackMsg& nack) { theSender->HandleNackMessage(nack); } - else + else if (nack.GetSenderId() != LocalNodeId()) { PLOG(PL_DEBUG, "NormSession::ReceiverHandleNackMessage() node>%lu heard NACK for unknown sender\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } } // end NormSession::ReceiverHandleNackMessage() @@ -3879,12 +4104,13 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) NormObject* obj = tx_table.Find(objectId); NormObjectTable::Iterator iterator(tx_table); NormObjectId nextId; - if (obj) + if (NULL != obj) { ASSERT(NormObject::STREAM == obj->GetType()); squelch->SetObjectId(objectId); - NormBlockId blockId = static_cast(obj)->StreamBufferLo(); - squelch->SetFecPayloadId(fec_id, blockId, 0, obj->GetBlockSize(blockId), fec_m); + //NormBlockId blockId = static_cast(obj)->StreamBufferLo(); + NormBlockId blockId = static_cast(obj)->RepairWindowLo(); + squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m); while ((obj = iterator.GetNextObject())) if (objectId == obj->GetId()) break; nextId = objectId + 1; @@ -3892,15 +4118,16 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) else { obj = iterator.GetNextObject(); - if (obj) + if (NULL != obj) { squelch->SetObjectId(obj->GetId()); NormBlockId blockId; if (obj->IsStream()) - blockId =static_cast(obj)->StreamBufferLo(); + //blockId =static_cast(obj)->StreamBufferLo(); + blockId = static_cast(obj)->RepairWindowLo(); else blockId = NormBlockId(0); - squelch->SetFecPayloadId(fec_id, blockId, 0, obj->GetBlockSize(blockId), fec_m); + squelch->SetFecPayloadId(fec_id, blockId.GetValue(), 0, obj->GetBlockSize(blockId), fec_m); nextId = obj->GetId() + 1; } else @@ -3924,24 +4151,25 @@ bool NormSession::SenderQueueSquelch(NormObjectId objectId) } nextId++; } + nextId++; } if (doEnqueue) { QueueMessage(squelch); PLOG(PL_DEBUG, "NormSession::SenderQueueSquelch() node>%lu sender queued squelch ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } else { PLOG(PL_DEBUG, "NormSession::SenderQueueSquelch() node>%lu sender updated squelch ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } return true; } else { - PLOG(PL_FATAL, " NormSession::SenderQueueSquelch() node>%lu message_pool exhausted! (couldn't squelch)\n", - LocalNodeId()); + PLOG(PL_FATAL, "NormSession::SenderQueueSquelch() node>%lu message_pool exhausted! (couldn't squelch)\n", + (unsigned long)LocalNodeId()); return false; } } // end NormSession::SenderQueueSquelch() @@ -3996,7 +4224,7 @@ bool NormSession::SenderQueueAppCmd() appMsg->SetContent(cmd_buffer, cmd_length, segment_size); QueueMessage(appMsg); PLOG(PL_DEBUG, "NormSession::SenderQueueAppCmd() node>%lu sender queued app-defined cmd ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); cmd_count--; if (0 != cmd_count) { @@ -4006,15 +4234,15 @@ bool NormSession::SenderQueueAppCmd() else { PLOG(PL_DEBUG, "NormSession::SenderQueueAppCmd() node>%lu cmd transmission completed ...\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); Notify(NormController::TX_CMD_SENT, NULL, NULL); } return true; } else { - PLOG(PL_FATAL, " NormSession::SenderQueueAppCmd() node>%lu message_pool exhausted!\n", - LocalNodeId()); + PLOG(PL_FATAL, "NormSession::SenderQueueAppCmd() node>%lu message_pool exhausted!\n", + (unsigned long)LocalNodeId()); return false; } } // end NormSession::SenderQueueAppCmd() @@ -4025,6 +4253,27 @@ bool NormSession::OnCmdTimeout(ProtoTimer& theTimer) return true; } // end NormSession::OnCmdTimeout() +bool NormSession::SenderSendAppCmd(const char* buffer, unsigned int length, const ProtoAddress& dst) +{ + // Build/immediately send a NORM_CMD(APPLICATION) message + NormCmdAppMsg appMsg; + appMsg.Init(); + appMsg.SetDestination(address); + appMsg.SetGrtt(grtt_quantized); + appMsg.SetBackoffFactor((unsigned char)backoff_factor); + appMsg.SetGroupSize(gsize_quantized); + // We use a surrogate segment_size in case sender not configured (e.g. server-listener) + appMsg.SetContent(buffer, length, segment_size ? segment_size : 64); + appMsg.SetDestination(dst); + if (MSG_SEND_OK != SendMessage(appMsg)) + PLOG(PL_ERROR, "NormSession::SenderSendAppCmd() node>%lu sender unable to send app-defined cmd ...\n", + (unsigned long)LocalNodeId()); + else + PLOG(PL_DEBUG, "NormSession::SenderSendAppCmd() node>%lu sender sending app-defined cmd len:%u...\n", + (unsigned long)LocalNodeId(), appMsg.GetLength()); + return true; +} // end NormSession::SenderSendAppCmd() + void NormSession::ActivateFlowControl(double delay, NormObjectId objectId, NormController::Event event) { flow_control_object = objectId; @@ -4107,7 +4356,7 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) UINT16 objectCount = 0; NormObjectTable::Iterator iterator(tx_table); NormObject* nextObject = iterator.GetNextObject(); - while (nextObject) + while (NULL != nextObject) { NormObject* currentObject = nextObject; nextObject = iterator.GetNextObject(); @@ -4120,7 +4369,7 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) } // Check for non-OBJECT level request or end - if (objectCount && (!repairEntireObject || !nextObject)) + if (objectCount && (!repairEntireObject || (NULL != nextObject))) { NormRepairRequest::Form form; switch (objectCount) @@ -4142,9 +4391,9 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) { if (0 == cmd.PackRepairRequest(req)) { + prevForm = NormRepairRequest::INVALID; PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case - prevForm = NormRepairRequest::INVALID; break; } } @@ -4181,22 +4430,21 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) } if (!repairEntireObject) { - if ((NormRepairRequest::INVALID != prevForm) && currentObject->IsRepairPending()) + if (currentObject->IsRepairPending()) { - prevForm = NormRepairRequest::INVALID; - if (0 == cmd.PackRepairRequest(req)) + if (NormRepairRequest::INVALID != prevForm)// && currentObject->IsRepairPending()) { - PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); - // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case - break; + prevForm = NormRepairRequest::INVALID; + if (0 == cmd.PackRepairRequest(req)) + { + PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); + // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case + break; + } } - currentObject->AppendRepairAdv(cmd); + if (!currentObject->AppendRepairAdv(cmd)) break; } - else - { - currentObject->AppendRepairAdv(cmd); - } - objectCount = 0; + objectCount = 0; // this is probably redundant } } // end while (nextObject) if (NormRepairRequest::INVALID != prevForm) @@ -4215,7 +4463,7 @@ bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/) { // NACK aggregation period has ended. (incorporate accumulated repair requests) PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu sender NACK aggregation time ended.\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); NormObjectTable::Iterator iterator(tx_table); NormObject* obj; while ((obj = iterator.GetNextObject())) @@ -4224,27 +4472,29 @@ bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/) if (tx_repair_mask.Test(objectId)) { PLOG(PL_TRACE, "NormSession::OnRepairTimeout() node>%lu tx reset obj>%hu ...\n", - LocalNodeId(), (UINT16)objectId); + (unsigned long)LocalNodeId(), (UINT16)objectId); if (obj->IsStream()) - obj->TxReset(((NormStreamObject*)obj)->StreamBufferLo()); + obj->TxReset(((NormStreamObject*)obj)->RepairWindowLo()); else obj->TxReset(); tx_repair_mask.Unset(objectId); if (!tx_pending_mask.Set(objectId)) + { PLOG(PL_ERROR, "NormSession::OnRepairTimeout() tx_pending_mask.Set(%hu) error (1)\n", - (UINT16)objectId); + (UINT16)objectId); + } } else { PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu activating obj>%hu repairs ...\n", - LocalNodeId(), (UINT16)objectId); + (unsigned long)LocalNodeId(), (UINT16)objectId); if (obj->ActivateRepairs()) { PLOG(PL_TRACE, "NormSession::OnRepairTimeout() node>%lu activated obj>%hu repairs ...\n", - LocalNodeId(), (UINT16)objectId); + (unsigned long)LocalNodeId(), (UINT16)objectId); if (!tx_pending_mask.Set(objectId)) - PLOG(PL_ERROR, "NormSession::OnRepairTimeout() tx_pending_mask.Set(%hu) error (2)\n", - (UINT16)objectId); + PLOG(PL_ERROR, "NormSession::OnRepairTimeout() node>%lu tx_pending_mask.Set(%hu) error (2)\n", + (unsigned long)LocalNodeId(), (UINT16)objectId); } } } // end while (iterator.GetNextObject()) @@ -4256,13 +4506,14 @@ bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/) double holdoffInterval = grtt_advertised; repair_timer.SetInterval(holdoffInterval); // repair holdoff interval = 1*GRTT PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu starting sender " - "NACK holdoff timer (%lf sec)...\n", LocalNodeId(), holdoffInterval); + "NACK holdoff timer (%lf sec)...\n", + (unsigned long)LocalNodeId(), holdoffInterval); } else { // REPAIR holdoff interval has now ended. PLOG(PL_DEBUG, "NormSession::OnRepairTimeout() node>%lu sender holdoff time ended.\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); } return true; } // end NormSession::OnRepairTimeout() @@ -4272,7 +4523,7 @@ bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/) // for more efficiency ... bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) { - NormMsg* msg; + NormMsg* msg; // Note: sometimes need RepairAdv even when cc_enable is false ... NormCmdRepairAdvMsg adv; @@ -4314,53 +4565,67 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) advertise_repairs = false; } - suppress_rate = -1.0; // reset cc feedback suppression rate - - if (msg) + if (NULL != msg) { - if (SendMessage(*msg)) + // Do "packet pairing of NORM_CMD(CC) and subsequent message (usually NORM_DATA), if any + //unsigned int msgLength = msg->GetLength(); + + unsigned int msgLength = tx_residual; + /* Uncomment this section of code to instate CMD(CC) / NORM_DATA "packet pairing" + tx_residual = 0; + if ((NormMsg::CMD == msg->GetType()) && + (NormCmdMsg::CC == static_cast(msg)->GetFlavor())) { - if (tx_rate > 0.0) - //tx_timer.SetInterval(msg->GetLength() / tx_rate); - tx_timer.SetInterval(GetTxInterval(msg->GetLength(), tx_rate)); - if (advertise_repairs) - advertise_repairs = false; - else - ReturnMessageToPool(msg); - // Pre-serve to allow pre-prompt for empty tx queue - // (TBD) do this in a better way ??? There is a slight chance - // that with this approach some new data may get pre-queued - // when an interim repair request should be serviced first - // instead ??? - //if (message_queue.IsEmpty() && IsSender()) Serve(); - return true; // reinstall tx_timer + tx_residual = msg->GetLength(); } else + */ { - // (TBD) should we check the type of error that occurred - // and take some smarter action here (e.g. re-open our sockets?) - // Requeue the message for another try - if (!advertise_repairs) - message_queue.Prepend(msg); - // Make sure the tx_timer interval is non-ZERO - // (this avoids a sort of infinite loop that can occur - // under certain conditions) - - // old code - time out and try again - /* - if (tx_rate > 0.0) - //tx_timer.SetInterval(msg->GetLength() / tx_rate); - tx_timer.SetInterval(GetTxInterval(msg->GetLength(), tx_rate)); - else if (0.0 == tx_timer.GetInterval()) - tx_timer.SetInterval(0.01); - return true; - */ - // new code - use async i/o to cue retry - //ASSERT(tx_timer.IsActive()); - if (tx_timer.IsActive()) - tx_timer.Deactivate(); - tx_socket->StartOutputNotification(); - return false; + msgLength += msg->GetLength(); + } + + switch (SendMessage(*msg)) + { + case MSG_SEND_OK: + if (tx_rate > 0.0) + tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate)); + if (advertise_repairs) + { + advertise_repairs = false; + suppress_rate = -1.0; // reset cc feedback suppression rate + } + else + { + ReturnMessageToPool(msg); + } + // Pre-serve to allow pre-prompt for empty tx queue + // (TBD) do this in a better way ??? There is a slight chance + // that with this approach some new data may get pre-queued + // when an interim repair request should be serviced first + // instead ??? + //if (message_queue.IsEmpty() && IsSender()) Serve(); + return true; // reinstall tx_timer + + case MSG_SEND_BLOCKED: + // Message was not sent due to to EWOULDBLOCK, so we invoke async i/o output notification + if (!advertise_repairs) + message_queue.Prepend(msg); + if (tx_timer.IsActive()) + tx_timer.Deactivate(); + //TRACE("starting output notification ...\n"); + tx_socket->StartOutputNotification(); + return false; // since timer was deactivated + + case MSG_SEND_FAILED: + // Message was not sent due to socket error (no route, etc), so so just timeout and try again + // (TBD - is there something smarter we should do) + //TRACE("MSG_SEND_FAILED\n"); + if (!advertise_repairs) message_queue.Prepend(msg); + if (tx_rate > 0.0) + tx_timer.SetInterval(GetTxInterval(msgLength, tx_rate)); + else if (0.0 == tx_timer.GetInterval()) + tx_timer.SetInterval(0.001); + return true; // timer will be reactivated } } else @@ -4384,10 +4649,12 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) return OnTxTimeout(tx_timer); } } + return true; // actually will never get here but compiler thinks it's needed } // end NormSession::OnTxTimeout() -bool NormSession::SendMessage(NormMsg& msg) +NormSession::MessageStatus NormSession::SendMessage(NormMsg& msg) { + //TRACE("sending message length %hu\n", msg.GetLength()); bool isReceiverMsg = false; bool isProbe = false; @@ -4400,20 +4667,27 @@ bool NormSession::SendMessage(NormMsg& msg) { case NormMsg::INFO: case NormMsg::DATA: - ((NormObjectMsg&)msg).SetInstanceId(instId); + { + NormObjectMsg& objMsg = static_cast(msg); + objMsg.SetInstanceId(instId); msg.SetSequence(tx_sequence++); // (TBD) set for session dst msgs + if (syn_status) objMsg.SetFlag(NormObjectMsg::FLAG_SYN); break; + } case NormMsg::CMD: { + NormCmdMsg& cmd = static_cast(msg); ((NormCmdMsg&)msg).SetInstanceId(instId); - switch (((NormCmdMsg&)msg).GetFlavor()) + switch (cmd.GetFlavor()) { case NormCmdMsg::CC: { + NormCmdCCMsg& ccMsg = static_cast(cmd); struct timeval currentTime; ProtoSystemTime(currentTime); - ((NormCmdCCMsg&)msg).SetSendTime(currentTime); + ccMsg.SetSendTime(currentTime); isProbe = true; + if (syn_status) ccMsg.SetSyn(); break; } case NormCmdMsg::SQUELCH: @@ -4446,8 +4720,11 @@ bool NormSession::SendMessage(NormMsg& msg) msg.SetSequence(0); // TBD - set per destination isReceiverMsg = true; NormAckMsg& ack = (NormAckMsg&)msg; - NormSenderNode* theSender = - (NormSenderNode*)sender_tree.FindNodeById(ack.GetSenderId()); + NormSenderNode* theSender; + if (IsServerListener()) + theSender = client_tree.FindNodeByAddress(ack.GetDestination()); + else + theSender = (NormSenderNode*)sender_tree.FindNodeById(ack.GetSenderId()); ASSERT(NULL != theSender); fecM = theSender->GetFecFieldSize(); instId = theSender->GetInstanceId(); @@ -4465,7 +4742,8 @@ bool NormSession::SendMessage(NormMsg& msg) msg.SetSourceId(local_node_id); UINT16 msgSize = msg.GetLength(); // Possibly drop some tx messages for testing purposes - bool drop = (UniformRand(100.0) < tx_loss_rate); + + bool drop = (tx_loss_rate > 0.0) ? (UniformRand(100.0) < tx_loss_rate) : false; if (isReceiverMsg && receiver_silent) { @@ -4474,7 +4752,7 @@ bool NormSession::SendMessage(NormMsg& msg) // never enqueue any receiver messages. But we // did this to make sure all integrity of timer // state interdependencies wasn't messed up - return true; // wasn't sent but wasn't supposed to + return MSG_SEND_OK; // we lie as it wasn't sent but it wasn't supposed to } else if (drop) { @@ -4492,7 +4770,9 @@ bool NormSession::SendMessage(NormMsg& msg) } else { - if (tx_socket->SendTo(msg.GetBuffer(), msgSize, msg.GetDestination())) + unsigned int numBytes = msgSize; + bool result = tx_socket->SendTo(msg.GetBuffer(), numBytes, msg.GetDestination()); + if (numBytes == msgSize) { // Separate send/recv tracing if (trace) @@ -4508,10 +4788,21 @@ bool NormSession::SendMessage(NormMsg& msg) } else { - PLOG(PL_ERROR, "NormSession::SendMessage() sendto(%s/%hu) error: %s\n", - msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); + // packet not sent tx_sequence--; - return false; + if (result) + { + // TBD - is PL_WARN too verbose here + PLOG(PL_WARN, "NormSession::SendMessage() sendto(%s/%hu) warning: %s\n", + msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); + return MSG_SEND_BLOCKED; + } + else + { + PLOG(PL_WARN, "NormSession::SendMessage() sendto(%s/%hu) warning: %s\n", + msg.GetDestination().GetHostString(), msg.GetDestination().GetPort(), GetErrorString()); + return MSG_SEND_FAILED; + } } } @@ -4526,7 +4817,7 @@ bool NormSession::SendMessage(NormMsg& msg) ActivateTimer(probe_timer); } } - else if (!isReceiverMsg) + else if (!isReceiverMsg && IsSender()) { probe_data_check = false; if (!probe_pending && probe_reset) @@ -4537,7 +4828,7 @@ bool NormSession::SendMessage(NormMsg& msg) ActivateTimer(probe_timer); } } - return true; + return MSG_SEND_OK; } // end NormSession::SendMessage() void NormSession::SetGrttProbingInterval(double intervalMin, double intervalMax) @@ -4708,7 +4999,7 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) { Notify(NormController::GRTT_UPDATED, (NormSenderNode*)NULL, (NormObject*)NULL); PLOG(PL_DEBUG, "NormSession::OnProbeTimeout() node>%lu decreased to new grtt to: %lf sec\n", - LocalNodeId(), grtt_advertised); + (unsigned long)LocalNodeId(), grtt_advertised); } grtt_response = false; // reset } @@ -4727,7 +5018,7 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) if (!cmd) { PLOG(PL_FATAL, "NormSession::OnProbeTimeout() node>%lu message_pool empty! can't probe\n", - LocalNodeId()); + (unsigned long)LocalNodeId()); ASSERT(0); return true; } @@ -4892,9 +5183,7 @@ void NormSession::AdjustRate(bool onResponse) { double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / (report_timer.GetInterval() - report_timer.GetTimeRemaining())); PLOG(PL_DETAIL, "NormSession::AdjustRate(slow start) clr>%lu newRate>%lf (oldRate>%lf sentRate>%lf clrRate>%lf\n", - clr->GetId(), 8.0e-03*txRate, 8.0e-03*tx_rate, sentRate, 8.0e-03*clr->GetRate()); - //TRACE("NormSession::AdjustRate(slow start) clr>%lu newRate>%lf (oldRate>%lf sentRate>%lf clrRate>%lf\n", - // clr->GetId(), 8.0e-03*txRate, 8.0e-03*tx_rate, sentRate, 8.0e-03*clr->GetRate()); + (unsigned long)clr->GetId(), 8.0e-03*txRate, 8.0e-03*tx_rate, sentRate, 8.0e-03*clr->GetRate()); } } else @@ -4918,9 +5207,7 @@ void NormSession::AdjustRate(bool onResponse) damper = 2.0; txRate *= damper; PLOG(PL_DETAIL, "NormSession::AdjustRate(stdy state) clr>%lu newRate>%lf (rtt>%lf loss>%lf)\n", - clr->GetId(), 8.0e-03*txRate, clr->GetRtt(), clr->GetLoss()); - //TRACE("NormSession::AdjustRate(stdy state) clr>%lu newRate>%lf (rtt>%lf loss>%lf clrRate>%9.3lf damper>%lf)\n", - // clr->GetId(), 8.0e-03*txRate, clr->GetRtt(), clr->GetLoss(), 8.0e-03*clr->GetRate(), damper); + (unsigned long)clr->GetId(), 8.0e-03*txRate, clr->GetRtt(), clr->GetLoss()); } } if (!address.IsMulticast()) @@ -5005,8 +5292,15 @@ void NormSession::AdjustRate(bool onResponse) if (txRate != tx_rate) { if (cc_adjust) SetTxRateInternal(txRate); - posted_tx_rate_changed = true; - Notify(NormController::TX_RATE_CHANGED, (NormSenderNode*)NULL, (NormObject*)NULL); + if (!posted_tx_rate_changed) + { + // TBD - make API notification filtering more consistent + // (e.g., "notify_on_rate_update" like for grtt, etc + // putting API code in charge of resetting these API + // state variables). + posted_tx_rate_changed = true; + Notify(NormController::TX_RATE_CHANGED, (NormSenderNode*)NULL, (NormObject*)NULL); + } } struct timeval currentTime; @@ -5046,21 +5340,25 @@ bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/) #endif // if/else _WIN32_WCE ASSERT(NULL != ct); - PLOG(PL_INFO, "REPORT time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", - ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, LocalNodeId()); + ProtoDebugLevel reportDebugLevel = PL_INFO; + PLOG(reportDebugLevel, "REPORT time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", + ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, (unsigned long)LocalNodeId()); if (IsSender()) { - PLOG(PL_INFO, "Local status:\n"); - double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / report_timer.GetInterval()); // kbps + PLOG(reportDebugLevel, "Local status:\n"); + double sentRate = 8.0e-03*sent_accumulator.GetScaledValue(1.0 / report_timer.GetInterval()); // kbps sent_accumulator.Reset(); - PLOG(PL_INFO, " txRate>%9.3lf kbps sentRate>%9.3lf grtt>%lf\n", + PLOG(reportDebugLevel, " txRate>%9.3lf kbps sentRate>%9.3lf grtt>%lf\n", 8.0e-03*tx_rate, sentRate, grtt_advertised); if (cc_enable) { const NormCCNode* clr = (const NormCCNode*)cc_node_list.Head(); if (clr) - PLOG(PL_INFO, " clr>%lu rate>%9.3lf rtt>%lf loss>%lf %s\n", clr->GetId(), - 8.0e-03*clr->GetRate(), clr->GetRtt(), clr->GetLoss(), cc_slow_start ? "(slow_start)" : ""); + { + PLOG(reportDebugLevel, " clr>%lu rate>%9.3lf rtt>%lf loss>%lf %s\n", + (unsigned long)clr->GetId(), 8.0e-03*clr->GetRate(), + clr->GetRtt(), clr->GetLoss(), cc_slow_start ? "(slow_start)" : ""); + } } } if (IsReceiver()) @@ -5069,27 +5367,39 @@ bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/) NormSenderNode* next; while ((next = (NormSenderNode*)iterator.GetNextNode())) { - PLOG(PL_INFO, "Remote sender>%lu\n", next->GetId()); + PLOG(reportDebugLevel, "Remote sender>%lu grtt>%lf sec loss>%lf\n", (unsigned long)next->GetId(), + next->GetGrttEstimate(), next->LossEstimate()); + // TBD - Output sender congestion control status if cc is enabled double rxRate = 8.0e-03*next->GetRecvRate(report_timer.GetInterval()); // kbps double rxGoodput = 8.0e-03*next->GetRecvGoodput(report_timer.GetInterval()); // kbps next->ResetRecvStats(); - PLOG(PL_INFO, " rxRate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput); - PLOG(PL_INFO, " rxObjects> completed>%lu pending>%lu failed>%lu\n", - next->CompletionCount(), next->PendingCount(), next->FailureCount()); - PLOG(PL_INFO, " fecBufferUsage> current>%lu peak>%lu overuns>%lu\n", next->CurrentBufferUsage(), - next->PeakBufferUsage(), - next->BufferOverunCount()); - PLOG(PL_INFO, " strBufferUsage> current>%lu peak>%lu overuns>%lu\n", next->CurrentStreamBufferUsage(), - next->PeakStreamBufferUsage(), - next->StreamBufferOverunCount()); - PLOG(PL_INFO, " resyncs>%lu nacks>%lu suppressed>%lu\n", - next->ResyncCount() ? next->ResyncCount() - 1 : 0, // "ResyncCount()" is reall "SyncCount()" - next->NackCount(), - next->SuppressCount()); + PLOG(reportDebugLevel, " rxRate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput); + PLOG(reportDebugLevel, " rxObjects> completed>%lu pending>%lu failed>%lu\n", + next->CompletionCount(), next->PendingCount(), next->FailureCount()); + PLOG(reportDebugLevel, " fecBufferUsage> current>%lu peak>%lu overuns>%lu\n", + next->CurrentBufferUsage(), next->PeakBufferUsage(), + next->BufferOverunCount()); + PLOG(reportDebugLevel, " strBufferUsage> current>%lu peak>%lu overuns>%lu\n", + next->CurrentStreamBufferUsage(), next->PeakStreamBufferUsage(), + next->StreamBufferOverunCount()); + PLOG(reportDebugLevel, " resyncs>%lu nacks>%lu suppressed>%lu\n", + next->ResyncCount() ? next->ResyncCount() - 1 : 0, // "ResyncCount()" is really "SyncCount()" + next->NackCount(), next->SuppressCount()); + // Some stream status for current receive stream (if applicable) + NormObject* obj = next->GetNextPendingObject(); + if ((NULL != obj) && obj->IsStream()) + { + NormStreamObject* stream = (NormStreamObject*)obj; + PLOG(reportDebugLevel, " stream_sync_id>%lu stream_next_id>%lu read_index:%lu.%hu\n", + (unsigned long)stream->GetSyncId().GetValue(), + (unsigned long)stream->GetNextId().GetValue(), + (unsigned long)stream->GetNextBlockId().GetValue(), + (UINT16)stream->GetNextSegmentId()); + } } } // end if (IsReceiver()) - PLOG(PL_INFO, "***************************************************************************\n"); + PLOG(reportDebugLevel, "***************************************************************************\n"); return true; } // end NormSession::OnReportTimeout() @@ -5105,7 +5415,7 @@ NormSessionMgr::NormSessionMgr(ProtoTimerMgr& timerMgr, ProtoSocket::Notifier& socketNotifier, ProtoChannel::Notifier* channelNotifier) : timer_mgr(timerMgr), socket_notifier(socketNotifier), channel_notifier(channelNotifier), - controller(NULL), top_session(NULL) + controller(NULL), data_free_func(NULL), top_session(NULL) { } @@ -5154,8 +5464,7 @@ NormSession* NormSessionMgr::NewSession(const char* sessionAddress, NormSession* theSession = new NormSession(*this, localNodeId); if (!theSession) { - PLOG(PL_ERROR, "NormSessionMgr::NewSession() new session error: %s\n", - GetErrorString()); + PLOG(PL_ERROR, "NormSessionMgr::NewSession() new session error: %s\n", GetErrorString()); return ((NormSession*)NULL); } theSession->SetAddress(theAddress); diff --git a/src/common/pcap2norm.cpp b/src/common/pcap2norm.cpp index a774dc3..43709c0 100644 --- a/src/common/pcap2norm.cpp +++ b/src/common/pcap2norm.cpp @@ -5,6 +5,7 @@ #include #include +#include // for PF_ types (protocol family) #include "protoPktETH.h" // for Ethernet frame parsing #include "protoPktIP.h" // for IP packet parsing #include "protoPktARP.h" @@ -68,8 +69,9 @@ int main(int argc, char* argv[]) return -1; } + int deviceType = pcap_datalink(pcapDevice); - UINT32 alignedBuffer[4096/4]; // 128 buffer for packet parsing + UINT32 alignedBuffer[4096/4]; // 4096 byte buffer for packet parsing UINT16* ethBuffer = ((UINT16*)alignedBuffer) + 1; unsigned int maxBytes = 4096 - 2; // due to offset, can only use 4094 bytes of buffer @@ -79,21 +81,47 @@ int main(int argc, char* argv[]) { unsigned int numBytes = maxBytes; if (hdr.caplen < numBytes) numBytes = hdr.caplen; - memcpy(ethBuffer, pktData, numBytes); - ProtoPktETH ethPkt((UINT32*)ethBuffer, maxBytes); - if (!ethPkt.InitFromBuffer(hdr.len)) + ProtoPktETH::Type ethType; + unsigned int payloadLength; + UINT32* payloadPtr; + if (DLT_NULL == deviceType) { - fprintf(stderr, "pcap2norm error: invalid Ether frame in pcap file\n"); - continue; - } + // pcap was captured from "loopback" device + memcpy(alignedBuffer, pktData, numBytes); + switch (alignedBuffer[0]) + { + case PF_INET: + ethType = ProtoPktETH::IP; + break; + case PF_INET6: + ethType = ProtoPktETH::IPv6; + break; + default: + continue; // not an IP packet + } + payloadLength = numBytes - 4; + payloadPtr = alignedBuffer + 1; + } + else + { + memcpy(ethBuffer, pktData, numBytes); + ProtoPktETH ethPkt((UINT32*)ethBuffer, maxBytes); + if (!ethPkt.InitFromBuffer(hdr.len)) + { + fprintf(stderr, "pcap2norm error: invalid Ether frame in pcap file\n"); + continue; + } + ethType = ethPkt.GetType(); + payloadLength = ethPkt.GetPayloadLength(); + payloadPtr = (UINT32*)ethPkt.AccessPayload(); + } + ProtoPktIP ipPkt; ProtoAddress srcAddr, dstAddr; - ProtoPktETH::Type ethType = ethPkt.GetType(); if ((ProtoPktETH::IP == ethType) || (ProtoPktETH::IPv6 == ethType)) { - unsigned int payloadLength = ethPkt.GetPayloadLength(); - if (!ipPkt.InitFromBuffer(payloadLength, (UINT32*)ethPkt.AccessPayload(), payloadLength)) + if (!ipPkt.InitFromBuffer(payloadLength, payloadPtr, payloadLength)) { fprintf(stderr, "pcap2norm error: bad IP packet\n"); continue; @@ -123,6 +151,10 @@ int main(int argc, char* argv[]) //PLOG(PL_ALWAYS, "pcap2norm IP packet dst>%s ", dstAddr.GetHostString()); //PLOG(PL_ALWAYS," src>%s length>%d\n", srcAddr.GetHostString(), ipPkt.GetLength()); } + else + { + fprintf(stderr, "eth type = %d\n", ethType); + } if (!srcAddr.IsValid()) continue; // wasn't an IP packet ProtoPktUDP udpPkt; @@ -153,7 +185,7 @@ void NormTrace2(const struct timeval& currentTime, const ProtoAddress& dstAddr) { - UINT8 fecM = 16; // NOTE - this assumes 16-bit RS code for fec_id == 2 + UINT8 fecM = 8; // NOTE - this assumes 16-bit RS code for fec_id == 2 static const char* MSG_NAME[] = { @@ -227,7 +259,7 @@ void NormTrace2(const struct timeval& currentTime, { const NormDataMsg& data = (const NormDataMsg&)msg; lastFecId = data.GetFecId(); - PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%lu seg>%hu ", + PLOG(PL_ALWAYS, "inst>%hu seq>%hu DATA obj>%hu blk>%u seg>%04hu ", data.GetInstanceId(), seq, //data.IsData() ? "DATA" : "PRTY", diff --git a/src/common/raft.cpp b/src/common/raft.cpp index 67bb57a..94db6ba 100644 --- a/src/common/raft.cpp +++ b/src/common/raft.cpp @@ -362,11 +362,17 @@ void RaftApp::OnInputReady() } else if (tx_msg_index == tx_msg_length) { - if (!tx_socket.SendTo(tx_msg_buffer+2, tx_msg_length-2, tx_address)) + unsigned int bytesSent = tx_msg_length-2; + if (!tx_socket.SendTo(tx_msg_buffer+2, bytesSent, tx_address)) { - PLOG(PL_ERROR, "raft: tx_socket.SendTo() error\n"); + PLOG(PL_ERROR, "raft: tx_socket.SendTo() error: %s\n", GetErrorString()); return; } + else if (0 == bytesSent) + { + PLOG(PL_WARN, "raft: tx_socket.SendTo() error: %s\n", GetErrorString()); + return; + } tx_msg_index = tx_msg_length = 0; } diff --git a/src/java/jni/normInstanceJni.cpp b/src/java/jni/normInstanceJni.cpp index 8fcd6c3..aaef916 100644 --- a/src/java/jni/normInstanceJni.cpp +++ b/src/java/jni/normInstanceJni.cpp @@ -179,6 +179,12 @@ JNIEXPORT jobject JNICALL PKGNAME(NormInstance_getNextEvent) // Get the event type jobjectArray array = (jobjectArray)env->CallStaticObjectMethod( (jclass)env->NewLocalRef(jw_NormEventType), mid_NormEventType_values); + + if (env->GetArrayLength(array) <= event.type) { + env->ThrowNew((jclass)env->NewLocalRef(jw_IOException), "Invalid NORM event type (NormEventType.java out of sync with NORM API event header?)"); + return false; + } + jobject type = env->GetObjectArrayElement(array, event.type); // Create the event diff --git a/src/java/jni/normNodeJni.cpp b/src/java/jni/normNodeJni.cpp index 8678308..3f1efce 100644 --- a/src/java/jni/normNodeJni.cpp +++ b/src/java/jni/normNodeJni.cpp @@ -57,7 +57,7 @@ JNIEXPORT jobject JNICALL PKGNAME(NormNode_getAddress) NormNodeHandle handle; char buffer[256]; unsigned int bufferLen; - unsigned short port; + UINT16 port; jobject address; jbyteArray array; jbyte *ptr; diff --git a/src/java/src/mil/navy/nrl/norm/enums/NormEventType.java b/src/java/src/mil/navy/nrl/norm/enums/NormEventType.java index 7d5d1c2..8ba2f56 100644 --- a/src/java/src/mil/navy/nrl/norm/enums/NormEventType.java +++ b/src/java/src/mil/navy/nrl/norm/enums/NormEventType.java @@ -7,30 +7,33 @@ package mil.navy.nrl.norm.enums; * * @author Jason Rush */ -public enum NormEventType { - NORM_EVENT_INVALID, - NORM_TX_QUEUE_VACANCY, - NORM_TX_QUEUE_EMPTY, - NORM_TX_FLUSH_COMPLETED, - NORM_TX_WATERMARK_COMPLETED, - NORM_TX_CMD_SENT, - NORM_TX_OBJECT_SENT, - NORM_TX_OBJECT_PURGED, - NORM_TX_RATE_CHANGED, - NORM_LOCAL_SENDER_CLOSED, - NORM_REMOTE_SENDER_NEW, - NORM_REMOTE_SENDER_RESET, - NORM_REMOTE_SENDER_ADDRESS, - NORM_REMOTE_SENDER_ACTIVE, - NORM_REMOTE_SENDER_INACTIVE, - NORM_REMOTE_SENDER_PURGED, - NORM_RX_CMD_NEW, - NORM_RX_OBJECT_NEW, - NORM_RX_OBJECT_INFO, - NORM_RX_OBJECT_UPDATED, - NORM_RX_OBJECT_COMPLETED, - NORM_RX_OBJECT_ABORTED, - NORM_GRTT_UPDATED, - NORM_CC_ACTIVE, - NORM_CC_INACTIVE; +public enum NormEventType { + NORM_EVENT_INVALID, + NORM_TX_QUEUE_VACANCY, + NORM_TX_QUEUE_EMPTY, + NORM_TX_FLUSH_COMPLETED, + NORM_TX_WATERMARK_COMPLETED, + NORM_TX_CMD_SENT, + NORM_TX_OBJECT_SENT, + NORM_TX_OBJECT_PURGED, + NORM_TX_RATE_CHANGED, + NORM_LOCAL_SENDER_CLOSED, + NORM_REMOTE_SENDER_NEW, + NORM_REMOTE_SENDER_RESET, + NORM_REMOTE_SENDER_ADDRESS, + NORM_REMOTE_SENDER_ACTIVE, + NORM_REMOTE_SENDER_INACTIVE, + NORM_REMOTE_SENDER_PURGED, + NORM_RX_CMD_NEW, + NORM_RX_OBJECT_NEW, + NORM_RX_OBJECT_INFO, + NORM_RX_OBJECT_UPDATED, + NORM_RX_OBJECT_COMPLETED, + NORM_RX_OBJECT_ABORTED, + NORM_GRTT_UPDATED, + NORM_CC_ACTIVE, + NORM_CC_INACTIVE, + NORM_ACKING_NODE_NEW, + NORM_SEND_ERROR, + NORM_USER_TIMEOUT } diff --git a/src/pynorm/core.py b/src/pynorm/core.py index d18a5a2..9c39b95 100644 --- a/src/pynorm/core.py +++ b/src/pynorm/core.py @@ -506,5 +506,5 @@ def get_libnorm(): return libnorm -# Global libnorm refrence that everyone should use +# Global libnorm reference that everyone should use libnorm = get_libnorm() diff --git a/src/pynorm/event.py b/src/pynorm/event.py index a9a17e5..c3b1ba9 100644 --- a/src/pynorm/event.py +++ b/src/pynorm/event.py @@ -91,5 +91,11 @@ class Event(object): return "NORM_CC_ACTIVE" elif self.type == c.NORM_CC_INACTIVE: return "NORM_CC_INACTIVE" + elif self.type == c.NORM_ACKING_NODE_NEW: + return "NORM_ACKING_NODE_NEW" + elif self.type == c.NORM_SEND_ERROR: + return "NORM_SEND_ERROR" + elif self.type == c.NORM_USER_TIMEOUT: + return "NORM_USER_TIMEOUT" else: return "Unknown event type" diff --git a/src/pynorm/node.py b/src/pynorm/node.py index a26d166..24c7d9c 100644 --- a/src/pynorm/node.py +++ b/src/pynorm/node.py @@ -29,14 +29,17 @@ class Node(object): return self._address except AttributeError: port = ctypes.c_uint16() - buffer = ctypes.create_string_buffer(50) + buf = ctypes.create_string_buffer(50) size = ctypes.c_uint(50) - if not libnorm.NormNodeGetAddress(self, buffer, ctypes.byref(size), + if not libnorm.NormNodeGetAddress(self, buf, ctypes.byref(size), ctypes.byref(port)): raise NormError("Node getAddress failed") - self._address = (buffer.value, port.value) + self._address = (buf.value, port.value) return self._address + def getCommand(self, buf): + return libnorm.NormNodeGetCommand(self, buf, len(buf)) + def getGrtt(self): grtt = libnorm.NormNodeGetGrtt(self) if grtt == -1.0: diff --git a/src/pynorm/session.py b/src/pynorm/session.py index 9c6924f..c648c59 100644 --- a/src/pynorm/session.py +++ b/src/pynorm/session.py @@ -133,6 +133,12 @@ class Session(object): def streamOpen(self, bufferSize, info=""): return Object(libnorm.NormStreamOpen(self, bufferSize, info, len(info))) + def sendCommand(self, cmdBuffer, robust=False): + return libnorm.NormSendCommand(self, cmdBuffer, len(cmdBuffer), robust) + + def cancelCommand(self): + libnorm.NormCancelCommand(self) + def setWatermark(self, normObject, overrideFlush=False): libnorm.NormSetWatermark(self, normObject, overrideFlush) diff --git a/makefiles/win32/win32PostProcess.cpp b/src/win32/win32PostProcess.cpp similarity index 100% rename from makefiles/win32/win32PostProcess.cpp rename to src/win32/win32PostProcess.cpp diff --git a/waf b/waf index f6e9616..2270b53 100755 --- a/waf +++ b/waf @@ -1,7 +1,7 @@ #!/usr/bin/env python -# encoding: ISO8859-1 -# Thomas Nagy, 2005-2015 - +# encoding: latin-1 +# Thomas Nagy, 2005-2018 +# """ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -32,13 +32,13 @@ POSSIBILITY OF SUCH DAMAGE. import os, sys, inspect -VERSION="1.8.11" -REVISION="930623f783bdf2f01355c2002e2b4462" -GIT="acf903268766428829e43ed44771876d72648dee" +VERSION="2.0.7" +REVISION="961a230889afde25409f04a102f7436c" +GIT="f78fbc32bb355a3291c9b5f79bbe0c8dfe81282a" INSTALL='' -C1='#.' -C2='#+' -C3='#)' +C1='#,' +C2='#*' +C3='#(' cwd = os.getcwd() join = os.path.join @@ -165,6 +165,5 @@ if __name__ == '__main__': Scripting.waf_entry_point(cwd, VERSION, wafdir) #==> -#BZh91AY&SY3Tk(#.AV`@"@l 8a|[#)#)#)#)#)#)#)#)#)#)#)#)#)#)#)#)#)#)#)<}ki5vh孛ۓ--}։i/}Ml6Zƞ[ϾjާJPzqv5QEtof:vohm^l}#)m;{Wi]}#)#)4J@E#)#)#){լloGܦC@l#)WU}(z3(^ǸBv (#.IT#.+P==tע;Mcn`6d\ZyTѷ}{ۚ\ַݳUo^޷wo;=oe^g^ι;ӽsv:uӻ4#.M`#.-{q7=L4>{}jkl#)#)-^Ɣ4 f׾;igoS>Y}wŲu;:|vۭ|{k#)ZBjCJnw<|zz4$}}|z<{vڥiB;oln8{wz{Wחf}۹iunO{n{뎚'vi{حn+UmmSLk7ۭL}UzXen.ۼw^9i+xm<#)]E펵GZڋh*91wMy{Ov5w^G{l.#.x|#)xy#)!eny|Uڧlvj^׫q VxO]Fq:"kޯ[=|[՞;|#+ooWdq|/tt8[}=ou1wzOw8vb`*>u\W}swX6pMW=]c@0UT}Cgw\vcS|¾w|͛Pｾ|o3@U\]ugoqX4A7k׼E* ׻zt{fn.4fHVWctMkzO{M>8QTvwmޣ^=c3ӗ#+w=ҲA}ta .s][ﶽޘ.}/ҝw־[">%4#)4#)h4ш2FCB!#4ɡh'b M a#+5Sا@O@j4#)#)#)#)#)$"!'J#bhA z #)#)#)#)#))"2dF)He=CMd=@4#+#+@#)#)#)#)@#) 2#)FFQ #)#)#)#)MDA#) 2&0 ~%<ҏOOԆz#)#)#)#)#)wVWp[jn߱ԹJB8fY T<( _#{$DZkv8TMJ1|K^^l[9~P?'xcAs[V~JvPsWYF9 |.a1A»l.qwJ\>V`)>]bR/|,y &ݾyG &m%ڋV #)U.@(4"0Ђ!!Rt(YPR$D S(#)#))($L0f%12!l%5"2&JQLSm%CS$(FԨDJllYfh4&Rƒ)6"ъ"R26ăARH& MPfYKR1AI(# 3bѨ66L&H"k)e6ؕ6RIDQlԌ֊Fɦh)QZ)Jlb`)L6 #)!@BI((Y6" !C$DBE$LmJRZcP XQE),&J$)D,LT&Shi őS0֐dQ,1&$$MԖDl%Ĕ3b $ԘHȄ@4f,4%F"B"B@M%ę4YJ`ٴFED)I(&jXe(Ĥ2id44K264ed$QhleYeAK3F64l%5hb44%&4RFʕL 2 ѠI!E A"d!,#.RMdFdPY(i``d"(llfj#RSTɢ*XŚEP$,E#.4Fb&06M2#+"#XImd Y2SMHe#,F IJED&PLImF-$dPcFCʍ! bj-)3V&ZL)0ȉBIQCfM[S@)#EdIk%FflF6mFMe#4imti +Hd%J2SXEbQ3*"i6KAbib)e) f62T2%)#+ңH,LKK1FJѦCVFȪmb(ʌV)IfjR#.)5MJ-26ٚ-̶-$jJ66 %4D*BѫF5!Md!ĔR 3RMTj6I6jBi6ֳjA,56e3f$M)-ZZJf3ZU+%T ښV1i45YJ3`ԖY#+SJ4Dj4i6B2! ALQ$JMe6ғM4JaE"AieHi)lʼn BfHcԖfY4L!#.R1`blDQؒQ31#.JƘL4dر0̵(LEjShYEE4#Di-4$R`)eHJFS`lBRMBUFLl6*SI6(IkHd0IDe$Е&̍!-lZ#.lDlblRi&S&V)h)(́J b&4cLE5M11RBEQa6-(Zi03DEؤłiI&0ɘi)fCL+F6El4&XV -%Pca@IRS3BJ̊-!,LhdZ1M)BJJh(TR[j(Q06Kɤ1IEJ[!#+$͒)6,4,ZfJ`I$m b)E[l2J4QƣhdTh1E&&((MVƱ[$bZ +EjJ,Sdd#+3TAFض)PDMEDhYFQk$d4*Z)*iFQM$YSbIQEdF5biL4F#QIFE-4eCI$ٔZ66i--c#-h-kbmmP&ZfM#i&0l,H٤TPLđ$I2"d1vs4eOCT0TZOቢe [4AILLŔ?<6E9tF:D>{9Cp<ܖ#.Y3XdIWͶZCwA.EL; vk-#Hhw៭as[3+qhHvǹQm5L)*c;[>vN{JTe0G%IlTIJĴZT#+P˅]Uص%urMR7dIi#.EQAe][ܰQ-0 kUI,b)c"ȫu'gmhDzea#.#.xgB9 .4U$K*CgeQ82,; IS3ļ͡6e_Xx#+qh^?<),d=Mws4W72i#.Mi _9`H)Ҽny$, l@4D.Czls IeD#. Woy{]Kofnk",.FU׳o/r䛺gu*y_;h~5١P5i+U#ꢵu0{[(fb0 1}Tm؍\&Kd޽#+[cD(d\g?&q#.m[7Dž\TSU!H?YNK>>^3߀c1PI1WIEMN~z}#)ohP@j^Td!>&R:RZ `,$^v[T^~t _8ۚ㬗<$g <4k*J;S>&rZf[LNꔂ=o^* -Q^{R b8HnIs'h)BC+z6=4}c*Dg㹂Trk|zm!&\4i4)U8U4izτѴ;/_0U²lQGZXew/>U5b2(!9Wֲ'lFSjBdfA[.﫳-]0(PR0ܧZmO1*Lv"A u(Q"kr#+@$v#+7яB_p 4MGH;b`´~LqzV7=쎶#+ɻqMۺVoЪf+N<" 79u8N]ي J4^Ӣ1>MتGM,)^Qh["Lį/<:~?v;J7۞UMP^e '@I#.Bމ`޹@[ ?>߷J=vn.FqDQg'7C?*FZ+Nzs~S48k~ߤ4RFȏŞvpݘk֮4dZ $@0lsZ?)ϖ@$Eŧ֦Qu` @a0]ɞCs^m?M)١3ǁsy< Hu8,I#.&x[FI==#?/Mv4 qεn:ըmw8fىQS@UTbcQt_Ҭ/\#.ovbfN/#hi4se(ƻRAv-5/t,naݪ#+֏W./20љ| cMBwyI\fsCt*,<ajCjC-~Le#.,y)IKIyLȕ4#.FF;o"._\;#+ Atkzz~5.f*I8 V^k%=<DE*rרAj5<.00* F{ NR#ނE(#.5Sش(J=eH*.!!F&ǐF'70S-ϧg,g<~_ 2Фg5sGL#+>pꎛ@/v I9߁(!2 Ø_'~>n^_#UYjt2Dy-p4>5-\gMz~qc)=z8}L&`jB;~/|(a٤~?*I}ON'r%0=rrHt@Ƹ"QJ:;GuUR%ӳצh&ttxaܜg'17ueEi՞#8mDDm#+!dr#K7i6ZIv#ŏt\Q:͏ٯk#$ zq:IEMޢeVC*uX"2Jk՟GIA Q/ tL#oV$ݐ~lhA"Ae!Oi #;$NGD1K-;{V"zB25HjT~ bZRY6yzh?V1lkyʞ0mRr[#.'֖Sڼ|n#+g4W-0liF1DYᖯ aifsfG6~1,N~0"޼Cn\X]ǫ#Ό-95dqu6~z=M޴WZ(mߒnч僑Dy(Mv/{ڿ ]jsYH\mCC#.()z?d9U #.f7@7uvڿB{BnE?d7Ap!l3CRmM71\}>#u'[yxQ5#"qk`MYKPz#.Vp98}$ N۱KTGYxC8^Y"ZIC+^7eOvR&$(1+:2,4?2J "?X7=w;rim]c'oĖ;C'/b"5\|32.[֥<ٞGvͨS5co #"0"iq{D/Oq9ZR0I7v4 kA|qzDVm37a0XpI=ݜd/V.|J?J4 YQj<>Q܁8Ӭq2%'/iȹSZh{)ݻx"4?@zB2vdvt:7:a[|unʻB9vA*hs(ɅS6NJ| RE D`|iBa\#bt8$!*@:t4nΞdqf~#+J=g\ZH@%E'>Qx n0}V/JL(51Eg3.A˸뮓qokl]6{n1Q9j2Q~cդP/#+Jbک`PPUE%-$=/vαdI#.ىɦV8ΩeM/#+59oQVÍxjRމjQ5F0ߛsbMd/ݱ S{%ET)rNN|v#SEJcL~Z›K5#mp̎k7;Z|cݭ8yW%"/d027)Ö!F{jh#+>Qh5ݾOHbsŸ8pVCt E{\FӹgsCoVM3ώ.p^k\,V5{ Wlq4cm4C7mTU M<+Rجv *&ARtkZșC9 {MhMg|ܷl2R;r#+$0=Zt$R0TÃyekQV!/ t>zaOqãsKA+ B},8b["p7c7,sފc#|PI<:4Z3LY]&}db"C-cHD˻\o!MӻiB׷oVxTW!;"i@Is Ӝ\2˻%f8 hKP0֨q}1ĥ "y'Xvh-QIusy㖃zyz#Τv8H;AgxsNzv4 ykٶr".J@E3/-É/ju& is?'i7Fhr孇Z+IpImPci ِ iH9w(͋,۴?y|klj:Muܒg^/=5t ÝcWA:/!KbdjFZ-Q'jM-Ɠs.ng+dbݩ^^.f)MMњΏ7sn(D]vLϡn港N-thtʧՌCὟ7ՎcmV,CƫJ̇+15D8-KaF2${ꤢ6ޟte}kݚtvL#.*<1#)c9IM±D>IgȠNmOS}p|7ߦcb}[;bH݇-ʋx>?OxgFyq}ݧÏ40REPmL aX5L@P=U+s.J'>Yae$Ba tSQRrIΊفcΜK8Xp#@Xf0frXsmvNZM#.sEdJETp.d*ꂝD\D@(ߓ#+挖s#+Mjup5RLMo=E'8H)FK]SM5q}yfHz.M$k\;3N)kO.Jƌ- MO& 4lhL_4#)#.YT8&ZW{wou7S"]F Y:ϊIX+n#.-'9WM2B!c_RH<}.#.pP #+f 5Qi.٧C`q*#+)$шzЖ35۱\?iX#YTc}*F4ϙ|:yB(=+/"٫LΏщ65x#.G {y|/J%iiJv 桉ibaE+V{kItOk{e=Mu>IWrP2JiZj$fi6 )2&F2EadhH#CjEe0i6$41'8eޟ#)BI{# E{*yj13_i8#+&LjЋopVaFa6 @Y_r!iBȹ9#)$=Xu,[(TL6x/Z"~5QQC%خr#."0#. mu3d-Ί?Z0_Gy a(",C/N.5{SP}p? \ݤnWTج& Maj6#j0[Emh3]P#.xU#.1P(\w`4L$]~|s>{cG{(@GI͚_J3QjO*ff#+Lz| sP~Fz}y ;tR} cw:zJꖲΡuUhIXP{uwKNoŽԎ~Vx^+=Ӈ_ѰײuF]GXީ0hrcSa_L:\gFPJ(}UUG_OW ;aD%yajщW?gvT5ΊcCHRrIS^d+:m`TEw+"~M@q*+=y," 4҆;XކZF@RF(J`8?bIx$0}Py!Uoժvf` $W./9᳇!΂hqom$̐Bb3vL%龎1!*X'14_mwq:fH3.M mX6akhR8KEO֗7{WwmKa=ኊEp@ӯ_vݑOWn1XE}>y|LV?WL|I.v珌-zt|>꿧̽+}ϫ=1i)ZT|@]}n|}2:D}LX[u$H_+ڴug\u:EQ^` A/UglN4avL_4#+y;-e`Cíďt#L몴<ݯQlu9P 82@S4ŦƜi!L~eKa舢jh˽ZSq׋Ȱa#3|;9zU>'?l,}1A7E^nY.Mۂ;j:ĬSP:$ ahӌwBhVM&@RufBґB#+ '?ع.7[ɿ^NS Ɨ$315'@[d^\QƤ=tz&Kogq=#.BXxR?^kƴ&˹ H@O#ዕ'ȨjugYў[_,2UQ9Dh}*=,R)ap@#/*_q :Kmo#)s?#)uۜw;DbEo8dkВ@:I[#.^fѶY$WsLϟVaUK)J8RbH)5),Ic$0|XИ#+(^h=f%M VO|#+o}\bߟ}(?8ԻϏ[{o0t_^}4Jl dOZ=׷FUuzR{TOyۻ9UW1<~Ӧd׎nB_Lcv)D_9wC־;A}<6}3p%ǎC5=SջWWcgQѧae߭Wz]_^>ѕZT,ò- .+O;z9ߵxqn {c\5U᠎@WayCe閨Y5r"U跧f\X)Hph{ d|5So-j*Uj ifaGno6[Oz`>tlEs>w,1g" >+c`>B[*ѾT.7njԕR|ngid9/K%^rJn{/z*^mVNʛ||Ή)=?{[(OWs}꿗|4CJ3{ꤴOMF0/5ʿ|_Fvï~I#t&ϲRkťUEAw}Zu>㤿v1v3MmgF#ϳ4bH/~^|lJe)ǟMU}~п_=1,4ݟtعU藇ו{(`/\>.\amb%\oM 4n.wķ3Fq0=m7?~{>3wٿjÁxsXksr-fd}[>ڿIbq.PcTx%Á;pgĩy\S9a'x{~IPh0֦-Lf# œN_GO}o_yn ~NXvWsٯnOZ7c㣨UzyHa:<&E߿;χ_̾zEmzo{Tڼ~۳W#.ۡy*%5K?OҼ[uo#O7HCd&#.0 ~܍~Xv68_"`߷^ϗ.[T掱z}=u񶴪xW}2Q_kj*Ns>}7_;kYge?-v> 8K!y'˥g?bV6UFV+5@o![,˨AmOJֺ:W?ΩIjծ9ݩ#o JrTnh[s#.vo4y04$rK_½,_ǿ{|4SPՙs'O#f#}.OtWG9heoY\3npz1E.bݑiJz#.Ce$#lKz`cFvLȈE'!&brw/pŮ[ȿ2ܵls2ۅngOowi}E9B (Ϻalkح\5LҸXq`.C2ui[7F#+U/vks\% Bq9Hf<S93tڳ)ف^uo JJ{wr~Y֤r]:~xW73N/-ӓx#+=Kt{Io/_N]Wrs/<<:fx^wsau7>nCi>!KFΒR OFáND]Z_? k3q$32FnB"TvikZ K9EhRLq4h UPqTETԉm5ַkVaIeM&t!HZIJʪ·HO=,P)PRo/۟]8H숇Bɴw7nFpALd?(4ԺOv;Ft-EE"YE Qt2sz.+\žDLĎ#+e=Fii+VD4RiڜE,fRH$QL6S&]ag}/ջ)B>#+ׅFļ^LKgMV};ls6S-z6G.'¾<5u~sD׋u8#.Z''2 +MZi!Y>IZ}ǟ}{ukk\c ڕ`޶}=ӻO_V4[.60?M)<(F/ <8K;#+U.o)q|pGTU,_Wwf_߯Y0n}erz>g jvO#)'9I#rbV1\a)ȵG ~4\*IC̸\BD`ZB$ĢjzIªM "Fqjܟ!鯓o~y9J f<=tO{.aU^iLjƋ'Pu ;}Wf5j8k8⨫olP_5f#ת@f ;#+f]'{n[wI%}#+{k<5sPǢ֌>[NP<(Gƻξ3Ựwx0mһ5",_[3XiVgx;A Bo}ëKt4< @QH2'*$[4gPJ4L12L L`P46DS]1!ѫb'rLr87#.21\ypq#Wri42X_CH;M#+rqyU@Q۽`DF)e`hp_FC4FFTJ# ]Lc#ig灲X2 fK/#)姭 TUoNӴ*g"#.#+A.$r`5SlkS)h\3AEIpmщYl39D(_g2IaΜ\b󼕦bPYU#.k4POAjۘi(0ex6LӊΘ!ќ8zHghx?ukrFܓ& >aYHX1D4_ԾX}woiL< KG}/<K^"XJ `4fl 9m1g€֩ҋ\az8h$0,NR#N@"F5#+flu "pi(Cvfӡ\)lԩF18.,H<QE#.,p C#+ ^ZՋ+뮶-fq˾YDAFO]~4_FUU+#M~f6fxg.۠" s.d:`wFEJ#sci1#.r1QKөri6Mhdcn ̃e:"4(ƴ"¸d+ӈbײNdą9L^4NerC0*rZyǤ+'G%uz`}4 $vL*8L{!ƚvgF}7!2:fQZRZBU|]>#.jyU^9^nywu[8vȬ49:gIjkHF]95Yv³]vAWEf]`v8U$ZɴKY^ + f{4I8]2dZ!ӸsJX#.xp>x4i]N#.-[MslD\H+AقdU>Ppl[czIm@$A TZ?zn޲YLc/bn͆^rZz L&Ւ_GCS)&ЅJSi$OTVx n #)Ѓ]/vҀ7mQ9ROno3'z;A{D=3ص/S'_tbЈ^do]3amtEtF#o>ƾ;G6F[1׍dv4A/gH媔?x􁔹LybxlZlzh"1I}hw2w|$|A(З &7IV:E)1 C(\:#+N,zK)9yƛ5x!F[&@^zlVa0n-k}lZap-宇;#.H n+ٶ/:26OU-4$C7n>L^< nDZU%  -r%WnZ0 ۭZ[ X?#+:]>mdcF o곫4 v]ZӹÌH\#.4Fb#v$ĎYz+:ŀEzTdHԞ%콻]5c?zO3=Ɨݾk-ZDZ]m|m5(D8Eç{3KJ6ELónm"fh>,I雋PvLIl\~$Da"L*k8?'#gstm lG$p;#.A=tl]#.Yi>r:ƒe#EzUus#.G4E8~#;mc6 G7m1MY\tˉE²[(=7͚hm\'a:c,Hv $ۢZ;DiJCR6E4ƺK'I?ir_~vZ?~.is9FFTAB1T̲{íe+Z1ϑrj (21,ۦb6.w ?3 4zrCgx)&3L/j:jDF~:|ɽeEXnS]Q;:+2A VaЌؚTxSkk簠߭*4#F#Ee/>ݎzW2oGȩ#+ܭvꖌ"QSDmHnֳWX%ADN=\HJ,.MsɼRϟgn>'rWX_*2wV}HpG@8'/fk$.#+'E/7R~T-\PNRE+qey=R𝔛Wвْџ9\gT+J.L EBpg=#|p/9N)ZstWezZXo|1L]y#+a1tsÜ65XR/~YZmX-5Eay-3ЎwBR#mNc&H؏剹߿:n?]jߎjyb>toXя2,m^񭢟ZpHByLjfq=-oZNq-κ(gx<%(GUwUȊR$M(YQy0s^j EؤM?u,l>_qzuj%E~zIܬh^بmQ'9ű;|q| O/`_^XPD&-}U⤴1ϭfTYEty^O<8m=|KqD} YD3wr{i;R2e7Ph$͖ssnC/0 ]Lfֈ-*#+d?9nZDU3 ^'7ʸ; FXL4|yn5xNZgԍ<3_ӭc<,/\$ΞwHi#.37E֦<72I]7'u%'u#,L_nw*!Cj=և֔n'N#ZEsw}^}ԍGҵj3l!fd嬔mo5eUYuGD4!#+fo[bn[-0)=@GEѤgEW]SHM*1ҸL_>ɸwt\[p/?eܪY#t㡡Z-Xj"bǣ:lTmyntZ8Iby d|c]SºU9W9^kN&:y? lFꌚ,ݧk>FQN[;jSz= su[h-diX6~1McUA``c<6[ik9U3Bob}GO5jy{=#uRORn}ٌ|=!y;5,-~j1H1\L5n9}L⎪}4BY;IZڞNcEvLI lt/781%˚$-w>><͝R8۶ !,R/nuQB6Q iW<2凿j<"7?;Eŋg ͣBHt#:fS0d2Kȝ뻸-]LvSg|[ƬdQĴK2\.|Ȑ5ǓMq_g'nvzu:-,}sK(w!+vb#(Z#.G:b]ڝ0^ryJF呶$9qnA'kB8|q*4(oQ>I7>(oD{-锦va@߲\pT%>wkq-*Z唢Q<|%*o}݌|:0/5/P_{RSiQW#%ʉ3'CuӦ:(HH:mNJۍև/6o&? o3Փ,;71Ub.s,}]. Sx(slTۊon+W;,UNNa] *GS{pҎԀ拎_cAu%8]#+X+ďEp\g$*;h1ㆭۣ}~Y[?uzi[!oU"xv걘<_!\wm3s>XqD|I޹T8\ƕik+tE Ϛ,JzjO_¡-lc#)g_-$$ zƴtq|]K"UT,sǺHacσ߆" 1vqxQJ\/IZMhoa9VYT:C|)+P"$A;B;snkj墫["Xdȋdy؇XaZ6[;Zo)Ahhpf hiUPKd J!ٲ4v.9#+'!D5Gt2eWB|Yƞ8C~ydZ+:vJiF3u}XWsծ !ۑFCNJ鄵#.XS;.,350XF,4I%MANǭVw4$oqޗӈ?;ZM֟iԪPyn#.WnUJ we' O}6{p9ˣYyLx>1ViX]Ӏ4c QKіq C E:N<5b_g nox{PdGuHg振̳_ σ`D>xJh %'^IyUOuxNEO36u1rk;a#.)e0u(Ks)Y'޾󟲜Gl*:rRYl{'8<䨄w,iR#{1Oo"M&+a熪t!#.-tfl0tvzI!#.]:h^׸9 Uv!567B989e:e0Nvǣv;o߱ջD@s4zt]utŃbe j5LJn`#.bba|1iךl>\mgHd #)MB#.exZ&8ȣcۻp[X[]u&icZ:1R lMBg WaǻgFhSɹӍ#+M_3Hs(]ܾxDZs;ؤ0"v۠ӋEQRv4:Y_ޝ,cM+l,]-CyW|9r<4j1bHJ]TH`>yuUBLkiLf-uUFm%gtNW>#+;Ymny NfpB#+ϝPc3mINg4}ק/G֚@>?W|ٻȬ>T|59/˚ϔWɣ EoݼuSoOVAnkNDI@_?AHf/M*BAP6deqL!2?sC"j cvRTv~j/6ָ='fXyb9'$({^q~gѦ^A#+#.ÞSNrb͠R=Ђ DϿI4c#)B~}އaung#.!q~1!ΟD&2Ct9_SBDN;(P#.[HI EHb$9FP!+L+[0nP; G雨ea ?bͺ}@H7>/}n~3}SC¦UޅIa8׵Ϻ?#+g}fbఆ%U+ZŁMsR֮{y 4sf3Ot*c@#+$PS}]JƱFܫg xy];^+N_| E #+#.n[.ɸIZ#)1w@P[wA!A7st`VRIh@&}v#+b&#:Mp>6xhRhjWvRhmSYOaFq#+\[tݷtȑPa|`kւY20Cb,)Mah*\3 }\JG UZ#9ԳtbkmAXθβßl.豯X&gR4jǛ$Km|RʮPPW;ct:~hٞ)8 -C$ Nwya,rFpc_tv=7 CF^w.fp3l`E)2vkY>?.}Şp{Yh0Edh;;r0d!{"0ǫ[i.恂iSHL$=+ſ({uŖ>tFSP&LiqZuO~?Tl^Z#l;Kt)QE+a* jюځO}ߦM ]ȥ:#+Օd7Tw͙Z!ۤ/f#.C9t^xщ[OzǾ=#1%P7vjDKV[*MvX~ nlS8j9eK@ƭܯw9MD8խ)zl*~Byĵ'XK5FpȋB^BxUK:TKIŊ4"ن0ݎ$ ohӻl w!;>$@Љj/޴._cțE<+;|flO:`쏁2d8"S35S#+U|3Čעvʭ\xaWZM<'}Ts!8L󉘒5mglO7hN*aj:#.3V v%7j #{, .~8.LI EtUNLTl E Cw-5#+CX!߃}>m%Ə1T>>L\j#+iC @&@ /lr3T ˋϷ>};ݲz#+0c&[N0[MC1[۬b>8B5:K!-zX;7Kvo(IQ#.Icp#+򺏂<|"3$91QxȖrY9Ds}6̑̂&[ *׌ϯl/w(<&DuQTD.aȘ9H+~>2ws쁲rP}Hh5R>h)_b2i1YIIzm)Fjካ4H,ɫCLaڠRj:'Qucjy'awSGC[/{B4iEkB֌h#+"U'lthmKIX*W3vT9a#+fØJmY#+i\,]AK|O(jtuƱwLQU;VñpZz\#. #+'F1SVPs:s~grgPM@gm(ٹgz֢h-%$6'քI&՟Pa-@hJ+"K]v꼼Nh$~؍tjG}57p;ŸDY0{~W>_ D#)w g>G)t]܂I)]N3@Ze0`,D77jæ$v- juݡά~ I?MAVt)KlsMP"tDܶ[?k^Z9va/9Ց Ϡy |jb`1TXT(J0TR͘:`TEJBO`c{ۉזa8,[wXz{.>9ys7@Ik9$$?\#)*ίG< eE ?,Wk'l:n$6{\y'ڒmh~p+/iM5.iɮ-#.مGԏw$9>'!Y4]Eh D,9gab!nA})7D#+`Z4K:$b :iꔷ Ғ `yȅ#.I54#) #)ښ)[F] 1DA` E.7(MAPB([gدy}x좌rW2^`Z~}uzg;k Uj+B*Dc[z"Yʡ#+392vixuLKPe Q*Abv\0SyՀ 4NmBDi A@o=[d+pQ W2&ytMt+c*a\{кn3GU*“3jC;D9Mop貐p`hҘxr/PpM &=qv KSWԺphb¢mb"D-40 *T)@,rӻ7#)mnv9u=wl% Y|Y#)pbYF$mΌE)PDa٩(y?c#)lp9 RNt#+{#+Si%cs[;ϟu*+#.B#+:{jъE0yW!m1 ln<$H6}տA3#.Pf"zc3= ⛛u~@hǼȁwigz'quP9# D"¶+UbZM4EKmE|etB!wьj-$jİݚ 萝)a즷U8Cj8zK9$3Y !X8kѦ7RdR>]@W,| 1cm!=!} (3:@ݷ|_kw}}'5[YUX|okx=(J|v>uȻPiml48|l)D`ܯbh)jցX#+g>~N\&Ϳ2BV~cϭF>i=|jRɌy"C3)Vu{w#+"37 ]T #.#+HD==ThN&.嫴N|0{#+KWb#+Fh|?y}M*BjV'R?E뮈Kqd9T_F'q0t,#[Y1!bu3pb"Fp;#Jp8:8-c8%z4#+:O_y!LHM5fAVmx$z~2&:U) ipPPcoFq:&^#l8ÅN98 hU%IG,%)=Q#+EBXx6*QT/Q4<) 09Jj;^ݗjwT~Ȑsg؀@UPP$aW AVqS ?` /فD_#.IZY*y~gYA7TÚ5Eɲl|/>8k!f0`fnlPf~?PyjX!B:{z̟>3u?;,Vܷ^ޯEJ]Ca8gLLDY)J3WjnJ0mSkүv4 u}O͡ii#.~~7φbq@{aKqg{.fyVe1WRBrY4vlW>M><.@B@>^PRp#)Tcgæ+-"%ODL[85W}뵷 5bH?mD|x⾁wc]<7N?BPcӘinoÁRgx&cIcr4chŠyW/~#+ϕI調|%cGn4XE<(*_dU<M1C62}g*XB dq?׀istP2NJq6N6Giqu1D N|N ҵ'.3R*PѯR%~6Lj\Nvg8kfbbHuz3+9J~NE.G%UN| 4a9}xzVx~7T[&sad;lo;uAPyG.pdz/!,oӬ5OmlwmRú/OOG^ޞ(8](jw}Xx[#f&#)t׏myNcC{3:S{?VaGEhL<̠[ا3c#+oϛTbޱ90{UJJonjU[,DnafМ6 x{w}[ vO^UpsMV*%ΰ[.yy[lt=4Пn"ΏqLb7_WwsrQȹV CJLWǛtv*ߥ(a7,b8ߗ##}6Q9IIbjc!*V'vGSæ]^kLfa#ퟳ~:/U:td]g!ZM 7(3) ^^q04AZ)WpŌW҆۸3-F{V@UEmdqD5 fDfY?r|Jmtcql5k|vg(]2sPP~۱E~#.#uQo5&=+Az#.*wO JTwpjm퇖Zzwjм+p~x,-4/r`M&<6疹뷎+a':kMmy?zffzxj>1#))uɽI0LoH% >HQ_|SD-T~epIf'8ɨ^ǃhwNUimtHvB!g8q3#+l1Yl'4F棂z#Ph힑bt>2q0Y$iuVgYvN[._l&*soꙂHQn_HG[BZ1?zJ#.dEK={1k [LNoBL~o:>*8o:qdgD^1iGK[qK[lLBAY/E%C߼7L]??ھ~#.ϗѶdT'a34Bt#!S!㗲#+bWsҞxzح{:n= c;pC-tɸ9D~hc{|S޲8'Wןt4a.=:x/s^D&8*'}M!,Ѿ?^AVq/ <~Lx$Nn|W̬枺W1Y%;:{g]S#.:Щ53C#dTzt*;]u5_n);gc|B÷üW=f!o[ǝkcݯ>եnLg %߯9}uF~۞|hч~}6s[GMT:+-Zz-L=/n1m9F~LP9ZC&K{U~ujIuI#+EDb;=t3Zgo_#tk₺m<9<40//oN!cX=띪B!#+ iHqA.o(]zS #)9/ɶ6~NB9Kh8/-Cv8ۘ5dAY{PR`3۬lw{cPd7OX/KDeɎ5L)Њ:/ԱKsVX&-04Oc|8_K։!943G#.'1e"ɐq/".kDN-WEyR^1v3sZ Z*r=T&9[ߟϋ Փ (﭅qB @Ux?}P|}]~Wxinl茜%N(#)Bkqͭ ;{=FC{IQ'VnsDk}{EI$ԠACU|}-|X nc?O']#+a;,cIʪGꢁַܢ#.Gۖsiy<b#.Y%* =s҉rL.rSoTzn*+'za #d3#J"OGz?F˴N}FGe\fQ"jWMgG8鎤5ѣvgUi&ED݋2U/oZkeI NQXɢϽ׃T#pac[e2.HEdPYi[70֍@$dV&L(?q 0b-ٮ8|^4-E?J4yB!(v_܍ouޖT5eV_y \ޏ.6[#+Qtk++/Tz(zT8Ka0Z Zr񾢁䎥esaz/(36#g!.N6g$tO QD*y xA\+P? YqV}\u7sXDa:s\yM*SMfan=ïm& dD'k7W*l9n7; 1?o-HaY,e(E*5U: JZhƚ϶7[!!.vp)0ؿzm5Ff5&#+S~5~Bs%+CdxXw-FbKΕ}(c=ꇑ\0g=&PT ?I De)OlE5>sc&RT%:iS;lY|GUDۖҟBr'ơޚcU:b2);si&#+gjJO#+5y#3:ۤR:Xʥt7r8:dL/( ."pR4lo)ϪQ#+H8|p^Ѡ)P(Ba鍇Nep.I~7xF2`x+TcL~^]8r^4Ǝ @@즱9Xsݮ#`(׍#wʆl&ƨO+0YL LaY]xi#+.liiÔULj\yz۸);q_d0]g#;Vx ~V!ycÔ=9ZLF'2?Dn\n%Դӻ (B;Pt ?ϧ^Q#+"#/՞+X[l1=YȽRi##.;7PR:+/f55&９,Ej_KfI_-#.Bil;2͠1ވ59A`wBM3j]Kmw6D*9Q.b+0}ڑrj#v#+]b ᲖQhUq%{;klJaF}]vRsaʠ2#+OvWqJ]"IoMV#+E#+o#+X(bFs[Ç\l' ȁ΂M9R#.s #NlsfTc@c:?miZI:Y.ELdN1 a451ݭvl-rUmUjYJ^VobEZcҲ0U^ƭ`bKt(})hTlV qq#),Fa,CvF#+lin͉dT;2ƦF3DBۺ#* A|\b=j]UJӶ#iuZNi Ycz NTn,3#)L~8Ĭ`IvžCbobUlA}muk.UG ;2jx(5d P:p+Or1ntt^Cn$u%8p*XuOUYrp_)mU<M#+`:Y;sO9vL?`H lt40v;i\z5r,j*-nQKh.]F*d|)KYK9x=RkPE4Uj[j\[µEtNUt٦#+,ɹa}'q8i5\1~%%t36xC5YgeW8#)V>:4OٖEaV3P𻝌%ۆJѿtO}y{C,zy\*ߪ맾4kR}hj./qGS08V;0ѫ >aϮ+]C7UplwJTUXQv3ocr`ҟW_$[st@ N5g;#.Ւ+,o/O_O?/mwa#+wceDy=H=xNx=>G9?JOt5Sl4ψ|>DSD:1{= ݆j[rQiCmtLrH}~]z4#.w-= ,!nhwl#+6k>67Z=N]ݤmFRRLtâH_{rT>XTRz6+~ab'!@B~,ͦ`aJft=?GH8;^PP%L 4_r2ӎ֪E)zbl*~aX9MH'aJP^HgĸJ}\!!~M5U*o`Qt kP6!@&#+iֆ#+V$؛;ĵF#)w%H~_6)ԴYz<CҔ{ىPu]:/Cfd+iwiXd*A8hp'CCA$"Op+ve!u$Q7@P`<@]M)\B)H72eAZKˆ#?N#]NMeRjĥ(^gyȶy<1Dh#^#+Tό7k\mH&/Z2];GzN&C!_]}WwޟH|:)M;5NC~7T\-!* I#)0"#+X1v\Ȇb:X}"g?tCoo!!)Nn7S`drC7b z,0$RD|6vJYEK\[fIR 0ƅtVnfr0=l#+&2 Fm};jB!]<ۖ8#+^; ig-u2a#+!{{}6 `#):BYّgwԃ$!>-#QVw,??n#꯲t"7u<>"ո]#.ݏT zd|BI*U&L6#hzZufǙs-NaN-8R|J7T\vfUYEY.6amG4zDeA@~wb'CcsoUc٘wV#c]!?\øpfK JAB;{'Cѽȓ0$*, ~U?jgM%Œ{foE٬bb/.2sM4KqkaH8m(wФLfx#.BI'VWaF-+Y6γJB)w>wzSӐ)4Q=4 gJ AZ 6zX2~=P}_}2|ɷ)͢["b}#+bL3v5ycP@ſZnGx#N aq||DbfmV"#)|t|5h7#Ǭ`l@v-iө&ݢp3=AADI HCwe?[C=E}:Nߡ 1d" :9Qƿ0 v^T aDb_]woL,un@~'/>LEMq+f+1}n@.$5-SCT \%M#@H ^ 3$܋6qmGpO֠2mg)˲GCo4zsCGSξRVW}ZFo.ϩUl}gr1O(WSB|Yƫ@p 73Ua#.ɺ 8k7&Ic ‰e^ŇUΦ#.E R{> fz(dEQ.˼eYuxŃ7N׬Ө7>ן~GI D>] #.qz(`h"_#(V.VU]mq$ϯux :>&}9Ww3#);͡ݱL4xQgd[-rWLucz{1#kLj(shm 8٩hHL41,L8M Vˠܖ=烛F"fTT۽ j=Ul8m;ׁ@#.>#Ć$py")*F ?@ frbۏaJu'Fed$Q5-7lAL̅nfH# J(DHEG|aixÿv,ߺ0+V4QJK3ڡ"L`S?ٵ5J׈bjwCAPf F؊E͋eI]aCWIDTq"f3ЇT `թk7H-l=;8 8q3(")aV nj"S45,DIڭGAP}uZتPL1EY?Q{5]*"FI#i2U\n-4f%mVyulc^1#+Hxf#.itt(#.sjbd!h}BO!ldێLC]S*FDY(#+&,ᔆ@aHC\ҏK Y!Tmxs#+Hz2՗n}Ё(15 ٸNOde~Рtj#Yh#) su t4Lofc#+n4#IZAa趿;wΐ#G A* F"~H}P2dQ#. g p y8ԄblEVpBZ6Eq|U{=D35ֆ0!)ڸ#+U`hC4\cl#+~ノ2-jsLجN Rs *jd j\ @aq:WWgk"%/fm&Ӂ1Tϗz)#.!M} )W4%dQߚdSr.Hb-dm7>>mp^C"E$wƓ0B##k3-ނ(|(RJ(3`k:OAg^ XRH)%0bbi0v[)5+RVnEVe".>.óxv>Hg5Z)`"t0BX#+OMp+pS>Ha_]HL9S={t#+τ:HD,?+*`Bjn  w77]8ǀ{z;#.EX`` `t3uo+K#.,sc<&t>22WUF{qeŘE0*C Svw$E@,R)H("GggP ZbfV* 0i5wk&`VfzEs'ЈAxۄGzTc@M3"Dz+H|#.?ܢ?#Giٕ|3WPm+6~إ-sS78lS;+ueE*Q@?C8CLlm44"d%$ F +M3K+V@`В *5%czjmo7p(ȆUE~&xQYĠoTS˗I'͒9 ˀ,X-D9`UPbʍffgog `Ó#.$pH{wJs.M̠;HEE" p]#)!V*c-3.ieo`z(t¯Y]ls鸫| ʳ5gѧ̟{ Y>3 _C'Ҁo!S#+Q`[!irjvvY ; |7zTfwYs4#.HwX;cw߉UxvuGDX(#.6,`#.pWC9WrMֵgKr%M(x*d1DΜq<>xSg{&=7j;&x纋}c?=k_6#rNK!5X!]08eYL|b!;곆\ q.;nlFE SLGt>>?>hΕ,ۯGX;nR6qu}}9^,P>α[:{"sh13uC:mɓ*XQMZQĨ4@Y9tN: BM#.xXd~#.&45nͷoٴpWQZCQ~g>M53A]O:#)YUs=>C92!L+0D/{qKI؋jfa@C!0殇!R]ӿ揠9F۶sbC"D߾)f>S#HM#?]ޑ߳<߄.GEBtiMvE\u'LydGfU,^t%S _̎!W+ŭde`?/b/#)b#+,ƘM0TzYl~o?aeEI,?DD AȳS]?C1?AŊs!twwM,ĉA F65\{aP[#)[pٕ C`Yݣ.#.BJ#&tŲ? n^;[1ZNa $^#.H`n܊QMqlK>DlBl4;0DsxQ~> A["\f?r;lUEUDuKUJU:CPhT4 oS#+qn#.NG=_m?΅0-#)HL=l< _n"{(Xz{#)q[R0ٽ%mflru(DNQct{^#)D#+#+#.*  454@@cgl *d!OZ]; VԞm[G/b0[M5[󣹹+]י;%ˣ~>eK=?'egIꡨSio#)1h~x82#.^ QBi`y#.K3_#)JJ!)cWWW[e,2S,"P2E%Qd*_Mʒ{:QZPȊ/M7Sk)4GDL[N ]!$ݰ̱͟Z\Br֎3I7C#.#. Cy$rIzHSȫCd(1AihYJcy!:GV&~Y <0d&Bqص V\H 1EϰԘ` Ƿ˾۽CCm[w%0c`;wmpiIYڂAi@1mE"#$N-f-ΰk;&ݏޖVa\,ݟQ@n#)s6^L#+-ěz< ͞ USB8TOAzlTWё|K^}QdQnjқ:t^tg.LnvxHGָVqYle,Pu\k9ho۰a[YZ˔$PZw~ VtvȞc#.CF2g47K[Қ+qlf5b,9*Ǟ}:nټ1#deٲ>3ߡIk#){<-;xIl6oC1#;1,| * -flZ,7<{#b`B #+8McBRC^0Zlo@C~]=Z'UWRBT0Zm\taki> ~Xj[uܹP^X(0cI75&PTR珂yD#.d64vݶεM*meA._4'׵_EWcRh5`#+ᦢnA'-rg3XޘȰA8|IDęASN_B]((USTC={aE9p3Y[cfd˙cl˕b˙Ǚ.#+:k[Cb(f.rDX, @C@9fo{q5ey;#)Th=q[ﾸudƶ޲@ҤIݸ*r<#+/t8 7uWWE2=OtՎD=!48(Ҋ"8؏A L`BQ!ơ!#%6DAV S;C/dutBG-D#+>de#+` RC3=UHBàz477F=`GtnMVULRk^)=/n1V!0`"1]M[/#)qx@4TCRJJ{@t`嶌@ZHcqqf )ѕ#+] l))V(712Vpne.S`a2=<.)jQ/sS^ϛܥaO5PԷL )0 YRHg30ih{ΪS^«s#)8e%d'mCpQA劋{u#p*1}}A)%.WUm#..UUVqˀdc^&Dv.Q] .{ma#ccLGE{epfaܵqjYԓbLٻ&&cEPr3;Z#.)p+PjvIB 51S r"2Lrps4tӷNB[X,";(բHJ\??V"!|{?B\KDLYb/3,X ZsW#+fppga`H˦-و8LLDe$@QP|$B(J9(zڢOh#. UB#.0>՟o{#?OPGed?hLV9#)̆WҪ͗""BB g=JHm-)#)J?~ZReA("Ȃ#.Bv#)Uo#+# 5b(MŢ.#)>dc=#) I=b;Vz$KE(j #+*U@<𣝐:/kbE։pCuʊȆHw2 @2#+2G! VDc4#)Th*;C|!h!ĉ$A`Uk %3.Jj-hF)wE3E"WV#_,[ FRc9Q̞d?Йbfnls*tXS̤-}fgPK ͦEXXM@FOkctP-#+HQ0ÅW0B}'iKURYRҭq#.d0 b"#.,#)L|od|eLDTFZ(aY[kKbu XSIrCn@;#)W2 ;XԪ)l#.#&(nHR&W5Kuq 5uw\csIY\uur{Ihφ$jw&d #)d"X̝ˋOd!Z+ttUks7 ˽N8*jC<\#.]8J<l]% TMgAѮLE$`8r$Rom#.x69P@T2P.1ȩ|9P=݊?&Dݗ<ɋz2.qסW~%AJp2*r)ɐ$De)aCKhgw#aI@P` (fE3zBqD1{ԏDB"i3UK+Ah$&?#e}~Xmla#)PZ]Q|žc?(;7\贮o'\2M ;Lmj~x|0#.*QIzq;QlOM&/aXA!+#34d/M N;\1!ˋ!4UUOMbR E9 m}b-1#)+.#+*x__> z`kjPq3tT#+P5Nad(,z`\o#.sm[Qs7tWqbѤFb"EAژh/`mIpN;JiF0v.̯xRv֋i@%B@ 1蜫bj^!57&E.FFհ&p|f4 olATD#+HP@+Cd>9pN߬N^Ciρ>:~#+3Oqqr٣*0rd;"!,̣ v[75:X@-#ZXj42kw[f;Wjk]jnlPi!iEԋ2q`&rÇR!yK#),!%'.+E0$w廈w#?+^gZǁ}=A<}=o/b#.,@,jYLk UR1:$C>3v+C?| #) !g˜L!K4IJvXsZ)sBјK ^hXkʬIvo@iL̆#)pGp02G&opU[aϦ{DH'4'σTT$zѶrn4\RS,`6@Ul% .|JB!}g Spt#$DD䂐 f4\0 G`ӀI UX[v|۪^ǜMmO"#+<\|?E藾:HPF%|#B^a9br_EG('fgeY]ݲpK[nh!JbicE ~u4I_+3=#+[HfaQ#+6+]O6zG~@o:.(uo|7,HE#.#+GݎIÚV21KRI$7 4FV0kV#+&(xPآ.՝+E#mCd>ae]\  jv @gp*J K dN/.x#.ӑ1|WedITZ@l+46)Am8Ĉ1#)Fe0A#.-1f&y0mFz3\MP\ cUCI3uJ#.51MH< Ogu*D@B:DyPvBXe>P#+"l kfbXnSh\ ^G<ϱ}'I#+JCS[XݤL3f P8{h$YQ(gPidf;I÷7P&n96£mJԪ6D?NdބNes( ]vo}ihM)|@cK0_Q!uɼe>n7*Spg&)F""(o21|unw~Q"Bv0`#.CKӃH5#hyG?dy T06=`#)vD lꆹ<|E=6 #)ǖߟ$GQOϜa@wyrMLf\0belhx6 !Iyd:n" '9#)C9"#+SH6gKϺ}ISC=&vMDu#.V#)#.Š6lR'@wxˏdISDF>R@asnScCR9e&`&gB p41Nx1c4j2C6s8ɾS5UId^]KRB5WP# 25FhZJJRH #3Yց'kC:.t@t& ͠Rl#.8glZ γN74ҏ ;o¬O7Wc#.$yzёJn#.NokCfAt݈\Iz/#.N5VMpg9'a. l؃CLI1%D c4ƙW%@YA&#.hcT)$;I@#ma1Due$`DpA*Jhc/BkL#+&iA40Z, #.(iiO .{oNtb36kQ1p7C&2iA&#.h|#+Nn'A,ʋyܓ.#+Ia.m#.Gg<)8Dt 0oGNp:n$4$@񈅢;s< HZ)߭0 V YiJ[[AT `}t32=j=t4a:J4ri1,Utܣ 50Kl4ş#!Xp#+RO @l-}U8~|Tq{t aga~"ׯ.n6G\iLY<\sF׼Ѡ/4,ͫX3ǣ|A<2AC,(~'["%Wo\(kcUƷ*mժ-KjUV[R>zh=ddHλDt@}gnYBHt;;=FGI HS%RfK1I-&l#.LHiBSDf$;tZM&LL$fEjbi4"whM#.DI),6$h%%(PJjLAHdSQ#+ (c#{{>|{;v;D}ۋL"}=_m#+aBz!QH)DuN^1Lo`aL QmpĀI1#+Pg'S餿#~o7fag=Te- *4K^]+H'xoi/ϳ9TLvJ@aCg~xYCli4DL$7i־U]a5#)Lϋ2#YUh(J1&Gi4H_Li$FЌSݧ߳3/ޥv/S XOu=?˟iD!bY#Bc>q1f<~ ީΑdnG`!;lB#+ yvpI we,5^,z;A6ym$aÏQNmϽiwigLa(Im{궜Gf]Vo8v~9E.%#)䃷[ GB4μ/rlxHϸGGffPstNa#CS8{1$rHV>UpOw.UwvzjCi'hpy-m<8^*ԟC%lںL+F66-j&uM&M7K|Lዎ0wStŒ6ó#+x`^n;wcWfɚf:\d%HT6`nfFNvdfG>eՆȈ3[NV8:0C/H3㭱ceD#+y[DSD0frQI#f\65reL)ׂ:,\CͣCd&!r0iXZ7 ! F:͸ )V1m2桋U:&EƉ18vgOۈtmµ9&6MPKvU[#+uZ9d4?#.Ξ+Яy"u.q0ASƸNB/lQ(@8[PA`(0yok*ihyh5(^"T$@b gt`uIL۳!}'8Q`:#/pG(K[h)$0wքR3"ha&(!P”mVd$ 4dCBvbS3[#jbaohp3`bMŕWbvfht"m*QBB͓J: $]apx qo Z9e4]FSi8#+ӂ`5j1 $p)1d;[rȔ:&x0;GP݅pbj:"i3nݔߑUeƝ]79 8R`uIΨp6KI-Zߦ꿅 #.k#)MKD#+#.jR(@kbwxfddmhϵ{v"#PP(ԺE*bKJj-:12m$YP.iH:i 2P釶}(5ɸu/8oX=mv{"ʁvK6998bm-!d!wi7LρjA#.m׵T4p-"9\H0R g08岔[4J)* `l!3#.l CaKӎ#+NgB#.TZyא5nF~_H]B#) +l#+ Zo/ riW]ٜŷKOOYk#6AO#.vPM#+($"8r2qc"qK5\F_qqc?4%?IwA/r~܏|WJ8^f`+fhDϣ},1`##+F59v6!Fr h_׳]#+%;G6ZF*NReaTN{Hȳ"Q#+Oy3E>jRDQ E$a"Ȭ!JVk$f*L[E_+ᶢ[w?$2^%y#)А(%3c#.X#)VՍT#)X;Ԩ{gM&A _A `.r@@W`5"eEF]QGo,*.q8 fM;qpEhqƥc]C_c$P&Kw+׭v-VjA@B0"4>LU;K;[N!j1G:b_a)kwG#+Pd:DD[̖TI,-];V)L>Sj Bv$`ә0-01ST82  gs LMAiԴ\=k1M:*8]XCHL9P1c:Au:a9JÜtvhg:;p88L|SŇH8)n45 dXT<Οq2W9|;Jq,w%q>t\$.a#vbPmMd\gLZ#)HܜY[EQ`j LC$фUԄanb2m.4SVYPg 4HP~0BolA1z -(-#)V#+! %iF)#.R,8qNtrNhCTMdv6/r0x~փO@A28q#Cjټ#. ZdGzA !JF?yJ9U`VtD:ʍ_֧3fKAdi0@ފ$ayh#./rӻw*|K8)wci.Ճ=Pr`d\#)ȸ e ,R%j~GZMU<(s&6(PD#)oBωl*q#e=ؒ]Wڭ^,ONf#YJ%I뮪 $o(Z-[IfԌJe2ieVILVTŚdTĚ-ZԱyQSmVDuآSeTbWn䊦i&+cRklJouuJ#+RMCV<#+4efl-ǍiSo:yULBWk5IYZ}Ty_*Jw eqZYϪ%`3ic7"(<, &ۼ|Eld?WOZ9!==(S EFH^hѵk{WLTM4Ze1m)*wm~&U!٭ZuWںTmh*@ R:J,($#)uR Q#) E51KIIjiB5+e-MLViM-(UMMJR%I)b(cRm%IIS(A6Jjbj$,6ZŨƓ i"f%6#.lJ(I$%&Rf#$̚)f-U6ȑIM+ZdŚS&=mUꄪE`1PB$TR&JZFjf*jl%UڈQD F8:#+,,V0$=6ûnx?nbs9`U@Dn$=^#+#)T$!矕e}FwoWo+ooL4."ֺ#rKIŁ`sp]Xy>|9rZ߻!oU8{5as@~<9p;m-1oHI?M ")6Lz,ɕƴXnWBT8^ZS) $ М!2X@! E"X#)c,P3P,dEC%X38#+Ù^خi3) #+-Y$(aAr+r0W^|Ej%oT< :Œ ") MRY5šVj}bO>2xVRFw05;0|=UbUB ,Ʌ #u UKS&igSPE|+f(#)]#+,IrB̩\ r9N]4ŖꑇZD?6[&_Z)@\CiDYrh!L=4ƨkW(lE͢guҖk}V#$WAd%c($Ц* D$"PoaBk s(YcLˮgZAd Bc:mI>4`DP\9jd| 8!CiH!f؟q5#+"p}і#.o58P?@<'=kF=- 6ڥͮUmZH5DFb^ L (#.JQ"j#)FtOULm.CHa+ATqi" %#.QhCcw=Ì^c5#)Á;nhV!׼wҨ,Huir xU%qb]Ttе6ZONȣjLD048׍qχ C3ma6(Oc9@!ېB#.B%cJ߯9y\*#+i`.+ʐں$*2V)JAp!Ljˢ,7aHPVi1ԊUţ!F!"לGUƴE#+x)"W)x΂.ŋbcZT/#)q#)"54Ti=@DMctt@tXjm%#.N \V)wqR3f\\DQȴi1`ccay9ȶ.UIwmj1Xp PD3#.M6g2 LN*Ѽ uE~#+2F#+q  44SYQ>jZf;97HD#.YuŢ"؆&Ӽ#)fñ(:"emUX'.ޚ#.&Lͪ#+Fd"FґJRX%#+4>W(t3#Պ|z|8z3sW=T7??L- B "e?~iޭdmkyjd%chu**m7me#)QhU ">6(Q(RG3`$2m:[R.nrfMCb(eD$@°`ZDKPe@6L& pUct&9#.?J@QLPXApQDyUy,Ґ*ZX&Z 0kY5tlWUQ3,\(iH!i"4i$қTZ2&TŒP#+#).0oQB~\;MCqz X-I Q =Q#)b"b#)(IRZHPi1%Ca$OQGDL ?6eƌaz7,8HCܾ"33<)h/LɉÜ{s6XQTj®LvsմkM$+%6hl/ރ#)O>s{ۈ[#T&F@ßgMxWήۺRb&֍YJiYO%vMdK"V꾟\63#.(.U,wM71B9VfLK0%пWLzhR45O8ݟU2(&c3]$.x́ B@m Gk1i"pEPdFMA 1z'`v$Mc'fLXxg^vl 1)Wiz; q665(KlDaDkkCѪufKźV4YO#)lMl1&A0nTQImt~i2DJl#+ћ0:Uy4@#))4JgeJLkMcFTvRu'N~#.&/J~;#)!r˞Q@EQM# BԚHJFmZj5#v$Е,:l̪lɶ)ԆkLk15ffSMi6lFhچm#+5feUEb)Peg~UM'-#+!A#)9(?xqr6ֻ.mUZi P"42A`@X=ol!# H' ` LPFFE̠#+Bɒ$ HuAnQivN~U‡]Axd#)#+Ed#)tx>$1BUT@q#."TZ}.AB@"*W/g;H3Gs%n7dS{*g(W@qU@ji21#7pG?h#++SD#+zL0zDk*gDKQ@ 0vycYȜ(AJ!Cd7Dzb*l!ēi3,$0lEZ ?gއDBk6O5_Qۺ@$iP)- !Tj+ShĈnk6j.KۦTJB$#.~>Є `u 0xArl#7j*UT@HU}USIⲔc"OQK2uZ5Uե&ٛR2khSU[Dm;s[r+!'ڞlC6@@s#)a 18OKLku6k$_Ձ,UV*Ҕ@D+ª)%t{ Hxf}D=E>#Й]u"Hު&HLݩ™L8i:^91JdM8Lr:bP6&LdX~ʾ65.9۬u[=9{Ik;Oq^(tNd բ=O& +#.b%Nfbiz1HVgyHmJ [x%%@e-Hi7uM*/H8fx|KHC~-  ּy,<6[SາRp"E_"D$ 8 "p5ˉtwV](U{n^[}D>\5oyw=9>6X*m@1_&ZzVȝ2LУ(uƥ؋hl:obF8#=$eva$ ÜiF QRVBsw jԷukYfiVYL2m6̫ncFJ^.IXN:Lau:ODT@&8Jf#b,G%k뢃`sp$c\ u3R{QtXT:4; _bmn.$m$պ6䀵&Ö yU:K(XB3`;?- nݸ#vx\;8!LY+N7lTXIڞjɥ&qc=\Y`,6Ucx=o%}=Wh*\oL+-?MlآA)Ӑ1e'8֢;LLNO]srbi#+^m\XqA{.[T G<&!bF!0RF~Z(ot$mnAAeMCT&8\H.YٚZ*}+s0{߆۞H?Gƚc4ζrɐ;'X<>wK6 t:69[?adP@w:K#+apHr1&4̍#; Pu/,/Ǵu2wO9֕dUn0(GR8 2E CaB;)=YˏdVwd/lм#(9"?E}$n#CGS6%lL3';Y€)!6%Av0tmX|iqzp81&R);.,;gn2Aw$Dכ lukqb$ -txګbEf&o֨#.v;tr0Di@~U ܁>{S_c8#ri#.(la^P!Ԭ!qHrۮBy#)c[G(鶌!o#$Q[4Ɛx@>$#}͇|iU Q(O_#+C#.#+ axāPxYPn#.CL#.% T(¨F3?s^SugىaᰓH73.9ȥ?BjJ-wFZ#n䞳#)jF/tTEa݄ѹ#p>h7SAzE!%DmQ_Z#+@5P:܂PGRhm6VbFGe#) @ՠJD,[HE"N 8wٟ,9>2b~mήۮ.>mAƘ.X؏uRjB9ej! :7}zY9_).webA:%r8;,FZ#. .$H<"8¿#NA%͆d\ocO6Wtmj|Hb#pA.‘-XqOtMw^9)p^.K-ԻlM]EɕLǹ`-L`cZX#4bdj\XVs`RP1&մnCPtkC2A "d""Ŏ#+v dO`9 HF[Y+oWe2tκwv]'ozi4<\ 3X` mPOP)i--fI(ɍ)6eUJZ^S뻻\nr-'^$R#.2Eb&\b%ҲR2V/E#+**#I\4) c#[ލC*40}GXHWl`3q]f 34D 5hn L82l:b)8E |/kz1jKk#+Abu9ᄭ0ә"[.M,J)" \.Bh I@͐"#.MKE+%"bhoqm.#.-*촀շɖ1#)Kha˛ oaqM&9ڵBIZ7s rg݁l*)#i0xC.&kuxfQvXa1x[HUXePؼYiق爅^A2#MP2߅/<&D-?9Jk^CCx=m #+1-o)&Pg5R*$R2&]mFL2ZICL3d1TaJĻhO0EFiC6!ut8ܿ Eƕ1U"ؠOo]c,1i( jq`8v`iu;cde#.)ߒb0BRi&0jI#) űM؛F!P1!i2It.0A9"8m6"x~!Su5ՇXU00G4Ɖ^ӵkUHl FH"I=e6)2A^Egi{^Hn)rHOnG~׉|&3yJ)p,M5Sf;{!ލٚgl\B2@ruoJ+y7 9E$t[4(XAf9:t|0i2Ze~sǏF 8]T0r% ~4CR$o>˖c<5"\gZ}N7SlW\Ul#.ODҜcV'`b4_¡-#t.>O|?/ӶWNZrPnFs lL)}E|ս66U-mF3 aDl-%Yꌀ#eu\#+!6qƷ.BJ/<<㥓x쭸[\VMnlZmnUݹ,EdY§uZ+ITVZۥZMTRAVYE p"+Tm锌!;U)gu$.Z#)if#dZ]U6Z/A0$ˑ8h) #.@baK!#.#./`y)VMH#. @Z+xS6C()McZ#6*2уQY+2H[4h֓I՚HŮu}'ÅFI2#&A.\$"Q%;?׏OlQ10\<=6c,t#)PH@"GZJeN^)FoĭW^+t݈SIrki)[]uvZmu֔BDM!z>c Oy*C(#)$d@$wu]E-$d@ gGġqA S #)@̻L9d6-K#.֧u}ݕ'#+Eaj@SuNQq٘c+x~6U$x1L!*CHS0f+"I'xFN-f# o83w&CsbO*xk(d:XّF|ɝmps6s(U[5]mY`ɠ #.y dw[<^ ?늂 <;()Kl*JI`8Čzr{H|\&dk&hG+jZBG?D}(}"=(0 B CPY?0 d6[&XEּ-vwrӐp\d*#):?0?~~p3 D*J $J>bQ2F,DQa=$G #) B\Pw$]e&1Fmط6;&ǦJ@ccbR/WR*HLIڅ{Jƫ‚Zf#+&ZIF#.Z 11DA",PP%bixr643*#)@&.#.$& /GlR ( 1J$!}n@lgGz(SAê/!Y#) D>B!G)7"`] "0XrWuS^q{Kzk#.~P^~H|wzqȋT`6_FW5ZߍHTJ) QwBqQ##)M'_ÕֽY+dՋ[d%{_rm+|9tT+u]ԩأ7u8**#ΪU5dm~־+ڧH#)8,#)3`i1ZɊeF^kb5 E#)Rf P- FA2C{pM „#.%iDĽJQ@S)T(#.l`3SX#+uExE5,mfl@0Hp-~":6YTl3f@$"H<UCs-&RE5mR^]mmPrm3sd]{"8b"f.G䙑:t$C+&vw덌}||1@?BČS$ d*~~sPy/,8  != .͕g3~X-؃KFܭ,@׹DHA5d+Yb0N< A(A#+ڲ0I#)∭#. F`#+7Ac$%D ڀaoݛk'ռBf󢩨[_(O e Ц!;1'ޕ,A&tӍ( XjL%3ݮ׊J6{`xiLtӥVι2a[f^0ȴxLm@|%SAqF%ECx#.ۖ'HoRr|L8x+!ῼξฆ"/JψX{6mwx .UGoM}m#Tv΅yÀ0P  F1t2Ma"±G#. bmGC6э j&R|ԒH16[2?9L9Arbp&`=\zrjz\Db|0c1O^B7ًLn>-oaNSP4&.̢V)$I3Ko#:l+HRU.$lP`e&BKs-7X/GKkY~xqckhhFzcM poL@X^S]}{zB+ p@QVvq6'X9:};@'#.uB9W#)oL Dr =?a_ S RMva-B-Lz{&6 m#.CMQQCS!(4a6`[y4W6| lP9wԮs'v99"o4Lcׯdp vhHa!JmfG WcKaQJ (Z*Pi|T4G\7y1@1Au[rL;#+o0f Yp43T3yǶkC5/ .?f;0eɕbW@cr$$U#. qa:gJ萛f"3S 1#+|GP<^"ڷ`ĉ #+#)?zC_+B0nPRA FBCqjF}S!;z15>ُ'l 8;q4i00P5f0N5ͫi*`Gs634#cCff`ڛUp)x2hşG#xQgd8ؙ29!!|s#.;P0jGE`3hs |Pݙh3[8" ogT#+$Uo}A~5t,p?L8hV&=9lQA#kk w1$$#+A"tqpSH/ۗlX܅&=fѠV+geJ5jٶҳSb}?;J܇ 1ߟǠT v Cw~%&̌XA%#+Hm fl%H҂HlԥM!N jܢOwsNJI66gc#+DmB"iL 7#.WꫛO2GUWOpuTà1 w.*.8ZDiBf|g!c)I͜Q]] lZB lZC!hbnC ` HQDH\{S upSQ&ac4mA_ ƿNzZ.pa!gmtj-$"2Zdcqdh F`` +bhHD2:۬uW-7]Jݜ7Wm'{]-:xT 00#)ZFBB>re.5ijLڙ!D2h_BTmR`C;|h,b#Gkͥʙ#+DdTdEUj#.*啐0ۑ8GdQ,ל5wͬYFf=\t37(a2mA>?#;wq?&^PYHw 2g}\NGd5޹F=tD^1Pzig;AgD];<'\J-dwVdx"B|@#+7G2ExFc\CQ*:QQY*Ks s˯n]2B0D`KJzDԓ>ხ{38m{c=r]38&Lrh:-̠䬜7X蹧y%` S9Hun<"\+zf-#+ߧIaJ+3bX\Pf:B#+6M ֌ F#SLjhRovIyV!D4튏-b%T†X2rNCV#.uٷsd/Z#}=߭˷sr{F|bRo}koO:>q-4&atB61jhvcj($=f{2\o,fQ|-iZV<ŕ2$w z;dW᙮9es)$h!PI!dDL%W^/!LM4s1ACWJ{ɉj珶 ɉF4ػ N dDsJ+XanBŗP/"]#):L0]@C3DH6"nZʵ;CsgU:,:] #+]G{[$#+kky jXޔR.c4=#. E@DdnkrL-<;:#)YҶT'4uc>ufcb Μt=SlH󀃊 lۑ3vrRj5G8=QAMjf;jm .6ncq;.J'\8P#F4aJ] q,"6 mayGzĨI滢g4Pdθ^ڌ 7A:ˈ?,Ce4N%&[)8-߱mףXtpʾ|Pyie,w\pfn8pdR ;3 Ga$ږ3o`#.$#)s۾7c=9ŝ/nn$mOyLQ|!ΙkEZ 2c4}:SlgM4u{>/k`Z4kS5A6 H(2*HȣUm2(rq/3m"T'+AB6k]3W,K2!T3i[YE#.0Wc;̕ζ֣ŶMJ8ir(#+TfK-FK[@c1woW''6#+#0BXFn<jXhi!("1ah]BlTOf&M$mh+RTLܵwZԑBӬ.ՄF0&@jB4ăe#+HSKIRj 2#.!:ij-Lp0vkcyL&m4)vԭݵvxi*JpM`ŭ3=Cp m5I@UE#.74`±Yp$n^xZ8aw"11g3i50um#+7I '7pKCmLG`;90q9:õP mCFk2Ū\""pn2FHqt2JQRAdBmFLF%ǡjcmO#+elկT옦ʔ\92dmJvXڔpT#.'8lsqP V"F{kǧ#.tUw'Ԗ㖜9koK fR3N6х}Vvw :wJmfM#+RhfwEX8e#."-$ V@ӄs8J,p bt`Æ?TUA0Y/ar602W;,Iݼ[m$n@$gk(9G4b #.NZˀi T2۳ $UUFJ2CT$^N5Tn>pWC| A"qm!7'Cޒ#++ga2-0MSA2}=cuK5\:xHԃ(S#+\cJXT\ KU*0EcYlE:aײ{܈@t8sC'()AlZ*U$~˫Z{8Hy#.ZN+uRRETfYڢKɶ媹U@""k>$3)6֊ҍDbM%mYei*lm4ZLY&4إ)4ҥ")FZ#+D(d4YRY2HБcZн~򽴃{!}mƃv4Q^CT"SB0m&(,}Y$G: ΊeF>L oM_#)ent|ŃygMzmE<k2 ,yuwDAm8a;H`I[Cŭ8>; 1[;}jEXVZ(Vlư1d}S*,P0A,wCBcf)ZEF`\MdDD4އ>4V:2|Rb$l:EP2L!#)i#+bIM#.53UP@0|k,5#`9&5DŃF1=hu5EYiXB$&FE`mϗOd m!1@!β'c`#+)!L)`K!ſ穴8]Hpq_hl[@xKn7It6UHĒCt.`B[$"I$w:!T40(IY,(~-Qs x?dzzF~䨞Ag{qK}^:V̽H#+SF{(ɀSLMA!BRQf$a:fhA)o&W`.+x|?*ۍԢ[xc%~^:ֺRew%Uk уu`wi48m!z|p|ƞw,TܕҼqc2cL\t*I@+5ּi2Ћ:Çvԟ$k_IR)*t $_#R\nO rC[SjՖRv Aq#~6D hnz!#)͞ΥClde[Q.!E~sthҌ*Q<:_ jmCEQ6E2ԽJ->F$֨(0XgQ#)hTfaAaE(c#.ܘg_ UGnT<(US^2RKx.!JGMokf@!`۝"p M[:*DV9IÛ(&hONȱLA:I0ΈlJIoPЪcunҚ2p~x*ǯ9*CF,e!Dck'wšX2SF܅, vB؝rknKwܘ$LxJĕ}(a$Mb3,&iۘSˌ]p޶|]6ErT })9.5-fSŕЙǕkd6:HM 5)n8f$8MsNkYfhP:( :+qi:m.#T OôX.aš#+اA! sg }1*#."F53s;$UF15^Csd14AZ0$JchP) $(* &aFQD(d:>d16!Х b7qiF2H:#p]LF#)&FsE97D2$T!ުlorͦ9J]R2#.K$8" @|(*1BoZDI3dEMaYiEȝ_w%_}r>GЉ&U}vǹDK)?e5!#.b ʼ2uEtִç hŘRȱYW^זH-u;^n*W=-Hy%˖@Z@64!K#.`+a@FwCdH§[r/"(X-AEf`+;=A>3/]>#)< >B"Tp RKs1U!eylMQ$phਞAJhGN$QӛgyQV:4*d?aE0g#)p#+&##.Od_-;`=Wc!qI,3}ə!MM]f"jv F1 @"sPKss CBH(T{7#p2V 2^P @DBI 4stg1n]袠ʪmmMK+SJmjjOvj-" V#)*2(Y"B #)HWp =bhCfO~(nxmFyî8;b "(D!1BGL~@`" g~NC5ژ/ѠtQkC~hB@hSPM6R!2PF*;PӍUuVWyaDA,)#."Q[S jTvkBۙ`:VVA9!zʨ 5琵z#.b ;C :yԭ&= Cw(! b`..B;Ta^_;"ܶ'US/Iq!n_p36X,̙wi*q#+`:% J4]v*#.ؑnƺ E_,ghW3 HܡsURY5!/"zd7V z;0l5!,DK6i R1DG~c>b`\t~>Пw8 d,#)r+HU#e4F`*mVf#++LW,k[RjҺ6SkX"`Wf.=NǬgb#)=tD$P4 du@<9^8s=T ״Cx4_~pEW$1OlXQB~e0a#xi%EDbV:s#+wKBi^%h:[f2#.ka#+ ,Cp݅}!_$$;[rLI ?,Q+b A*{^T`ϖf)BČ`7'ރ6J;`,8=iaLNDA}kא{]ړW#+ɵR)Zj;vhKmL #.YuP`s!!an3{xޠ|t|O2XYﰶuf#bs_tj HjyBVmLDS%>)\eHjpf6~gU#.A_)~N:(*f#+b,T[5x`J6Ĕ6=K 0kcR42cGhrbU#.#+~hwmD` (H&ԡa- `H0a#<#+g3o;wYd!W݇-]-~i!"$o#)!MuގhS,E}\ b^d)[Zl,#+j\y+٫sZnI TY!H!-R R1#+(#)~" (.gIs#D9,K;xL{*gmFo%Ԥ?&QA@Y\J|'X% lVRTL#.mHUybݚUiJ#+z9VTc[튎2 1DډJҚe-tڥPJi#.PAD'"Ш VNzkd@ܫMpa!̣D nw,Ĭǯ#)Xd%lZvp!)a4{/q63ES>(xӎ5G$媤efĤ`^C;9|ȃ;pÞ%qMT3wE.+L ƨú.|X%iOC#+^e#.cNG[6^~y  jI7֬O;Lctõq7G.G@N5Us4!#)B?Nׇ{l$X^"J#A(H"38[/b,`!3>9J D2z?2P?c*!H31ӡb'/#):M_JԒqf#)/YեeV[%Bi@N|tic8Cx<2Mx7Cc s֭;Fl'o}! . 4kpHP5LY3ˢuIb\άhj#.qDpz ߹&qſ$N&Z>v9MBu:?}za( (~<@Bo@{c)}FD%s Y;I0~? o|:#)d ?~*#.af_/n?oO}_|_WwwOO=__?GZGN߫Vg?e#.1QA&b+!3Sz@0T#.4{FF42d1<ԠMЈܖS)#+Ѝ!#+0m!s1QV#+(kIJi뒥X#.V9!u8aĸ~h\콳.bV8R30`pֆG{ r#%sVoUy._XwA$lvwFgi s<d7`.PyfhO#.B׻ Fmʻ&@/^?bQ"5m"2)BIKN$#.l۶tMq@|DKd#.PbH#.#5D笚6#)x|8CS9UNgqE#.RBjs_}oHS)-q**#.[JfkY+Ijd1ce} P""a'`@ YeZw)BD1#+%T@b({uPÿ]pdc$q;S[DS`k.}~lս@N%ΦUcf5rnW`ѧ1n ?9e](,#+Ʉt.,dQL Q 7I|E{8O*7x^ˆY[R@ݕс}I-yQhۖ+x,61iKmxUjMc@Da#)$8kw1;Cpc4@YE(vw3wUdBR+_opݟ1GTxUpXSPz>va$YBq05"%{ވG7O.eCX0#.!>F Gy5]j[R5kOCe3^UkjYo}mڹsSjsȓBA~]ZOLJ ϑI}*?gt$`_M6v~)A!;{Ed2>uv0qiao#+~}T3[Wٟ2:V9ʑDCA(gpK桯Wo;{Q@qhPbݹ: h5J#..kcm)_;r;53blnwnf^*#,[wkowMyrp']}#|{>㻹|o;Ogu6˧:Gg8zћ0lCwlkNaލ#*NܧwVT9vm{h6 ]]M]޺}yi}N>]h|P#({.Buu׸\5q/wѾ(iK16ɻ۽^7۸C#(R@S{^knZwrup)#*ݲwҠ(\˶m!ݷs=ׯ8n[=2y ^]϶̓OGك&}54,u۬z]z5W6yg #(#(M#(MOM&F#(C"y)QH FAƠ!#A42MxFڦjy&iPiQh#(#(#(#*#(#(H$B24MJj~)5joT~A(h#(@#(#($JH&LCQOO=G#*#( #*#(#(#(#(h#(#(I @#(# M 10&M4&i#(P#(#($D24 BfM#(7M#(#(#(#(#(Omakjmy҄LQ;mNDY&BD AV_o쿳M~˼-캩u,LɆVh7՚n#J|:#(hP`VH@wvA/ϞțaEՖbnB%&ަnSvKxݜ>O<2yHh m2\[QT*$ #,#,2 uPUcj_cyj4|n%UT "D b(%~V鶼֦+ (H*cHd3P)l&0H"fT"XҔiE6l13@d FKSQlh0Y4#DB%F)R#(m4,i"dmDPҚmD BHlLFfcFQ&I(#*YiRѩ[Me3bM&BQdKm4Ԕl-6e&"`2,֘,PS""ЉiE&ED6)*aDAR6!PH(2#$!X!fhdDa"SJf$@Y)dѱ0!BFJR %&EDdУ&$%D4d$X&e-1cY!&3Jf0RXV$$QAI b R(,PIHF,2I64k jHb iM$ "l@IYR,(3"I2"fȘ&U4Đ)AfY HXFdFŕ!2)jDD@BJYFPcQ4)&bK(4FB1dȚ R%Fee#,LTaBA1 &TLbJĢ&d5#1#,DfŔ+)d()$dlc&#S4XIYYV"0DQ&M52i#,CDRhآJɚ) #$dȦY#FRBXbI4di(ib1fI13HM&Qd,ȒSdE(&Y2VlՒdY"CTUl)d1 2IcQM(U0Г2,"2$Re5 I!kjmkF (MLєҊIԥ)f1Ie e6hS+ e6#)DdKka͢Tj&S)CF6MFDai1d)$k" -mRڊBѓL"XXб)iLRj#*i*leTVaM[,ɦVDSk%Yl)FFHڋ%2RjFJ6h"db-bhŀ0[`)R cF454m1+Aj[VXHLjjɢM,VmRMT6mS"-Y3T hILƆ&V1Pʍ$HX,XF"Dm!Mh3KQY&6L(!")FȊ Jl`b#,-!Md)fEH#!4LQDȦ4i@#(4 JdZ2fjQ)$ hS1e+hƤ%h,e(hIEMIK22V), jE54ECEJi6ce̢h2&2#*16$*L$lʍAeM%c4dͫ"41)`B#*0D1l6(Y)X! Z-Q4RI#cأ`MAYfd3(FBK)*dPXME$ԕF B#*R5biM2QF"ّh*b2ZQTkLMLdJ*eEŚZ&)+)fm#*d2 H6,lY4I%! Vd#**4̩%ddi#,i&(M6e$b1i0dEIkF2m%l!TmIEE4&E23"cMHFl6,mdfJЛPljB$lS65QlkcԙVV&V,͍l*f*#*,%FX,ʒ-I[M-FiѴVUDʅLIDbD4lSI6*ضŦj1eRɶ j4Չ(#FFJ(6I36YQQLU&S+c'?vb7x_OLZvS׻_3俎fW-&ϑKUƿc4{Sg>G #*.`RpI m{m?zUe]I k[sUa;}87>#, gwQm)6I#*9(gNĿkUFLI"/NDɣwH3U3I2-$iHįKFg5r ~1[7{/K@)(Hkbt5RUX2"Ľ74#*2QzךwSqݲ,wp1_.Ƽkɣq}]r،bPx=1Zb(ФY;E1x+)cܹє}L+A40 #z,8̒M !Z ]RX.,Zi(RIͺi#*Uͺyhب5Z4E~~mr)r5'֊F= aa*#6{71joOuUMӐ49<$oRFW4j&wعGJ̔et9̅/:v_.ElzΓle'X+=54.4^JE&9`*8}"34 xmhmhWgo+\Ke1⛽> ؔWs=ٻ[&U#,`H,՝3[q#*n#elad~yeXT)b#}_*tW)D,IJE&}ˬ,j[=bޝ13&)|u葎7mZ(UFxZX-'3{iX<_Sz]M!QE^7,jc{ɍ?6LT~æh%$V*\&1Ovw% CI):'n( rRER&[yߗzy2d{R{1(MuS߻cbѲmxƠ[:h/||70_őh;G.ekfL h|蒣DdPTB#,罗F|6󫓗LrάL#=9d#PcbVR+ӾdoW<+)/Nb*0QB0BVg:2ZGKMZҗuE;(0,c#(8!= 5{s Ii6l-7be RYQ 5THtq=;tdm85lyE!EAB%kn5{&۶,ck#;}Q_Lz8-qK9">>FYgD%"cƼE>r׮;((A#,a>еTEaJ+gu?m>}㹯^1>DF7AM/n$Pљ 2 ?C'GչY$km_|:n%ۈzR0CU55 ǍM/ep0k4O-n#*3?R#4YAyB|k\ 0y'ʰU+rb焿}pS߮$Q0 w X|=XY#GQ#*fvciq6NfYty҄k[bR+1~,6'Q#x1H0욺UTž\KE&^fEE¡UR @uN+E3RJ{6;2CG9}9EKP9+*א fo*8A<|ay(skX.#(ӎ䫑) $A5swͨk`Fzjcq-:hSb;*(K q&wso)oX%nA}ʥ|1G]S4ydf\iȉU F8T;u:+/#*Ԭ试TT?#,e8T!_ĒTG9tuT~bhUlQ%udp"Z_^y}LU+;X9wjODz1ů7+x< փ7TNNyn'Uy{cÞ/W"Ķv g^V{3e><(fTbYzfĪU&E].lu1ϋ@h%D86[SZ,4[CZč e  N|P lZ#*![9O#(_=#"?ؘc6oYOw#*z>X-:baU/SݜNڕ MRPQ?RZM8ҋMeU WƋX"4+j_}h. Afv2 b>6{phx<>ڴpsߗG}c~p'Lj7u_iSӳ R$ꮩlkm#(CtTf(X0b]K#*WBfjzRY~N5OG~"ԑ3E+m"q')"efk$MAN7?V.j=.w\g]|Qtĸ~Fף\CGbJ6(>S.%0yAq(2}֔M3;Qk#(ED& ʐ#*AԨpiS1Z$yjԔ+_k]R5xݹN"W\uH"KZb7Kgr~p߅>Cۿ'$$(\q&OlChr.qT5U<:p/iB#,Uw 3brB";c?I̘7ETQo! }7[,&{C#׼eJc\pgfӳ5xF&_XdoTAR}֐ J^X{W[DY+J6죛-0aX{W`,QCc+m}ƄE+`Ʌ>Ӛu5gT@c=<F-U ȳ>*Iõ|Z}ngmI˸|WXv_mMuZ3կO~q#q}:gy'Sd@kc8T@ᓒW9pڸ>8#*#*SvnyG޳8ntl fӿܮ+R6NRwSn0gOs ^3]~綸:XwT.#,u)h-U'f0ؗg|{fH}KӣAxs!dyB$(:?P81RAKQ)sƑ`Й2JETc+RBb,mr)Os($!! QT`HX{࣯SY~\ul&>:,%PLȡuCoL-Ui|0i`NԃǂR<p-N61ngԠ'ZEMːa'gLV[+,#GoY ~Ir,NǴ_fZgz:nxP<#,$_uyeNCCw: y fsӿ~qĕ{r4Inmy0۴-Kʂwǟh,}[ٰ#*heG-/o93f,x+ʎ3Ӳu:b-,]n#*ְKya;Y?;\' ȋ@\i(]r[0K-r!x.vVAvmMytMQcLr9Tc%*3Ibq6yZJ^m6_$CȲ-G%nE9E !_3JM]3!ެ\CO`zBP{jnwgaW2|ܿ^DdjU8FRÖ]eޭTӁAm3E+N}#*aDgDEDBÞ,FԚ䁌BZU{_a5"mW`,Q4? R`2S30F<(?m?ǧMtHGXOɥ߭A1,T= |x#,\Rr#("E5YkoFs8xk8FƷϼρch#tִ-{/0dYq=%7pq bG&wOnfbA1dy$UsSfdbrТܧ&i}\2#(rکDGspCJy`az؝mY&ϬU1xŅz_Y&O--0)]\qk̝3Dc\w2 I[7S|'MOO^ym#=l6NqCXh(%yol.w6wk7e7>}y\nB _PۆkcO! i1@*RGWa;H#*a!{[C7b*𘊡s^=lzGv#*En)Pt&d9^ooNiƲQay.~/\ ̓F7`ךVv] Muc|>835d-SGa{X}_t;J#)&#*N~bӇ( PKK RdnAIsF9㍇suB:YQ6Çbia/t3 RdMDq5:ûAj4[W~[TDRb9aEP%\u6#G H_˜0kWf»c*>Y^(T.QBB_b" x#*B #(^JLSڱy?z]n_wBl(zAvUJH&uC8aV#,2lmȵ8>;BQŅ.#(h:1#,EX j[gR TY|5c]%^xna͓-,e„g,e&|#,IhL3938SJ>uPF`;fdEE̻\Zx3)LPk:֫sN䆰2e01RH6LcLV45׮ ن*dc7a'5ϣƹUzqX8hԁ$l;pjrpIXgtlJ>v#*l ɤ+c; *x;osvwC8]v4@5N4*YO!w\wf)6@wFY*%[b.FƔ#*%Ʒ_>srșʻU>[;/֤g`T%ių~KP[a fh )c|g8w~M:}z#('ML#*&HjOIn9׳[$kJ&=ވ9;'8IacrF i*~a K>A!&@o0]{lSw ӥ\5h'UBZz#dgn{P48{y mO+p+b%$MMnCʼnhb.U1>/*'?K#*D4F!e+ ˥3S= #,(dY$֋hNik%"t(෍#* C=B&Cb+J,F t(E#*V,Md%!zb1Ǩ\TAB js/.s΍pF)7YE3\եkQ=cΙ)`_(#,5yq0]׏P6.2e|W9Ir]93OG}S(-qgq :6K.һ5X-+o.ӟ8f8c7dcG~m@nP$\GgGK %f8x뗃-2>/j(dvMw rnC#,6^S墔jB;OVkOYb, 6OrZ5WnuK^>m13N5) vKǶ#\>JoF043ɕi"1E#,T1E ^jʷq|T%L"u97VC#,7rOv(OM7.gxKfWg_ ;+g,;߾#,0Y<[fu'hK=ww58m~=)H$ˋԈKQ[kͺQ#*RL`jQP1#,#}l8Dݚ陵x\AָLI; Hh xʶ=p~l#*>M}PX5{.Mg]dHFY.7Z3tͱ@qKHO4"_&=[,n4D?{SV*AE#*QˬND—VI|gQ~-*R#,epm㇉x#Gb"P`Dz Ǘ<#(FchJDIlvm}|g~0Zֻ/ADc`:O{N3$#Hw͆/fޞ֟KyV-!QC*֕j'?RG_PVQ6}7Ų[#( *RN{npa񏌨wxHhuC폝dI(#=Nւ)RY4I RĢM^۵k-K\)bPi/4m_ Jؤ҉5FŁ!C}}^oeh"V[A<+>o+Q-?Υ2At?cIQQ#thoѯ)񿒷GFd`=? zIjZoWuPaY>mKNlӱH=sa|w?\pW2dߣZs+ӔdO}*tV _<]G\ar@%lUip`P;`"nH磇q8OhC,>_*3O|.rȥQK*#(hÐL/ܖh`=:wk#ʅ a&lmg(z#,2GP=%^#,*4XsyDRMf**JOȀ$$ %oqPz)8Ѿ8o>;O>!~_+_ʞhr}9F}swߟ#̌(ؕiaQpÖp LG؏o="E7e,vw9ܺ(!?F}i#*xR_ ܈eo0)a1V/zy?#10E|tG?afwJZ#*;ˊab/w#*AIayM"N?-nmT 6M--<#,κNTե72ULՙ)8\sآ#Y9@\tZ`#,^Ud+ʂJ0sֱ;Rv`A<^e/UC&,`Hs#([a>~i4 @UW]ʢ<-l1Q|GyjX ޿fvA9YįXDCAp_-؟ǡr#(%Є=rzn>}{v'KFWqCuK)-BJH ~pɆA#_fyf$ou&}^S|̺uaͯt{n4|3ٷ#,=oyZS_vi`],Ƃ=\YS"O^I7lWf#nlBו懖piÓ\<yKv;vrJo"~>͞yMf.{!Quy1-r $so"Yˏ%%="ݏDhٔ(>WZ$Wu8x|+]՝ֶ~NW6P4hom4468#ì#,x=6|^[>{/Og?3j`#,u?/OvY#,.}EHW4|Z#,(F1g(PJTj3~ǡ?HTK[" Ǝk#,u~x+lZKL99Y:0G6U? Jy9#*_Wo%s==Yc3jwc垗;~:D>}:s_>|.~APg;}\0~X:G_ ۝(ݣm/O2M?U}:0E>jp7tj=7Ńg8]߳3צ=kpKxhO?A=Z~v?w}Ijv$9U~`:O!{LBj"'f!6#M 7'-}zNގ/ywGX]>;;y?+)ɒǾvhx? #BF#,{F:0L_w#*ӇO"GKDj`,*) l*FcA5Q[4sIqZL<#($9h, ʄNGA]rtM ͎1{⸽ٳa^{x:#*#,R":YR#4LWrv\=33$!K9- @C&~9=!7?ӻ;4CDvn:[0}᠄Sx0*VEp< g͎t#*5u/1c;YMkc"19yǎu3a/6?a7Ӝ*6*dP<u֥M*憈zO t6vK#*p½>!GVg$"H!%'i(յô ^˘!s/ͦ~Y6V6z> 1{?=YgC`ͬu{.աt]e*S$Iǘg (tub:hG6{; waٍ_iR9v;e%2.bdKs6OKh^Tû57E{63?v8C?߇ɒkkOktwD'1{c9R/z3zFAQ 7Mz"I+ɐY=3dS0Ȏ>%|5MΰXCit`ؤioZ&r36B{Z8du["/((>1,?iLr`<xt3PrZ/n!;601!V`='h9:;SD@;f.zŅJUdl5j liai$H?HPYso/ō2)LxuK{#64 -'lDjEuem?º]:.F/U<^:F?77&ߴwA7 㔏;xE[ߛ1VU&겷9-Œ2bQ$3au`|_>^>҉jAW8jmptyE1f9CdIUqsrNk C=zpTԧt>vzzim6g٣b._՞7=uXCp ^P~5p^v&1[[VSKTkD42AY;SI7Y#kA}~b]#,#,$eUVG- %,O6!qCC/ GǕ:{wX>.$J/#,nRTH>B2#г6.nK`G-Ӯȗ՟Ä/;.AOV9%UE[heU3U *%LV'yxtv)R(m_Twށ4gx:L#,0XF3\XzS$Es)~MFg/cC#bԆaH).apABz,ugǝ9m,Y%ө Hȑ#u0+*#*#*a4IxjGzt(qPGcE|oF4RhD(hТvb79tB!S?d2f,#,oq!*r wt߳8x@-=$1uс O">tnϾM[1(>&;\&\#,HCm4>L$t#(qiNyf#L+/ Z}~&(yq9āZރ6uNbڭ. "ڧm[xj b79YC#(U![iKxJsU=Mg2mrhS`t+UUJU-1h1d;|@ 5L]w{5A"5x.h"Z\(i6Ab>A5W[m#*K f)b×wDe69j#*:M-ɱ#(cF6hfqFB#5B9 #,|_yX#,#*o)p$;GTb͝l竂AE/##*Q;f}a]]aG7fˮ^whXַ.gMk~门n8rS/BJU8C9cqv#,wQLp8 ؁;F2iJ-8Mk]+tƆQ/ vu(j:;#,ɼIJVlsyl|uE.V}5#*k[l:1]"c"tb3͕2FvjN19b"0NrcZ673B(Khv:0lr9$6&Dm<͎c/r>36ͳ?RAB߳yJvReEK<*=ڜloGA9{aFJ߾)SU#*%dIF6$,5\m4a=Wmr"aY;\/QY˕$ײ-7D:3Y1 D1ʛtqZFUB@oB]O#,Δ͵Q[);G$&I77ENJo0c&ota^ٽvN_lV ~a7h[:괶J^7N[Pq1MrI_NE60r+X)zm ,' ᫲iz+^ٶ^f=M0Er3oʟ)t v'X[#*ȇЦTb)=kҁ_']3(°m@fb3GQQb;}ּwx/7afz/ZhzcZfSP?#&$>C5:eN6fh6H-% R1bȄkƆq=v!3ih/ܽC`s9rN(w/&!EU=F)#*+pDH``tsO7фa}Q0kgCv;0q}и{2 2sӿi%Ik>AC5ƕ' /`!D 8BXyNnO B=~kZP HQEAsdI`#(|ѣ[۰ Xk `P#(uG#(h0nƂĘRVi@9:%ÞHanVv.S>ꈈxXЦgkUO<#(G?PhcSڠ8#,R nrFS?۰r$@j~?\XInG^&LS1ewdR,#.\(`a$ɋaGhZ+RG,KqY{cgwP1qዘoYLeXqy4˰JFQ\BWm5=23 RS.BU(,tZIO8"nRJMP<ws݌AG! C!v4ϵb_Qv /G<'3€O\}[uY4vbGm# #,1& XsᚐâД穭s>zhB(K'3lq-S9l7ѢX^x[B컋LRB#*:]!cAxDRz/}k`}0lcՔH"h\%B[oN< Vͨk ؓ>_ u vˣC7ٔ93Ed7Z7w3LuGY>S 2TO;q+r`qLB:d|<ݼSa"VquR׏ו}8u'D&#*PtUO65"FX*!vPctR`L*hv=:/$:h(%uCu'B-{T\ic9_Cɞ9\`>)6QD`[31w{DGM$導 aկH?ӕ %drV&뢕@)էjS>xՒ'lļpHUjռ2M/{m a>y65NQV$t]2\\fu!J{ F"řqk4\:|fHttIy=#,7TljwE#*mug;E[26/#*mȎfNemEa'o9W)>vm'~#*b~}tDm zyv.qf2{T_(ցAj\#Eg,CmF2\jx#owAMI3G2yxm̖%3>Lv{8؊5cs=y/#*>J_zIۢ͟ 1٬yno@XW~m6IR ͥݙ بzi?gWrf<)|Ig:$qCntPb[4ID>9OPPy\26zTh K,GW^ 6pS6CG_*8\_:*5K&/&u5N3{,0y`TQO*K$K$|wɏq}:=³wbe'~3YNxǯE?Pdꋍ͝իq"3SG/Hx}ͪXauMU$Œ.`UIwS?{MttQj?n]YڱZD7tP?L4]QIM)'8p9=DݹR4)vB%%(el]xB/T#,"V'#=p2dN!of8#*6K[~}q3w<y=/Sp:hӵftkr9Ix]q-]6W3,D3ڶ,7p]=]oj#,/o$W݂IP6XJoi:#,ǕHtzo`,+XȇVEЀx3,쬯 AU`3#,_`s6VGJj7n.T׺/a PPNe ṷ̈4k#*,aiX't:ɼU7f.-{ɿAUILQ{}|c0l6@NM0Ӧ"I,uc9U^L;I$iTx&p`u=oWLgRQ#(dFأ#*hT3(;kAlkAM1W&UN=%:U{x73LScPk1:B1]<f0Gh:<Ԩ0- XѺfh}OJ`V_0ߤ;N޳1'('ayL:p~ ^^)~#,7{U,PB2IT(#8RZC`*Bk7+KML6.sm$H:ߦFPp;}"G(GNe\;l]um*zCSq q#*aEwKKM)U;,= xSg>O%o]MH, æT؄/%l#*|2y"MhLTgUاws{H(eڝ ;5IjJ#fع,Z^pZ=ԪIa)NK/ԜFQ򈉴#zĠes>^w2aN·XG;BH?á.bH/#3#,jQ:Nk"oyL&S  T3{S@^Rlٟ9u;7T^W53͢]{G6i#*R_1lS|+46k96T` T+1hR-!Z>_on|%/K=]Q/|BY75!yWms#(%rd&{|46*Úcv#*qtʼnGtD*-̓?enZ!'`Mw>uWW~͊rվ 3;zMwֻ,r#(!~XocJ3]_sB+&}c:v+F:ȁܬ)&#*Mw'|/$}Gc,,P#,;, Y hs7ak,niK6.PiNnЁrrm,3[=_DJ#,`~6_{)?Ikg=窵nq#,:rbwtl5cUsiDB/z(:\Ʋ93ËtŃ%@w#,5ՖN#*e{8/o^eVT$ڀd:GPϥ3uiwEZ˹A\‰Y!#,U qCsiQ,6J 46((tf æ<#cPZP^J KnTQ)f(㗓,EF9`)Nm骬¬H}E"[]ǝbWE#*Vt %ñLKJ5ﱂz޺I ;(i.ihJh:4b[-Ls#4iMF##*rTܖؓlyk%@͆<0(=~ne D:8>Cj惴=6IC>;meyL@g*r|c?JZ=OXם#[,#(٢l9.Y|;͝:.D [w τԤo#,xNF;^<$=@i҄'ռ1jb<|MKdKhHFa#*nND ߽Ћ挢p5( ٴxВ;sݳP6{\%Zqg4'x;SJJ7r*J1&7:wlncqNe;<-bIiF-n>8OiM=QsIg5t&K(FJ&4 C=f= q/2՟/ A*} &w S+4Alp#,iOkmPMNIi. 4PA4d펹 '+U<:$$_ #*ZCcF"s+LW#(){7|t<{î:sCR:mVФt|Rt'X9 w_I'xNVLuVJ껩c6uUُ.[/ (2jrpu:)%(2dX#,:#,Jz |Āt+T 'Fbd4I,C9C4c꺄 &C(<R};0v: }z|Vc#u1__;b}|z~/w'#,|=#,#(xQ\c{x/ڴpǓ ~n. `G\J[ Kp7%C#,hIKI{e.jKdI#,(4=J"/߽Cb=\Dts!U"2,A`av# ScGI1گg.Glfx$Ʉ v)P<@yv}DŨt/}~??i5,Rڮʿ YTT8)#,ec}DzRezAyktҚZ0Q )oΨ;(H= U,*<*/#*wGϽu[_7oFr߻~7t,Sco&9#*R&g:SZi_388ʠ 0A\>RB$yiŸ,=a!L$B(5.k5p1Gct#=Ɏ@;8A-:V#(x /ӣgU,#T4};b#*B'kgrZ(d eQ!fnpw{GYm0 lobu @@QE2,J膨ބR@lP3j 'e(!Vnp#,c!|t:fe n]I0!ar9mHA%#(eh#K.*D2m\mõ :|7І0GgΊRE06M/w~4KQ4$q0/<v5I‰5v8:je`a"ʞR#(F 4І;VcL4=I{N6joX!)=WܰtЁOdR#(vQp.qqݏmM.L me#*fqpt&4g]u0fFP9BE.); 1xNJDA0Z`γHT+ӷV"#(t0:cCĄڅ %#,a|C"V¼y``0߿ߑpav%W@A `S7zyIp)%&$*yDB۳\ldxP3nAީFэȠb"QE!l`r)m6Ȗ #*5AId-./Dy|uFø0sw8LI/ӿyX湨Y9kCv:_WD~7UY9z(h3'@xJoqWy͎Z@YCqŅц0#(Cͦ1IsmgGnu`?NH;̱'G^uZgO~x:f,I}(m^Y0z d[#,`Mޅ=&h,Rm!ݍmVڇn4c4Iiz x:2V#*rmYl¶kG.>A;z4N Ӈ"[׌LS}/zط7#*1>Zr~[#z'ExvCTvaP7)ek܂;\Mp,wX%\)E<.̙"Ɇ2$I1Gx]=N3@xYf[:䀠aKˑ輨;gs<ז;bC#+2#,80[#(5=)H8A3\vCϟ,9|js.B Znx 7F#(^h^E8iqIIh 2+ ,2 tqp#(P g^#ku\*V"jTS|#X8I7ug fz ~'#(X8$~JN:B+77:Hk\Sᷤe:IysSCzM*%#,g o#*C}蹖+ @1 I<nR[iZ#(:zf*9/*dU1y8:meZKʝj?OQ=+{bfrP`\#*bG:RBpc=[1q%~o kup$U^]<X':c'ǃ^;dJg^ྪ@5\K&Mn*Imt- 78B : :`a%A"͇j/2M4mvؾDd$8O|G4(eӍuַ9ߡMl򍰧_KO3/eh&C#(CCX}816_T7#,y$<#(0$"ePmvHD^l(paICJ-$F:tٸtu&wiX|'#,z#,#(#,-A|Ҕ'wfO\nT>)Lg9#,GGl#*ЦrehIOJ՗*/}_?zZ: ť+vߡ;v !0~zꦉPyڟ7`P׀#({ۿ gCA1~CKe3-FB-D#, ]!f*&C#}҅Y";!`ct#*@"ll(V#*SaIc2#,H# YHRP%1}8 #` EVɪQwb0F% &0YJ2$̐&DAPC8PaU?fH#(Fa(*Ń#(6V̇WA0#*AB;jtELFҕ6FϥC$ˢD*e!Bz)%Ϭ+QHJQO={6uV܏9#($ۚ{\:97+&Ђa 12Q*Ab#*3ץIG;q=t\T:.mVr=}ٲWTDxN-XL%nє5Ų L|uPݓ3Tjzjvs6aX2͡|;øڕBB{rɸıXn,Y֊f,3%1c<3뻹9T6<D37YyEBm2$ T b<@ȯ%-)6QS4;M4F>tj8i7q#(7Z- M1I#*)i;l ֆhh=d#,UeX"U(gMpQ>p՟4)vw!9I]VQ{:GS 9xC膨,wւvοbP:iVbVzh +{8wKﮕ[mʭ5Vm%F6]-E.nNHȈ#^gE& dMʹk>(;>QzpCYLGZ#, 5ʲ$'D2LA9 ){-OV<*y>>7S}Y6llʱW6 JfO_&ǃqbG aPEܚ4 zʸ9jRdF[I)-|*3ZO#*&ak~iiF!QAI#(Mַy6."B ڄצdICyNTE*^pVMy(-Qr0m :sc"6Hjexr'U]|SS#(iqEKlPC^4,HTa Ov9C}:$9#*|ėĢ@2wr SS^_Gmv湱jY.顣 EY"2ǷL#('lwۖ+1#*#(RR<9mRFoq\N>DTUOr#,##,#,I\jB]Kzqf$йڍO۞ِyv(r2z|*7 綝#, xJsm 3D"Ș:! Aw(y;Q&fjs8ӋçC_:逳oN#64[#*uk$ 811|yMqϣnw#*;qG#7N:P_4|!Ɋ#,\Đgmwwuʎ¦} XӹU'vc٭>#,,-oܵ |N.ˏ#o(铍`wLzE#,;*~sHxCb1z{/y(jb6L\m܉j4 Q8򚝢obAA?T*tgHL<'Miyl=6YC$G4m[edϣG~}ATdvW_aHuFexNO{N7]G(ʈ2E|=DD'vv-.Acd3"WCI݁X`M'Sc+|lEh]edDyFn?9е!s4J@8=pOmj1OOmZdË}< ɐY#,vZwYQU&x̋J%E^sTxH\N2\ރ7?~#,oY/p80@qgtLqӎd#% ".bb7(ZJ/gxX\#*??S3֤1!Q"ňeٚ :}Da{';3-) :=k$ #,7l3i.jWePע"р(1:#,^7|CN;P7Gd* ^ԅ5dʎ{>1jAslƤmblg4Ϛtؾtv1v>:>Kq Ҿvꍔv5@"~0x.Ƚ*!(#('| qOMK袄'9|tW}:Э)' NZXV@Ok|e=~BX@]'v'mC=/~Ƭ/~ASNOa8t#(&8E~.9J>LeY-ڙ _O 'O("\zBgC/'A$F-㧵?zRΈnc&gu7K&HsH!M\5IT'4Wny+l| :Omtc7Ӗ-gp"N=~b sz-99Q#( y{^~Ԋpva&{ϴlxG^Q%L1>N)Rث狊/݋:Xr4?-#,#((<}{4|c Ƞkl9~V^Y U#*갨^#*>X֙xc\J/v;.ydb|rdO!#,iQa׭ŰiֱSXljw%x$vMHO ,݇W $+a 3P\ʵauj"q$XsNw;YmgSOtq#,1SC$ޱ?|C@$a08nj5DȈ9wewDȀlm0\eXn/Ȕk6ܒaOĠ`#,^#UmQ -/>itRHGKTӢ/oZ_yʾń&ftk}Տ%hHzlhuA_Sw6kq-B/LIJ0R\{#[Z[ɶ:|~Vl0qu*6L:' [𬝏+7WE@ޢ4'f#*#*+bC,ܘq 糖+tZpWO)HTFGOW"Û 䛒s buxmcHH*3k%mo@ka;B2L/ݩY.wۅ~=xj,wHW~Wd2M\PcN~S#,w}gFcm sw7?&'w=}՘zk#,λ\ꞠǴ3N=}w~Xu=Z}kፃ(0 N%kk$I$#f#*T buUz#5ŚI]Q^#(0eg-]*[4#*חzjs?E^=]$~smACxmO*)QjRw)ƭS9Q?ُ~bGoN|š?4)D]8=KcQ9蘬!H_64EaT D8 "s3-,tbwl;q(Nubq ІB5gt[i(NIF2qM{0Ue;BW={ge'j_gɵra$!ly߸9FV_?LCLXɵqq4[DWy*3RSkipVQ!*$R_/"MHO7G&Zhw,#,NjBq8'Za>߳7`qO?#,߭c IL ̉[ۥXr{b?/̮;8iߩWMJ] Y]g#(U~#NH$ 3E(:X}Bf4Fb^1~kq}yy]{)x;˼?.zvh/KyNi6\r#*;>S'h`p9q`ѴJ`[/z_+û '8 /ua&"qW:'h2,*[(A`򧉓#gsy1fUjO ̒sz2dS}?Od:R{zy#,'TyT8*܈aL;ܦ!x#(V) i6Yc^?T>1p'>u{dXXy~v.#Q,5E#*C#(;@ JRV($!/cKF"AuP:#(P Eޠ6zqiWߕ񙔅L +3#,JWPmf[H-^d,7߱OG5|<P\."?9h>fp~XgRE~2Q$'&$&FG t;H?Xp8R,xB](3n)"@+56qKpPy-WC#,Mg nF)7x~,;Ey>NoOy]g^òYΥsW76qR@cH QåsiD%5@T?4OqYM"{>[o#*c(۾ r#@$XF(D}oW\4rSb`?,Su0OSD_S{ˉaBJx0_՝Xn!KFuua!Fċh;ӕS~IØ  ot}\0`ӱHĮF+LUr> }+u7f8J;r.ܡgr&auk$INb$&2a}nt# #(X˖Ĩ(u꼗@x2v8g,e喑 \B D+}ګR|˫YnȊ`A/pF$bvj% LSa`>W;9CTr#(N&Ok@6ǝWspCouZ{w+ j$kͽ,QzMCѝup8OImFX8g&wF#*#Yj~ieׂl3 aÌ㲸Jݎ'נuNUjrԮIc04[:DUތ+on7`d1@~ͲG!v٣Bd0Dg_nCl,&x~TBô z +c*Áb-OOSҨMTe5%_d񥋋, 20>_/-^'-o2>8}ޑgN6.s{ZCkNip䩞c2jEtwӘތpx2zɑ{÷i"z@IIElˀ}K:cԆ:dL7<:fTl\E3>/=6%qulv< #*njkxXEE`>?эԦ!هDA撇=Qb8pX> @#JozW^u_Ic + )#,#*TxVh`roڨyp4>vʢU4CÙ_ܹvpUFg#,UjXj#M_|l@h"j&qS{˯ 1A@\nL#= k5S`5d! p᩻唄03">W90c]LR,U/AS0uIAN% %kÝӄt=w{U.?$lغnuY;Nrt @K$`;Kڣ\vPAM@rY Dr(f3JPȌK;ڱx[f6_F֒g w6ڻ;92y_AfaIۦ _\0ѱ\DCBRI;r_Shms#x^s=@-MǙI6kzs,oŝѳiaϑ{A֥t'[8qɝ;-]pA' ؑz?_B'qoRvmOW_lC_xdK}Vc*c1|A*#*8s S5@^ӻMSh"*TC3L;R3lzsd<-30Q4@ו/FqϙV#}˯#* ?[t -;]c G6&#,vMd ZwBr׮6x_,zne_xtkjEqp}eTg~ՆeŴ^[zn2d+$s-ݏ%;;xx:Pxj(\ pkQ9r-u8ϻ,m.#(䶐, ߝ t;gE嬞 #,uɚ];[ /XCV}92o#,)iS5"gd5. ـQmcQiڳې /;$ƨbRp˂-oJ0^:4roT83{#cښ@|~4]A_zr)pр=1$n#*cPl߿F'Qn#(x&;D(#*' d#*ADOtcOE[#* 3,80p`_MRӹNM#(;0p.VU)}y>w{A߭O!:?ξsaD#*. ` yٔ(p%"8H_8G[k;F ?uh8}[;ḩ\ٿ%-#KP#(hB޺}:y[ ܿu"L˔aD5#*K7t|-~Un\?·溣6*PN9cYmяIHxAP6hi*OU#,]K#(w&}0/lJ,p9"#(hM 0;u3yg #*pj#,?Pdr9.a}=SQ#*cp"eW~JpXxbPgþU ^?B`m6-K]@YU;#(Nj]"=&:c5TM#Q_R"v Yi^l3J?U=€!:VIr?Ð>]ouN*:s!n`=`w Zd|%*z>M'pp?G+q/C@ٔ%u/ MZp=[A%Ui.83qDtVV p2 p0ƞAëwQcU,)q:1ln&P4sf;8)H#*N.6JOzCf;IGi?LB>tmhϧ:Obv r^OknM@ghmQF<#*s5oelu,%`.t#(o{x-)F,=ٲD/79l;g~#(I T  !ö"hv]dJAJBRxxc#*|0E1& Hcñ:JV=E^{{ME~#(lOl3ΑHB`,)$"G@]q!AwY?W5?<ˀkz)#*AF#剡1KFZA{ƓnqiiUHq)Zu<?8 p_R)X^F1}xAL wy+ #,f'#(>vSq`He h7`X:np˩\(2mmf36c148C.k,hspp6A!#(Z#*#(]D(IGS 6>#, ߷gWqE#*2+֦z}*u Hue_j`AvH(.#*d#(Q"gKﲠ05,>^E!l<FDYA SqR3S y335E"3fdm}H)Py6$ M؋.Yug$:=#`3aQu0!l$}g?gP OWjVK.O^ތz$HQWg湈iw"S4J#k4k4DuǎHX7̮R#*nG^I$f4)1S.5%fj^Yu.aGꙬ #*#*~%k 2=f:aumD2/ak-5[BRLefIvW͹<b=͓Qx,Z#yhH;{O!ԁĴ>VD9Tk!'1RO!~W͚\okh]0X2t*Ֆk " -Rm(2d?ۍc94,I$Bpuv^xOk MPRWC* P#(m(PcQQ~̬Tz'3`Zś'FRn$$X#*0?MRE\A (+U$/vu,;#*OmSdcgdDv#(~K_p}6ߜ60B/;{e_2QRR"0hHnju&ˈ[5[ݛʅ`ĥK:Rd~oӥһTmÊbr0YxΈq#([PѸ=bifBآwCz-$ #pb`^Axw;_$#,58~o*q(r"9#(@H}M?d_ w>k-vov(/)?V/a-zRϢ%<"ODGk'0:>Kɳe?XV:BSA[}sEiGams럤ko~'aN*PDǴA&&iF zxtWwNcz:7[w*Z`{B-v$$FB?XYد!q{CdP [479Z>k9Z ވvrA5_PߗQ~}c}rlV7H@%lx~|e|>p}m lCԦjI'>!`h#(}>_HW`Zt=\9~#*i3V"!،#*re/~F=#20Uʒ#*Cp,D]aU񊿍BQttCtל#,nq`=ᰰtϘ;Xq2Y،In(xDF)(d܈ .&T҉ÆưbF_D㳱Cn}wʽo$ͥKXۜ|Dģ ŮnF#( l/=izNJ8B =&~d'h6+f0L{Z'>yaخgHl qx~)m%6^珿#,/8j/ط*4?r 2J#,/.HfԩP`8~oK=+}y?a83J~҉\k|;jڤrLj#*B#,ϒBu"x)c*X LZJ)@0PԨGQHvN#*nLV?R#, Yjo8utX,)BBh"!539Gǟ/4@ۻ>GA 0}L#kxXjrxoQoZQqS9#(6L bo#*CAHK#(@e€P(End]KRA($#U Ds]D3?FJp #(K_هh|oy#,4#c#Dee$,đ7#($u_rv?jfX; B?9_rUH 6ﲡ wڽXZCTh#,4XFm|kJ*S#,D)JӐy^9A}s=y7]=2@%I s}~o4aڑ4a!ҡ܅&*F1B( &,Ƥ2n4B}(䥅Ry:#(QI:ypb--/@@F ׽?T(p tQG۸wlBCIϘ;\'vۦewͅ@a}l{?5ˑwuvlzy1DDW~>!'%I+ϴO>qqz Zdɗd5!xXF4|5 /67C3njB5H0%$(ET"꣞IJ#Y9!|zc B5]T`eBS\#,oG-!7faކHDi1ϯ% #,( !Ia~d}=:<Ɣ !WP8#*#*$cha*`٥ T A|O+?#(s#`j3Y,0"TGFpuS\ JEQ3BfG~!_>L¡4NCaI38׻e E JIpH}N:zί+J5ZE)`\\P~`wǤ<#*md]:7x##(|#,FGdU0 讧NzLRr$[6fu'0$t \h*ü ,7!2?}=?Glo8UP!g|e}ŭ,cuͯ(}I!x6a|˕[R^$U$M*êC&[b%\Trż_ݞqq`Ɏ;VurT#+%&])w#,Tnˬ<^>9X<{wdRE^:O\!Ni$*pΤ$_eډGj`3>ZuZA+T'2#(}Ӑv@<~Ͷi:= iC"t<1!H5x #)z# 7wA:6p`J%[9_ve*f:Fs:ީ?y1}#FJ NQw̕܍lE.^M{~):Dɘ|sAe#,kZLn ~t͞}~AST{/9m`IÑ6(ϼ%93L56vuqg#(9p]N^v#*T9#(:B쬦#,#*;`G/b8B A $ChY a֪?l}#,!όT 0YG:N2Ba A^"`8AQzRʱy(bWgUL+qZr@ŲP;[=İFQCP60`-6#,=O>gϲC ^JF6yۨx !1g)8!>GssIMB>!P>5a^L1$mf bX{: KwuyjAnO>y굇z==Љ `]U%==gӑ@M'2u)g;\JEBwԜ y3I"߉Pڇ**"jUUD>ʕ1-D?Lhq*k51 hzOQ(0MT]ݤ̦yVP~DF,o;UJh; 4H;$_o6Q*F%Jj67wtiL[0(k~$"p^UaЮ v.l?\$aȷ< 9KU۩h!eC:UO״Mit}/N^^{X*}.B4'K"(Haa:îT[oCڮ,ED~c[ϳ5Ĭj7kIlw u#*YT9S2fAryzB\aW~Y(|W$4?} Dƌ?sA%Zo9Ld6~#,@ !2o}C05"H}&/ua-Upם3긠i< BӰU4)~ s30Pz Hoqݶ<1Kt2HxS5ObhјOFM3i$tGϫgb+ ?a}uM6m#(<5:M]][) L!MocS19,2P]A1!o32n:TiE.KArãƃJ"n{?yb*FXxI=h4Gj3]DVƒ'o#*d/':=R;ԣ}|_A]B35M v`CΝ_?¨uC%Q e/_#(hGU}nSO55r&\yG7qEç<_=(lrUe#*nvixA}8 t "fqqc邯8qa鎵g*( ߌqq߉lm87>#(>܇PA¶mgy@|e~u;`$GaDP3:+ķ:'nQ!;I9N_7l͕@YC.i+ިrа"NZrX7V;-uHi#*kC<:a=̶ Z&͐8S3n^{F̩hS C#,QAE`ӽvw iX9AëZLҘ͔>Y>]"<%b#(xi^>./XBu5'lϋyĥ𹅭Km#*#ī7 tw7vb{ ~&Y̼yOwݿ*ΚRTELт#*(d@RqgqצjsFo,!4azh 3!.I#*353oL#*"Q_?;mrn"cfp>6]8 u{^/Ca%Bq<)Б#^pMڤ0lZ$s`Ù\OI*)u fh,yFA>L>}-ȄN7ܸM/?4tqCCKx#(~bWS("x+l+G#76ьz*?zXSm]iȭ l?ͣ?]Qi )1B#ʼnp8{W(ˍLU6Y1PbhF# .PfÃd#*v8n>B V2iMؽ#*tVT"r>[#Oa'P<[I#,ufNsmO~#rȰ<{Z-" a=8pa$EhJ-dc#m•6ɖQAVqPeK2J&E;{[YѢV!#,B(?Ca}9C$@F{WvݞA i0d$[5H%jJDGv,{ ֗2ƃ9F#*&9Y Pv+mX C̸rxdG8C w9KMf#*$(li.:)!i@rmz<m42/QȌ=@Qs "FUǼ4Hd>:[w{<]mg*LgLns8F|+Wݕx'+Xf3Ķll:o?`Syh)&D0aBUhjG0xNs+6?] &vɅV0)\v&g9< ;X8:Hݻ;TZ.JPl[S@&W=aFak_6Z!6Fg:ҧ(8!EW<8NȢ]hfC4ax ǩSvaPbfTa;tX(PhgYJYÌyӧ[94׫ۗG0;v(s^#huɃ㸜sڙ#*m -#,"aPUt(FehvowMz6~TGr.:|2R2o(nt a,-Kܱ-]NMB#ȡ,C=e]43Iɞ%JM\fspo:zI@#($E%0L:Ⱀ85tM]<s|8 HKmABPT,=)ĔXwtj&$GD(#*nL:'~'27"`ST$xtrnںN@&+ӚQ#,fb^fd˙clƲ3$cg潕q۞R*SI\T#H- Jy%XY^lml#(!w2#,}s)ADeQS~#,JזԨN:K3tpX3֨d:$ ᠱJ3l+;Ѯ<cN"zۮd4h>]3*]Kyo#Ħ>bҼ;N"V2 tȣߵjFfgI-}9c[6úèr3JlEVș`vui!F%HZJ|!U#,{D$QHmTT׍p*hZJ`xr*ȃXp6@s-dY#*LOV#x!OF,܀IO,6SF@rD.,:Vsrx鮂%u΍:cgGg6#,Wҷ7a"\˕$MIAdHĒ{⊳Le$z1,t'߻4\0D҄йV6j`(I:[yB1LU)a 6dEPϗ`jxw[mӊz;n@ws=?̏W!LpZn#( *IK! MnթzR[3dgjg]Co?tӥ_RT*[ۺ$!F0 H4ېJU2Ub_, D}^V-Ϫ#@ V nD{!!%-{wʟ'#*RA*4>}#(5{:YO*J:<)"P>N)p̄"#("ա! \MŌTh 83 d ##(=A@pgKƭϯ\fwV:$坻{m|#%<]L$nf"-q#*(YCRra=vUvyѴB3%\$͕>*fP#jޚd#*1|Ad+m^lFmϣk#(]s8D":n<UB#(A!)#("wƭV-֊QVEN&9?0'$۝\f#,(ij#,\R:#*a)`` #*#*@аXe?I#,e5z*RIwoަ75ؔ]ݠ-xc%ewh˧[{֐ ʏ1Ԃb 1XAMkojx#*,g? ET`z 2=Q<9Ĺe|d(ӿٍAGYivU{td#*a <p˳Yg֣-%I!|VFK_ E<[EL*&*#(PdIvBc OGAV/@{Kvh_*#*Y=ayйŀ cʹ 6d2k '*HBP†}*pt0SK#,Kas( dC.Gp t:}nDBCc*j 1F/ '!؊ȥbabMcHmkܢֈqHcQ#({Xhx"iҾU744dpp|Ņ#(W9սnS@])UQNn蝎*d[mK#* 2b% WP^:F,`HU!˥DHkN:!`F\bMZ6f•2f`TYiPX,3>#*;d@'T!U2U;TWTt{o覚8KHda&+#*!qD"dY/#,) !+i =Ќ5B!E0*!:G{ytXuSJg"߳M0f+hKa~PL~%HutsOܡs6niTχ_,w|m!!aJ5S2#*Ou *}V9,3tɴ8 (&Tc Ns(nօ7@y"atPeCgA?rلk_W?#!ܽ/#*_]H?)hy#ע{~vblIn~ʡ#,Otق[D=ۿ׎H3ylw07ۿȗ*#,銎ΰAj|C!`:J f2R%.#a0&|t;ְ0$q֜i zsE(T_כι7b m<п1XJJu@d#(!`X#,3CJy].cD83|@41)z(:VA4f @ט#lID3־R#n4`5ɻ.֡AC*|i [Y_A`e+#%KM.L=˂_"Ac !X۝`lLXLNPXQ6x(GB犇d@6\Q-ad]+/nÜq1вf[dM r,ZJ˚IAkDexZ"" xTq̐=p&}7:דlAWA7`GpScƦj0<BR-jS<H#cxLDy^8x #,<~'6zo>pؙ#*1C\Y=y8GTP v"M( FzNwt*|FBcZ>)2= -ӣ;ϭ\Bϑ^$1Cn#,pȞ_^5+AVJ(#*|NHkgWWhrۯFݣh|~Ih1E&]Vdvպ  lKpAsR@^Nִ.+oq@72Yi/͹Bcg8 /Pv ȋUR"!>84O׊Ȍ4VK*Xw1<@RW#* ^Uƶ.2&8#,8{Ihr'. o,t]p}Kvx>ؚsHBfniLTW;(rVTZ=SR6Ȏ溒$NtTHǚt%vfmފfm92m!Kz/= P$d^T'0:+)(^GyyZijUn)dhfɭtl\2@"jdarQV8hd!|9.Cev,T"#(d&8&NPP(1$]q(›x;V>^^|JTAnR,6`ddr>v{tC=  2HP&!Y=WX~d@Qݡ je HM؝=aqPnItۂZAkb.o4hf[+'ʑYUc{^[-w$=C#*meQR (H)Fa) R #,BǴݤ止jx>g H`;NE*B  @&.57Bc0| ofc-˛п&=#,`"^%Jn~fy*:yzֵ !4ďn8pվU`Gk/'m*vnj%3DNJ`w* bs.0P1C%#* ɻi-7[hcFHX|~W?(RhlHTsIĨB|ˡ2&,EZ(D9tLK*Fcmi28$ +'.N7ԃլ;~ոG@=Kcfrc=cj7kgo}]˜#Ґl83t4h@FF#*mXUN6?]p Yԡ3m>;eّ#(G#*&(!זPba9u )LB#* uG(pD&,#*FA@PuiЗ9xT$,8|;0YjĂWXG2=`+ EVj#,lP< ؑ`BFW#*#*0dL0Pqg{"^rrHR_RAbCX^t^oTZHj!`D$Hp4`2F?>[7}ݝGSQ=pS.,2n'#,0TƽP2"#*a'ْmgHqPNή{͊}I(z{)#/Idfak&#($І#* 0w葿cbB)/X9/'SSgvqdtbBBTé*C($(Di6(HU#,4;]EX#*A"u\A-rd9'l6*#(xQ雕*2\RuUX(#DI) l"|PF$L)60DR(VMBsR.U1 B#*ұ/u+ 9xb#(!M^|9"l#("Gs9[^U0ɻ/jm}),UgDm1 "yZc!%s1Yehwa5D!I$.*z~UN0ff#8n]v(+M7TF0_\y鮑_(ml6Qǟ,2K-IT+hTw_3#*ZbYd#(I&_z'G™#*ZmdcQʰiM'lc26vvqAgl APڟ֎QU%(Zj#,2j!Fue@Q $YdEA( cJ7_\|+R602L8k {AcNliyCr6#(=WNr`դ3=KrJ5G"iK Xh6ĐJ.<6;#-'=&5;wZLLkmZ) .(lA@QBLñ\W⩾YrVZ9-Uࢃm'hb=OFv,9nKa3Lu6g);Y.Et ByxyxwGY- ߟQv<:F@ⁱmz:Џ[vw `QbshI|ʈ.aI#,PH\I#4!Fl9ola#(! /KM J)s1J jAL(op[6`>BA?w3Wm/.HAV8 nNT>o6FeOU)wSrFHBd_H`ܤߩI?Ui៴M*jezhŁB H#*edxySկԤ*\mj[^*5rֶ5kQkW+nkc[hڮ[>GϴȈ_#:d7[W'I@8X>Ocg'Ha VBS&JfiRLRjIu&e!J2SDL(4k߷0ɭ&,LfLL2FdQFi}DTY()IIfUɣQiQELFQd1Mcd&Q)2S)24S4bAiw$lZEImݩ"Ѻm!_G?/}~F&'v8s)w\a7`4&֜B#*ē_E[Ũ%e 1mGNܯioԉx]:p+(p:whׂ]6mx&"=L};j@,=uA{2u#*d" L'WmPPP4썘Hv3E!$q/zH#(Q=RnXQ?Gkd9yݝ#(HCĝe[竨wO`=?cJT=O6tB/@<\#(F3¡2'u c*X@w!=\gVeI#*Ȗ/ݚ `GpÁװzmU٪xEك%lL隓x`Q#*Y5V\0>x uM3@C^tXd+NM &bdReC3džػFu`Bkց=,Y?9@íRP&]-o ]8igx^LJqaϓڦZ0MG~?3^5|(N^q,ڋz"VnfvѬc5SqK2I3;ckCeb688 \p\#*>3HONfLw|yH#]F,o5sʷf ی}cC`6z٪(LiΘ o1"$kfM(|QAl͜Z##(z;(tʴk _ F3uQFTr9?jAln5'Ɇ%+a\f*ɾ]U쌠SY#cfz2x.5M}']%4:߈vC=^K kKc2$Lu&Dɷ pU Skɕc<ݢ>p\5NKMzɵQ3SpP\*[ WM8놺A΃yQ%hK4kj'[dOæq,gz^l#*#O<0;bYlҢEa[^G:pe#,M2NSja5_8糥fsj^#*68rN 53S ]aQ{ 5+jXt0*9jidžwifómUzLoZ-ޘ &E.-GLlM(GEIIAJ?*fā[d*2fv" mZ*R n)b7f(AvHlb8c͜.2\nwEl!d'$N'T=  a/Bd\2~ „nlPu"#*m0iN"g¨ c8z̹Lڇ6@f&ӡ2=@Ԑ&ULIŪHEhBVK4mWligKd99T#E3AJ.cZi+&l34de\KYcx Y455gxp58Pia!kD&(z'wcwcFY @n@iM1˸c ީ(e)10h0Th0q1b v*f.fѤ&3,cm#K[.UqMϖR'#,2R+,?/p̐Pѯ&vQC(Cb[8Kb fpÍhqy2CfL.cQ67#̽l]a+*P(5b$cdf̤l<89#EƎpUC=N B9#*b^a3CKlja3nBKF, M=Ai#t( r3inf;#*˙t6!2u\90j8gtى2#(.iCNa .XPU`s $Gk&hɚVЖڿ]$B(&"hgx#**nP9D| _G7!}ov*%#,y2 :A"@ٛ_L|Wc a,*I*Aݖ최O-L(CēNUI>l'y'yس7RM߬i5z#*Bc'T&e#v@gOP̳$7aru6 e<7B(ff ڟ(#,t.B$GzP?**4*DPHFA@ X(l Ce@m0D!~0ІCإj.]) Edo#*dx! ֫ߎ #,#(#,#,1veMl)p?\&R{ NK'cb;1ڢ<#(#(~39!}C'"-Z:({m~껊3@cS&E߸4F;`^#*p[N߃;1̺!ĤeSn؅K(]oǸ-zLlY& [UJhx+0\)3\ˍdϖCYЂT6#*Ȇ~~_r8d`rh+fr>GƏ+ShIIƍEQ&503- dn$R OpҴTE㷫rL%]Nl7>>e Dָ$,DAaL2Щ86Nww u4Zn0 iAȜe2OtZ? Xo$*Fzbd74WGe#8+I,u!qЂ PnɁ0ɅE#(bXE)K 6 iM3D#0Lq] CX*~ .#,xU1#"EUPm& R3oQ ǒw2v桡TcC&-y2CL|w붶.Q ijHTJ\rjID:ȧ3W9'MD4\fV5!B   K9쑚yyz1;N"]AYB#,KҔ7K`k%Ɠ  T#(~]P3@ADCŅ #(};þ&NzAzbHI%E`֙7p7\*'mȸ23)M9"Hcl[:퍌}kߊ9ڋ#B)A(a]CfBi#554CT_ppCfPf,v}ä2BHbt+6鱽/-k$IÑ̜NeoAQt3L[/<'GiCFMU UmÐBRK#,j,ڔ\%(č('<;XB&[M"n(=XXjH b\vaf$0@R@&#(Q&TI6(҃cS!0NKTT ]y63Y @A]x?PvW|m/zSN)XQen}(OΆ|hl!ֶC7DGA|bi%G#, *TwZ'DFKEjmZRBD\fykyUX^ej/|'F#*-h⸚-J1ՕUrr$E#*{#*gxy2T:ʕ.)euI j1E])DM ؆ 5$iLi""ۣ˼$%ҁQܑ2M9&y#*I"nd@jQL|cv;i }6R:xsxMpg´,&.nْfck3.ja:iyXǩ 5fYJg'}gL$Z[]LR$!*tlP9 4`ubȚP/.k۲T6BH$r8ڥER0XF#* Z4$yErNmPMP\)R\$"i"P[rt?9vBP($ s[v:]6,wtcVeH! d^5Y#,[wnɼW-zreiʼlWfhTe?ۡ##,QMf -H8VI J&֛"eMiflkZb* )$`2DNEljE5CrK9 #,q)v%(U@#(=#*_8!;m՞'ap:tf^a߱sQA}  fX#*mk0aݣۯ=,l#*LXI!I*jXŪm8昪u7Q ԛLR$_ŽRhH-@,BьVe#(TehA]!;0t4߭},n#(٬#(5( #, ZLr" Eh8i#*kqAx㑞fYwm?2s>>o}WPEȀ.8'Ty)b |`-LQ̒.~m Qgb0 䜮Hg3.W@#*QQ%hW8^bm.^:H@$#( @W|}_"{?(22.]7sAmUV6i{-ԢjOnTD&EKDݫ\m+Ki6QM2e2R,OȺ–aiHmVD⍤[ew]-B_{kW]1iD)Ik*}kr%^$mjJmuId;›iFaSV[-&uSkţ-W^w$2hk75xbSm#U5؊͵#c^ӞRVVb[l)P:+ 3mP_)e~q>Q"ޘL(Eۨ﷽2kޚǡtYˉACt5S#+2Re˟ˏBq1*LPT_ʡ*T##q+3?_9 Jpn)'%Wow֛n|k}ٮЌ൨}/XT8L)X#(Bp#(#([cUF#**jEp4Z ̐#,7dC ΗEl骴Id N%#,TCA E1DyJTAFh@nx@]8y=ǯ~ǝub.47q#(x@PI`UdP2O.?)!D`вPk[C+) E&i&Fh4=!&ph,0$4hbkX# /,"lƠmE͉D6&b2m6ƫ $I6KKVE&1wpyw~a?V!- N҃ā҃4CR4j˅L-"ku+ټ%.m'W"@aLIÓNmHbp!V/J -AXA;q0}U^"C #('RcwVݭ|^gB O;pZ}PcGs$L¼Z$!#,lj?h\,D; _&9"98ЫGvZJʙP!+~y7Ƈzَ@DG"zyt+o<\Ut]@1%O #,55Lz`fd1ꂆ iY_0N){+`2kL4n@&4`RͣᨊhnP#,92eu ei٦]gNn ¡T2H =G"}f), ׺X[CXݲ^LIw\C0m@q19ǜN76{u>#,"v!fT*z28ZJ9#LEi#*ȘA#(L`>02HsOe"Fo EVt0C!ؤ5XF&wgNZ}`lC=QP@G-ҔN$aSUjwQIVs2ŃC؃cӿFH~#,D*)%U$F"B) JEZU#*۾W`a#d#,M.YのX=0DC Vm}<`px Z(3` i(xCrՆ́\*;"l r-u'?oی(4dj#*z+ϺwFXnq O@m#Y#*z()#(A#ADh%ᘆm4'Α觀W9#,兞D~./_D7Q Ƽz1 *?C!ϼP:͎!5*Aޯ}!g̀cZmiݵ͵ؑa%@E$P!<f .B+MCݍfe`\n0fJXqXޫ&a#N=~_ڬ}C ܞAh+8+a$$TCDhwGjBR+Dg,11Yyrp,Ya6]=]_{yz4xySّ! "'Jai+ե$aBEi#*E?h?,ᆌ3TEP v(@ TRXaXZ? 0uZ^e#(Rg6#Dk2%{Q,dIi]9AG г(ɐ#,Τeı#4,hO J&&:AC̬R5 mV754B"*$F_o-ZwFr5*i*4uTbd[0XUu+FQXA"E"dE" -eA7V-@*?d}iXw(jEDHXHA>糙`5?\"R<>VQc/h!P1#(R@V(F㊅i/B(Ə8PeCl*H@@uײַHw&È&@(7!"HM ^-D^idɯ:eK6+x,:"dalt,WUQkW޼zvj׍^n(Mj+TK%WT[4Ҝ褽K竊'nɄbQ8ɸia*ȬW䑕E.*zf`BLA,e qab/,Z#(n;dp,OɩT\cAWy$"(j;4K^gŔ#(6O'W2P8ދC{W]qIRҔM{5YuA%ey=_Io0㷗}[U7{TQAj01@6ߴ|x8ѐs'#btX|MɶU"qpܼsf$-a`zT5H%0I rCLFR/u&*nID/U#*|Gϟ ox0N-*{$ҿ 8a)t QiR /7MGMN#,?44_<{ô:E@4hZGv$>^qrD(a#MuY#,#,HV#ld4a%#(PAb$%%ʙa*>ٶgzs !l!мytvjCYA!*WʺJ@5<#u;7cQԟT5d枕#(>@:+%@ gP/{ TRE#*e%f56+FF2tA&2չmLSdZ(YV*5+M-f[e m(MYiZ-m`#-iޫ~v~T'ͤ[$"hG S!҈@""mbSkW57o0 "^(}MN(DkmunUF#'RA¯\C]Ƚ+ABR1YTzK= [pO"f!"EࣳyvKąt!OGcBEC#(BE_lWCC؝mbH$ E"P*/A/j[ qOb/g;3@>U8;gɒ%zhykZssT-M=8A(G$˩+P wf9h۩G^YtATTO-#,:P{ZD 7,tK@#*5~&=>gq٦dh=GP}@_Sbު xI͖vz3Ȗc^+:nWxКͬjT H)B%bZ(}.QAl[.#,4UU _U1 QE1Z#gJ֝b kJ5&{_ô)wV7t\-2qc P)B\K`efn43nIxu6<-Z@ "grw1tǝ EM`[jT>P)N]GG.a9F$a22'_U^YEXativhxp#*%;jtݎ q/0La{}vLjz#*0°HЗБՓp88DY!ɶ'ϧ 1`X=L;f3#`1̷yf hZ$;_r]msm <#*ڎ=d0#* ^: YF.ogg.L='pi`@Ry7oc"xm+ٛ^t#*@Z#,XCcEXjTj'κILΞ&`X6oOC9D G Pʨ9łgHtIj7(閵unPDl [bax*Ca`ZWC֡py#o&`#*H(+`.;a%Ӷ,_ :9mrSP1hm0z#*"8#xFP5t:mxsxH `weRI%ݛ!k39gpfͻ1m9hCB&WsczfL_)``#,txh {m#*1DagrmDƖ+a~ZJL#,3(ä}jv=:ТIe)¥¹3^I2 ᚏ%D#ƶjzбqC:lK܃ 1jy ś$IZ]D0""q'fM1@gPDz21׌vO=l/TWOk/H 0"\Q#,D<8 ϯe#(\/El[V65ƍcTX֯cZ60\S1E#,"0# (~ W*H0|z߻ =ǨVbMF>Ö7vЛ*~\ x]5t;C~Qd[M`tLa/ؼ!vm;_تBFJn= !HH7$Ayw[L:#*N̈́B#*sd8KHyCC[|C1J/;cFF7|PY&[4G1DM3#*u4Q#(U4ORWt]̽"g FM  KC4l(es1pH1]\ (E*E#,68=HMPc!1#$ٕR T*<O^+ρHy1b2#*(U5ET6Pbd&=i^ʙ#&C*dGqaZ9ʈK$$22r93N iY89ȱh/`vX,@&(*ǓV%<5fуԍ.i\6-PbU6y`B#"_Iw$pn 9Ouel!8L^KE"0P(F#,-eY!][HD :2MPVL!P̅`H@]<(|0].J^EET5Hꢛ(13JT[EFƊ4cL&fP̘MFHWP9O >ߺ"F?!n8#c!Gy*"A,l\IrЌGXCqψ[TLPn+m!90j`<3 SBi=#,}j|cɯD $>QQ #,!a>y!(9t>i͸fc1Ycc+iLcL$,4id8w&Nq[])ۜyxBMhC&+'Mo/cS5]hŒa1%O28%%L~5mv)]p{eclje;8`Y0%<E𓶅c,DRtvVC7^WGj'-OCJdž#,Ɋo̲RV(ztڊ<.>3"# Ũ%ʗֵ]6?E[8؄XDQBX(+c#(JzZ~ax w>T b Zqàjb.1 `E#($Daiō[ۥƼ* 4(1k7Sbڒ-T0"#,wHDQfIl֮/J;˼UDF漼gzI4b1kYcI#(`KkZHDCՅZdHTlm L(Q" UPXbiy 4tmf@R^$#,\P#* ^?=뉣  N##(W Xgb*@RltCG}bQmi1zu]v`wl%ݻOw 5ʪZVXA(Dxi|H@#,Pgj2?? d^ `m8P@Q~ iJU0J[%cNA@-]k>NTrgLp#*x5]rHE< c*zu` i#*Hmi#*<A#mo3\Fl*0H⊭IAP#*R_υ֯8wUZLTE!:> YA iydeW#*(6%(e)$l&ʖ۴&II\ەng]َt.Wwn\[Ǔlr[W.,hv*nwH,w'6>YBOtڅx"1뚵oƫ\Z ?(,br 26ɫBj*jAf8#, $ԕ=jqwR͢kԵª64|H`NߑP l&PtIb2LlEkuͫխz*J4bмD (iHd[0Bc Apꟷz&AJ}#()Yל({+Hc#* +>~VP7t/W ,#(","E("A!v>Fbsk[_ [ʤF "@:|&#*UB?E}2jKL#,"I1B#(?L͖ve.}PRF)ԟAm#*J=^w#*ImDWd` d*~r^~Rդ#( ]#*7&Nq#* L#*4|lDHI'oSC9%*Ǩ\-%I?.Ѡ)I 0@%KEҢq$HPҨ #,Z)F\a,!/+5tU%*7ZԄ"IFkn`- hm`zmʒܽ춵շXvfQVT $ 'XĞkM32r$.#,j #(K=3,~B#*kҡ.X#lYL*0ZlAi@BEDn*T(@ h?FIAےg'5]&h\ ƏB?7@WhSI { V7h Tr"BH%XN~->`Ah$c nt*uEAeR"H4JPT#,~xe u$b@JXP#PZB),bF"0Q%?6z!{&hRh .k*3ƌ280@M"md $iudxm$"t#,|٥" w#QU#,l-26 ]0PhDhv؃#,@~Fʀ4ӟ7"@I롦 d#(j"A#,h@vMt4A8>pmwbݿI$уHX*f?N&ϔsSSSP~<cn17NWJJ6OV2kqj($MBBriц8ZCD"2Ah2Z[&7RĪFV1&A<慊ri:<qL7x$L0:.[UPgpp3 &4hȉi }9wx o8olQێq[\(v쒂B\>H++cudG>sj[cӖIavG#,scH_%1L},jۄ {r=RDпBcN]#(H45ԑ8D0BU5yMWI嘞h1ǢM&Tw>" Ϻ_\R30pEɴg8F[fkE(=gO4֓́[_\H 0}]);r-Vg׍h@`FvTM^)@æjmI7} h:oZ$ٌ#*j -l2KxmL'7XVD1#*TRBYi›Fs6Q~W Wi-;)JULKO[\#*2.ԁx U@2#5F7:DufũocX"QIEsk%W[-˻W+b]5mm!lZmrk-͝r4mjTW* UFv=YKht,}EwdiN73Bm;ȶ3nD#*Q#(w7}Oar"KwoU|#*h9<*//BZQ9@#(6/5$S2wd#( Gq1(IBHJ!UP¾Q-.P<lF@#(E#,QH$P-KXPd%*.^p@(k$h%[}#*P*T5>s!4hD#*fZ: s)x*#(v̾?H1"wzMY7C|@CjTPp$(#,( )T$-,&ŚRQj6R$F6֊mZjԴ3ljeJ-ڍMo?=F1Z̈A74'ۅHzz:%@`\je+(9vlKi\yF^3PJ_E{f w^:v]s߳9o㙭!ݖp94#,d{.ױ lfBVZ:PNj8r56SatFuz@`>"Egh7C*mٞU/@:X|pg=uv qD-K(CtSI$&lȸ8ԇ$V~ n)ܕv Gsj{.e jԵc/$ U۹](/%ÚԦb-Ģpq// b4rId/׆Zq\gۇ*K7m860΂+"}e*9~zt7tML2WS՛ڇ22xq;ڋ8h@1. njrmL1:-sz1%4 Ehq+.]v}B~<<[F"/K0ͪ}!ۺ#*O9wao)D(ϻ#(1.tX]Pr:J&f!1/6r!FϽrNKxq iǫpfhxD鷋 fRq @iXhpf,cy;>gCmƅ&0Y Ղ-6ŹkD[ut F ֧PKO0!425oa#,#X$Yr ڪ)o(^lME "HQȂ!ؒ[8ފB8!#,k,Z]r1R6BxZA9O@_gy7yOK|)Wž?C{++]9a2*O !51Tq^H7#Bx DUĝ N[956p~/ܧ}4F̪f|f=6cxw#M'wra9*bƪ7Kӹya9*UfE e5FJ]αuڗ~h $&2&haѹS(BYaXUSq}}Bs!5"Oկ q@*fտjMlF&QZ-)51]ڹe(A*.;o*#,$T}fn,z|W R;#,-CLacD>OUpdSRG)1@كA.%f-,⨓ I@/>)-C4o&BNW3F2췴 QbS~#*). a#(!cH+#,8 ddV$F>{϶$XBr`SfZDty/ZwF3[JQ#*L׀.kBڷR'GX{h]=zCԆ0:LI!U,ї lPUl!i0X ar-"I$6*|T40+m-= ZSݎ>#=03䨞Cfu^_bn HX@έӸ{#* P5DS*Li[Zg#(}NJd^Z[ci^^sI!!G|Aw$g|#Iiaqb f2o2D}~QG%7jJst&12i&AQi|#(.^(ZtZ}~`'Wzy.6ԝ k|}Q\hE2d '#*9;.fM2Ц TBDwqqbC/Tg2Qc/4?wFǖ#A "е~S:&3Bb$#(02Ox2EABDDAU0iB(&%?]nPԇtwx$}YcN#*ޠ+3d*hHaeMx~hg{ G2#(sϣx~r?O"?dZJL$E]%ǞvR$tSFrX7Ӛz9blw=!wMvU'f!K#(")Ș8( OnİdB**-0'*.ܚѾl.#qʄrNbG#,Th`ByPs|憓0e3\v2ÂK[vRc|~I`&jĥe@xBC',D* bt&H2)R%a+& "ƃBן[4e+c=RM<"TJ*"-\hD`Pv^ cw̫4aEG9XwWdE|kc>xD^PMLa\%XN|d3#*m<#, ITaf1&`7mX)z:ub3-)+|ʣFegl62'4&#F7^M) WMmNM.0S9m8d;Kܺ{՗[>!milc#4Kiƥ{#,-;EmnoQddh6vw`>%L3/HI|2iQ+^*fbC&݉]}qʼ\ɸD)ZBŠF՗$O6ڨ&Xnu I R B|֢~E0Vlh<ؘP=o]qr*rk@Xj7f$m6Cv<tҥg`us"[7N&2ᾄA%qK{eGkph6vF4x$43@ 6`+aBa6@t%.!ttWIƳxC jtk i΄ƾP#*6\q2柁]l;4Ԍ٩5˦IĶ qz`-Qnc<3g3ks5hkw.|ABL6kG/ NO"#,ljw+>]69"噑s#Zy|b5lhl2:2YÔT&(>ml>fv@:aXz8th:j&#,1iu]p3CE\%15bS$6p#*!7nf{nUAf'"JNkqŇI挧z&`iQ:bSc-9hn9Zbmi-koLW0Klv#vdr; MNNpl)9M{=/GX Fl\;3#*&Q01(oY![l&E 0,a74U@VB +l*K⍡4;q$3mt8i d .|3J",c+HgE"b >t9EAQQ@c^#*owIacEP`j( 6 (TC?Gz6#,uuy] v2SWZDgș rZA˧2o--b9t+(#,oR qUrH2#_YoJe\TD_ ̢{i8T2lVAE#4;xc(L4x4.(ʑ8r2ŝhq_(چ4GV}$N%blqb\'֕Ԉpa@ZB !\R3r٥^Tpdi]ng1YVtՌ@@_Z~'N.&Ur}uߓ3KDA#*#%RޖDA(FC9JXx,X[#,[|h=}* ۉ٤TFi>%C3pnb1P !="x䐁ӕ8f>}9@?G4nlXMu=qXF+=fx`3eF e\{@B@EUmQ%j%je}SY+Q&8""`#(@KŅj,6HtငH "lAzT'/NTR _U *j7b , Ȳ+"HB#({xt5U`ߎfуV@FDTP6v߿*x򣎾W8۷7 E8#(uq:_A%{bjBQI.`ȰP13HUY\Q$QHq4ܛ\.wVܥ-fj)BZ4#,,@u59pq6#*Dk y$Q(#*0s4 =<^ia 1V=>u36TƂCz(C V3;:['hwT b@1@#*Xp"/~i>!&Ւl?5kX؟+u,I춺o}ݡ0?jR=UZaš#,D(=2Lw6*{\Bvbc5WD#,4$@+بEJ՘8`,%|s.PtWO\$@aDD,0JJPμ8pc&Icԟavޭ"^/$khm&,6J#,ֱ0&Qړoh\g5LX!x8Uё;#aNj #*#*ޞbeۂlHϤy^-;#,Oq"fA:#((x4A$×uM.vsΫosI@g &SmiZ'mT:8)8tL 57M'w G˹,:"lpy1pC)J,pw0lde5|q!:#,M )kRBU\я¶5FNNtʌjv`h#,@[D*PކZBCclwzb#(@1-F2J5V0>w-ְ/X VkdYSt,0DR/ pWz )@w| ?dH8;sb]2*'0E#(MLy,oPW3I% Mb)O72J!C) e.Yrʳ "F_^({@'w+v.4Ѵ8n5@`(\{,ٯ5c߁EC#,|H 9ȠP.tٲU 6BɌ]F4i%]}kzyEe(y7s\#,6KEW清+Ikyo㆕Q`!)"n2ݷ;%U}|}WzEyHuAX(FA 9#*<E@U)F*5?a 7`|v(΁G<2E=(y=Ux" PRBA>CWB f硆#0HqS#(!{Sbѐ"`OrBAyk`"Th#,jM(3o)Y0jf`@|P5PRyRdX{-fY{u2Gkak Ni`MǡBٜKJFs8Alh{,3>m p`j=261~D=734CMBc?=Wݶeq%)&ȨI–o5xViUPN㟟e$#l*Q# B.nu#J#&z(qFR|g.wUHPD," cZ RQ*͖$$`Ah6__dyiJR@~}P џ2σP5$iIw3~cS-KelQae$lAn,p'%yt:%&TWm;wexכT,ޕrT[fnʮj*jl-^]Di6T2%64+ *+ TXH`[ܷ7/Mˈ5&YWkyמu񨬣)K^wrݮeMt]|^&a *Ѝr^5UAhXnBQEF*dT2@%Cj?LfY\`ƚa)ܛxU~Ӧ=.f\{Qv9`#(|Qq;z>zXt( !30`R66;~07"^GM#*UJ]G/lr$aB1 ̦ցD۷A(^&7KcCDkM^nqaI/#4F}I+.I'܆#,*--Q4Ho3UbٛE h)Q#*U>3nN¡; 0¿~H(&aMf #l%o`p(v8pܙ ۥdH( Gn⺕#$oyvvu'ח9?,erqG'˥+W72҂뿷5W)%{㿵w8q2֘:V#<3\]ojTԑ#*l{Y6548H$ a%LP4D UW֚D;u⑝گ}nC TK({yMꢎիnÇVJTmƶ9TI^2{ %ܑC 6<bӛ#gw95#uqV ud4| c(W9v`Ip^/~&zqC`P:!$q3>)Ȧ]#*ٻ3:FS]bjb #*'Nw-&boh u6;XpLMIz|pp$hO#,V]߹ۆsUWWuƷ1,gS5TczR@_{l3#(9BFA6nwˆ14#(Y"%aCEVD"C.ᔇ'C5ʊ g <=w^2wC0UN.; "aCe#,ئ#*#*k`,DVґ1^7Q(j#*M&khM3I,(.ڻ[mus)D[KQVQxM]WjeZ_BpA~o;}Sas#_39ܪ>"U֓88n_Z#,gU6Ϩf{~ #9cw`ZN*)*0Lceҧ~!x?fK65@MY٨#*%8a[t\/!eڍvzwO4 RjqeD֐+}msRm&ۛXm(oSEM`KLLΕ o$#('A,9ILB:\n#(Y#]ܹ yubRQ(<ć#=7툺#*E sewqPѨ"ŀj;AJPE fkvvZRb$REANoCX"DD|}, 6b ^!WV!dOPB;?;5=PDdE%SI0G#(H*C(Owmo&-^g/nG ^#;p#@{aw7#6<8_#*ٔ!?RsaXD= BYM?U%JH?#eJ?p:Wo)|\.2`þGm:v: ~rbKks i s3Xʿ{%^kvvKH&6#,T K;j1$(A