diff --git a/common/normApi.cpp b/common/normApi.cpp new file mode 100644 index 0000000..b919200 --- /dev/null +++ b/common/normApi.cpp @@ -0,0 +1,1131 @@ + +#include "normApi.h" +#include "normSession.h" + +// const defs +const NormInstanceHandle NORM_INSTANCE_INVALID = ((NormInstanceHandle)0); +const NormSessionHandle NORM_SESSION_INVALID = ((NormSessionHandle)0); +const NormNodeHandle NORM_NODE_INVALID = ((NormNodeHandle)0); +const NormNodeId NORM_NODE_NONE = ((NormNodeId)0x00000000); +const NormNodeId NORM_NODE_ANY = ((NormNodeId)0xffffffff); +const NormObjectHandle NORM_OBJECT_INVALID = ((NormObjectHandle)0); + +class NormInstance : public NormController +{ + public: + NormInstance(); + virtual ~NormInstance(); + + void Notify(NormController::Event event, + class NormSessionMgr* sessionMgr, + class NormSession* session, + class NormServerNode* server, + class NormObject* object); + + bool Startup(); + void Shutdown(); + + bool WaitForEvent(); + bool GetNextEvent(NormEvent* theEvent); + bool SetCacheDirectory(const char* cachePath); + + bool NotifyQueueIsEmpty() const + {return notify_queue.IsEmpty();} + + void PurgeObjectNotifications(NormObjectHandle objectHandle); + + ProtoDispatcher::Descriptor GetDescriptor() const + { +#ifdef WIN32 + return notify_event; +#else + return notify_fd[0]; +#endif // if/else WIN32/UNIX + } + + static NormInstance* GetInstanceFromSession(NormSessionHandle sessionHandle) + { + NormSession* session = (NormSession*)sessionHandle; + return static_cast(session->GetSessionMgr().GetController()); + } + static NormInstance* GetInstanceFromObject(NormObjectHandle objectHandle) + { + NormSession& session = ((NormObject*)objectHandle)->GetSession();; + return static_cast(session.GetSessionMgr().GetController()); + } + + class Notification + { + public: + NormEvent event; + + void Append(Notification* n) {next = n;} + + Notification* GetNext() const {return next;} + + class Queue + { + public: + Queue(); + ~Queue(); + bool IsEmpty() const {return (NULL == head);} + void Destroy(); + void Append(Notification* n) + { + n->Append(NULL); + if (tail) + tail->Append(n); + else + head = n; + tail = n; + } + Notification* RemoveHead() + { + Notification* n = head; + if (n) + { + head = n->GetNext(); + tail = head ? tail : NULL; + } + return n; + } + Notification* GetHead() {return head;} + void SetHead(Notification* n) {head = n;} + private: + Notification* head; + Notification* tail; + + }; // end class NormInstance::Notification::Queue + + private: + Notification* next; + }; // end class NormInstance::Notification + + ProtoDispatcher dispatcher; + NormSessionMgr session_mgr; + + private: + Notification::Queue notify_pool; + Notification::Queue notify_queue; + Notification* previous_notification; + + const char* rx_cache_path; + +#ifdef WIN32 + HANDLE notify_event; +#else + int notify_fd[2]; +#endif // if/else WIN32/UNIX +}; // end class NormInstance + +//////////////////////////////////////////////////// +// NormInstance::Notification::Queue implementation +NormInstance::Notification::Queue::Queue() + : head(NULL), tail(NULL) +{ +} + +NormInstance::Notification::Queue::~Queue() +{ + Destroy(); +} + +void NormInstance::Notification::Queue::Destroy() +{ + Notification* n; + while ((n = RemoveHead())) delete n; +} // end NormInstance::Notification::Queue::Destroy() + +//////////////////////////////////////////////////// +// NormInstance implementation +NormInstance::NormInstance() + : session_mgr(static_cast(dispatcher), + static_cast(dispatcher)), + previous_notification(NULL), rx_cache_path(NULL) +{ +#ifdef WIN32 + notify_event = NULL; +#else + notify_fd[0] = notify_fd[1] = -1; +#endif // if/else WIN32/UNIX + session_mgr.SetController(this); +} + +NormInstance::~NormInstance() +{ + Shutdown(); +} + +bool NormInstance::SetCacheDirectory(const char* cachePath) +{ + // (TBD) verify that we can _write_ to this directory! + bool result = false; + if (dispatcher.SuspendThread()) + { + int length = strlen(cachePath); + if (PROTO_PATH_DELIMITER != cachePath[length-1]) + length += 2; + else + length += 1; + length = (length < PATH_MAX) ? length : PATH_MAX; + char* pathStorage = new char[length]; + if (pathStorage) + { + strncpy(pathStorage, cachePath, length); + pathStorage[length - 2] = PROTO_PATH_DELIMITER; + pathStorage[length - 1] = '\0'; + if (rx_cache_path) delete[] (char*)rx_cache_path; + rx_cache_path = pathStorage; + result = true; + } + else + { + DMSG(0, "NormInstance::SetCacheDirectory() new pathStorage error: %s\n", + GetErrorString()); + } + dispatcher.ResumeThread(); + } + return result; +} // end NormInstance::SetCacheDirectory() + +void NormInstance::Notify(NormController::Event event, + class NormSessionMgr* sessionMgr, + class NormSession* session, + class NormServerNode* server, + class NormObject* object) +{ + Notification* n = notify_pool.RemoveHead(); + if (!n) + { + if (!(n = new Notification)) + { + DMSG(0, "NormInstance::Notify() new Notification error: %s\n", + GetErrorString()); + return; + } + } + + switch (event) + { + case RX_OBJECT_NEW: + { + // recv object "Accept()" policy implemented here + switch (object->GetType()) + { + case NormObject::STREAM: + { + NormStreamObject* stream = static_cast(object); + // (TBD) implement silent_client accept differently + NormObjectSize size = stream->GetSize(); + if (!stream->Accept(size.LSB())) + { + DMSG(0, "NormInstance::Notify() stream accept error\n"); + notify_pool.Append(n); + return; + } + break; + } + case NormObject::FILE: + { + if (NULL != rx_cache_path) + { + char fileName[PATH_MAX]; + strncpy(fileName, rx_cache_path, PATH_MAX); + strcat(fileName, "normTempXXXXXX"); +#ifdef WIN32 + if (!_mktemp(fileName)) +#else + int fd = mkstemp(fileName); + if (fd >= 0) + { + close(fd); + } + else +#endif // if/else WIN32 + { + DMSG(0, "NormInstance::Notify(RX_OBJECT_NEW) Warning: mkstemp() error: %s\n", + strerror(errno)); + } + if (!((NormFileObject*)object)->Accept(fileName)) + { + DMSG(0, "NormInstance::Notify(RX_OBJECT_NEW) file object accept error!\n"); + } + } + else + { + // we're ignoring files + return; + } + break; + } + default: + // (TBD) support other object types + return; + } // end switch(object->GetType()) + break; + } // end case RX_OBJECT_NEW + + case RX_OBJECT_COMPLETED: + case TX_OBJECT_PURGED: + case RX_OBJECT_ABORTED: + object->Retain(); + break; + + default: + break; + } // end switch(event) + + bool doNotify = notify_queue.IsEmpty(); + n->event.type = (NormEventType)event; + n->event.session = session; + n->event.sender = server; + n->event.object = object; + notify_queue.Append(n); + + if (doNotify) + { +#ifdef WIN32 + if (0 == SetEvent(notify_event)) + { + DMSG(0, "NormInstance::Notify() SetEvent() error: %s\n", + GetErrorString()); + } +#else + char byte; + while (1 != write(notify_fd[1], &byte, 1)) + { + if ((EINTR != errno) && (EAGAIN != errno)) + { + DMSG(0, "NormInstance::Notify() write() error: %s\n", + GetErrorString()); + break; + } + } +#endif // if/else WIN32/UNIX + } +} // end NormInstance::Notify() + +void NormInstance::PurgeObjectNotifications(NormObjectHandle objectHandle) +{ + Notification* next = notify_queue.GetHead(); + while (next) + { + if (objectHandle == next->event.object) + next->event.type = NORM_EVENT_INVALID; + next = next->GetNext(); + } +} // end NormInstance::PurgeObjectNotifications() + +// NormInstance::dispatcher MUST be suspended _before_ calling this +bool NormInstance::GetNextEvent(NormEvent* theEvent) +{ + // First, do any garbage collection for "previous_notification" + if (NULL != previous_notification) + { + // (TBD) "Release" purged/completed/aborted objects + switch(previous_notification->event.type) + { + case NORM_RX_OBJECT_COMPLETED: + case NORM_TX_OBJECT_PURGED: + case NORM_RX_OBJECT_ABORTED: + { + NormObject* obj = (NormObject*)(previous_notification->event.object); + obj->Release(); + break; + } + default: + break; + } + notify_pool.Append(previous_notification); + previous_notification = NULL; + } + Notification* n; + while ((n = notify_queue.RemoveHead())) + { + if (notify_queue.IsEmpty()) + { +#ifdef WIN32 + if (0 == ResetEvent(notify_event)) + { + DMSG(0, "NormInstance::GetNextEvent() ResetEvent error: %s\n", + GetErrorString()); + } +#else + char byte; + while (read(notify_fd[0], &byte, 1) > 0); // TBD - error check +#endif // if/else WIN32/UNIX + } + switch (n->event.type) + { + case NORM_EVENT_INVALID: + continue; + case NORM_RX_OBJECT_UPDATE: + // reset update event notification + ((NormObject*)n->event.object)->SetNotifyOnUpdate(true); + break; + default: + break; + } + if (theEvent) *theEvent = n->event; + previous_notification = n; // keep dispatched event for garbage collection + return true; + } + return false; +} // end NormInstance::GetNextEvent() + +bool NormInstance::WaitForEvent() +{ +#ifdef WIN32 + WaitForSingleObject(notify_event, INFINITE); +#else + fd_set fdSet; + FD_ZERO(&fdSet); + FD_SET(notify_fd[0], &fdSet); + while (1) + { + if (0 > select(notify_fd[0] + 1, &fdSet, (fd_set*)NULL, + (fd_set*)NULL, (struct timeval*)NULL)) + { + if (EINTR != errno) + { + DMSG(0, "NormInstance::WaitForEvent() select() error: %s\n", + GetErrorString()); + return false; + } + } + else + { + break; + } + } +#endif + return true; +} // end NormInstance::WaitForEvent() + + +bool NormInstance::Startup() +{ + // 1) Create descriptor to use for event notification +#ifdef WIN32 + // Create initially non-signalled, manual reset event + notify_event = CreateEvent(NULL, TRUE, FALSE, NULL); + if (NULL == notify_event) + { + DMSG(0, "NormInstance::Startup() CreateEvent() error: %s\n", GetErrorString()); + return false; + } +#else + if (0 != pipe(notify_fd)) + { + DMSG(0, "NormInstance::Startup() pipe() error: %s\n", GetErrorString()); + return false; + } + // make reading non-blocking + if(-1 == fcntl(notify_fd[0], F_SETFL, fcntl(notify_fd[0], F_GETFL, 0) | O_NONBLOCK)) + { + DMSG(0, "NormInstance::Startup() fcntl(F_SETFL(O_NONBLOCK)) error: %s\n", GetErrorString()); + close(notify_fd[0]); + close(notify_fd[1]); + notify_fd[0] = notify_fd[1] = -1; + return false; + } +#endif // if/else WIN32/UNIX + // 2) Start thread + return dispatcher.StartThread(); +} // end NormInstance::Startup() + +void NormInstance::Shutdown() +{ + dispatcher.Stop(); +#ifdef WIN32 + if (NULL != notify_event) + { + CloseHandle(notify_event); + notify_event = NULL; + } +#else + if (notify_fd[0] > 0) + { + close(notify_fd[0]); // close read end of pipe + close(notify_fd[1]); // close write end of pipe + notify_fd[0] = notify_fd[1] = -1; + } +#endif // if/else WIN32/UNIX + if (rx_cache_path) + { + delete[] (char*)rx_cache_path; + rx_cache_path = NULL; + } + + // Garbage collect our "previous_notification" + if (NULL != previous_notification) + { + switch(previous_notification->event.type) + { + case NORM_RX_OBJECT_COMPLETED: + case NORM_TX_OBJECT_PURGED: + case NORM_RX_OBJECT_ABORTED: + { + NormObject* obj = (NormObject*)(previous_notification->event.object); + obj->Release(); + break; + } + default: + break; + } + notify_pool.Append(previous_notification); + previous_notification = NULL; + } + + Notification* n; + while ((n = notify_queue.RemoveHead())) + { + switch (n->event.type) + { + case NORM_RX_OBJECT_NEW: + { + NormObject* obj = (NormObject*)n->event.object; + switch (obj->GetType()) + { + case NormObject::DATA: + delete (char*)static_cast(obj)->GetData(); + break; + case NormObject::FILE: + // (TBD) unlink temp file? + break; + default: + break; + } + break; + } + // Garbage collect retained objects + case NORM_RX_OBJECT_COMPLETED: + case NORM_TX_OBJECT_PURGED: + case NORM_RX_OBJECT_ABORTED: + ((NormObject*)n->event.object)->Release(); + break; + default: + break; + } + delete n; + } + + notify_pool.Destroy(); +} // end NormInstance::Shutdown() + + +////////////////////////////////////////////////////////////////////////// +// NORM API FUNCTION IMPLEMENTATIONS +// + +NormInstanceHandle NormCreateInstance() +{ + NormInstance* normInstance = new NormInstance; + if (normInstance) + { + if (normInstance->Startup()) + return ((NormInstanceHandle)normInstance); + else + delete normInstance; + } + return NORM_INSTANCE_INVALID; +} // end NormCreateInstance() + +void NormDestroyInstance(NormInstanceHandle instanceHandle) +{ + delete ((NormInstance*)instanceHandle); +} // end NormDestroyInstance() + +NormSessionHandle NormCreateSession(NormInstanceHandle instanceHandle, + const char* sessionAddr, + unsigned short sessionPort, + NormNodeId localNodeId) +{ + // (TBD) wrap this with SuspendThread/ResumeThread ??? + NormInstance* instance = (NormInstance*)instanceHandle; + if (instance) + { + NormSession* session = + instance->session_mgr.NewSession(sessionAddr, sessionPort, localNodeId); + if (session) return ((NormSessionHandle)session); + } + return NORM_SESSION_INVALID; +} // end NormCreateSession() + +void NormDestroySession(NormSessionHandle sessionHandle) +{ + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance) + { + if (instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + { + session->Close(); + session->GetSessionMgr().DeleteSession(session); + } + instance->dispatcher.ResumeThread(); + } + } +} // end NormDestroySession() + +void NormSetTransmitRate(NormSessionHandle sessionHandle, + double bitsPerSecond) +{ + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) session->SetTxRate(bitsPerSecond); + instance->dispatcher.ResumeThread(); + } +} // end NormSetTransmitRate() + +void NormSetGrttEstimate(NormSessionHandle sessionHandle, + double grttEstimate) +{ + // (TBD) suspend instance if timer reschedule is + // added to ServerSetGrtt() method. + NormSession* session = (NormSession*)sessionHandle; + if (session) session->ServerSetGrtt(grttEstimate); +} // end NormSetGrttEstimate() + +NormNodeId NormGetLocalNodeId(NormSessionHandle sessionHandle) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) + return session->LocalNodeId(); + else + return NORM_NODE_NONE; +} // end NormGetLocalNodeId() + +bool NormSetLoopback(NormSessionHandle sessionHandle, bool state) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) + return session->SetLoopback(state); + else + return false; +} // end NormSetLoopback() + +bool NormStartSender(NormSessionHandle sessionHandle, + unsigned long bufferSpace, + unsigned short segmentSize, + unsigned char numData, + unsigned char numParity) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + result = session->StartServer(bufferSpace, segmentSize, numData, numParity); + else + result = false; + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormStartSender() + +void NormStopSender(NormSessionHandle sessionHandle) +{ + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) session->StopServer(); + instance->dispatcher.ResumeThread(); + } +} // end NormStopSender() + +bool NormStartReceiver(NormSessionHandle sessionHandle, + unsigned long bufferSpace) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + result = session->StartClient(bufferSpace); + else + result = false; + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormStartReceiver() + +void NormStopReceiver(NormSessionHandle sessionHandle) +{ + + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) session->StopClient(); + instance->dispatcher.ResumeThread(); + } +} // end NormStopReceiver() + +void NormSetMessageTrace(NormSessionHandle sessionHandle, bool state) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) session->SetTrace(state); +} // end NormSetMessageTrace() + +void NormSetTxLoss(NormSessionHandle sessionHandle, double percent) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) session->SetTxLoss(percent); +} // end NormSetTxLoss() + +void NormSetRxLoss(NormSessionHandle sessionHandle, double percent) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) session->SetRxLoss(percent); +} // end NormSetRxLoss() + + + +NormNodeId NormGetNodeId(NormNodeHandle nodeHandle) +{ + NormNode* node = (NormNode*)nodeHandle; + if (node) + return node->GetId(); + else + return NORM_NODE_NONE; +} // end NormGetNodeId() + +NormRepairBoundary NormGetNodeRepairBoundary(NormNodeHandle nodeHandle) +{ + NormServerNode* node = static_cast((NormNode*)nodeHandle); + if (node) + return ((NormRepairBoundary)(node->GetRepairBoundary())); + else + return NORM_BOUNDARY_BLOCK; // return default value +} // end NormGetNodeRepairBoundary() + +void NormSetNodeRepairBoundary(NormNodeHandle nodeHandle, + NormRepairBoundary repairBoundary) +{ + NormServerNode* node = static_cast((NormNode*)nodeHandle); + if (node) + node->SetRepairBoundary((NormServerNode::RepairBoundary)repairBoundary); +} // end NormSetNodeRepairBoundary() + +void NormSetDefaultRepairBoundary(NormSessionHandle sessionHandle, + NormRepairBoundary repairBoundary) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) + session->ClientSetDefaultRepairBoundary((NormServerNode::RepairBoundary)repairBoundary); +} // end NormSetDefaultRepairBoundary() + + +NormObjectType NormGetObjectType(NormObjectHandle objectHandle) +{ + if (NORM_OBJECT_INVALID != objectHandle) + return ((NormObjectType)(((NormObject*)objectHandle)->GetType())); + else + return NORM_OBJECT_NONE; +} // end NormGetObjectType() + +NormObjectTransportId NormGetObjectTransportId(NormObjectHandle objectHandle) +{ + // Like many other API calls, these should be changed + // to provide an error code result + if (NORM_OBJECT_INVALID != objectHandle) + return ((NormObjectTransportId)(((NormObject*)objectHandle)->GetId())); + else + return 0; +} // end NormGetFileName() + +bool NormGetObjectInfo(NormObjectHandle objectHandle, + char* infoBuffer, + unsigned short* infoLen) +{ + if ((NORM_OBJECT_INVALID != objectHandle) && infoLen) + { + bool result = true; + NormObject* object = (NormObject*)objectHandle; + if (object->HaveInfo()) + { + unsigned short bufferLen = *infoLen; + *infoLen = object->GetInfoLength(); + if (bufferLen >= *infoLen) + bufferLen = *infoLen; + else + result = false; // incomplete copy + if (infoBuffer) + memcpy(infoBuffer, object->GetInfo(), bufferLen); + } + else + { + *infoLen = 0; + } + return result; + } + return false; +} // end NormGetObjectInfo() + +void NormCancelObject(NormObjectHandle objectHandle) +{ + if (NORM_OBJECT_INVALID != objectHandle) + { + NormInstance* instance = NormInstance::GetInstanceFromObject(objectHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormObject* obj = (NormObject*)objectHandle; + NormServerNode* server = obj->GetServer(); + if (server) + server->DeleteObject(obj, 8); + else + obj->GetSession().DeleteTxObject(obj); + instance->PurgeObjectNotifications(objectHandle); + instance->dispatcher.ResumeThread(); + } + } +} // end NormCancelObject() + +void NormRetainObject(NormObjectHandle objectHandle) +{ + if (NORM_OBJECT_INVALID != objectHandle) + { + NormInstance* instance = NormInstance::GetInstanceFromObject(objectHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormObject* obj = (NormObject*)objectHandle; + obj->Retain(); + instance->dispatcher.ResumeThread(); + } + } +} // end NormRetainObject() + +void NormReleaseObject(NormObjectHandle objectHandle) +{ + if (NORM_OBJECT_INVALID != objectHandle) + { + NormInstance* instance = NormInstance::GetInstanceFromObject(objectHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormObject* obj = (NormObject*)objectHandle; + obj->Release(); + //instance->PurgeObjectNotifications(objectHandle); + instance->dispatcher.ResumeThread(); + } + } +} // end NormReleaseObject() + +NormNackingMode NormGetObjectNackingMode(NormObjectHandle objectHandle) +{ + NormObject* object = (NormObject*)objectHandle; + if (object) + return ((NormNackingMode)object->GetNackingMode()); + else + return NORM_NACK_NONE; +} // end NormGetObjectNackingMode() + +void NormSetObjectNackingMode(NormObjectHandle objectHandle, + NormNackingMode nackingMode) +{ + NormObject* object = (NormObject*)objectHandle; + if (object) object->SetNackingMode((NormObject::NackingMode)nackingMode); +} // end NormSetObjectNackingMode() + +void NormSetNodeNackingMode(NormNodeHandle nodeHandle, + NormNackingMode nackingMode) +{ + NormServerNode* node = (NormServerNode*)nodeHandle; + if (node) node->SetDefaultNackingMode((NormObject::NackingMode)nackingMode); +} // end NormSetNodeNackingMode() + +void NormSetDefaultNackingMode(NormSessionHandle sessionHandle, + NormNackingMode nackingMode) +{ + NormSession* session = (NormSession*)sessionHandle; + if (session) session->ClientSetDefaultNackingMode((NormObject::NackingMode)nackingMode); +} // end NormSetDefaultNackingMode() + +bool NormSetWatermark(NormSessionHandle sessionHandle, + NormObjectHandle objectHandle) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + NormObject* obj = (NormObject*)objectHandle; + if (session && obj) + { + // (TBD) set stream watermark differently + // (segmentId doesn't matter for non-stream) + if (obj->IsStream()) + { + NormStreamObject* stream = static_cast(obj); + session->ServerSetWatermark(stream->GetId(), + stream->FlushBlockId(), + stream->FlushSegmentId()); + } + else + { + NormBlockId blockId = obj->GetFinalBlockId(); + NormSegmentId segmentId = obj->GetBlockSize(blockId) - 1; + session->ServerSetWatermark(obj->GetId(), + blockId, + segmentId); + } + result = true; + } + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormSetWatermark() + +bool NormAddAckingNode(NormSessionHandle sessionHandle, + NormNodeId nodeId) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + result = session->ServerAddAckingNode(nodeId); + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormAddAckingNode() + +void NormRemoveAckingNode(NormSessionHandle sessionHandle, + NormNodeId nodeId) +{ + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) session->ServerRemoveAckingNode(nodeId); + instance->dispatcher.ResumeThread(); + } +} // end NormAddAckingNode() + +NormObjectHandle NormOpenStream(NormSessionHandle sessionHandle, + unsigned long bufferSize) +{ + NormObjectHandle objectHandle = NORM_OBJECT_INVALID; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + { + NormObject* obj = + static_cast(session->QueueTxStream(bufferSize)); + if (obj) objectHandle = (NormObjectHandle)obj; + } + instance->dispatcher.ResumeThread(); + } + return objectHandle; +} // end NormOpenStream() + +void NormCloseStream(NormObjectHandle streamHandle) +{ + NormCancelObject(streamHandle); +} // end NormCloseStream() + + +// (TBD) Some stream i/o performance improvement can be realized +// if the option to make "NormWriteStream()" and +// "NormReadStream()" _blocking_ calls is implemented. +// Right now, the stream read/write are non-blocking +// so applications should use "NormGetNextEvent()" to +// know when to read or write ... This results in the +// underlying NORM thread to be suspended/resumed _twice_ +// per read or write ... It would be more efficient to +// "suspend" the NORM thread while the application +// processes all pending events and _then_ "resume" the +// NORM thread ... an approach to this would be enable +// the app to install an event handler callback and dispatch +// events with a "NormDispatchEvents()" call ... +// + +void NormSetStreamFlushMode(NormObjectHandle streamHandle, + NormFlushMode flushMode) +{ + NormStreamObject* stream = + static_cast((NormObject*)streamHandle); + if (stream) + stream->SetFlushMode((NormStreamObject::FlushMode)flushMode); +} // end NormSetStreamFlushMode() + +void NormSetStreamPushMode(NormObjectHandle streamHandle, bool state) +{ + NormStreamObject* stream = + static_cast((NormObject*)streamHandle); + if (stream) stream->SetPushMode(state); +} // end NormSetStreamPushMode() + +void NormFlushStream(NormObjectHandle streamHandle) +{ + NormInstance* instance = NormInstance::GetInstanceFromObject(streamHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormStreamObject* stream = + static_cast((NormObject*)streamHandle); + if (stream) stream->Flush(); + instance->dispatcher.ResumeThread(); + } +} // end NormFlushStream() + +void NormMarkStreamEom(NormObjectHandle streamHandle) +{ + NormInstance* instance = NormInstance::GetInstanceFromObject(streamHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormStreamObject* stream = + static_cast((NormObject*)streamHandle); + if (stream) stream->Write(NULL, 0, true); + instance->dispatcher.ResumeThread(); + } +} // end NormMarkStreamEom() + +unsigned int NormWriteStream(NormObjectHandle streamHandle, + const char* buffer, + unsigned int numBytes) +{ + unsigned int result = 0; + NormInstance* instance = NormInstance::GetInstanceFromObject(streamHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormStreamObject* stream = + static_cast((NormObject*)streamHandle); + if (stream) + result = stream->Write(buffer, numBytes, false); + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormWriteStream() + +bool NormReadStream(NormObjectHandle streamHandle, + char* buffer, + unsigned int* numBytes) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromObject(streamHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormStreamObject* stream = + static_cast((NormObject*)streamHandle); + if (stream) + { + result = stream->Read(buffer, numBytes); + } + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormReadStream() + +NormObjectHandle NormQueueFile(NormSessionHandle sessionHandle, + const char* fileName, + const char* infoPtr, + unsigned int infoLen) +{ + NormObjectHandle objectHandle = NORM_OBJECT_INVALID; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + { + NormObject* obj = session->QueueTxFile(fileName, infoPtr, infoLen); + if (obj) objectHandle = (NormObjectHandle)obj; + } + instance->dispatcher.ResumeThread(); + } + return objectHandle; +} // end NormQueueFile() + +bool NormSetCacheDirectory(NormInstanceHandle instanceHandle, + const char* cachePath) +{ + NormInstance* instance = (NormInstance*)instanceHandle; + if (instance) + return instance->SetCacheDirectory(cachePath); + else + return false; +} // end NormSetCacheDirectory() + +bool NormSetFileName(NormObjectHandle fileHandle, + const char* fileName) +{ + bool result = false; + NormInstance* instance = NormInstance::GetInstanceFromObject(fileHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + // (TBD) verify "fileHandle" is a NORM_FILE ?>?? + NormFileObject* file = + static_cast((NormObject*)fileHandle); + result = file->Rename(fileName); + instance->dispatcher.ResumeThread(); + } + return result; +} // end NormGetFileName() + +bool NormGetFileName(NormObjectHandle fileHandle, + char* nameBuffer, + unsigned int bufferLen) +{ + bool result = false; + if (NORM_OBJECT_INVALID != fileHandle) + { + // (TBD) verify "fileHandle" is a NORM_FILE ?>?? + NormFileObject* file = + static_cast((NormObject*)fileHandle); + bufferLen = (bufferLen < PATH_MAX) ? bufferLen : PATH_MAX; + strncpy(nameBuffer, file->GetPath(), bufferLen); + nameBuffer[bufferLen=1] = '\0'; + result = true; + } + return result; +} // end NormGetFileName() + +NormObjectHandle NormQueueData(NormSessionHandle sessionHandle, + const char* dataPtr, + unsigned long dataLen, + const char* infoPtr = NULL, + unsigned int infoLen = 0) +{ + NormObjectHandle objectHandle = NORM_OBJECT_INVALID; + NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); + if (instance && instance->dispatcher.SuspendThread()) + { + NormSession* session = (NormSession*)sessionHandle; + if (session) + { + NormObject* obj = session->QueueTxData(dataPtr, dataLen, infoPtr, infoLen); + if (obj) objectHandle = (NormObjectHandle)obj; + } + instance->dispatcher.ResumeThread(); + } + return objectHandle; +} // end NormQueueData() + + +bool NormGetNextEvent(NormInstanceHandle instanceHandle, NormEvent* theEvent) +{ + bool result = false; + NormInstance* instance = (NormInstance*)instanceHandle; + if (instance) + { + if (instance->NotifyQueueIsEmpty()) //(TBD) perform this check??? + instance->WaitForEvent(); + if (instance->dispatcher.SuspendThread()) + { + result = instance->GetNextEvent(theEvent); + instance->dispatcher.ResumeThread(); + } + } + return result; +} // end NormGetNextEvent() + diff --git a/common/normApi.h b/common/normApi.h new file mode 100644 index 0000000..0984944 --- /dev/null +++ b/common/normApi.h @@ -0,0 +1,258 @@ +#ifndef _NORM_API +#define _NORM_API + +#ifdef WIN32 +#include +#include +#endif // WIN32 + +//////////////////////////////////////////////////////////// +// IMPORTANT NOTICE +// The NORM API is _very_ much in a developmental phase +// right now. So, although this code is available in +// source code distribution, it is very much subject +// to change in future revisions. The goal of the NORM +// API _will_ be to provide a stable base of function calls +// for applications to use, even as the underlying NORM +// C++ code continues to evolve. But, until this notice +// is removed, the API shouldn't be considered final. + + +typedef const void* NormInstanceHandle; +extern const NormInstanceHandle NORM_INSTANCE_INVALID; + +NormInstanceHandle NormCreateInstance(); +void NormDestroyInstance(NormInstanceHandle instanceHandle); + +// NORM session creation and control +typedef const void* NormSessionHandle; +extern const NormSessionHandle NORM_SESSION_INVALID; +typedef unsigned long NormNodeId; +extern const NormNodeId NORM_NODE_NONE; +extern const NormNodeId NORM_NODE_ANY; + +NormSessionHandle NormCreateSession(NormInstanceHandle instanceHandle, + const char* sessionAddress, + unsigned short sessionPort, + NormNodeId localNodeId); + +void NormDestroySession(NormSessionHandle sessionHandle); + +// Session management and parameters + +void NormSetTransmitRate(NormSessionHandle sessionHandle, + double bitsPerSecond); + +void NormSetGrttEstimate(NormSessionHandle sessionHandle, + double grttEstimate); + +NormNodeId NormGetLocalNodeId(NormSessionHandle sessionHandle); + +bool NormSetLoopback(NormSessionHandle sessionHandle, bool state); + +// Debug parameters +void NormSetMessageTrace(NormSessionHandle sessionHandle, bool state); +void NormSetTxLoss(NormSessionHandle sessionHandle, double percent); +void NormSetRxLoss(NormSessionHandle sessionHandle, double percent); + +// Sender control & parameters +bool NormStartSender(NormSessionHandle sessionHandle, + unsigned long bufferSpace, + unsigned short segmentSize, + unsigned char numData, + unsigned char numParity); + +void NormStopSender(NormSessionHandle sessionHandle); + +bool NormAddAckingNode(NormSessionHandle sessionHandle, + NormNodeId nodeId); + +void NormRemoveAckingNode(NormSessionHandle sessionHandle, + NormNodeId nodeId); + + +// Receiver control & parameters +bool NormStartReceiver(NormSessionHandle sessionHandle, + unsigned long bufferSpace); + +void NormStopReceiver(NormSessionHandle sessionHandle); + +void NormSetDefaultUnicastNack(NormSessionHandle sessionHandle, + bool state); + +typedef const void* NormNodeHandle; +extern const NormNodeHandle NORM_NODE_INVALID; +typedef const void* NormObjectHandle; +extern const NormObjectHandle NORM_OBJECT_INVALID; +typedef unsigned short NormObjectTransportId; + +enum NormNackingMode +{ + NORM_NACK_NONE, + NORM_NACK_INFO_ONLY, + NORM_NACK_NORMAL +}; + +enum NormRepairBoundary +{ + NORM_BOUNDARY_BLOCK, + NORM_BOUNDARY_OBJECT +}; + + +NormNodeId NormGetNodeId(NormNodeHandle nodeHandle); + +NormRepairBoundary NormGetNodeRepairBoundary(NormNodeHandle nodeHandle); + +void NormSetNodeRepairBoundary(NormNodeHandle nodeHandle, + NormRepairBoundary repairBoundary); + +void NormSetDefaultRepairBoundary(NormSessionHandle sessionHandle, + NormRepairBoundary repairBoundary); + +// General NormObject functions + +enum NormObjectType +{ + NORM_OBJECT_NONE, + NORM_OBJECT_DATA, + NORM_OBJECT_FILE, + NORM_OBJECT_STREAM +}; + + +NormObjectType NormGetObjectType(NormObjectHandle objectHandle); + + +bool NormGetObjectInfo(NormObjectHandle objectHandle, + char* infoBuffer, + unsigned short* infoLen); + +NormObjectTransportId NormGetObjectTransportId(NormObjectHandle objectHandle); + +void NormCancelObject(NormObjectHandle objectHandle); + + +void NormRetainObject(NormObjectHandle objectHandle); +void NormReleaseObject(NormObjectHandle objectHandle); + + +// Receiver-only NormObject functions +NormNackingMode NormGetObjectNackingMode(NormObjectHandle objectHandle); + +void NormSetObjectNackingMode(NormObjectHandle objectHandle, + NormNackingMode nackingMode); + +void NormSetNodeNackingMode(NormNodeHandle nodeHandle, + NormNackingMode nackingMode); + +void NormSetDefaultNackingMode(NormSessionHandle sessionHandle, + NormNackingMode nackingMode); + +// Sender-only NormObject functions +bool NormSetWatermark(NormSessionHandle sessionHandle, + NormObjectHandle objectHandle); + + +// NormStreamObject functions + +NormObjectHandle NormOpenStream(NormSessionHandle sessionHandle, + unsigned long bufferSize); + +void NormCloseStream(NormObjectHandle streamHandle); + +enum NormFlushMode +{ + NORM_FLUSH_NONE, + NORM_FLUSH_PASSIVE, + NORM_FLUSH_ACTIVE +}; + +void NormSetStreamFlushMode(NormObjectHandle streamHandle, + NormFlushMode flushMode); + +void NormSetStreamPushMode(NormObjectHandle streamHandle, + bool state); + +unsigned int NormWriteStream(NormObjectHandle streamHandle, + const char* buffer, + unsigned int numBytes); + +void NormFlushStream(NormObjectHandle streamHandle); + +void NormMarkStreamEom(NormObjectHandle streamHandle); + +bool NormReadStream(NormObjectHandle streamHandle, + char* buffer, + unsigned int* numBytes); + +// NormFileObject Functions +NormObjectHandle NormQueueFile(NormSessionHandle sessionHandle, + const char* fileName, + const char* infoPtr = (const char*)0, + unsigned int infoLen = 0); + +bool NormSetCacheDirectory(NormInstanceHandle instanceHandle, + const char* cachePath); + +bool NormGetFileName(NormObjectHandle fileHandle, + char* nameBuffer, + unsigned int bufferLen); + +bool NormSetFileName(NormObjectHandle fileHandle, + const char* fileName); + +// NormDataObject Functions +bool NormQueueData(const char* dataPtr, + unsigned long dataLen, + const char* infoPtr = (const char*)0, + unsigned int infoLen = 0); + +// NORM Event Notification Routines + +enum NormEventType +{ + NORM_EVENT_INVALID = 0, + NORM_TX_QUEUE_VACANCY, + NORM_TX_QUEUE_EMPTY, + NORM_TX_OBJECT_SENT, + NORM_TX_OBJECT_PURGED, + NORM_LOCAL_SERVER_CLOSED, + NORM_REMOTE_SERVER_NEW, + NORM_REMOTE_SERVER_INACTIVE, + NORM_REMOTE_SERVER_ACTIVE, + NORM_RX_OBJECT_NEW, + NORM_RX_OBJECT_INFO, + NORM_RX_OBJECT_UPDATE, + NORM_RX_OBJECT_COMPLETED, + NORM_RX_OBJECT_ABORTED +}; + +typedef struct +{ + NormEventType type; + NormSessionHandle session; + NormNodeHandle sender; + NormObjectHandle object; +} NormEvent; + + +// This call blocks until the next NormEvent is ready unless asynchronous +// notification is used (see below) +bool NormGetNextEvent(NormInstanceHandle instanceHandle, NormEvent* theEvent); + +// The "NormGetDescriptor()" function returns a HANDLE (WIN32) or +// a file descriptor (UNIX) which can be used for async notification +// of pending NORM events. On WIN32, the returned HANDLE can be used +// with system calls like "WaitForSingleEvent()", and on Unix the +// returned descriptor can be used in a "select()" call. If this type +// of asynchronous notification is used, calls to "NormGetNextEvent()" will +// not block. + +#ifdef WIN32 +HANDLE NormGetDescriptor(NormInstanceHandle instanceHandle); +#else +int NormGetDescriptor(NormInstanceHandle instanceHandle); +#endif // if/else WIN32/UNIX + +#endif // _NORM_API diff --git a/common/normApp.cpp b/common/normApp.cpp index 7c71f0c..fbf31e7 100644 --- a/common/normApp.cpp +++ b/common/normApp.cpp @@ -11,6 +11,11 @@ #include #include // for "isspace()" +#ifdef UNIX +#include +#include // for "waitpid()" +#endif // UNIX + // Command-line application using Protolib EventDispatcher class NormApp : public NormController, public ProtoApp { @@ -64,8 +69,8 @@ class NormApp : public NormController, public ProtoApp unsigned int input_index; unsigned int input_length; bool input_active; - bool push_stream; - NormStreamObject::FlushType msg_flush_mode; + bool push_mode; + NormStreamObject::FlushMode msg_flush_mode; bool input_messaging; // stream input mode UINT16 input_msg_length; UINT16 input_msg_index; @@ -79,6 +84,7 @@ class NormApp : public NormController, public ProtoApp char* address; // session address UINT16 port; // session port number UINT8 ttl; + bool loopback; char* interface_name; // for multi-home hosts double tx_rate; // bits/sec bool cc_enable; @@ -90,6 +96,8 @@ class NormApp : public NormController, public ProtoApp UINT8 auto_parity; UINT8 extra_parity; double backoff_factor; + double grtt_estimate; // initial grtt estimate + double group_size; unsigned long tx_buffer_size; // bytes NormFileList tx_file_list; double tx_object_interval; @@ -118,13 +126,15 @@ NormApp::NormApp() session_mgr(GetTimerMgr(), GetSocketNotifier()), session(NULL), tx_stream(NULL), rx_stream(NULL), input(NULL), output(NULL), input_index(0), input_length(0), input_active(false), - push_stream(false), msg_flush_mode(NormStreamObject::FLUSH_PASSIVE), + push_mode(false), msg_flush_mode(NormStreamObject::FLUSH_PASSIVE), input_messaging(false), input_msg_length(0), input_msg_index(0), output_index(0), output_messaging(false), output_msg_length(0), output_msg_sync(false), - address(NULL), port(0), ttl(3), interface_name(NULL), + address(NULL), port(0), ttl(3), loopback(false), interface_name(NULL), tx_rate(64000.0), cc_enable(false), segment_size(1024), ndata(32), nparity(16), auto_parity(0), extra_parity(0), backoff_factor(NormSession::DEFAULT_BACKOFF_FACTOR), + grtt_estimate(NormSession::DEFAULT_GRTT_ESTIMATE), + group_size(NormSession::DEFAULT_GSIZE_ESTIMATE), tx_buffer_size(1024*1024), tx_object_interval(0.0), tx_repeat_count(0), tx_repeat_interval(2.0), tx_repeat_clear(true), rx_buffer_size(1024*1024), rx_cache_path(NULL), unicast_nacks(false), silent_client(false), @@ -153,16 +163,33 @@ NormApp::~NormApp() if (post_processor) delete post_processor; } +// NOTE on message flushing mode: +// +// "none" - messages are aggregated to make NORM_DATA +// payloads a full "segmentSize" +// +// "passive" - At the end-of-message, short NORM_DATA +// payloads are sent as needed so that +// the message is immediately sent +// (i.e. not held for aggregation) +// "active" - Same as "passive", plus NORM_CMD(FLUSH) +// messages are generated until new +// message is written to stream +// (This helps keep receivers in sync +// with sender when intermittent message +// traffic is sent) + const char* const NormApp::cmd_list[] = { "+debug", // debug level "+log", // log file name - "-trace", // message tracing on + "+trace", // message tracing on "+txloss", // tx packet loss percent (for testing) "+rxloss", // rx packet loss percent (for testing) "+address", // session destination address "+ttl", // multicast hop count scope + "+loopback", // "off" or "on" to recv our own packets "+interface", // multicast interface name to use "+cc", // congestion control on/off "+rate", // tx date rate (bps) @@ -184,18 +211,19 @@ const char* const NormApp::cmd_list[] = "+auto", // Number of FEC packets to proactively send (<= nparity) "+extra", // Number of extra FEC packets sent in response to repair requests "+backoff", // Backoff factor to use + "+grtt", // Set sender's initial GRTT estimate + "+gsize", // Set sender's group size estimate "+txbuffer", // Size of sender's buffer "+rxbuffer", // Size receiver allocates for buffering each sender "-unicastNacks", // unicast instead of multicast feedback messages "-silentClient", // "silent" (non-nacking) client (EMCON mode) "+processor", // receive file post processing command - "+instance", // specify norm instance name for commands + "+instance", // specify norm instance name for remote control commands NULL }; void NormApp::OnControlEvent(ProtoSocket& /*theSocket*/, ProtoSocket::Event theEvent) { - TRACE("NormApp::OnControlEvent() ...\n"); switch (theEvent) { case ProtoSocket::RECV: @@ -205,7 +233,7 @@ void NormApp::OnControlEvent(ProtoSocket& /*theSocket*/, ProtoSocket::Event theE if (control_pipe.Recv(buffer, len)) { buffer[len] = '\0'; - TRACE("norm: received command \"%s\"\n", buffer); + DMSG(0, "norm: received command \"%s\"\n", buffer); char* cmd = buffer; char* val = NULL; for (unsigned int i = 0; i < len; i++) @@ -214,6 +242,7 @@ void NormApp::OnControlEvent(ProtoSocket& /*theSocket*/, ProtoSocket::Event theE { buffer[i++] = '\0'; val = buffer + i; + break; } } if (!OnCommand(cmd, val)) @@ -288,8 +317,16 @@ bool NormApp::OnCommand(const char* cmd, const char* val) } else if (!strncmp("trace", cmd, len)) { - tracing = true; - if (session) session->SetTrace(true); + if (!strcmp("on", val)) + tracing = true; + else if (!strcmp("off", val)) + tracing = false; + else + { + DMSG(0, "NormApp::OnCommand(trace) invalid argument!\n"); + return false; + } + if (session) session->SetTrace(tracing); } else if (!strncmp("txloss", cmd, len)) { @@ -346,12 +383,38 @@ bool NormApp::OnCommand(const char* cmd, const char* val) else if (!strncmp("ttl", cmd, len)) { int ttlTemp = atoi(val); - if ((ttlTemp < 1) || (ttlTemp > 255)) + if ((ttlTemp < 0) || (ttlTemp > 255)) { DMSG(0, "NormApp::OnCommand(ttl) invalid value!\n"); return false; } - ttl = ttlTemp; + bool result = session ? session->SetTTL((UINT8)ttlTemp) : true; + ttl = result ? ttlTemp : ttl; + if (!result) + { + DMSG(0, "NormApp::OnCommand(ttl) error setting socket ttl!\n"); + return false; + } + } + else if (!strncmp("loopback", cmd, len)) + { + bool loopTemp = loopback; + if (!strcmp("on", val)) + loopTemp = true; + else if (!strcmp("off", val)) + loopTemp = false; + else + { + DMSG(0, "NormApp::OnCommand(loopback) invalid argument!\n"); + return false; + } + bool result = session ? session->SetLoopback(loopTemp) : true; + loopback = result ? loopTemp : loopback; + if (!result) + { + DMSG(0, "NormApp::OnCommand(loopback) error setting socket loopback!\n"); + return false; + } } else if (!strncmp("interface", cmd, len)) { @@ -580,12 +643,34 @@ bool NormApp::OnCommand(const char* cmd, const char* val) double backoffFactor = atof(val); if (backoffFactor < 0) { - DMSG(0, "NormSimAgent::OnCommand(backoff) invalid txRate!\n"); + DMSG(0, "NormApp::OnCommand(backoff) invalid value!\n"); return false; } backoff_factor = backoffFactor; if (session) session->SetBackoffFactor(backoffFactor); } + else if (!strncmp("grtt", cmd, len)) + { + double grttEstimate = atof(val); + if (grttEstimate < 0) + { + DMSG(0, "NormApp::OnCommand(grtt) invalid value!\n"); + return false; + } + grtt_estimate = grttEstimate; + if (session) session->ServerSetGrtt(grttEstimate); + } + else if (!strncmp("gsize", cmd, len)) + { + double groupSize = atof(val); + if (groupSize < 0) + { + DMSG(0, "NormApp::OnCommand(gsize) invalid value!\n"); + return false; + } + group_size = groupSize; + if (session) session->ServerSetGroupSize(groupSize); + } else if (!strncmp("txbuffer", cmd, len)) { if (1 != sscanf(val, "%lu", &tx_buffer_size)) @@ -614,7 +699,8 @@ bool NormApp::OnCommand(const char* cmd, const char* val) } else if (!strncmp("push", cmd, len)) { - push_stream = true; + push_mode = true; + if (tx_stream) tx_stream->SetPushMode(push_mode); } else if (!strncmp("flush", cmd, len)) { @@ -630,6 +716,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val) DMSG(0, "NormApp::OnCommand(flush) invalid msg flush mode!\n"); return false; } + if (tx_stream) tx_stream->SetFlushMode(msg_flush_mode); } else if (!strncmp("processor", cmd, len)) { @@ -644,12 +731,10 @@ bool NormApp::OnCommand(const char* cmd, const char* val) // First, try to connect to see if instance is already running if (control_pipe.Connect(val)) { - TRACE("connected to \"%s\"\n", val); control_remote = true; } else if (control_pipe.Listen(val)) { - TRACE("listening as \"%s\"\n", val); control_remote = false; } else @@ -740,7 +825,7 @@ void NormApp::OnInputReady() { //DMSG(0, "NormApp::OnInputReady() ...\n"); - NormStreamObject::FlushType flushType = NormStreamObject::FLUSH_NONE; + bool endOfStream = false; // write to the stream while input is available _and_ // the stream has buffer space for input while (input) @@ -816,7 +901,8 @@ void NormApp::OnInputReady() } if (stdin != input) fclose(input); input = NULL; - flushType = NormStreamObject::FLUSH_ACTIVE; + endOfStream = true; + tx_stream->SetFlushMode(NormStreamObject::FLUSH_ACTIVE); } else if (ferror(input)) { @@ -839,11 +925,10 @@ void NormApp::OnInputReady() unsigned int writeLength = input_length;// ? input_length - input_index : 0; - if (writeLength || (NormStreamObject::FLUSH_NONE != flushType)) + if (writeLength || endOfStream) { unsigned int wroteLength = tx_stream->Write(input_buffer+input_index, - writeLength, flushType, false, - push_stream); + writeLength, false); input_length -= wroteLength; if (0 == input_length) input_index = 0; @@ -857,7 +942,9 @@ void NormApp::OnInputReady() input_msg_index = 0; input_msg_length = 0; // Mark EOM _and_ flush - tx_stream->Write(NULL, 0, msg_flush_mode, true, false); + tx_stream->Write(NULL, 0, true); + // No need to explicitly flush with "flush mode" set. + //tx_stream->Flush(); } } if (wroteLength < writeLength) @@ -890,7 +977,7 @@ void NormApp::OnInputReady() #endif // if/else WIN32/UNIX input_active = true; else - DMSG(0, "NormApp::Notify(TX_QUEUE_EMPTY) error adding input notification!\n"); + DMSG(0, "NormApp::OnInputReady() error adding input notification!\n"); } break; } @@ -992,6 +1079,9 @@ void NormApp::Notify(NormController::Event event, case NormObject::DATA: DMSG(0, "NormApp::Notify() FILE/DATA objects not _yet_ supported...\n"); break; + + case NormObject::NONE: + break; } break; } @@ -1032,6 +1122,7 @@ void NormApp::Notify(NormController::Event event, } case NormObject::DATA: case NormObject::STREAM: + case NormObject::NONE: break; } // end switch(object->GetType()) break; @@ -1042,6 +1133,8 @@ void NormApp::Notify(NormController::Event event, { case NormObject::FILE: // (TBD) update reception progress display when applicable + // Call object->SetNotifyOnUpdate(true) here to keep + // the update notifications coming. Otherwise they stop! break; case NormObject::STREAM: @@ -1161,17 +1254,22 @@ void NormApp::Notify(NormController::Event event, case NormObject::DATA: DMSG(0, "NormApp::Notify() DATA objects not _yet_ supported...\n"); break; + case NormObject::NONE: + break; } // end switch (object->GetType()) break; - case RX_OBJECT_COMPLETE: + case RX_OBJECT_COMPLETED: { + // (TBD) if we're not archiving files we should + // manage our cache, deleting the cache + // on shutdown ... //DMSG(0, "NormApp::Notify(RX_OBJECT_COMPLETE) ...\n"); switch(object->GetType()) { case NormObject::FILE: { - const char* filePath = ((NormFileObject*)object)->Path(); + const char* filePath = ((NormFileObject*)object)->GetPath(); //DMSG(0, "norm: Completed rx file: %s\n", filePath); if (post_processor->IsEnabled()) { @@ -1189,9 +1287,15 @@ void NormApp::Notify(NormController::Event event, case NormObject::DATA: ASSERT(0); break; + case NormObject::NONE: + break; } break; } + default: + { + DMSG(4, "NormApp::Notify() unhandled event: %d\n", event); + } } // end switch(event) } // end NormApp::Notify() @@ -1315,14 +1419,17 @@ bool NormApp::OnStartup(int argc, const char*const* argv) session->SetTrace(tracing); session->SetTxLoss(tx_loss); session->SetRxLoss(rx_loss); - session->SetBackoffFactor(backoff_factor); - + session->SetTTL(ttl); + session->SetLoopback(loopback); + if (input || !tx_file_list.IsEmpty()) { NormObjectId baseId = (unsigned short)(rand() * (65535.0/ (double)RAND_MAX)); session->ServerSetBaseObjectId(baseId); - session->SetCongestionControl(cc_enable); + session->SetBackoffFactor(backoff_factor); + session->ServerSetGrtt(grtt_estimate); + session->ServerSetGroupSize(group_size); if (!session->StartServer(tx_buffer_size, segment_size, ndata, nparity, interface_name)) { DMSG(0, "NormApp::OnStartup() start server error!\n"); @@ -1341,6 +1448,8 @@ bool NormApp::OnStartup(int argc, const char*const* argv) session_mgr.Destroy(); return false; } + tx_stream->SetFlushMode(msg_flush_mode); + tx_stream->SetPushMode(push_mode); } } @@ -1407,6 +1516,10 @@ void NormApp::SignalHandler(int sigNum) { NormApp* app = static_cast(ProtoApp::GetApp()); if (app->post_processor) app->post_processor->OnDeath(); + // The use of "waitpid()" here is a work-around for + // an IRIX SIGCHLD issue + int status; + while (waitpid(-1, &status, WNOHANG) > 0); signal(SIGCHLD, SignalHandler); break; } diff --git a/common/normBitmask.cpp b/common/normBitmask.cpp index de7cba6..a143ad3 100644 --- a/common/normBitmask.cpp +++ b/common/normBitmask.cpp @@ -378,7 +378,9 @@ bool NormBitmask::Add(const NormBitmask& b) if (b.num_bits > num_bits) return false; for(unsigned int i = 0; i < b.mask_len; i++) mask[i] |= b.mask[i]; - if (b.first_set < first_set) first_set = b.first_set; + if ((b.first_set < first_set) && + (b.first_set < b.num_bits)) + first_set = b.first_set; return true; } // end NormBitmask::Add() @@ -406,13 +408,14 @@ bool NormBitmask::XCopy(const NormBitmask& b) for (unsigned int i = begin; i < len; i++) mask[i] = b.mask[i] & ~mask[i]; if (len < mask_len) memset(&mask[len], 0, mask_len - len); - if (b.first_set < first_set) + UINT32 theFirst = (b.first_set < b.num_bits) ? b.first_set : num_bits; + if (theFirst < first_set) { first_set = b.first_set; } else { - first_set = b.first_set; + first_set = theFirst; if (!GetNextSet(first_set)) first_set = num_bits; } return true; @@ -425,6 +428,7 @@ bool NormBitmask::Multiply(const NormBitmask& b) for(unsigned int i = 0; i < len; i++) mask[i] |= b.mask[i]; if (len < mask_len) memset(&mask[len], 0, mask_len - len); + if (b.first_set > first_set) { first_set = b.first_set; @@ -557,7 +561,7 @@ bool NormSlidingMask::CanSet(UINT32 index) const bool NormSlidingMask::Set(UINT32 index) { - ASSERT(CanSet(index)); + //ASSERT(CanSet(index)); if (IsSet()) { // Determine position with respect to current start diff --git a/common/normEncoder.cpp b/common/normEncoder.cpp index 7dfd8ad..5103e5f 100644 --- a/common/normEncoder.cpp +++ b/common/normEncoder.cpp @@ -218,8 +218,8 @@ void NormEncoder::Encode(const char *data, char **pVec) */ NormDecoder::NormDecoder() - : npar(0), vector_size(0), - Lambda(NULL), sVec(NULL), oVec(NULL) + : npar(0), vector_size(0), Lambda(NULL), + sVec(NULL), oVec(NULL), scratch(NULL) { } diff --git a/common/normFile.cpp b/common/normFile.cpp index 45942ac..a209e50 100644 --- a/common/normFile.cpp +++ b/common/normFile.cpp @@ -1,4 +1,3 @@ - #include "normFile.h" #include // for strerror() @@ -63,7 +62,7 @@ bool NormFile::Open(const char* thePath, int theFlags) if (mkdir(tempPath, 0755)) #endif // if/else WIN32/UNIX { - DMSG(0, "NormFile::Rename() mkdir(%s) error: %s\n", + DMSG(0, "NormFile::Open() mkdir(%s) error: %s\n", tempPath, strerror(errno)); return false; } @@ -159,10 +158,21 @@ bool NormFile::Rename(const char* oldName, const char* newName) if (!strcmp(oldName, newName)) return true; // no change required // Make sure the new file name isn't an existing "busy" file // (This also builds sub-directories as needed) - if (NormFile::IsLocked(newName)) return false; + if (NormFile::IsLocked(newName)) + { + DMSG(0, "NormFile::Rename() error: file is locked\n"); + return false; + } #ifdef WIN32 // In Win32, the new file can't already exist - if (NormFile::Exists(newName)) _unlink(newName); + if (NormFile::Exists(newName)) + { + if (0 != _unlink(newName)) + { + DMSG(0, "NormFile::Rename() _unlink() error: %s\n", GetErrorString()); + return false; + } + } // In Win32, the old file can't be open int oldFlags = 0; if (IsOpen()) @@ -171,7 +181,7 @@ bool NormFile::Rename(const char* oldName, const char* newName) oldFlags &= ~(O_CREAT | O_TRUNC); // unset these Close(); } -#else +#endif // WIN32 // Create sub-directories as needed. char tempPath[PATH_MAX]; strncpy(tempPath, newName, PATH_MAX); @@ -198,29 +208,44 @@ bool NormFile::Rename(const char* oldName, const char* newName) { ptr = strchr(ptr, PROTO_PATH_DELIMITER); if (ptr) *ptr = '\0'; +#ifdef WIN32 + if (0 != _mkdir(tempPath)) +#else if (mkdir(tempPath, 0755)) +#endif // if/else WIN32/UNIX { DMSG(0, "NormFile::Rename() mkdir(%s) error: %s\n", - tempPath, strerror(errno)); + tempPath, GetErrorString()); return false; } if (ptr) *ptr++ = PROTO_PATH_DELIMITER; - } -#endif // if/else WIN32 + } + if (rename(oldName, newName)) { DMSG(0, "NormFile::Rename() rename() error: %s\n", strerror(errno)); #ifdef WIN32 - //if (oldFlags) Open(oldName, oldFlags); + if (oldFlags) + { + if (Open(oldName, oldFlags)) + offset = 0; + else + DMSG(0, "NormFile::Rename() error re-opening file w/ old name\n"); + } #endif // WIN32 return false; - } else { #ifdef WIN32 // (TBD) Is the file offset OK doing this??? - //if (oldFlags) Open(newName, oldFlags); + if (oldFlags) + { + if (Open(newName, oldFlags)) + offset = 0; + else + DMSG(0, "NormFile::Rename() error opening file w/ new name\n"); + } #endif // WIN32 return true; } @@ -234,7 +259,7 @@ int NormFile::Read(char* buffer, int len) #else int result = read(fd, buffer, len); #endif // if/else WIN32 - if (result > 0) offset += result; + if (result > 0) offset += (off_t)result; return result; } // end NormFile::Read() @@ -246,7 +271,7 @@ int NormFile::Write(const char* buffer, int len) #else int result = write(fd, buffer, len); #endif // if/else WIN32 - if (result > 0) offset += result; + if (result > 0) offset += (off_t)result; return result; } // end NormFile::Write() diff --git a/common/normMessage.cpp b/common/normMessage.cpp index 81580db..57676d4 100644 --- a/common/normMessage.cpp +++ b/common/normMessage.cpp @@ -29,14 +29,14 @@ bool NormMsg::InitFromBuffer(UINT16 msgLength) case DATA: // (TBD) look at "fec_id" to determine "fec_payload_id" size // (we _assume_ "fec_id" == 129 here - if ((unsigned char)buffer[NormDataMsg::FEC_ID_OFFSET] == 129) + if ((unsigned char)buffer[NormObjectMsg::FEC_ID_OFFSET] == 129) { header_length_base = 24; } else { DMSG(0, "NormMsg::InitFromBuffer(DATA) unknown fec_id value: %u\n", - buffer[NormDataMsg::FEC_ID_OFFSET]); + buffer[NormObjectMsg::FEC_ID_OFFSET]); return false; } break; @@ -54,7 +54,7 @@ bool NormMsg::InitFromBuffer(UINT16 msgLength) else { DMSG(0, "NormMsg::InitFromBuffer(FLUSH|SQUELCH) unknown fec_id value: %u\n", - buffer[NormDataMsg::FEC_ID_OFFSET]); + buffer[NormCmdFlushMsg::FEC_ID_OFFSET]); return false; } break; @@ -420,7 +420,7 @@ unsigned char NormQuantizeRtt(double rtt) else if (rtt < NORM_RTT_MIN) rtt = NORM_RTT_MIN; if (rtt < 3.3e-05) - return ((unsigned char)ceil((rtt*NORM_RTT_MIN)) - 1); + return ((unsigned char)((rtt/NORM_RTT_MIN)) - 1); else return ((unsigned char)(ceil(255.0 - (13.0*log(NORM_RTT_MAX/rtt))))); } // end NormQuantizeRtt() diff --git a/common/normMessage.h b/common/normMessage.h index 2589ea8..21c6146 100644 --- a/common/normMessage.h +++ b/common/normMessage.h @@ -20,7 +20,6 @@ const UINT8 NORM_PROTOCOL_VERSION = 1; const int NORM_ROBUST_FACTOR = 20; // default robust factor - // Pick a random number from 0..max inline double UniformRand(double max) { @@ -46,7 +45,7 @@ const double NORM_RTT_MAX = 1000.0; inline double NormUnquantizeRtt(unsigned char qrtt) { return ((qrtt < 31) ? - (((double)(qrtt+1))/(double)NORM_RTT_MIN) : + (((double)(qrtt+1))*(double)NORM_RTT_MIN) : (NORM_RTT_MAX/exp(((double)(255-qrtt))/(double)13.0))); } unsigned char NormQuantizeRtt(double rtt); @@ -95,6 +94,7 @@ inline double NormUnquantizeRate(unsigned short rate) } // This class is used to describe object "size" and/or "offset" +// (TBD) This hokey implementation should use "off_t" class NormObjectSize { public: @@ -103,6 +103,13 @@ class NormObjectSize NormObjectSize(UINT32 size) : msb(0), lsb(size) {} NormObjectSize(UINT16 size) : msb(0), lsb(size) {} NormObjectSize(const NormObjectSize& size) : msb(size.msb), lsb(size.lsb) {} + NormObjectSize(off_t size) + { + lsb = size & 0x00000000ffffffff; + size >>= 32; + ASSERT(0 == (size & 0xffff0000)); + msb = size & 0x0000ffff; + } UINT16 MSB() const {return msb;} UINT32 LSB() const {return lsb;} @@ -150,15 +157,24 @@ class NormObjectSize } NormObjectSize operator*(const NormObjectSize& b) const; NormObjectSize operator/(const NormObjectSize& b) const; + off_t GetOffset() const + { + off_t offset = msb; + offset <<= 32; + offset += lsb; + return offset; + } private: UINT16 msb; UINT32 lsb; }; // end class NormObjectSize -typedef UINT32 NormNodeId; -const NormNodeId NORM_NODE_INVALID = 0x00000000; -const NormNodeId NORM_NODE_ANY = 0xffffffff; +#ifndef _NORM_API +typedef unsigned long NormNodeId; +const NormNodeId NORM_NODE_NONE = 0x00000000; +const NormNodeId NORM_NODE_ANY = 0xffffffff; +#endif // !_NORM_API class NormObjectId { @@ -167,18 +183,12 @@ class NormObjectId NormObjectId(UINT16 id) {value = id;} NormObjectId(const NormObjectId& id) {value = id.value;} operator UINT16() const {return value;} - int operator-(const NormObjectId& id) const - { - int result = value - id.value; - return ((0 == (result & 0x00008000)) ? - (result & 0x0000ffff) : - (((result != 0x00008000) || (value < id.value)) ? - result | 0xffff0000 : result)); - } + INT16 operator-(const NormObjectId& id) const + {return ((INT16)(value - id.value));} bool operator<(const NormObjectId& id) const - {return ((*this - id) < 0);} + {return (((short)(value - id.value)) < 0);} bool operator>(const NormObjectId& id) const - {return ((*this - id) > 0);} + {return (((INT16)(value - id.value)) > 0);} bool operator<=(const NormObjectId& id) const {return ((value == id.value) || (*this=(const NormObjectId& id) const @@ -187,7 +197,7 @@ class NormObjectId {return (value == id.value);} bool operator!=(const NormObjectId& id) const {return (value != id.value);} - NormObjectId& operator++(int ) {value++; return *this;} + NormObjectId& operator++(int) {value++; return *this;} private: UINT16 value; @@ -204,17 +214,17 @@ class NormBlockId {return (value == (UINT32)id);} bool operator!=(const NormBlockId& id) const {return (value != (UINT32)id);} - long operator-(const NormBlockId& id) const - {return ((long)value - id.value);} + INT32 operator-(const NormBlockId& id) const + {return ((INT32)value - id.value);} bool operator<(const NormBlockId& id) const - {return ((*this - id) < 0);} + {return (((INT32)(value-id.value)) < 0);} bool operator>(const NormBlockId& id) const - {return ((*this - id) > 0);} + {return (((INT32)(value-id.value)) > 0);} NormBlockId& operator++(int ) {value++; return *this;} private: UINT32 value; -}; // end class NormObjectId +}; // end class NormBlockId typedef UINT16 NormSymbolId; typedef NormSymbolId NormSegmentId; @@ -361,7 +371,7 @@ class NormMsg void ExtendHeaderLength(UINT16 len) { header_length += len; - length = header_length; + length += len; buffer[HDR_LEN_OFFSET] = header_length >> 2; } @@ -624,6 +634,7 @@ class NormDataMsg : public NormObjectMsg return (length - dataIndex); } + // These routines are only applicable to messages containing NORM_OBJECT_STREAM content // Some static helper routines for reading/writing embedded payload length/offsets static UINT16 GetStreamPayloadHeaderLength() {return (PAYLOAD_DATA_OFFSET);} @@ -1049,7 +1060,7 @@ class NormRepairRequest void SetForm(NormRepairRequest::Form theForm) {form = theForm;} void ResetFlags() {flags = 0;} void SetFlag(NormRepairRequest::Flag theFlag) {flags |= theFlag;} - void UnsetFlag(NormRepairRequest::Flag theFlag) {flags &= ~theFlag;} + void ClearFlag(NormRepairRequest::Flag theFlag) {flags &= ~theFlag;} // Returns length (each repair item requires 8 bytes of space) bool AppendRepairItem(const NormObjectId& objectId, @@ -1417,6 +1428,7 @@ class NormAckMsg : public NormMsg { UINT16 len = MIN(payloadLen, segmentSize); memcpy(buffer+header_length, payload, len); + length += len; return (payloadLen <= segmentSize); } @@ -1439,7 +1451,7 @@ class NormAckMsg : public NormMsg UINT16 GetPayloadLength() const {return (length - header_length);} const char* GetPayload() const {return (buffer + header_length);} - private: + protected: enum { SERVER_ID_OFFSET = MSG_OFFSET, @@ -1451,6 +1463,75 @@ class NormAckMsg : public NormMsg }; }; // end class NormAckMsg +class NormAckFlushMsg : public NormAckMsg +{ + public: + void Init() + { + SetType(ACK); + SetBaseHeaderLength(ACK_HEADER_LEN); + SetAckType(NormAck::FLUSH); + SetFecId(129); // only one supported for the moment + buffer[RESERVED_OFFSET] = 0; + length = ACK_HEADER_LEN+PAYLOAD_LENGTH; + } + + // Note: must apply any header exts _before_ the payload is set + void SetFecId(UINT8 fecId) + {buffer[header_length+FEC_ID_OFFSET] = fecId;} + void SetObjectId(NormObjectId objectId) + {*((UINT16*)(buffer+header_length+OBJ_ID_OFFSET)) = htons((UINT16)objectId);} + void SetFecBlockId(const NormBlockId& blockId) + { + UINT32 temp32 = htonl((UINT32)blockId); + memcpy(buffer+header_length+BLOCK_ID_OFFSET, &temp32, 4); + } + void SetFecBlockLen(UINT16 blockLen) + { + blockLen = htons(blockLen); + memcpy(buffer+header_length+BLOCK_LEN_OFFSET, &blockLen, 2); + } + void SetFecSymbolId(UINT16 symbolId) + { + symbolId = htons(symbolId); + memcpy(buffer+header_length+SYMBOL_ID_OFFSET, &symbolId, 2); + } + + // Message processing + UINT8 GetFecId() {return buffer[header_length+FEC_ID_OFFSET];} + NormObjectId GetObjectId() const + { + return (ntohs(*((UINT16*)(buffer+header_length+OBJ_ID_OFFSET)))); + } + NormBlockId GetFecBlockId() const + { + return (ntohl(*((UINT32*)(buffer+header_length+BLOCK_ID_OFFSET)))); + } + UINT16 GetFecBlockLen() const + { + return (ntohs(*((UINT16*)(buffer+header_length+BLOCK_LEN_OFFSET)))); + } + UINT16 GetFecSymbolId() const + { + return (ntohs(*((UINT16*)(buffer+header_length+SYMBOL_ID_OFFSET)))); + } + + private: + // These are the payload offsets for "fec_id" = 129 + // "fec_payload_id" field + enum + { + FEC_ID_OFFSET = 0, + RESERVED_OFFSET = FEC_ID_OFFSET + 1, + OBJ_ID_OFFSET = RESERVED_OFFSET + 1, + BLOCK_ID_OFFSET = OBJ_ID_OFFSET + 2, + BLOCK_LEN_OFFSET = BLOCK_ID_OFFSET + 4, + SYMBOL_ID_OFFSET = BLOCK_LEN_OFFSET + 2, + PAYLOAD_LENGTH = SYMBOL_ID_OFFSET + 2 + }; + +}; // end class NormAckFlushMsg + class NormReportMsg : public NormMsg { // (TBD) diff --git a/common/normNode.cpp b/common/normNode.cpp index 53b10f9..6896da5 100644 --- a/common/normNode.cpp +++ b/common/normNode.cpp @@ -4,8 +4,8 @@ #include -NormNode::NormNode(class NormSession* theSession, NormNodeId nodeId) - : session(theSession), id(nodeId), +NormNode::NormNode(class NormSession& theSession, NormNodeId nodeId) + : session(theSession), id(nodeId), reference_count(0), parent(NULL), right(NULL), left(NULL) { @@ -15,10 +15,25 @@ NormNode::~NormNode() { } -const NormNodeId& NormNode::LocalNodeId() const {return session->LocalNodeId();} +void NormNode::Retain() +{ + reference_count++; +} // end NormNode::Retain() + +void NormNode::Release() +{ + if (reference_count) + reference_count--; + else + DMSG(0, "NormNode::Release() releasing non-retained node?!\n"); + if (0 == reference_count) delete this; +} // end NormNode::Release() -NormCCNode::NormCCNode(class NormSession* theSession, NormNodeId nodeId) +const NormNodeId& NormNode::LocalNodeId() const {return session.LocalNodeId();} + + +NormCCNode::NormCCNode(class NormSession& theSession, NormNodeId nodeId) : NormNode(theSession, nodeId) { @@ -29,14 +44,23 @@ NormCCNode::~NormCCNode() } -NormServerNode::NormServerNode(class NormSession* theSession, NormNodeId nodeId) - : NormNode(theSession, nodeId), session_id(0), synchronized(false), - is_open(false), segment_size(0), ndata(0), nparity(0), erasure_loc(NULL), - cc_sequence(0), cc_enable(false), cc_rate(0.0), rtt_confirmed(false), is_clr(false), is_plr(false), +NormServerNode::NormServerNode(class NormSession& theSession, NormNodeId nodeId) + : NormNode(theSession, nodeId), session_id(0), synchronized(false), sync_id(0), + max_pending_range(256), is_open(false), segment_size(0), ndata(0), nparity(0), + repair_boundary(BLOCK_BOUNDARY), erasure_loc(NULL), + cc_sequence(0), cc_enable(false), cc_rate(0.0), + rtt_confirmed(false), is_clr(false), is_plr(false), slow_start(true), send_rate(0.0), recv_rate(0.0), recv_accumulator(0), recv_total(0), recv_goodput(0), resync_count(0), nack_count(0), suppress_count(0), completion_count(0), failure_count(0) { + + + repair_boundary = session.ClientGetDefaultRepairBoundary(); + default_nacking_mode = session.ClientGetDefaultNackingMode(); + unicast_nacks = session.UnicastNacks(); + // (TBD) get "max_pending_range" value from NormSession parameter + repair_timer.SetListener(this, &NormServerNode::OnRepairTimeout); repair_timer.SetInterval(0.0); repair_timer.SetRepeat(1); @@ -49,6 +73,10 @@ NormServerNode::NormServerNode(class NormSession* theSession, NormNodeId nodeId) cc_timer.SetInterval(0.0); cc_timer.SetRepeat(1); + ack_timer.SetListener(this, &NormServerNode::OnAckTimeout); + ack_timer.SetInterval(0.0); + ack_timer.SetRepeat(0); + grtt_send_time.tv_sec = 0; grtt_send_time.tv_usec = 0; grtt_quantized = NormQuantizeRtt(NormSession::DEFAULT_GRTT_ESTIMATE); @@ -73,27 +101,48 @@ NormServerNode::~NormServerNode() Close(); } -bool NormServerNode::Open(UINT16 sessionId, UINT16 segmentSize, UINT16 numData, UINT16 numParity) +bool NormServerNode::Open(UINT16 sessionId) { session_id = sessionId; - if (!rx_table.Init(256)) + if (!rx_table.Init(max_pending_range)) { DMSG(0, "NormServerNode::Open() rx_table init error\n"); Close(); return false; } - if (!rx_pending_mask.Init(256, 0x0000ffff)) + if (!rx_pending_mask.Init(max_pending_range, 0x0000ffff)) { DMSG(0, "NormServerNode::Open() rx_pending_mask init error\n"); Close(); return false; } - if (!rx_repair_mask.Init(256, 0x0000ffff)) + if (!rx_repair_mask.Init(max_pending_range, 0x0000ffff)) { DMSG(0, "NormServerNode::Open() rx_repair_mask init error\n"); Close(); return false; } + is_open = true; + synchronized = false; + return true; +} // end NormServerNode::Open() + +void NormServerNode::Close() +{ + if (activity_timer.IsActive()) activity_timer.Deactivate(); + if (repair_timer.IsActive()) repair_timer.Deactivate(); + if (cc_timer.IsActive()) cc_timer.Deactivate(); + FreeBuffers(); + rx_repair_mask.Destroy(); + rx_pending_mask.Destroy(); + rx_table.Destroy(); + synchronized = false; + is_open = false; +} // end NormServerNode::Close() + +bool NormServerNode::AllocateBuffers(UINT16 segmentSize, UINT16 numData, UINT16 numParity) +{ + ASSERT(IsOpen()); // Calculate how much memory each buffered block will require UINT16 blockSize = numData + numParity; unsigned long maskSize = blockSize >> 3; @@ -102,7 +151,7 @@ bool NormServerNode::Open(UINT16 sessionId, UINT16 segmentSize, UINT16 numData, blockSize * sizeof(char*) + 2*maskSize + numData * (segmentSize + NormDataMsg::GetStreamPayloadHeaderLength()); - unsigned long bufferSpace = session->RemoteServerBufferSize(); + unsigned long bufferSpace = session.RemoteServerBufferSize(); unsigned long numBlocks = bufferSpace / blockSpace; if (bufferSpace > (numBlocks*blockSpace)) numBlocks++; if (numBlocks < 2) numBlocks = 2; @@ -143,14 +192,11 @@ bool NormServerNode::Open(UINT16 sessionId, UINT16 segmentSize, UINT16 numData, nominal_packet_size = (double)segmentSize; ndata = numData; nparity = numParity; - is_open= true; - Activate(); return true; -} // end NormServerNode::Open() +} // end NormServerNode::AllocateBuffers() -void NormServerNode::Close() +void NormServerNode::FreeBuffers() { - if (activity_timer.IsActive()) activity_timer.Deactivate(); decoder.Destroy(); if (erasure_loc) { @@ -158,16 +204,15 @@ void NormServerNode::Close() erasure_loc = NULL; } NormObject* obj; - while ((obj = rx_table.Find(rx_table.RangeLo()))) DeleteObject(obj); + while ((obj = rx_table.Find(rx_table.RangeLo()))) + { + session.Notify(NormController::RX_OBJECT_ABORTED, this, obj); + DeleteObject(obj, 6); + } segment_pool.Destroy(); block_pool.Destroy(); - rx_repair_mask.Destroy(); - rx_pending_mask.Destroy(); - rx_table.Destroy(); - segment_size = ndata = nparity = 0; - is_open = false; - synchronized = false; -} // end NormServerNode::Close() + segment_size = ndata = nparity = 0; +} // end NormServerNode::FreeBuffers() void NormServerNode::HandleCommand(const struct timeval& currentTime, @@ -197,22 +242,21 @@ void NormServerNode::HandleCommand(const struct timeval& currentTime, switch (flavor) { case NormCmdMsg::SQUELCH: - if (IsOpen()) + { + const NormCmdSquelchMsg& squelch = (const NormCmdSquelchMsg&)cmd; + // 1) Sync to squelch + NormObjectId objectId = squelch.GetObjectId(); + Sync(objectId); + // 2) Prune stream object if applicable + NormObject* obj = rx_table.Find(objectId); + if (obj && (NormObject::STREAM == obj->GetType())) { - const NormCmdSquelchMsg& squelch = (const NormCmdSquelchMsg&)cmd; - // 1) Sync to squelch - NormObjectId objectId = squelch.GetObjectId(); - Sync(objectId); - // 2) Prune stream object if applicable - NormObject* obj = rx_table.Find(objectId); - if (obj && (NormObject::STREAM == obj->GetType())) - { - NormBlockId blockId = squelch.GetFecBlockId(); - ((NormStreamObject*)obj)->Prune(blockId); - } - // 3) (TBD) Discard any invalidated objects + NormBlockId blockId = squelch.GetFecBlockId(); + ((NormStreamObject*)obj)->Prune(blockId); } + // 3) (TBD) Discard any invalidated objects break; + } case NormCmdMsg::ACK_REQ: // (TBD) handle ack requests @@ -263,7 +307,7 @@ void NormServerNode::HandleCommand(const struct timeval& currentTime, is_clr = is_plr = false; } double maxBackoff; - if (is_clr || is_plr || !session->Address().IsMulticast()) + if (is_clr || is_plr || !session.Address().IsMulticast()) { // Respond immediately maxBackoff = 0.0; @@ -306,15 +350,60 @@ void NormServerNode::HandleCommand(const struct timeval& currentTime, cc_timer.SetInterval(backoffTime); DMSG(6, "NormServerNode::HandleCommand() node>%lu begin CC back-off: %lf sec)...\n", LocalNodeId(), backoffTime); - session->ActivateTimer(cc_timer); + session.ActivateTimer(cc_timer); } // end if (CC_RATE == ext.GetType()) } // end while (GetNextExtension()) break; } case NormCmdMsg::FLUSH: + { // (TBD) should we force synchronize if we're expected // to positively acknowledge the FLUSH - + const NormCmdFlushMsg& flush = (const NormCmdFlushMsg&)cmd; + bool doAck = false; + UINT16 nodeCount = flush.GetAckingNodeCount(); + NormNodeId localId = LocalNodeId(); + for (UINT16 i = 0; i < nodeCount; i++) + { + if (flush.GetAckingNodeId(i) == localId) + { + doAck = true; + break; + } + } + if (!synchronized) + { + if (doAck) + { + // Force sync since we're expected to ACK + Sync(flush.GetObjectId()); + } + else + { + // (TBD) optionally sync on any flush ? + } + } + if (0 != nodeCount) // this was a watermark flush + { + if (!PassiveRepairCheck(flush.GetObjectId(), + flush.GetFecBlockId(), + flush.GetFecSymbolId())) + { + if (doAck) + { + watermark_object_id = flush.GetObjectId(); + watermark_block_id = flush.GetFecBlockId(); + watermark_segment_id = flush.GetFecSymbolId(); + if (!ack_timer.IsActive()) + { + double ackBackoff = UniformRand(grtt_estimate); + ack_timer.SetInterval(ackBackoff); + session.ActivateTimer(ack_timer); + } + } + break; // no pending repairs, skip regular "RepairCheck" + } + } if (synchronized) { const NormCmdFlushMsg& flush = (const NormCmdFlushMsg&)cmd; @@ -325,7 +414,7 @@ void NormServerNode::HandleCommand(const struct timeval& currentTime, flush.GetFecSymbolId()); } break; - + } case NormCmdMsg::REPAIR_ADV: { const NormCmdRepairAdvMsg& repairAdv = (const NormCmdRepairAdvMsg&)cmd; @@ -401,7 +490,7 @@ void NormServerNode::HandleCCFeedback(UINT8 ccFlags, double ccRate) { if (cc_timer.IsActive()) cc_timer.Deactivate(); cc_timer.SetInterval(grtt_estimate*backoff_factor); // (TBD) ??? - session->ActivateTimer(cc_timer); + session.ActivateTimer(cc_timer); cc_timer.DecrementRepeatCount(); } } @@ -596,13 +685,12 @@ void NormServerNode::CalculateGrttResponse(const struct timeval& currentTime, } } // end NormServerNode::CalculateGrttResponse() -void NormServerNode::DeleteObject(NormObject* obj) +void NormServerNode::DeleteObject(NormObject* obj, int which) { - // (TBD) Notify app of object's closing/demise? + if (rx_table.Remove(obj)) + rx_pending_mask.Unset(obj->GetId()); obj->Close(); - rx_table.Remove(obj); - rx_pending_mask.Unset(obj->GetId()); - delete obj; + obj->Release(); } // end NormServerNode::DeleteObject() NormBlock* NormServerNode::GetFreeBlock(NormObjectId objectId, NormBlockId blockId) @@ -610,7 +698,7 @@ NormBlock* NormServerNode::GetFreeBlock(NormObjectId objectId, NormBlockId block NormBlock* b = block_pool.Get(); if (!b) { - if (session->ClientIsSilent()) + if (session.ClientIsSilent()) //if (1) { // forward iteration to find oldest older object with resources @@ -700,53 +788,6 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) } backoff_factor = (double)msg.GetBackoffFactor(); - - if (IsOpen()) - { - if (msg.GetSessionId() != session_id) - { - DMSG(2, "NormServerNode::HandleObjectMessage() node>%lu server>%lu sessionId change - resyncing.\n", - LocalNodeId(), GetId()); - Close(); - resync_count++; - } - } - - if (!IsOpen()) - { - DMSG(4, "NormServerNode::HandleObjectMessage() node>%lu opening server>%lu ...\n", - LocalNodeId(), GetId()); - // Currently,, our implementation requires the FEC Object Transmission Information - // to properly allocate resources - NormFtiExtension fti; - while (msg.GetNextExtension(fti)) - { - if (NormHeaderExtension::FTI == fti.GetType()) - { - // (TBD) pass "fec_id" to Open() method too - if (!Open(msg.GetSessionId(), - fti.GetSegmentSize(), - fti.GetFecMaxBlockLen(), - fti.GetFecNumParity())) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu open error\n", - LocalNodeId(), GetId()); - // (TBD) notify app of error ?? - return; - } - break; - } - } - if (!IsOpen()) - { - DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu - no FTI provided!\n", - LocalNodeId(), GetId()); - // (TBD) notify app of error ?? - return; - } - } - - NormMsg::Type msgType = msg.GetType(); NormObjectId objectId = msg.GetObjectId(); NormBlockId blockId; @@ -761,7 +802,52 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) const NormDataMsg& data = (const NormDataMsg&)msg; // (TBD) verify source block length per new spec blockId = data.GetFecBlockId(); - segmentId = data.GetFecSymbolId(); + segmentId = data.GetFecSymbolId(); + + // The current NORM implementation assumes senders maintain a fixed, common + // set of FEC coding parameters for its transmissions. The buffers (on a + // "per-remote-server basis") for receiver FEC processing are allocated here + // when: + // 1) A NORM_DATA message is received and the buffers have not + // been previously allocated, or + // 2) When the FEC parameters have changed (TBD) + // + if (!BuffersAllocated()) + { + DMSG(4, "NormServerNode::HandleObjectMessage() node>%lu allocating server>%lu buffers ...\n", + LocalNodeId(), GetId()); + // Currently,, our implementation requires the FEC Object Transmission Information + // to properly allocate resources + NormFtiExtension fti; + while (msg.GetNextExtension(fti)) + { + if (NormHeaderExtension::FTI == fti.GetType()) + { + // (TBD) pass "fec_id" to Open() method too + if (!AllocateBuffers(fti.GetSegmentSize(), + fti.GetFecMaxBlockLen(), + fti.GetFecNumParity())) + { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu buffer allocation error\n", + LocalNodeId(), GetId()); + // (TBD) notify app of error ?? + return; + } + break; + } + } + if (!BuffersAllocated()) + { + DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu - no FTI provided!\n", + LocalNodeId(), GetId()); + // (TBD) notify app of error ?? + return; + } + } + else + { + // (TBD) make sure FEC parameters are still the same. + } } ObjectStatus status; @@ -780,7 +866,11 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) } else { - DMSG(0, "NormServerNode::HandleObjectMessage() waiting to sync ...\n"); + if (0 == sync_id) + { + DMSG(0, "NormServerNode::HandleObjectMessage() waiting to sync ...\n"); + sync_id = 1; + } return; } } @@ -815,8 +905,11 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) } else { - obj = NULL; - DMSG(0, "NormServerNode::HandleObjectMessage() NORM_OBJECT_DATA not yet supported!\n"); + if (!(obj = new NormDataObject(session, this, objectId))) + { + DMSG(0, "NormServerNode::HandleObjectMessage() new NORM_OBJECT_DATA error: %s\n", + strerror(errno)); + } } if (obj) @@ -826,27 +919,33 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) { if (NormHeaderExtension::FTI == fti.GetType()) { - // Open receive object and notify app for accept. - if (obj->Open(fti.GetObjectSize(), msg.FlagIsSet(NormObjectMsg::FLAG_INFO))) + // Pre-open receive object and notify app for accept. + if (obj->Open(fti.GetObjectSize(), + msg.FlagIsSet(NormObjectMsg::FLAG_INFO), + fti.GetSegmentSize(), + fti.GetFecMaxBlockLen(), + fti.GetFecNumParity())) { - session->Notify(NormController::RX_OBJECT_NEW, this, obj); + session.Notify(NormController::RX_OBJECT_NEW, this, obj); if (obj->Accepted()) { - if (obj->IsStream()) - ((NormStreamObject*)obj)->StreamUpdateStatus(blockId); rx_table.Insert(obj); + obj->Retain(); + // (TBD) Do I _need_ to call "StreamUpdateStatus()" here? + if (obj->IsStream()) + (static_cast(obj))->StreamUpdateStatus(blockId); DMSG(8, "NormServerNode::HandleObjectMessage() node>%lu server>%lu new obj>%hu\n", LocalNodeId(), GetId(), (UINT16)objectId); } else { - DeleteObject(obj); + DeleteObject(obj, 7); obj = NULL; } } else { - DeleteObject(obj); + DeleteObject(obj, 1); obj = NULL; } break; @@ -856,7 +955,7 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) { DMSG(0, "NormServerNode::HandleObjectMessage() node>%lu server>%lu " "new obj>%hu - no FTI provided!\n", LocalNodeId(), GetId(), (UINT16)objectId); - DeleteObject(obj); + DeleteObject(obj, 2); obj = NULL; } } @@ -884,14 +983,31 @@ void NormServerNode::HandleObjectMessage(const NormObjectMsg& msg) #endif // !SIMULATE if (NormObject::STREAM != obj->GetType()) { - // Streams neveer complete - session->Notify(NormController::RX_OBJECT_COMPLETE, this, obj); - DeleteObject(obj); + // Streams never complete + session.Notify(NormController::RX_OBJECT_COMPLETED, this, obj); + DeleteObject(obj, 3); completion_count++; } } } - RepairCheck(NormObject::TO_BLOCK, objectId, blockId, segmentId); + + switch (repair_boundary) + { + case BLOCK_BOUNDARY: + // Normal FEC "block boundary" repair check + // (checks for repair needs for objects/blocks _prior_ to current objectId::blockId) + RepairCheck(NormObject::TO_BLOCK, objectId, blockId, segmentId); + break; + case OBJECT_BOUNDARY: + // Optional "object boundary repair check (non-streams only!) + // (checks for repair needs for objects _prior_ to current objectId) + // (also requests "info" for current objectId) + if (obj && (NormObject::STREAM == obj->GetType())) + RepairCheck(NormObject::TO_BLOCK, objectId, blockId, segmentId); + else + RepairCheck(NormObject::THRU_INFO, objectId, blockId, segmentId); + break; + } } // end NormServerNode::HandleObjectMessage() bool NormServerNode::SyncTest(const NormObjectMsg& msg) const @@ -910,6 +1026,23 @@ bool NormServerNode::SyncTest(const NormObjectMsg& msg) const } // end NormServerNode::SyncTest() + +// This method establishes the sync point "sync_id" +// objectId. The sync point is the first ordinal +// object id for which the receiver is maintaining +// reliability. Objects prior to the "sync point" +// are ignored. +// The related member variables and their purpose: +// "sync_id" - sync point object id, gets rolled upward +// in NormServerNode::SetPending() to deal with wrap +// +// "next_id" - id of next expected pending object +// (set in NormServerNode::SetPending()) +// +// "max_pending_object" - max object id heard from sender +// (inited in NormServerNode::Sync() on +// initial sync, update in NormServerNode::RepairCheck() +// void NormServerNode::Sync(NormObjectId objectId) { if (synchronized) @@ -919,12 +1052,13 @@ void NormServerNode::Sync(NormObjectId objectId) { NormObjectId lastPending; GetLastPending(lastPending); - if ((objectId > lastPending) || ((next_id - objectId) > 256)) + if ((objectId > lastPending) || ((next_id - objectId) > max_pending_range)) { NormObject* obj; while ((obj = rx_table.Find(rx_table.RangeLo()))) { - DeleteObject(obj); + session.Notify(NormController::RX_OBJECT_ABORTED, this, obj); + DeleteObject(obj, 4); failure_count++; } rx_pending_mask.Clear(); @@ -935,14 +1069,15 @@ void NormServerNode::Sync(NormObjectId objectId) while ((obj = rx_table.Find(rx_table.RangeLo())) && (obj->GetId() < objectId)) { - DeleteObject(obj); + session.Notify(NormController::RX_OBJECT_ABORTED, this, obj); + DeleteObject(obj, 5); failure_count++; } unsigned long numBits = (UINT16)(objectId - firstPending) + 1; rx_pending_mask.UnsetBits(firstPending, numBits); } } - if ((next_id < objectId) || ((next_id - objectId) > 256)) + if ((next_id < objectId) || ((next_id - objectId) > max_pending_range)) { max_pending_object = next_id = objectId; } @@ -1005,7 +1140,7 @@ NormServerNode::ObjectStatus NormServerNode::GetObjectStatus(const NormObjectId& { if (objectId < sync_id) { - if ((sync_id - objectId) > 256) + if ((sync_id - objectId) > max_pending_range) { return OBJ_INVALID; } @@ -1061,7 +1196,32 @@ NormServerNode::ObjectStatus NormServerNode::GetObjectStatus(const NormObjectId& } } // end NormServerNode::ObjectStatus() -// (TBD) mod repair check to do full server flush? +// This is a "passive" THRU_SEGMENT repair check +// (used to for watermark ack check) +bool NormServerNode::PassiveRepairCheck(NormObjectId objectId, + NormBlockId blockId, + NormSegmentId segmentId) +{ + NormObjectId nextId; + if (GetFirstPending(nextId)) + { + if (nextId < objectId) + { + return true; + } + else if (nextId == objectId) + { + NormObject* obj = rx_table.Find(nextId); + if (obj) + return obj->PassiveRepairCheck(blockId, segmentId); + else + return true; // entire object pending + } + } + return false; +} // end NormServerNode::PassiveRepairCheck() + +// This is the "active" repair check, which may activate NACKing void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, NormObjectId objectId, NormBlockId blockId, @@ -1071,6 +1231,7 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, if (objectId > max_pending_object) max_pending_object = objectId; if (!repair_timer.IsActive()) { + // repair timer inactive bool startTimer = false; NormObjectId nextId; if (GetFirstPending(nextId)) @@ -1084,9 +1245,13 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, { NormObject::CheckLevel level; if (nextId < objectId) + { level = NormObject::THRU_OBJECT; + } else + { level = checkLevel; + } startTimer |= obj->ClientRepairCheck(level, blockId, segmentId, false); } @@ -1101,13 +1266,13 @@ void NormServerNode::RepairCheck(NormObject::CheckLevel checkLevel, { // BACKOFF related code double backoffInterval = - (session->Address().IsMulticast() && (backoff_factor > 0.0)) ? + (session.Address().IsMulticast() && (backoff_factor > 0.0)) ? ExponentialRand(grtt_estimate*backoff_factor, gsize_estimate) : 0.0; repair_timer.SetInterval(backoffInterval); DMSG(4, "NormServerNode::RepairCheck() node>%lu begin NACK back-off: %lf sec)...\n", LocalNodeId(), backoffInterval); - session->ActivateTimer(repair_timer); + session.ActivateTimer(repair_timer); } } } @@ -1180,7 +1345,7 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) if (repairPending) { // We weren't completely suppressed, so build NACK - NormNackMsg* nack = (NormNackMsg*)session->GetMessageFromPool(); + NormNackMsg* nack = (NormNackMsg*)session.GetMessageFromPool(); if (!nack) { DMSG(0, "NormServerNode::OnRepairTimeout() node>%lu Warning! " @@ -1189,6 +1354,7 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) return false; } nack->Init(); + bool nackAppended = false; if (cc_enable) { @@ -1225,12 +1391,13 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) { cc_timer.Deactivate(); cc_timer.SetInterval(grtt_estimate*backoff_factor); - session->ActivateTimer(cc_timer); + session.ActivateTimer(cc_timer); cc_timer.DecrementRepeatCount(); } - } + } // end if (cc_enable) - // Iterate through rx pending object list, appending repair requests as needed + // Iterate through rx pending object list, + // appending repair requests as needed NormRepairRequest req; NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; bool iterating = GetFirstPending(nextId); @@ -1266,15 +1433,27 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) } if (prevForm != nextForm) { - if (NormRepairRequest::INVALID != prevForm) - nack->PackRepairRequest(req); // (TBD) error check + if ((NormRepairRequest::INVALID != prevForm) && + (NormObject::NACK_NONE != default_nacking_mode)) + { + if (0 == nack->PackRepairRequest(req)) + { + DMSG(0, "NormServerNode::OnRepairTimeout() warning: full NACK msg\n"); + break; + } + nackAppended = true; + } if (NormRepairRequest::INVALID != nextForm) { nack->AttachRepairRequest(req, segment_size); // (TBD) error check req.SetForm(nextForm); req.ResetFlags(); - // (TBD) An "INFO-ONLY" repair policy would be enforced here - req.SetFlag(NormRepairRequest::OBJECT); + // Set flags for missing objects according to + // default "nacking mode" + if (NormObject::NACK_INFO_ONLY == default_nacking_mode) + req.SetFlag(NormRepairRequest::INFO); + else + req.SetFlag(NormRepairRequest::OBJECT); } prevForm = nextForm; } @@ -1297,11 +1476,19 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) { if (obj->IsPending(nextId != max_pending_object)) { - if (NormRepairRequest::INVALID != prevForm) - nack->PackRepairRequest(req); // (TBD) error check + if ((NormRepairRequest::INVALID != prevForm) && + (NormObject::NACK_NONE != default_nacking_mode)) + { + if (0 == nack->PackRepairRequest(req)) + { + DMSG(0, "NormServerNode::OnRepairTimeout() warning: full NACK msg\n"); + break; + } + nackAppended = true; + } prevForm = NormRepairRequest::INVALID; bool flush = (nextId != max_pending_object); - obj->AppendRepairRequest(*nack, flush); // (TBD) error check + nackAppended |= obj->AppendRepairRequest(*nack, flush); } consecutiveCount = 0; } @@ -1317,22 +1504,41 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) } // end if (appendRequest) nextId++; iterating = GetNextPending(nextId); - } while (iterating || (0 != consecutiveCount)); - // Now that all repair requests have been appended, pack the nack - if (NormRepairRequest::INVALID != prevForm) - nack->PackRepairRequest(req); // (TBD) error check + } // end while (iterating || (0 != consecutiveCount)) + // Pack in repair request "req" if it's outstanding + if ((NormRepairRequest::INVALID != prevForm) && + (NormObject::NACK_NONE != default_nacking_mode)) + { + if (0 != nack->PackRepairRequest(req)) + nackAppended = true; + else + DMSG(0, "NormServerNode::OnRepairTimeout() warning: full NACK msg\n"); + } // Queue NACK for transmission nack->SetServerId(GetId()); nack->SetSessionId(session_id); // GRTT response is deferred until transmit time - // (TBD) support unicast nacking - if (session->UnicastNacks()) + if (unicast_nacks) nack->SetDestination(GetAddress()); else - nack->SetDestination(session->Address()); - session->QueueMessage(nack); - nack_count++; + nack->SetDestination(session.Address()); + // Debug check to make sure NACK has content + if (nackAppended) + { + ASSERT(nack->GetRepairContentLength() > 0); + session.QueueMessage(nack); + nack_count++; + } + else + { + // The nack had no repair request content, + // perhaps because of our "nacking mode" + // even though there were pending objects + DMSG(4, "NormServerNode::OnRepairTimeout() node>%lu zero content nack ...\n", + LocalNodeId()); + session.ReturnMessageToPool(nack); + } } else { @@ -1342,7 +1548,7 @@ bool NormServerNode::OnRepairTimeout(ProtoTimer& /*theTimer*/) } // end if/else(repairPending) // BACKOFF related code double holdoffInterval = - session->Address().IsMulticast() ? grtt_estimate*(backoff_factor + 2.0) : + session.Address().IsMulticast() ? grtt_estimate*(backoff_factor + 2.0) : grtt_estimate; holdoffInterval = (backoff_factor > 0.0) ? holdoffInterval : 1.01*grtt_estimate; @@ -1402,20 +1608,33 @@ void NormServerNode::UpdateRecvRate(const struct timeval& currentTime, unsigned void NormServerNode::Activate() { - if (activity_timer.IsActive()) + if (!activity_timer.IsActive()) { - activity_timer.ResetRepeat(); + activity_timer.SetInterval(grtt_estimate*NORM_ROBUST_FACTOR); + session.ActivateTimer(activity_timer); + server_active = false; } else { - activity_timer.SetInterval(grtt_estimate*NORM_ROBUST_FACTOR); - session->ActivateTimer(activity_timer); + server_active = true; } } // end NormServerNode::Activate() bool NormServerNode::OnActivityTimeout(ProtoTimer& /*theTimer*/) { - if ((activity_timer.GetRepeat() - activity_timer.GetRepeatCount()) > 1) + if (server_active) + { + activity_timer.ResetRepeat(); + } + else if (0 == activity_timer.GetRepeatCount()) + { + // Serve completely inactive? + DMSG(0, "NormServerNode::OnActivityTimeout() node>%lu server>%lu gone inactive?\n", + LocalNodeId(), GetId()); + FreeBuffers(); + // (TBD) Notify application + } + else { DMSG(4, "NormServerNode::OnActivityTimeout() node>%lu for server>%lu\n", LocalNodeId(), GetId()); @@ -1426,23 +1645,26 @@ bool NormServerNode::OnActivityTimeout(ProtoTimer& /*theTimer*/) { NormObject* objMax = rx_table.Find(max_pending_object); if (NULL != objMax) - RepairCheck(NormObject::THRU_SEGMENT, - max_pending_object, - objMax->GetMaxPendingBlockId(), - objMax->GetMaxPendingSegmentId()); + { + NormSegmentId segMax = objMax->GetMaxPendingSegmentId(); + if (0 != segMax) + RepairCheck(NormObject::THRU_SEGMENT, + max_pending_object, + objMax->GetMaxPendingBlockId(), + objMax->GetMaxPendingSegmentId() - 1); + else + RepairCheck(NormObject::TO_BLOCK, + max_pending_object, + objMax->GetMaxPendingBlockId(), + 0); + } else - RepairCheck(NormObject::THRU_SEGMENT, // (TBD) thru object??? + RepairCheck(NormObject::TO_BLOCK, // (TBD) thru object??? max_pending_object, 0, 0); } - } - if (0 == activity_timer.GetRepeatCount()) - { - DMSG(0, "NormServerNode::OnActivityTimeout() node>%lu server>%lu gone inactive?\n", - LocalNodeId(), GetId()); - Close(); - } - return true; + server_active = false; + return true; } // end NormServerNode::OnActivityTimeout() bool NormServerNode::UpdateLossEstimate(const struct timeval& currentTime, @@ -1463,6 +1685,40 @@ bool NormServerNode::UpdateLossEstimate(const struct timeval& currentTime, return result; } +void NormServerNode::AttachCCFeedback(NormAckMsg& ack) +{ + // GRTT response is deferred until transmit time + NormCCFeedbackExtension ext; + ack.AttachExtension(ext); + if (is_clr) + { + ext.SetCCFlag(NormCC::CLR); + } + else if (is_plr) + ext.SetCCFlag(NormCC::PLR); + if (rtt_confirmed) + ext.SetCCFlag(NormCC::RTT); + ext.SetCCRtt(rtt_quantized); + double ccLoss = LossEstimate(); + UINT16 lossQuantized = NormQuantizeLoss(ccLoss); + ext.SetCCLoss(lossQuantized); + if (slow_start) + { + ext.SetCCFlag(NormCC::START); + ext.SetCCRate(NormQuantizeRate(2.0 * recv_rate)); + } + else + { + double ccRate = NormSession::CalculateRate(nominal_packet_size, + rtt_estimate, ccLoss); + ext.SetCCRate(NormQuantizeRate(ccRate)); + } + //DMSG(0, "NormServerNode::OnCCTimeout() node>%lu sending ACK rate:%lf kbps (rtt:%lf loss:%lf s:%lf recvRate:%lf) slow_start:%d\n", + // LocalNodeId(), NormUnquantizeRate(ext.GetCCRate()) * (8.0/1000.0), + // rtt_estimate, ccLoss, nominal_packet_size, recv_rate*(8.0/1000.), slow_start); + ext.SetCCSequence(cc_sequence); +} // end + bool NormServerNode::OnCCTimeout(ProtoTimer& /*theTimer*/) { // Build and queue ACK() @@ -1475,7 +1731,7 @@ bool NormServerNode::OnCCTimeout(ProtoTimer& /*theTimer*/) case 1: { // We weren't suppressed, so build an ACK(RTT) and send - NormAckMsg* ack = (NormAckMsg*)session->GetMessageFromPool(); + NormAckMsg* ack = (NormAckMsg*)session.GetMessageFromPool(); if (!ack) { DMSG(0, "NormServerNode::OnCCTimeout() node>%lu Warning! " @@ -1489,48 +1745,21 @@ bool NormServerNode::OnCCTimeout(ProtoTimer& /*theTimer*/) ack->SetAckType(NormAck::CC); ack->SetAckId(0); - // GRTT response is deferred until transmit time - NormCCFeedbackExtension ext; - ack->AttachExtension(ext); - if (is_clr) - { - ext.SetCCFlag(NormCC::CLR); - } - else if (is_plr) - ext.SetCCFlag(NormCC::PLR); - if (rtt_confirmed) - ext.SetCCFlag(NormCC::RTT); - ext.SetCCRtt(rtt_quantized); - double ccLoss = LossEstimate(); - UINT16 lossQuantized = NormQuantizeLoss(ccLoss); - ext.SetCCLoss(lossQuantized); - if (slow_start) - { - ext.SetCCFlag(NormCC::START); - ext.SetCCRate(NormQuantizeRate(2.0 * recv_rate)); - } - else - { - double ccRate = NormSession::CalculateRate(nominal_packet_size, - rtt_estimate, ccLoss); - ext.SetCCRate(NormQuantizeRate(ccRate)); - } - //DMSG(0, "NormServerNode::OnCCTimeout() node>%lu sending ACK rate:%lf kbps (rtt:%lf loss:%lf s:%lf recvRate:%lf) slow_start:%d\n", - // LocalNodeId(), NormUnquantizeRate(ext.GetCCRate()) * (8.0/1000.0), - // rtt_estimate, ccLoss, nominal_packet_size, recv_rate*(8.0/1000.), slow_start); - ext.SetCCSequence(cc_sequence); - if (session->UnicastNacks()) + AttachCCFeedback(*ack); // cc feedback extension + + if (unicast_nacks) ack->SetDestination(GetAddress()); else - ack->SetDestination(session->Address()); + ack->SetDestination(session.Address()); if (is_clr || is_plr) { - session->SendMessage(*ack); - session->ReturnMessageToPool(ack); + // Don't rate limit clr or plr reps. + session.SendMessage(*ack); + session.ReturnMessageToPool(ack); } else { - session->QueueMessage(ack); + session.QueueMessage(ack); } // Begin cc_timer "holdoff" phase @@ -1546,6 +1775,59 @@ bool NormServerNode::OnCCTimeout(ProtoTimer& /*theTimer*/) return true; } // end NormServerNode::OnCCTimeout() +bool NormServerNode::OnAckTimeout(ProtoTimer& /*theTimer*/) +{ + NormAckFlushMsg* ack = (NormAckFlushMsg*)session.GetMessageFromPool(); + if (ack) + { + ack->Init(); + ack->SetServerId(GetId()); + ack->SetSessionId(session_id); + ack->SetAckId(0); + AttachCCFeedback(*ack); + ack->SetObjectId(watermark_object_id); + ack->SetFecBlockId(watermark_block_id); + ack->SetFecBlockLen(ndata); // yuk + ack->SetFecSymbolId(watermark_segment_id); + if (unicast_nacks) + ack->SetDestination(GetAddress()); + else + ack->SetDestination(session.Address()); + + if (is_clr || is_plr) + { + // Don't rate limit clr or plr reps. + session.SendMessage(*ack); + session.ReturnMessageToPool(ack); + } + else + { + session.QueueMessage(ack); + // Install cc feedback holdoff + if (cc_timer.IsActive()) cc_timer.Deactivate(); + cc_timer.SetInterval(grtt_estimate*backoff_factor); + session.ActivateTimer(cc_timer); + cc_timer.DecrementRepeatCount(); + } + } + else + { + DMSG(0, "NormServerNode::OnAckTimeout() warning: message pool exhausted!\n"); + } + return true; +} // end NormServerNode::OnAckTimeout() + + +NormAckingNode::NormAckingNode(class NormSession& theSession, NormNodeId nodeId) + : NormNode(theSession, nodeId), ack_received(false), req_count(NORM_ROBUST_FACTOR) +{ + +} + +NormAckingNode::~NormAckingNode() +{ +} + NormNodeTree::NormNodeTree() : root(NULL) { diff --git a/common/normNode.h b/common/normNode.h index d1a5832..553243b 100644 --- a/common/normNode.h +++ b/common/normNode.h @@ -14,8 +14,10 @@ class NormNode friend class NormNodeListIterator; public: - NormNode(class NormSession* theSession, NormNodeId nodeId); + NormNode(class NormSession& theSession, NormNodeId nodeId); virtual ~NormNode(); + void Retain(); + void Release(); const ProtoAddress& GetAddress() const {return addr;} void SetAddress(const ProtoAddress& address) {addr = address;} @@ -24,12 +26,13 @@ class NormNode inline const NormNodeId& LocalNodeId() const; protected: - class NormSession* session; + class NormSession& session; private: NormNodeId id; ProtoAddress addr; - // We keep NormNodes in a binary tree + unsigned int reference_count; + // We keep NormNodes in a binary tree (TBD) make this a ProtoTree NormNode* parent; NormNode* right; NormNode* left; @@ -99,24 +102,24 @@ class NormLossEstimator2 unsigned int LastLossInterval() {return history[1];} private: - enum {DEPTH = 8}; + enum {DEPTH = 8}; // Members - bool init; - unsigned long lag_mask; - unsigned int lag_depth; - unsigned long lag_test_bit; - unsigned short lag_index; + bool init; + unsigned long lag_mask; + unsigned int lag_depth; + unsigned long lag_test_bit; + unsigned short lag_index; - unsigned short event_window; - unsigned short event_index; - double event_window_time; - double event_index_time; - bool seeking_loss_event; + unsigned short event_window; + unsigned short event_index; + double event_window_time; + double event_index_time; + bool seeking_loss_event; - bool no_loss; - double initial_loss; + bool no_loss; + double initial_loss; - double loss_interval; // EWMA of loss event interval + double loss_interval; // EWMA of loss event interval unsigned long history[9]; // loss interval history double discount[9]; @@ -137,11 +140,33 @@ class NormLossEstimator2 }; // end class NormLossEstimator2 +class NormAckingNode : public NormNode +{ + public: + NormAckingNode(class NormSession& theSession, NormNodeId nodeId); + ~NormAckingNode(); + bool IsPending() const + {return (!ack_received &&( req_count > 0));} + void Reset(unsigned int maxAttempts = NORM_ROBUST_FACTOR) + { + ack_received = false; + req_count = maxAttempts; + } + void DecrementReqCount() {if (req_count > 0) req_count--;} + unsigned int GetReqCount() const {return req_count;} + bool AckReceived() const {return ack_received;} + void MarkAckReceived() {ack_received = true;} + + private: + bool ack_received; // was ack received? + unsigned int req_count; // remaining request attempts + +}; // end NormAckingNode class NormCCNode : public NormNode { public: - NormCCNode(class NormSession* theSession, NormNodeId nodeId); + NormCCNode(class NormSession& theSession, NormNodeId nodeId); ~NormCCNode(); bool IsClr() const {return is_clr;} @@ -182,9 +207,26 @@ class NormServerNode : public NormNode public: enum ObjectStatus {OBJ_INVALID, OBJ_NEW, OBJ_PENDING, OBJ_COMPLETE}; - NormServerNode(class NormSession* theSession, NormNodeId nodeId); + enum RepairBoundary {BLOCK_BOUNDARY, OBJECT_BOUNDARY}; + + NormServerNode(class NormSession& theSession, NormNodeId nodeId); ~NormServerNode(); + // Parameters + NormObject::NackingMode GetDefaultNackingMode() const + {return default_nacking_mode;} + void SetDefaultNackingMode(NormObject::NackingMode nackingMode) + {default_nacking_mode = nackingMode;} + + NormServerNode::RepairBoundary GetRepairBoundary() const + {return repair_boundary;} + // (TBD) force an appropriate RepairCheck on boundary change??? + void SetRepairBoundary(RepairBoundary repairBoundary) + {repair_boundary = repairBoundary;} + + bool UnicastNacks() {return unicast_nacks;} + void SetUnicastNacks(bool state) {unicast_nacks = state;} + bool UpdateLossEstimate(const struct timeval& currentTime, unsigned short theSequence, bool ecnStatus = false); @@ -201,10 +243,15 @@ class NormServerNode : public NormNode void HandleNackMessage(const NormNackMsg& nack); void HandleAckMessage(const NormAckMsg& ack); - bool Open(UINT16 sessionId, UINT16 segmentSize, UINT16 numData, UINT16 numParity); - void Activate(); + bool Open(UINT16 sessionId); + UINT16 GetSessionId() {return session_id;} + bool IsOpen() const {return is_open;} void Close(); - bool IsOpen() const {return is_open;} + bool AllocateBuffers(UINT16 segmentSize, UINT16 numData, UINT16 numParity); + bool BuffersAllocated() {return (0 != segment_size);} + void FreeBuffers(); + void Activate(); + bool SyncTest(const NormObjectMsg& msg) const; void Sync(NormObjectId objectId); @@ -234,7 +281,7 @@ class NormServerNode : public NormNode } void SetPending(NormObjectId objectId); - void DeleteObject(NormObject* obj); + void DeleteObject(NormObject* obj, int which); UINT16 SegmentSize() {return segment_size;} UINT16 BlockSize() {return ndata;} @@ -287,6 +334,9 @@ class NormServerNode : public NormNode private: + bool PassiveRepairCheck(NormObjectId objectId, + NormBlockId blockId, + NormSegmentId segmentId); void RepairCheck(NormObject::CheckLevel checkLevel, NormObjectId objectId, NormBlockId blockId, @@ -295,66 +345,80 @@ class NormServerNode : public NormNode bool OnActivityTimeout(ProtoTimer& theTimer); bool OnRepairTimeout(ProtoTimer& theTimer); bool OnCCTimeout(ProtoTimer& theTimer); + bool OnAckTimeout(ProtoTimer& theTimer); + + void AttachCCFeedback(NormAckMsg& ack); void HandleRepairContent(const char* buffer, UINT16 bufferLen); - UINT16 session_id; - bool synchronized; - NormObjectId sync_id; // only valid if(synchronized) - NormObjectId next_id; // only valid if(synchronized) + UINT16 session_id; + bool synchronized; + NormObjectId sync_id; // only valid if(synchronized) + NormObjectId next_id; // only valid if(synchronized) + NormObjectId max_pending_object; // index for NACK construction + NormObjectId current_object_id; // index for suppression + UINT16 max_pending_range; // max range of pending objs allowed - bool is_open; - UINT16 segment_size; - UINT16 ndata; - UINT16 nparity; + bool is_open; + UINT16 segment_size; + UINT16 ndata; + UINT16 nparity; - NormObjectTable rx_table; - NormSlidingMask rx_pending_mask; - NormSlidingMask rx_repair_mask; - NormBlockPool block_pool; - NormSegmentPool segment_pool; - NormDecoder decoder; - UINT16* erasure_loc; + NormObjectTable rx_table; + NormSlidingMask rx_pending_mask; + NormSlidingMask rx_repair_mask; + RepairBoundary repair_boundary; + NormObject::NackingMode default_nacking_mode; + bool unicast_nacks; + NormBlockPool block_pool; + NormSegmentPool segment_pool; + NormDecoder decoder; + UINT16* erasure_loc; - ProtoTimer activity_timer; - ProtoTimer repair_timer; - NormObjectId current_object_id; // index for suppression - NormObjectId max_pending_object; // index for NACK construction + bool server_active; + ProtoTimer activity_timer; + ProtoTimer repair_timer; + + // Watermark acknowledgement + ProtoTimer ack_timer; + NormObjectId watermark_object_id; + NormBlockId watermark_block_id; + NormSegmentId watermark_segment_id; // Remote server grtt measurement state - double grtt_estimate; - UINT8 grtt_quantized; - struct timeval grtt_send_time; - struct timeval grtt_recv_time; - double gsize_estimate; - UINT8 gsize_quantized; - double backoff_factor; + double grtt_estimate; + UINT8 grtt_quantized; + struct timeval grtt_send_time; + struct timeval grtt_recv_time; + double gsize_estimate; + UINT8 gsize_quantized; + double backoff_factor; // Remote server congestion control state - NormLossEstimator2 loss_estimator; - UINT16 cc_sequence; - bool cc_enable; - double cc_rate; // ccRate at start of cc_timer - ProtoTimer cc_timer; - double rtt_estimate; - UINT8 rtt_quantized; - bool rtt_confirmed; - bool is_clr; - bool is_plr; - bool slow_start; - double send_rate; // sender advertised rate - double recv_rate; // measured recv rate - struct timeval prev_update_time; // for recv_rate measurement - unsigned long recv_accumulator; // for recv_rate measurement - double nominal_packet_size; + NormLossEstimator2 loss_estimator; + UINT16 cc_sequence; + bool cc_enable; + double cc_rate; // ccRate at start of cc_timer + ProtoTimer cc_timer; + double rtt_estimate; + UINT8 rtt_quantized; + bool rtt_confirmed; + bool is_clr; + bool is_plr; + bool slow_start; + double send_rate; // sender advertised rate + double recv_rate; // measured recv rate + struct timeval prev_update_time; // for recv_rate measurement + unsigned long recv_accumulator; // for recv_rate measurement + double nominal_packet_size; // For statistics tracking - unsigned long recv_total; // total recvd accumulator - unsigned long recv_goodput; // goodput recvd accumulator - unsigned long resync_count; - unsigned long nack_count; - unsigned long suppress_count; - unsigned long completion_count; - unsigned long failure_count; // usually due to re-syncs + unsigned long recv_total; // total recvd accumulator + unsigned long recv_goodput; // goodput recvd accumulator + unsigned long resync_count; + unsigned long nack_count; + unsigned long suppress_count; + unsigned long completion_count; + unsigned long failure_count; // usually due to re-syncs }; // end class NormServerNode diff --git a/common/normObject.cpp b/common/normObject.cpp index dcd5f40..cae4677 100644 --- a/common/normObject.cpp +++ b/common/normObject.cpp @@ -4,23 +4,23 @@ #include NormObject::NormObject(NormObject::Type theType, - class NormSession* theSession, + class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& transportId) - : type(theType), session(theSession), server(theServer), + : type(theType), session(theSession), server(theServer), reference_count(0), transport_id(transportId), segment_size(0), pending_info(false), repair_info(false), current_block_id(0), next_segment_id(0), max_pending_block(0), max_pending_segment(0), - info(NULL), info_len(0), accepted(false) + info(NULL), info_len(0), accepted(false), notify_on_update(true) { + if (theServer) + nacking_mode = theServer->GetDefaultNackingMode(); + else + nacking_mode = NACK_NORMAL; // it doesn't really matter if !theServer } NormObject::~NormObject() { - /*if (server) - server->DeleteObject(true); - else - session->DeleteTxObject(true);*/ Close(); if (info) { @@ -29,26 +29,46 @@ NormObject::~NormObject() } } +void NormObject::Retain() +{ + reference_count++; + if (server) server->Retain(); +} // end NormObject::Retain() + +void NormObject::Release() +{ + if (server) server->Release(); + if (reference_count) + reference_count--; + else + DMSG(0, "NormObject::Release() releasing non-retained object?!\n"); + if (0 == reference_count) delete this; +} // end NormObject::Release() + // This is mainly used for debug messages NormNodeId NormObject::LocalNodeId() const { - return session->LocalNodeId(); -} + return session.LocalNodeId(); +} // end NormObject::LocalNodeId() + +NormNodeId NormObject::GetServerNodeId() const +{ + return server ? server->GetId() : NORM_NODE_NONE; +} // end NormObject::GetServerNodeId() bool NormObject::Open(const NormObjectSize& objectSize, const char* infoPtr, - UINT16 infoLen) + UINT16 infoLen, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity) { // Note "objectSize" represents actual total object size for // DATA or FILE objects, buffer size for STREAM objects // In either case, we need our sliding bit masks to be of // appropriate size. - UINT16 segmentSize, numData, numParity; if (server) { - segmentSize = server->SegmentSize(); - numData = server->BlockSize(); // max source symbols per FEC block - numParity = server->NumParity(); // max parity symbols per FEC block if (infoLen > 0) { pending_info = true; @@ -62,9 +82,6 @@ bool NormObject::Open(const NormObjectSize& objectSize, } else { - segmentSize = session->ServerSegmentSize(); - numData = session->ServerBlockSize(); // max source symbols per FEC block - numParity = session->ServerNumParity(); // max parity symbols per FEC block if (infoPtr) { if (info) delete []info; @@ -178,7 +195,7 @@ void NormObject::Close() if (server) server->PutFreeBlock(block); else - session->ServerPutFreeBlock(block); + session.ServerPutFreeBlock(block); } repair_mask.Destroy(); pending_mask.Destroy(); @@ -186,14 +203,15 @@ void NormObject::Close() segment_size = 0; } // end NormObject::Close(); +// Used by server bool NormObject::HandleInfoRequest() { - // (TBD) immediately make info pending? bool increasedRepair = false; if (info) { if (!repair_info) { + pending_info = true; repair_info = true; increasedRepair = true; } @@ -251,7 +269,7 @@ bool NormObject::TxReset(NormBlockId firstBlock) { increasedRepair |= block->TxReset(GetBlockSize(blockId), nparity, - session->ServerAutoParity(), + session.ServerAutoParity(), segment_size); } } @@ -261,7 +279,7 @@ bool NormObject::TxReset(NormBlockId firstBlock) bool NormObject::TxResetBlocks(NormBlockId nextId, NormBlockId lastId) { bool increasedRepair = false; - UINT16 autoParity = session->ServerAutoParity(); + UINT16 autoParity = session.ServerAutoParity(); lastId++; while (nextId != lastId) { @@ -294,10 +312,10 @@ bool NormObject::ActivateRepairs() { NormBlockId lastId; ASSERT(GetLastRepair(lastId)); - DMSG(6, "NormObject::ActivateRepairs() node>%lu obj>%hu activating blk>%lu->%lu repairs\n", + DMSG(6, "NormObject::ActivateRepairs() node>%lu obj>%hu activating blk>%lu->%lu block repairs ...\n", LocalNodeId(), (UINT16)transport_id, (UINT32)nextId, (UINT32)lastId); repairsActivated = true; - UINT16 autoParity = session->ServerAutoParity(); + UINT16 autoParity = session.ServerAutoParity(); do { NormBlock* block = block_buffer.Find(nextId); @@ -332,9 +350,9 @@ bool NormObject::ActivateRepairs() bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) { // Determine range of blocks possibly pending repair - NormBlockId nextId; + NormBlockId nextId = 0; GetFirstRepair(nextId); - NormBlockId endId; + NormBlockId endId = 0; GetLastRepair(endId); if (block_buffer.IsEmpty()) { @@ -356,10 +374,12 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) } endId++; } + // Instantiate a repair request for BLOCK level repairs NormRepairRequest req; + bool requestAppended = false; req.SetFlag(NormRepairRequest::BLOCK); - if (repair_info) req.SetFlag(NormRepairRequest::INFO); + if (repair_info) req.SetFlag(NormRepairRequest::INFO); NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; // Iterate through the range of blocks possibly pending repair NormBlockId firstId; @@ -396,7 +416,14 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) if (form != prevForm) { if (NormRepairRequest::INVALID != prevForm) - cmd.PackRepairRequest(req); // (TBD) error check + { + if (0 == cmd.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairAdv() warning: full msg\n"); + return requestAppended; + } + requestAppended = true; + } cmd.AttachRepairRequest(req, segment_size); // (TBD) error check req.SetForm(form); prevForm = form; @@ -428,18 +455,79 @@ bool NormObject::AppendRepairAdv(NormCmdRepairAdvMsg& cmd) { if (NormRepairRequest::INVALID != prevForm) { - cmd.PackRepairRequest(req); // (TBD) error check + if (0 == cmd.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairAdv() warning: full msg\n"); + return requestAppended; + } prevForm = NormRepairRequest::INVALID; } block->AppendRepairAdv(cmd, transport_id, repair_info, GetBlockSize(currentId), segment_size); // (TBD) error check + requestAppended = true; } } } // end while(nextId < endId) if (NormRepairRequest::INVALID != prevForm) - cmd.PackRepairRequest(req); // (TBD) error check + { + if (0 == cmd.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairAdv() warning: full msg\n"); + return requestAppended; + } + requestAppended = true; + } + else if (repair_info && !requestAppended) + { + // make the "req" an INFO-only request + req.ClearFlag(NormRepairRequest::BLOCK); + req.SetForm(NormRepairRequest::ITEMS); + req.AppendRepairItem(transport_id, 0, 0, 0); + if (0 == cmd.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairAdv() warning: full msg\n"); + return requestAppended; + } + } return true; } // end NormObject::AppendRepairAdv() +// This is used by server for watermark check +bool NormObject::FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) const +{ + if (repair_info) + { + blockId = 0; + segmentId = 0; + return true; + } + NormBlockBuffer::Iterator iterator(block_buffer); + NormBlock* block; + while ((block = iterator.GetNextBlock())) + if (block->IsRepairPending()) break; + if (GetFirstRepair(blockId)) + { + if (!block || (blockId <= block->GetId())) + { + segmentId = 0; + return true; + } + } + if (block) + { +#ifdef PROTO_DEBUG + ASSERT(block->GetFirstRepair(segmentId)); +#else + block->GetFirstRepair(segmentId); +#endif // if/else PROTO_DEBUG + // The segmentId must < block length for watermarks + if (segmentId >= GetBlockSize(block->GetId())) + segmentId = GetBlockSize(block->GetId()) - 1; + return true; + } + return false; +} // end NormObject::FindRepairIndex() + + // Called by server only bool NormObject::IsRepairPending() const { @@ -509,6 +597,46 @@ bool NormObject::IsPending(bool flush) const } } // end NormObject::IsPending() +// This is a "passive" THRU_SEGMENT repair check +// (used to for watermark ack check) +bool NormObject::PassiveRepairCheck(NormBlockId blockId, + NormSegmentId segmentId) +{ + if (pending_info) return true; + NormBlockId firstPendingBlock; + if (GetFirstPending(firstPendingBlock)) + { + if (firstPendingBlock < blockId) + { + return true; + } + else if (firstPendingBlock == blockId) + { + + NormBlock* block = block_buffer.Find(firstPendingBlock); + if (block) + { + NormSegmentId firstPendingSegment; + if (block->GetFirstPending(firstPendingSegment)) + { + if (segmentId > firstPendingSegment) + return true; + else + return false; + } + else + { + ASSERT(0); + } + } + else + { + return true; // entire block was pending + } + } + } + return false; +} // end NormObject::PassiveRepairCheck() bool NormObject::ClientRepairCheck(CheckLevel level, NormBlockId blockId, @@ -520,6 +648,8 @@ bool NormObject::ClientRepairCheck(CheckLevel level, // and "next_segment_id" to be > blockSize switch (level) { + case TO_OBJECT: + return false; case THRU_INFO: break; case TO_BLOCK: @@ -676,7 +806,8 @@ bool NormObject::ClientRepairCheck(CheckLevel level, return needRepair; } // end NormObject::ClientRepairCheck() -// Note this clears "repair_mask" state (called on client repair_timer timeout) +// Note this clears "repair_mask" state +// (called on client repair_timer timeout) bool NormObject::IsRepairPending(bool flush) { ASSERT(server); @@ -722,150 +853,179 @@ bool NormObject::AppendRepairRequest(NormNackMsg& nack, { // If !flush, we request only up _to_ max_pending_block::max_pending_segment. NormRepairRequest req; + bool requestAppended = false; // is set to true when content added to "nack" NormRepairRequest::Form prevForm = NormRepairRequest::INVALID; + // First iterate over any pending blocks, appending any requests NormBlockId nextId; bool iterating = GetFirstPending(nextId); - if (iterating) + NormBlockId prevId = nextId; + iterating = iterating && (flush || (nextId <= max_pending_block)); + UINT32 consecutiveCount = 0; + while (iterating || (0 != consecutiveCount)) { - iterating = iterating && (flush || (nextId <= max_pending_block)); - NormBlockId prevId = nextId; - UINT32 consecutiveCount = 0; - while (iterating || (0 != consecutiveCount)) + NormBlockId lastId; + ASSERT(GetLastPending(lastId)); + DMSG(6, "NormObject::AppendRepairRequest() node>%lu obj>%hu, blk>%lu->%lu (maxPending:%lu)\n", + LocalNodeId(), (UINT16)transport_id, + (UINT32)nextId, (UINT32)lastId, (UINT32)max_pending_block); + bool appendRequest = false; + NormBlock* block = iterating ? block_buffer.Find(nextId) : NULL; + if (block) + appendRequest = true; + else if (iterating && ((UINT32)(nextId - prevId) == consecutiveCount)) + consecutiveCount++; + else + appendRequest = true; + if (appendRequest) { - NormBlockId lastId; - ASSERT(GetLastPending(lastId)); - DMSG(6, "NormObject::AppendRepairRequest() node>%lu obj>%hu, blk>%lu->%lu (maxPending:%lu)\n", - LocalNodeId(), (UINT16)transport_id, - (UINT32)nextId, (UINT32)lastId, (UINT32)max_pending_block); - bool appendRequest = false; - NormBlock* block = iterating ? block_buffer.Find(nextId) : NULL; - if (block) - appendRequest = true; - else if (iterating && ((UINT32)(nextId - prevId) == consecutiveCount)) + NormRepairRequest::Form nextForm; + switch(consecutiveCount) { - consecutiveCount++; - } - else - appendRequest = true; - if (appendRequest) + case 0: + nextForm = NormRepairRequest::INVALID; + break; + case 1: + case 2: + nextForm = NormRepairRequest::ITEMS; + break; + default: + nextForm = NormRepairRequest::RANGES; + break; + } // end switch(reqCount) + if (prevForm != nextForm) { - NormRepairRequest::Form nextForm; - switch(consecutiveCount) + if ((NormRepairRequest::INVALID != prevForm) && + (NACK_NONE != nacking_mode)) { - case 0: - nextForm = NormRepairRequest::INVALID; - break; - case 1: - case 2: - nextForm = NormRepairRequest::ITEMS; - break; - default: - nextForm = NormRepairRequest::RANGES; - break; - } // end switch(reqCount) - if (prevForm != nextForm) - { - if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - if (NormRepairRequest::INVALID != nextForm) + if (0 == nack.PackRepairRequest(req)) { - nack.AttachRepairRequest(req, segment_size); - req.SetForm(nextForm); - req.ResetFlags(); - req.SetFlag(NormRepairRequest::BLOCK); - if (pending_info) req.SetFlag(NormRepairRequest::INFO); + DMSG(0, "NormObject::AppendRepairRequest() warning: full NACK msg\n"); + return requestAppended; } - prevForm = nextForm; + requestAppended = true; } if (NormRepairRequest::INVALID != nextForm) - DMSG(6, "NormObject::AppendRepairRequest() BLOCK request\n"); - switch (nextForm) { - case NormRepairRequest::ITEMS: - req.AppendRepairItem(transport_id, prevId, GetBlockSize(prevId), 0); // (TBD) error check - if (2 == consecutiveCount) - { - prevId++; - req.AppendRepairItem(transport_id, prevId, GetBlockSize(prevId), 0); // (TBD) error check - } - break; - case NormRepairRequest::RANGES: - { - NormBlockId lastId = prevId+consecutiveCount-1; - req.AppendRepairRange(transport_id, prevId, GetBlockSize(prevId), 0, - transport_id, lastId, GetBlockSize(lastId), 0); // (TBD) error check - break; - } - default: - break; - } // end switch(nextForm) - if (block) - { - bool blockPending = false; - if (nextId == max_pending_block) - { - NormSymbolId firstPending = 0; - if (!block->GetFirstPending(firstPending)) ASSERT(0); - if (firstPending < max_pending_segment) blockPending = true; - } - else - { - blockPending = true; - } - if (blockPending) - { - UINT16 numData = GetBlockSize(nextId); - if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - if (flush || (nextId != max_pending_block)) - { - block->AppendRepairRequest(nack, numData, nparity, transport_id, - pending_info, segment_size); // (TBD) error check - } - else - { - if (max_pending_segment < numData) - block->AppendRepairRequest(nack, max_pending_segment, 0, transport_id, - pending_info, segment_size); // (TBD) error check - else - block->AppendRepairRequest(nack, numData, nparity, transport_id, - pending_info, segment_size); // (TBD) error check - } - } - consecutiveCount = 0; - prevForm = NormRepairRequest::INVALID; + nack.AttachRepairRequest(req, segment_size); + req.SetForm(nextForm); + req.ResetFlags(); + if (NACK_NORMAL == nacking_mode) + req.SetFlag(NormRepairRequest::BLOCK); + if (pending_info) + req.SetFlag(NormRepairRequest::INFO); } - else if (iterating) + prevForm = nextForm; + } + if (NormRepairRequest::INVALID != nextForm) + DMSG(6, "NormObject::AppendRepairRequest() BLOCK request\n"); + switch (nextForm) + { + case NormRepairRequest::ITEMS: + req.AppendRepairItem(transport_id, prevId, GetBlockSize(prevId), 0); // (TBD) error check + if (2 == consecutiveCount) + { + prevId++; + req.AppendRepairItem(transport_id, prevId, GetBlockSize(prevId), 0); // (TBD) error check + } + break; + case NormRepairRequest::RANGES: { - consecutiveCount = 1; + NormBlockId lastId = prevId+consecutiveCount-1; + req.AppendRepairRange(transport_id, prevId, GetBlockSize(prevId), 0, + transport_id, lastId, GetBlockSize(lastId), 0); // (TBD) error check + break; + } + default: + break; + } // end switch(nextForm) + if (block) + { + bool blockIsPending = false; + if (nextId == max_pending_block) + { + NormSymbolId firstPending = 0; + if (!block->GetFirstPending(firstPending)) ASSERT(0); + if (firstPending < max_pending_segment) blockIsPending = true; } else { - consecutiveCount = 0; // we're all done + blockIsPending = true; } - prevId = nextId; - } // end if (appendRequest) - nextId++; - iterating = GetNextPending(nextId); - //DMSG(0, "got next pending>%lu result:%d\n", (UINT32)nextId, iterating); - iterating = iterating && (flush || (nextId <= max_pending_block)); - } // end while (iterating || (0 != consecutiveCount)) - //DMSG(0, "bailed ...\n"); - } - else + if (blockIsPending && (NACK_NONE != nacking_mode)) + { + UINT16 numData = GetBlockSize(nextId); + if (NormRepairRequest::INVALID != prevForm) + { + if (0 == nack.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairRequest() warning: full NACK msg\n"); + return requestAppended; + } + } + if (flush || (nextId != max_pending_block)) + { + block->AppendRepairRequest(nack, numData, nparity, transport_id, + pending_info, segment_size); // (TBD) error check + } + else + { + if (max_pending_segment < numData) + block->AppendRepairRequest(nack, max_pending_segment, 0, transport_id, + pending_info, segment_size); // (TBD) error check + else + block->AppendRepairRequest(nack, numData, nparity, transport_id, + pending_info, segment_size); // (TBD) error check + } + requestAppended = true; + } + consecutiveCount = 0; + prevForm = NormRepairRequest::INVALID; + } + else if (iterating) + { + consecutiveCount = 1; + } + else + { + consecutiveCount = 0; // we're all done + } + prevId = nextId; + } // end if (appendRequest) + nextId++; + iterating = GetNextPending(nextId); + //DMSG(0, "got next pending>%lu result:%d\n", (UINT32)nextId, iterating); + iterating = iterating && (flush || (nextId <= max_pending_block)); + } // end while (iterating || (0 != consecutiveCount)) + + // This conditional makes sure any outstanding requests constructed + // are packed into the nack message. + if ((NormRepairRequest::INVALID != prevForm) && + (NACK_NONE != nacking_mode)) + { + if (0 == nack.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairRequest() warning: full NACK msg\n"); + return requestAppended; + } + requestAppended = true; + prevForm = NormRepairRequest::INVALID; + } + if (!requestAppended && pending_info && (NACK_NONE != nacking_mode)) { // INFO_ONLY repair request - ASSERT(pending_info); nack.AttachRepairRequest(req, segment_size); - prevForm = NormRepairRequest::ITEMS; req.SetForm(NormRepairRequest::ITEMS); req.ResetFlags(); req.SetFlag(NormRepairRequest::INFO); req.AppendRepairItem(transport_id, 0, 0, 0); // (TBD) error check - } // end if/else (iterating) - if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - return true; + if (0 == nack.PackRepairRequest(req)) + { + DMSG(0, "NormObject::AppendRepairRequest() warning: full NACK msg\n"); + return requestAppended; + } + requestAppended = true; + } + return requestAppended; } // end NormObject::AppendRepairRequest() @@ -889,7 +1049,7 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, } memcpy(info, infoMsg.GetInfo(), info_len); pending_info = false; - session->Notify(NormController::RX_OBJECT_INFO, server, this); + session.Notify(NormController::RX_OBJECT_INFO, server, this); } else { @@ -902,6 +1062,8 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, else // NORM_MSG_DATA { const NormDataMsg& data = (const NormDataMsg&)msg; + UINT16 numData = GetBlockSize(blockId); + // For stream objects, a little extra mgmt is required if (STREAM == type) { @@ -915,7 +1077,6 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, // ??? Ignore this new packet and try to fix stream ??? //return; server->IncrementResyncCount(); - while (!stream->StreamUpdateStatus(blockId)) { // Server is too far ahead of me ... @@ -937,7 +1098,6 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, } } } - UINT16 numData = GetBlockSize(blockId); if (pending_mask.Test(blockId)) { NormBlock* block = block_buffer.Find(blockId); @@ -993,12 +1153,17 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, segment[payloadMax] = 0; block->AttachSegment(segmentId, segment); block->UnsetPending(segmentId); + bool objectUpdated = false; // 2) Write segment to object (if it's data) if (segmentId < numData) { block->DecrementErasureCount(); if (WriteSegment(blockId, segmentId, segment)) + { + objectUpdated = true; + // For statistics only (TBD) #ifdef NORM_DEBUG server->IncrementRecvGoodput(segmentLength); + } } else { @@ -1052,6 +1217,7 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, { if (WriteSegment(blockId, sid, block->Segment(sid))) { + objectUpdated = true; // For statistics only (TBD) #ifdef NORM_DEBUG server->IncrementRecvGoodput(segmentLength); } @@ -1070,7 +1236,11 @@ void NormObject::HandleObjectMessage(const NormObjectMsg& msg, // Notify application of new data available // (TBD) this could be improved for stream objects // so it's not called unnecessarily - session->Notify(NormController::RX_OBJECT_UPDATE, server, this); + if (objectUpdated && notify_on_update) + { + notify_on_update = false; + session.Notify(NormController::RX_OBJECT_UPDATE, server, this); + } } else { @@ -1215,24 +1385,29 @@ bool NormObject::NextServerMsg(NormObjectMsg* msg) return true; } NormBlockId blockId; - if (!GetFirstPending(blockId)) return false; + if (!GetFirstPending(blockId)) + { + if (!IsStream()) + DMSG(0, "NormObject::NextServerMsg() pending object w/ no pending blocks?\n"); + return false; + } NormDataMsg* data = (NormDataMsg*)msg; UINT16 numData = GetBlockSize(blockId); NormBlock* block = block_buffer.Find(blockId); if (!block) { - if (!(block = session->ServerGetFreeBlock(transport_id, blockId))) + if (!(block = session.ServerGetFreeBlock(transport_id, blockId))) { - //DMSG(2, "NormObject::NextServerMsg() node>%lu Warning! server resource " - // "constrained (no free blocks).\n", LocalNodeId()); + DMSG(2, "NormObject::NextServerMsg() node>%lu Warning! server resource " + "constrained (no free blocks).\n", LocalNodeId()); return false; } // Load block with zero initialized parity segments UINT16 totalBlockLen = numData + nparity; for (UINT16 i = numData; i < totalBlockLen; i++) { - char* s = session->ServerGetFreeSegment(transport_id, blockId); + char* s = session.ServerGetFreeSegment(transport_id, blockId); if (s) { UINT16 payloadMax = segment_size + NormDataMsg::GetStreamPayloadHeaderLength(); @@ -1244,28 +1419,42 @@ bool NormObject::NextServerMsg(NormObjectMsg* msg) } else { - //DMSG(2, "NormObject::NextServerMsg() node>%lu Warning! server resource " - // "constrained (no free segments).\n", LocalNodeId()); - session->ServerPutFreeBlock(block); + DMSG(12, "NormObject::NextServerMsg() node>%lu Warning! server resource " + "constrained (no free segments).\n", LocalNodeId()); + session.ServerPutFreeBlock(block); return false; } } - block->TxInit(blockId, numData, session->ServerAutoParity()); + block->TxInit(blockId, numData, session.ServerAutoParity()); if (!block_buffer.Insert(block)) { ASSERT(STREAM == type); ASSERT(blockId > block_buffer.RangeLo()); - NormBlock* b = block_buffer.Find(block_buffer.RangeLo()); - ASSERT(b); - block_buffer.Remove(b); - session->ServerPutFreeBlock(b); - bool success = block_buffer.Insert(block); - ASSERT(success); + //if (blockId > block_buffer.RangeLo()) + { + NormBlock* b = block_buffer.Find(block_buffer.RangeLo()); + ASSERT(b); + block_buffer.Remove(b); + session.ServerPutFreeBlock(b); + bool success = block_buffer.Insert(block); + ASSERT(success); + } + /*else + { + DMSG(0, "NormObject::NextServerMsg() node>%lu Warning! can't repair old block\n", LocalNodeId()); + session.ServerPutFreeBlock(block); + pending_mask.Unset(blockId); + return false; + }*/ } } NormSegmentId segmentId = 0; - if (!block->GetFirstPending(segmentId)) ASSERT(0); + if (!block->GetFirstPending(segmentId)) + { + DMSG(0, "NormObject::NextServerMsg() nothing pending!?\n"); + ASSERT(0); + } // Try to read segment if (segmentId < numData) @@ -1277,7 +1466,8 @@ bool NormObject::NextServerMsg(NormObjectMsg* msg) { // (TBD) deal with read error //(for streams, it currently means the stream is non-pending) - if (!IsStream()) DMSG(0, "NormObject::NextServerMsg() ReadSegment() error\n"); + if (!IsStream()) + DMSG(0, "NormObject::NextServerMsg() ReadSegment() error\n"); return false; } data->SetPayloadLength(payloadLength); @@ -1314,7 +1504,7 @@ bool NormObject::NextServerMsg(NormObjectMsg* msg) if (payloadLength < payloadMax) memset(buffer+payloadLength, 0, payloadMax-payloadLength); // (TBD) the encode routine could update the block's parity readiness - session->ServerEncode(data->AccessPayload(), block->SegmentList(numData)); + session.ServerEncode(data->AccessPayload(), block->SegmentList(numData)); block->IncreaseParityReadiness(); } } @@ -1377,14 +1567,14 @@ void NormStreamObject::StreamAdvance() } else { - DMSG(4, "NormStreamObject::StreamAdvance() node>%lu Pending segment repairs (blk>%lu) " + DMSG(0, "NormStreamObject::StreamAdvance() node>%lu Pending segment repairs (blk>%lu) " "delaying stream advance ...\n", LocalNodeId(), (UINT32)block->GetId()); } } } else { - DMSG(0, "NormStreamObject::StreamAdvance() Pending block repair delaying stream advance ...\n"); + DMSG(0, "NormStreamObject::StreamAdvance() pending block repair delaying stream advance ...\n"); } } // end NormStreamObject::StreamAdvance() @@ -1402,8 +1592,8 @@ bool NormObject::CalculateBlockParity(NormBlock* block) payloadMax = MIN(payloadMax, SIM_PAYLOAD_MAX); #endif // SIMULATE if (payloadLength < payloadMax) - memset(buffer+payloadLength, 0, payloadMax-payloadLength); - session->ServerEncode(buffer, block->SegmentList(numData)); + memset(buffer+payloadLength, 0, payloadMax-payloadLength+1); + session.ServerEncode(buffer, block->SegmentList(numData)); } else { @@ -1416,7 +1606,7 @@ bool NormObject::CalculateBlockParity(NormBlock* block) NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) { - NormBlock* block = session->ServerGetFreeBlock(transport_id, blockId); + NormBlock* block = session.ServerGetFreeBlock(transport_id, blockId); if (block) { UINT16 numData = GetBlockSize(blockId); @@ -1426,7 +1616,7 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) UINT16 totalBlockLen = numData + nparity; for (UINT16 i = numData; i < totalBlockLen; i++) { - char* s = session->ServerGetFreeSegment(transport_id, blockId); + char* s = session.ServerGetFreeSegment(transport_id, blockId); if (s) { UINT16 payloadMax = segment_size + NormDataMsg::GetStreamPayloadHeaderLength(); @@ -1440,7 +1630,7 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) { //DMSG(2, "NormObject::ServerRecoverBlock() node>%lu Warning! server resource " // "constrained (no free segments).\n", LocalNodeId()); - session->ServerPutFreeBlock(block); + session.ServerPutFreeBlock(block); return (NormBlock*)NULL; } } @@ -1449,7 +1639,7 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) { if (!block_buffer.Insert(block)) { - session->ServerPutFreeBlock(block); + session.ServerPutFreeBlock(block); DMSG(4, "NormObject::ServerRecoverBlock() node>%lu couldn't buffer recovered block\n"); return NULL; } @@ -1457,7 +1647,7 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) } else { - session->ServerPutFreeBlock(block); + session.ServerPutFreeBlock(block); return (NormBlock*)NULL; } } @@ -1473,7 +1663,7 @@ NormBlock* NormObject::ServerRecoverBlock(NormBlockId blockId) // // NormFileObject Implementation // -NormFileObject::NormFileObject(class NormSession* theSession, +NormFileObject::NormFileObject(class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId) : NormObject(FILE, theSession, theServer, objectId), @@ -1520,10 +1710,16 @@ bool NormFileObject::Open(const char* thePath, // We're sending this file if (file.Open(thePath, O_RDONLY)) { - UINT32 size = file.GetSize(); + off_t size = file.GetSize(); if (size) { - if (!NormObject::Open(size, infoPtr, infoLen)) + NormObjectSize osize((off_t)size); + if (!NormObject::Open(NormObjectSize(size), + infoPtr, + infoLen, + session.ServerSegmentSize(), + session.ServerBlockSize(), + session.ServerNumParity())) { DMSG(0, "NormFileObject::Open() send object open error\n"); Close(); @@ -1590,9 +1786,9 @@ bool NormFileObject::WriteSegment(NormBlockId blockId, // Determine segment offset from blockId::segmentId NormObjectSize segmentOffset; NormObjectSize segmentSize = NormObjectSize(segment_size); - if ((UINT32)blockId < large_block_count) + if (((UINT32)blockId) < large_block_count) { - segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + segmentOffset = (large_block_length*(UINT32)blockId) + (segmentSize*segmentId); } else { @@ -1601,8 +1797,7 @@ bool NormFileObject::WriteSegment(NormBlockId blockId, segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } - off_t offsetScaleMSB = 0xffffffff + 1; // yuk! can't we do better? - off_t offset = (off_t)segmentOffset.LSB() + ((off_t)segmentOffset.MSB() * offsetScaleMSB); + off_t offset = segmentOffset.GetOffset(); if (offset != file.GetOffset()) { if (!file.Seek(offset)) return false; @@ -1644,8 +1839,9 @@ UINT16 NormFileObject::ReadSegment(NormBlockId blockId, segmentOffset = segmentOffset + small_block_length*smallBlockIndex + segmentSize*segmentId; } - off_t offsetScaleMSB = 0xffffffff + 1; // yuk! can't we do better - off_t offset = (off_t)segmentOffset.LSB() + ((off_t)segmentOffset.MSB() * offsetScaleMSB); + //off_t offsetScaleMSB = 0xffffffff + 1; // yuk! can't we do better + //off_t offset = (off_t)segmentOffset.LSB() + ((off_t)segmentOffset.MSB() * offsetScaleMSB); + off_t offset = segmentOffset.GetOffset(); if (offset != file.GetOffset()) { if (!file.Seek(offset)) @@ -1655,17 +1851,168 @@ UINT16 NormFileObject::ReadSegment(NormBlockId blockId, return (len == nbytes) ? len : 0; } // end NormFileObject::ReadSegment() +///////////////////////////////////////////////////////////////// +// +// NormDataObject Implementation +// +NormDataObject::NormDataObject(class NormSession& theSession, + class NormServerNode* theServer, + const NormObjectId& objectId) + : NormObject(DATA, theSession, theServer, objectId), + large_block_length(0,0), small_block_length(0,0), + data_ptr(NULL), data_max(0) +{ + +} + +NormDataObject::~NormDataObject() +{ + Close(); +} + +// Assign data object to data ptr +bool NormDataObject::Open(char* dataPtr, + UINT32 dataLen, + const char* infoPtr, + UINT16 infoLen) +{ + if (server) + { + // We're receiving this data object + ASSERT(NULL == infoPtr); + } + else + { + // We're sending this data object + if (!NormObject::Open(dataLen, + infoPtr, + infoLen, + session.ServerSegmentSize(), + session.ServerBlockSize(), + session.ServerNumParity())) + { + DMSG(0, "NormDataObject::Open() send object open error\n"); + Close(); + return false; + } + } + data_ptr = dataPtr; + data_max = dataLen; + large_block_length = NormObjectSize(large_block_size) * segment_size; + small_block_length = NormObjectSize(small_block_size) * segment_size; + return true; +} // end NormDataObject::Open() + +bool NormDataObject::Accept(char* dataPtr, UINT32 dataMax) +{ + ASSERT(NULL == server); + if (Open(dataPtr, dataMax)) + { + NormObject::Accept(); + return true; + } + else + { + return false; + } +} // end NormDataObject::Accept() + +void NormDataObject::Close() +{ + NormObject::Close(); +} // end NormDataObject::Close() + +bool NormDataObject::WriteSegment(NormBlockId blockId, + NormSegmentId segmentId, + const char* buffer) +{ + UINT16 len; + if (blockId == final_block_id) + { + if (segmentId == (GetBlockSize(blockId)-1)) + len = final_segment_size; + else + len = segment_size; + } + else + { + len = segment_size; + } + // Determine segment offset from blockId::segmentId + NormObjectSize segmentOffset; + NormObjectSize segmentSize = NormObjectSize(segment_size); + if ((UINT32)blockId < large_block_count) + { + segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + } + else + { + segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this + UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + segmentOffset = segmentOffset + small_block_length*smallBlockIndex + + segmentSize*segmentId; + } + ASSERT(0 == segmentOffset.MSB()); + if (data_max <= segmentOffset.LSB()) + return true; + else if (data_max <= (segmentOffset.LSB() + len)) + len -= (segmentOffset.LSB() + len - data_max); + memcpy(data_ptr + segmentOffset.LSB(), buffer, len); + return true; +} // end NormDataObject::WriteSegment() + + +UINT16 NormDataObject::ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer) +{ + // Determine segment length from blockId::segmentId + UINT16 len; + if (blockId == final_block_id) + { + if (segmentId == (GetBlockSize(blockId)-1)) + len = final_segment_size; + else + len = segment_size; + } + else + { + len = segment_size; + } + + // Determine segment offset from blockId::segmentId + NormObjectSize segmentOffset; + NormObjectSize segmentSize = NormObjectSize(segment_size); + if ((UINT32)blockId < large_block_count) + { + segmentOffset = large_block_length*(UINT32)blockId + segmentSize*segmentId; + } + else + { + segmentOffset = large_block_length*large_block_count; // (TBD) pre-calc this + UINT32 smallBlockIndex = (UINT32)blockId - large_block_count; + segmentOffset = segmentOffset + small_block_length*smallBlockIndex + + segmentSize*segmentId; + } + ASSERT(0 == segmentOffset.MSB()); + ASSERT(data_max >= (segmentOffset.LSB() + len)); + memcpy(buffer, data_ptr + segmentOffset.LSB(), len); + return true; +} // end NormDataObject::ReadSegment() + ///////////////////////////////////////////////////////////////// // // NormStreamObject Implementation // -NormStreamObject::NormStreamObject(class NormSession* theSession, +NormStreamObject::NormStreamObject(class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId) : NormObject(STREAM, theSession, theServer, objectId), - stream_sync(false), flush_pending(false), msg_start(true) + stream_sync(false), sync_offset_valid(false), + flush_pending(false), msg_start(true), + flush_mode(FLUSH_NONE), push_mode(false) { } @@ -1674,9 +2021,9 @@ NormStreamObject::~NormStreamObject() Close(); } -bool NormStreamObject::Open(UINT32 bufferSize, - const char* infoPtr, - UINT16 infoLen) +bool NormStreamObject::Open(UINT32 bufferSize, + const char* infoPtr, + UINT16 infoLen) { if (!bufferSize) { @@ -1687,13 +2034,14 @@ bool NormStreamObject::Open(UINT32 bufferSize, UINT16 segmentSize, numData; if (server) { - segmentSize = server->SegmentSize(); - numData = server->BlockSize(); + // receive streams have already be pre-opened + segmentSize = segment_size; + numData = ndata; } else { - segmentSize = session->ServerSegmentSize(); - numData = session->ServerBlockSize(); + segmentSize = session.ServerSegmentSize(); + numData = session.ServerBlockSize(); } NormObjectSize blockSize((UINT32)segmentSize * (UINT32)numData); @@ -1730,13 +2078,22 @@ bool NormStreamObject::Open(UINT32 bufferSize, write_offset = read_offset = 0; if (!server) { - if (!NormObject::Open(NormObjectSize((UINT32)bufferSize), infoPtr, infoLen)) + if (!NormObject::Open(NormObjectSize((UINT32)bufferSize), + infoPtr, + infoLen, + session.ServerSegmentSize(), + session.ServerBlockSize(), + session.ServerNumParity())) { DMSG(0, "NormStreamObject::Open() object open error\n"); Close(); return false; } + stream_next_id = pending_mask.Size(); } + + stream_sync = false; + sync_offset_valid = false; flush_pending = false; msg_start = true; return true; @@ -1771,8 +2128,17 @@ void NormStreamObject::Close() block_pool.Destroy(); } // end NormStreamObject::Close() -bool NormStreamObject::LockBlocks(NormBlockId nextId, NormBlockId lastId) +bool NormStreamObject::LockBlocks(NormBlockId firstId, NormBlockId lastId) { + NormBlockId nextId = firstId; + // First check that we _can_ lock them all + while (nextId <= lastId) + { + NormBlock* block = stream_buffer.Find(nextId); + if (NULL == block) return false; + nextId++; + } + nextId = firstId; while (nextId <= lastId) { NormBlock* block = stream_buffer.Find(nextId); @@ -1861,8 +2227,7 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) if (delta > NormBlockId(2*pending_mask.Size())) stream_sync_id = blockId; return true; - } - + } } } } @@ -1876,6 +2241,7 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) stream_sync_id = blockId; stream_next_id = blockId + pending_mask.Size(); read_index.block = blockId; + read_index.segment = 0; return true; } } // end NormStreamObject::StreamUpdateStatus() @@ -1895,7 +2261,8 @@ UINT16 NormStreamObject::ReadSegment(NormBlockId blockId, if ((blockId == write_index.block) && (segmentId >= write_index.segment)) { - //DMSG(0, "NormStreamObject::ReadSegment() stream starved (2)\n"); + //DMSG(0, "NormStreamObject::ReadSegment(blk>%lu seg>%hu) stream starved (2) (write_index>%lu:%hu)\n", + // (UINT32)blockId, (UINT16)segmentId, (UINT32)write_index.block, (UINT16)write_index.segment); return false; } block->UnsetPending(segmentId); @@ -1920,20 +2287,20 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* segment) { - /*if ((blockId < read_index.block) || + if ((blockId < read_index.block) || ((blockId == read_index.block) && (segmentId < read_index.segment))) { - //DMSG(0, "NormStreamObject::WriteSegment() block/segment < read_index!?\n"); + DMSG(4, "NormStreamObject::WriteSegment() block/segment < read_index!?\n"); return false; - }*/ + } UINT32 segmentOffset = NormDataMsg::ReadStreamPayloadOffset(segment); // if (segmentOffset < read_offset) UINT32 diff = segmentOffset - read_offset; if ((diff > 0x80000000) || ((0x80000000 == diff) && (segmentOffset > read_offset))) { - DMSG(0, "NormStreamObject::WriteSegment() diff:%lu segmentOffset:%lu < read_offset:%lu \n", + DMSG(4, "NormStreamObject::WriteSegment() diff:%lu segmentOffset:%lu < read_offset:%lu \n", diff, segmentOffset, read_offset); return false; } @@ -1956,14 +2323,15 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, block->GetFirstPending(read_index.segment); NormBlock* tempBlock = block; UINT32 tempOffset = read_offset; - session->Notify(NormController::RX_OBJECT_UPDATE, server, this); + session.Notify(NormController::RX_OBJECT_UPDATE, server, this); block = stream_buffer.Find(stream_buffer.RangeLo()); if (tempBlock == block) { if (tempOffset == read_offset) { // App didn't want any data here, purge segment - DMSG(0, "NormStreamObject::WriteSegment() app didn't want data ?!\n"); + DMSG(0, "NormStreamObject::WriteSegment() app didn't want data ?!, blockId:%lu\n", + (UINT32)block->GetId()); ASSERT(0); char* s = block->DetachSegment(read_index.segment); segment_pool.Put(s); @@ -2020,6 +2388,12 @@ bool NormStreamObject::WriteSegment(NormBlockId blockId, block->SetPending(segmentId); ASSERT(block->Segment(segmentId) == s); } + if (!sync_offset_valid) + { + // Set "sync_offset" on first received data buffered. + sync_offset = segmentOffset; + sync_offset_valid = true; + } return true; } // end NormStreamObject::WriteSegment() @@ -2058,14 +2432,16 @@ void NormStreamObject::Prune(NormBlockId blockId) // Sequential (in order) read/write routines (TBD) Add a "Seek()" method bool NormStreamObject::Read(char* buffer, unsigned int* buflen, bool findMsgStart) { + SetNotifyOnUpdate(true); // reset notification when streams are read unsigned int bytesRead = 0; unsigned int bytesToRead = *buflen; - while (bytesToRead > 0) + //while (bytesToRead > 0) + do { NormBlock* block = stream_buffer.Find(read_index.block); if (!block) { - //DMSG(0, "NormStreamObject::Read() stream buffer empty (1)\n"); + // DMSG(0, "NormStreamObject::Read() stream buffer empty (1) (sbEmpty:%d)\n", stream_buffer.IsEmpty()); *buflen = bytesRead; return true; } @@ -2083,7 +2459,8 @@ bool NormStreamObject::Read(char* buffer, unsigned int* buflen, bool findMsgStar UINT32 diff = read_offset - segmentOffset; if ((diff > 0x80000000) || ((0x80000000 == diff) && (read_offset > segmentOffset))) { - DMSG(0, "NormStreamObject::Read() node>%lu broken stream!\n", LocalNodeId()); + DMSG(4, "NormStreamObject::Read() node>%lu obj>%hu blk>%lu seg>%hu broken stream! (read_offset>%lu segmentOffset>%lu)\n", + LocalNodeId(), (UINT16)transport_id, (UINT32)read_index.block, read_index.segment, read_offset, segmentOffset); read_offset = segmentOffset; *buflen = bytesRead; return false; @@ -2093,8 +2470,9 @@ bool NormStreamObject::Read(char* buffer, unsigned int* buflen, bool findMsgStar UINT16 length = NormDataMsg::ReadStreamPayloadLength(segment); if (index >= length) { - DMSG(0, "NormStreamObject::Read() node>%lu mangled stream! index:%hu length:%hu\n", - LocalNodeId(), index, length); + DMSG(0, "NormStreamObject::Read() node>%lu obj>%hu blk>%lu seg>%hu mangled stream! index:%hu length:%hu " + "read_offset:%lu segmentOffset:%lu\n", + LocalNodeId(), (UINT16)transport_id, (UINT32)read_index.block, read_index.segment, index, length, read_offset, segmentOffset); read_offset = segmentOffset; *buflen = bytesRead; ASSERT(0); @@ -2163,13 +2541,12 @@ bool NormStreamObject::Read(char* buffer, unsigned int* buflen, bool findMsgStar read_index.segment = 0; } } - } // end while (len > 0) + } while (bytesToRead > 0); // end while (len > 0) *buflen = bytesRead; return true; } // end NormStreamObject::Read() -UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, - FlushType flushType, bool eom, bool push) +UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) { UINT32 nBytes = 0; do @@ -2181,7 +2558,7 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, { block = stream_buffer.Find(stream_buffer.RangeLo()); ASSERT(block); - if (push) + if (push_mode) { NormBlockId blockId = block->GetId(); pending_mask.Unset(blockId); @@ -2190,8 +2567,13 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, if (b) { block_buffer.Remove(b); - session->ServerPutFreeBlock(b); - } + session.ServerPutFreeBlock(b); + } + if (!pending_mask.IsSet()) + { + pending_mask.Set(write_index.block); + stream_next_id = write_index.block + 1; + } } else if (block->IsPending()) { @@ -2205,6 +2587,7 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, block->ClearPending(); bool success = stream_buffer.Insert(block); ASSERT(success); + } char* segment = block->Segment(write_index.segment); if (!segment) @@ -2213,16 +2596,21 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, { NormBlock* b = stream_buffer.Find(stream_buffer.RangeLo()); ASSERT(b != block); - if (push) + if (push_mode) { - NormBlockId blockId = block->GetId(); + NormBlockId blockId = b->GetId(); pending_mask.Unset(blockId); repair_mask.Unset(blockId); NormBlock* c = FindBlock(blockId); if (c) { block_buffer.Remove(c); - session->ServerPutFreeBlock(c); + session.ServerPutFreeBlock(c); + } + if (!pending_mask.IsSet()) + { + pending_mask.Set(write_index.block); + stream_next_id = write_index.block + 1; } } else if (b->IsPending()) @@ -2278,7 +2666,9 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, nBytes += count; write_offset += count; // Is the segment full? or flushing - if ((count == space) || ((FLUSH_NONE != flushType) && (0 != index) && (nBytes == len))) + //if ((count == space) || ((FLUSH_NONE != flush_mode) && (0 != index) && (nBytes == len))) + if ((count == space) || + ((FLUSH_NONE != flush_mode) && (nBytes == len) && ((0 != index) || (0 != len)))) { block->SetPending(write_index.segment); if (++write_index.segment >= ndata) @@ -2294,16 +2684,18 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, { if (eom) msg_start = true; - if (FLUSH_ACTIVE == flushType) + if (FLUSH_ACTIVE == flush_mode) flush_pending = true; else flush_pending = false; + if ((0 != nBytes) || (FLUSH_NONE != flush_mode)) + session.TouchServer(); } else { - flushType = FLUSH_NONE; + if (0 != nBytes) + session.TouchServer(); } - if (nBytes || (FLUSH_NONE != flushType)) session->TouchServer(); return nBytes; } // end NormStreamObject::Write() @@ -2313,7 +2705,7 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, // NormSimObject Implementation (dummy NORM_OBJECT_FILE or NORM_OBJECT_DATA) // -NormSimObject::NormSimObject(class NormSession* theSession, +NormSimObject::NormSimObject(class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId) : NormObject(FILE, theSession, theServer, objectId) @@ -2404,7 +2796,8 @@ NormObject* NormObjectTable::Find(const NormObjectId& objectId) const { if ((objectId < range_lo) || (objectId > range_hi)) return (NormObject*)NULL; NormObject* theObject = table[((UINT16)objectId) & hash_mask]; - while (theObject && (objectId != theObject->GetId())) theObject = theObject->next; + while (theObject && (objectId != theObject->GetId())) + theObject = theObject->next; return theObject; } else diff --git a/common/normObject.h b/common/normObject.h index 9b71d2f..c58475e 100644 --- a/common/normObject.h +++ b/common/normObject.h @@ -14,6 +14,7 @@ class NormObject public: enum Type { + NONE, DATA, FILE, STREAM @@ -21,6 +22,7 @@ class NormObject enum CheckLevel { + TO_OBJECT, THRU_INFO, TO_BLOCK, THRU_SEGMENT, @@ -28,7 +30,21 @@ class NormObject THRU_OBJECT }; + enum NackingMode + { + NACK_NONE, + NACK_INFO_ONLY, + NACK_NORMAL + }; + virtual ~NormObject(); + void Retain(); + void Release(); + unsigned int GetReferenceCount() {return reference_count;} + + // This must be reset after each update + void SetNotifyOnUpdate(bool state) + {notify_on_update = state;} // Object information NormObject::Type GetType() const {return type;} @@ -39,19 +55,33 @@ class NormObject UINT16 GetInfoLength() const {return info_len;} bool IsStream() const {return (STREAM == type);} + class NormSession& GetSession() const {return session;} NormNodeId LocalNodeId() const; - class NormServerNode* GetServer() {return server;} + class NormServerNode* GetServer() const {return server;} + NormNodeId GetServerNodeId() const; bool IsOpen() {return (0 != segment_size);} // Opens (inits) object for tx operation bool Open(const NormObjectSize& objectSize, const char* infoPtr, - UINT16 infoLen); + UINT16 infoLen, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity); // Opens (inits) object for rx operation - bool Open(const NormObjectSize& objectSize, bool hasInfo) - {return Open(objectSize, (char*)NULL, hasInfo ? 1 : 0);} + bool Open(const NormObjectSize& objectSize, + bool hasInfo, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity) + { + return Open(objectSize, (char*)NULL, hasInfo ? 1 : 0, + segmentSize, numData, numParity); + } void Close(); + + virtual bool WriteSegment(NormBlockId blockId, NormSegmentId segmentId, const char* buffer) = 0; @@ -59,9 +89,17 @@ class NormObject NormSegmentId segmentId, char* buffer) = 0; + NackingMode GetNackingMode() const {return nacking_mode;} + void SetNackingMode(NackingMode nackingMode) + { + nacking_mode = nackingMode; + // (TBD) initiate an appropriate NormServerNode::RepairCheck + // to prompt repair process if needed + } + // These are only valid after object is open NormBlockId GetFinalBlockId() const {return final_block_id;} - UINT32 GetBlockSize(NormBlockId blockId) + UINT32 GetBlockSize(NormBlockId blockId) const { return (((UINT32)blockId < large_block_count) ? large_block_size : small_block_size); @@ -114,7 +152,8 @@ class NormObject return result; } - + bool FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) const; + // Methods available to server for transmission bool NextServerMsg(NormObjectMsg* msg); NormBlock* ServerRecoverBlock(NormBlockId blockId); @@ -141,6 +180,7 @@ class NormObject NormBlock* FindBlock(NormBlockId blockId) {return block_buffer.Find(blockId);} bool ActivateRepairs(); bool IsRepairSet(NormBlockId blockId) {return repair_mask.Test(blockId);} + bool IsPendingSet(NormBlockId blockId) {return pending_mask.Test(blockId);} bool AppendRepairAdv(NormCmdRepairAdvMsg& cmd); NormBlockId GetMaxPendingBlockId() const {return max_pending_block;} @@ -161,7 +201,8 @@ class NormObject // Used by receiver for resource management scheme NormBlock* StealNewestBlock(bool excludeBlock, NormBlockId excludeId = 0); NormBlock* StealOldestBlock(bool excludeBlock, NormBlockId excludeId = 0); - + bool PassiveRepairCheck(NormBlockId blockId, + NormSegmentId segmentId); bool ClientRepairCheck(CheckLevel level, NormBlockId blockId, NormSegmentId segmentId, @@ -178,15 +219,16 @@ class NormObject protected: NormObject(Type theType, - class NormSession* theSession, + class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId); void Accept() {accepted = true;} NormObject::Type type; - class NormSession* session; + class NormSession& session; class NormServerNode* server; // NULL value indicates local (tx) object + unsigned int reference_count; NormObjectId transport_id; NormObjectSize object_size; @@ -194,9 +236,9 @@ class NormObject UINT16 ndata; UINT16 nparity; NormBlockBuffer block_buffer; - bool pending_info; + bool pending_info; // set when we need to send or recv info NormSlidingMask pending_mask; - bool repair_info; + bool repair_info; // client: set when NormSlidingMask repair_mask; NormBlockId current_block_id; // for suppression NormSegmentId next_segment_id; // for suppression @@ -208,11 +250,14 @@ class NormObject UINT32 small_block_size; NormBlockId final_block_id; UINT16 final_segment_size; - + NackingMode nacking_mode; char* info; UINT16 info_len; + // Here are some members used to let us know + // our status with respect to the rest of the world bool accepted; + bool notify_on_update; NormObject* next; }; // end class NormObject @@ -221,7 +266,7 @@ class NormObject class NormFileObject : public NormObject { public: - NormFileObject(class NormSession* theSession, + NormFileObject(class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId); ~NormFileObject(); @@ -231,7 +276,8 @@ class NormFileObject : public NormObject UINT16 infoLen = 0); bool Accept(const char* thePath); void Close(); - const char* Path() {return path;} + + const char* GetPath() {return path;} bool Rename(const char* newPath) { bool result = file.Rename(path, newPath); @@ -254,11 +300,44 @@ class NormFileObject : public NormObject NormObjectSize small_block_length; }; // end class NormFileObject +class NormDataObject : public NormObject +{ + // (TBD) allow support of greater than 4GB size data objects + public: + NormDataObject(class NormSession& theSession, + class NormServerNode* theServer, + const NormObjectId& objectId); + ~NormDataObject(); + + bool Open(char* dataPtr, + UINT32 dataLen, + const char* infoPtr = NULL, + UINT16 infoLen = 0); + bool Accept(char* dataPtr, UINT32 dataMax); + void Close(); + + const char* GetData() {return data_ptr;} + + virtual bool WriteSegment(NormBlockId blockId, + NormSegmentId segmentId, + const char* buffer); + + virtual UINT16 ReadSegment(NormBlockId blockId, + NormSegmentId segmentId, + char* buffer); + + private: + NormObjectSize large_block_length; + NormObjectSize small_block_length; + char* data_ptr; + UINT32 data_max; +}; // end class NormDataObject + class NormStreamObject : public NormObject { public: - NormStreamObject(class NormSession* theSession, + NormStreamObject(class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId); ~NormStreamObject(); @@ -269,19 +348,35 @@ class NormStreamObject : public NormObject void Close(); bool Accept(UINT32 bufferSize); - enum FlushType + enum FlushMode { FLUSH_NONE, // no flush action taken FLUSH_PASSIVE, // pending queued data is transmitted, but no CMD(FLUSH) sent FLUSH_ACTIVE // pending queued data is transmitted, _and_ active CMD(FLUSH) }; + + void SetFlushMode(FlushMode flushMode) {flush_mode = flushMode;} + void Flush(bool eom = false) + { + FlushMode oldFlushMode = flush_mode; + SetFlushMode(FLUSH_ACTIVE); + Write(NULL, 0, eom); + SetFlushMode(oldFlushMode); + } + void SetPushMode(bool state) {push_mode = state;} + bool Read(char* buffer, unsigned int* buflen, bool findMsgStart = false); - UINT32 Write(const char* buffer, UINT32 len, FlushType flushType, bool eom, bool push); + UINT32 Write(const char* buffer, UINT32 len, bool eom = false); + bool SyncOffsetIsValid() {return sync_offset_valid;} + UINT32 GetSyncOffset() {return sync_offset;} + UINT32 GetCurrentReadOffset() {return read_offset;} bool StreamUpdateStatus(NormBlockId blockId); void StreamResync(NormBlockId nextBlockId) - {stream_next_id = nextBlockId;} + { + stream_next_id = nextBlockId; + } void StreamAdvance(); virtual bool WriteSegment(NormBlockId blockId, @@ -308,7 +403,11 @@ class NormStreamObject : public NormObject NormSegmentId FlushSegmentId() {return (write_index.segment ? (write_index.segment-1) : (ndata-1));} - + + NormBlockId GetNextBlockId() const + {return (server ? read_index.block : write_index.block);} + NormSegmentId GetNextSegmentId() const + {return (server ? read_index.segment : write_index.segment);} private: class Index { @@ -320,6 +419,8 @@ class NormStreamObject : public NormObject bool stream_sync; NormBlockId stream_sync_id; NormBlockId stream_next_id; + UINT32 sync_offset; + bool sync_offset_valid; NormBlockPool block_pool; NormSegmentPool segment_pool; @@ -330,6 +431,8 @@ class NormStreamObject : public NormObject UINT32 read_offset; bool flush_pending; bool msg_start; + FlushMode flush_mode; + bool push_mode; }; // end class NormStreamObject #ifdef SIMULATE @@ -338,7 +441,7 @@ class NormStreamObject : public NormObject class NormSimObject : public NormObject { public: - NormSimObject(class NormSession* theSession, + NormSimObject(class NormSession& theSession, class NormServerNode* theServer, const NormObjectId& objectId); ~NormSimObject(); diff --git a/common/normPostProcess.cpp b/common/normPostProcess.cpp index b0b1166..95df810 100644 --- a/common/normPostProcess.cpp +++ b/common/normPostProcess.cpp @@ -56,7 +56,6 @@ void NormPostProcessor::GetCommand(char* buffer, unsigned int buflen) bool NormPostProcessor::SetCommand(const char* cmd) { - // 1) Delete old command resources if (process_argv) { @@ -81,10 +80,10 @@ bool NormPostProcessor::SetCommand(const char* cmd) { argCount++; while (!isspace(*ptr) && ('\0' != *ptr)) ptr++; + while (isspace(*ptr) && ('\0' != *ptr)) ptr++; } if (!argCount) return true; // post processing disabled - // 3) Allocate new process_argv array (2 extra slots, one for "target", // and one for terminating NULL pointer. if (!(process_argv = new char*[argCount+2])) @@ -115,7 +114,8 @@ bool NormPostProcessor::SetCommand(const char* cmd) } strncpy(arg, argStart, argLength); arg[argLength] = '\0'; - process_argv[index] = arg; + process_argv[index++] = arg; + while (isspace(*ptr) && ('\0' != *ptr)) ptr++; } return true; } // end NormPostProcessor::SetCommand() diff --git a/common/normSegment.cpp b/common/normSegment.cpp index a2ab0c6..1d49149 100644 --- a/common/normSegment.cpp +++ b/common/normSegment.cpp @@ -257,6 +257,7 @@ bool NormBlock::ActivateRepairs(UINT16 numParity) if (repair_mask.IsSet()) { pending_mask.Add(repair_mask); + ASSERT(pending_mask.IsSet()); repair_mask.Clear(); return true; } @@ -390,7 +391,7 @@ bool NormBlock::HandleSegmentRequest(NormSegmentId nextId, NormSegmentId lastId, return increasedRepair; } // end NormBlock::HandleSegmentRequest() - +// (TBD) this should return true is something is appending, false otherwise bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, NormObjectId objectId, bool repairInfo, @@ -433,7 +434,13 @@ bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, if (form != prevForm) { if (NormRepairRequest::INVALID != prevForm) - cmd.PackRepairRequest(req); // (TBD) error check + { + if (0 == cmd.PackRepairRequest(req)) + { + DMSG(0, "NormBlock::AppendRepairAdv() warning: full msg\n"); + break; + } + } req.SetForm(form); cmd.AttachRepairRequest(req, segmentSize); // (TBD) error check prevForm = form; @@ -460,12 +467,16 @@ bool NormBlock::AppendRepairAdv(NormCmdRepairAdvMsg& cmd, } } // end while (nextId < totalSize) if (NormRepairRequest::INVALID != prevForm) - cmd.PackRepairRequest(req); // (TBD) error check + { + if (0 == cmd.PackRepairRequest(req)) + DMSG(0, "NormBlock::AppendRepairAdv() warning: full msg\n"); + } } return true; } // end NormBlock::AppendRepairAdv() // Called by client +// (TBD) this should return true iff something appended, false otherwise bool NormBlock::AppendRepairRequest(NormNackMsg& nack, UINT16 numData, UINT16 numParity, @@ -527,7 +538,13 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, if (form != prevForm) { if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check + { + if (0 == nack.PackRepairRequest(req)) + { + DMSG(0, "NormBlock::AppendRepairRequest() warning: full NACK msg\n"); + break; + } + } nack.AttachRepairRequest(req, segmentSize); // (TBD) error check req.SetForm(form); prevForm = form; @@ -554,76 +571,10 @@ bool NormBlock::AppendRepairRequest(NormNackMsg& nack, } } // end while (nextId < lastId) if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - - /* - // old code begins here - NormSegmentId prevId = nextId; - while ((nextId <= lastId) || (segmentCount > 0)) { - // force break of possible ending consec. series - if (nextId == lastId) nextId++; - if (segmentCount && (segmentCount == (nextId - prevId))) - { - segmentCount++; // consecutive series continues - } - else - { - NormRepairRequest::Form nextForm; - switch(segmentCount) - { - case 0: - nextForm = NormRepairRequest::INVALID; - break; - case 1: - case 2: - nextForm = NormRepairRequest::ITEMS; - break; - default: - nextForm = NormRepairRequest::RANGES; - break; - } // end switch(reqCount) - if (prevForm != nextForm) - { - if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - if (NormRepairRequest::INVALID != nextForm) - { - nack.AttachRepairRequest(req, segmentSize); - req.SetForm(nextForm); - req.SetFlag(NormRepairRequest::SEGMENT); - if (pendingInfo) req.SetFlag(NormRepairRequest::INFO); - } - prevForm = nextForm; - } - if (NormRepairRequest::INVALID != nextForm) - DMSG(6, "NormBlock::AppendRepairRequest() SEGMENT request\n"); - switch (nextForm) - { - case NormRepairRequest::ITEMS: - req.AppendRepairItem(objectId, id, prevId); // (TBD) error check - if (2 == reqCount) - req.AppendRepairItem(objectId, id, prevId+1); // (TBD) error check - break; - case NormRepairRequest::RANGES: - req.AppendRepairItem(objectId, id, prevId); // (TBD) error check - req.AppendRepairItem(objectId, id, prevId+reqCount-1); // (TBD) error check - break; - default: - break; - } // end switch(nextForm) - prevId = nextId; - if (nextId < lastId) - segmentCount = 1; - else - segmentCount = 0; - } // end if/else (reqCount && (reqCount == (nextId - prevId))) - nextId++; - if (nextId <= lastId) nextId = pending_mask.NextSet(nextId); - } // end while(nextId <= lastId) - if (NormRepairRequest::INVALID != prevForm) - nack.PackRepairRequest(req); // (TBD) error check - */ + if (0 == nack.PackRepairRequest(req)) + DMSG(0, "NormBlock::AppendRepairRequest() warning: full NACK msg\n"); + } return true; } // end NormBlock::AppendRepairRequest() diff --git a/common/normSession.cpp b/common/normSession.cpp index 0c51fd8..094361f 100644 --- a/common/normSession.cpp +++ b/common/normSession.cpp @@ -19,13 +19,15 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) : session_mgr(sessionMgr), notify_pending(false), tx_socket(ProtoSocket::UDP), rx_socket(ProtoSocket::UDP), local_node_id(localNodeId), - ttl(DEFAULT_TTL), tx_rate(DEFAULT_TRANSMIT_RATE/8.0), + ttl(DEFAULT_TTL), loopback(false), + tx_rate(DEFAULT_TRANSMIT_RATE/8.0), backoff_factor(DEFAULT_BACKOFF_FACTOR), is_server(false), session_id(0), ndata(DEFAULT_NDATA), nparity(DEFAULT_NPARITY), auto_parity(0), extra_parity(0), next_tx_object_id(0), tx_cache_count_min(8), tx_cache_count_max(256), tx_cache_size_max((UINT32)20*1024*1024), flush_count(NORM_ROBUST_FACTOR+1), - posted_tx_queue_empty(false), advertise_repairs(false), + posted_tx_queue_empty(false), + acking_node_count(0), watermark_pending(false), advertise_repairs(false), suppress_nonconfirmed(false), suppress_rate(-1.0), suppress_rtt(-1.0), probe_proactive(true), probe_pending(false), probe_reset(false), grtt_interval(0.5), @@ -36,6 +38,8 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) grtt_response(false), grtt_current_peak(0.0), grtt_age(0.0), cc_enable(false), cc_sequence(0), cc_slow_start(true), is_client(false), unicast_nacks(false), client_silent(false), + default_repair_boundary(NormServerNode::BLOCK_BOUNDARY), + default_nacking_mode(NormObject::NACK_NORMAL), trace(false), tx_loss_rate(0.0), rx_loss_rate(0.0), next(NULL) { @@ -73,7 +77,7 @@ NormSession::NormSession(NormSessionMgr& sessionMgr, NormNodeId localNodeId) // (It may be used to trigger transmission of report messages // in the future for debugging, etc report_timer.SetListener(this, &NormSession::OnReportTimeout); - report_timer.SetInterval(60.0); + report_timer.SetInterval(10.0); report_timer.SetRepeat(-1); } @@ -86,6 +90,7 @@ NormSession::~NormSession() bool NormSession::Open(const char* interfaceName) { ASSERT(address.IsValid()); + // (TBD) support single socket option if (!tx_socket.IsOpen()) { if (!tx_socket.Open()) @@ -118,6 +123,12 @@ bool NormSession::Open(const char* interfaceName) Close(); return false; } + if (!tx_socket.SetLoopback(loopback)) + { + DMSG(0, "NormSession::Open() tx_socket set loopback error\n"); + Close(); + return false; + } if (interfaceName) { rx_socket.SetMulticastInterface(interfaceName); @@ -160,7 +171,33 @@ void NormSession::Close() } // end NormSession::Close() -bool NormSession::StartServer(unsigned long bufferSpace, +void NormSession::SetTxRate(double txRate) +{ + txRate /= 8.0; // convert to bytes/sec + if (tx_timer.IsActive()) + { + tx_timer.Deactivate(); + if (txRate > 0.0) + { + double adjustInterval = (tx_rate/txRate) * tx_timer.GetTimeRemaining(); + tx_timer.SetInterval(adjustInterval); + ActivateTimer(tx_timer); + } + tx_rate = txRate; + } + else if (0.0 == tx_rate) + { + tx_rate = txRate; + tx_timer.SetInterval(0.0); + ActivateTimer(tx_timer); + } + else + { + tx_rate = txRate; + } +} // end NormSession::SetTxRate() + +bool NormSession::StartServer(UINT32 bufferSpace, UINT16 segmentSize, UINT16 numData, UINT16 numParity, @@ -283,31 +320,109 @@ void NormSession::Serve() { // Only send new data when no other messages are queued for transmission if (!message_queue.IsEmpty()) return; - - NormObject* obj = NULL; + // Queue next server message NormObjectId objectId; + NormObject* obj = NULL; if (ServerGetFirstPending(objectId)) { obj = tx_table.Find(objectId); ASSERT(obj); } - else + + if (watermark_pending) { - if (!posted_tx_queue_empty) + // Determine next message (objectId::blockId::segmentId) to be sent + NormObject* nextObj; + NormObjectId nextObjectId; + NormBlockId nextBlockId = 0; + NormSegmentId nextSegmentId = 0; + if (obj) { - posted_tx_queue_empty = true; - Notify(NormController::TX_QUEUE_EMPTY, - (NormServerNode*)NULL, - (NormObject*)NULL); - // (TBD) Was session deleted? - return; - } - } + // Use current transmit pending object + nextObj = obj; + nextObjectId = objectId; + if (nextObj->IsPending()) + { + if(nextObj->GetFirstPending(nextBlockId)) + { + NormBlock* block = nextObj->FindBlock(nextBlockId); + if (block) + { +#ifdef PROTO_DEBUG + ASSERT(block->GetFirstPending(nextSegmentId)); +#else + block->GetFirstPending(nextSegmentId); +#endif // if/else PROTO_DEBUG + // Adjust so watermark segmentId < block length + if (nextSegmentId >= nextObj->GetBlockSize(nextBlockId)) + nextSegmentId = nextObj->GetBlockSize(nextBlockId) - 1; + + } + } + else + { + // info only pending; so blockId = segmentId = 0 (as inited) + } + } + else + { + // Must be an active, but non-pending stream object + nextBlockId = static_cast(nextObj)->GetNextBlockId(); + nextSegmentId = static_cast(nextObj)->GetNextSegmentId(); + } + } + else + { + // Nothing transmit pending, check for repair pending object + NormObjectTable::Iterator iterator(tx_table); + while ((nextObj = iterator.GetNextObject())) + if (nextObj->IsRepairPending()) break; + if (ServerGetFirstRepairPending(nextObjectId)) + { + if (nextObj && (nextObj->GetId() < nextObjectId)) + { + nextObjectId = nextObj->GetId(); + } + else + { + nextObj = tx_table.Find(nextObjectId); + ASSERT(nextObj); + } + } + if (nextObj) + { +#ifdef PROTO_DEBUG + ASSERT(nextObj->FindRepairIndex(nextBlockId, nextSegmentId)); +#else + nextObj->FindRepairIndex(nextBlockId, nextSegmentId); +#endif + } + else + { + nextObjectId = next_tx_object_id; + } + } + if ((nextObjectId > watermark_object_id) || + ((nextObjectId == watermark_object_id) && + ((nextBlockId > watermark_block_id) || + (((nextBlockId == watermark_block_id) && + (nextSegmentId > watermark_segment_id)))))) + { + if (ServerQueueWatermarkFlush()) + { + return; + } + else + { + // (TBD) optionally return here to have ack collection temporarily suspend data transmission + return; + } + } + } // end if (watermark_pending) if (obj) { - NormObjectMsg* msg = (NormObjectMsg*)GetMessageFromPool(); if (msg) { @@ -319,7 +434,7 @@ void NormSession::Serve() msg->SetGroupSize(gsize_quantized); QueueMessage(msg); flush_count = 0; - if (flush_timer.IsActive()) flush_timer.Deactivate(); + //if (flush_timer.IsActive()) flush_timer.Deactivate(); if (!obj->IsPending()) { if (obj->IsStream()) @@ -333,25 +448,35 @@ void NormSession::Serve() ReturnMessageToPool(msg); if (obj->IsStream()) { - NormStreamObject* stream = (NormStreamObject*)obj; - if (stream->IsFlushPending() && - (flush_count < NORM_ROBUST_FACTOR)) + NormStreamObject* stream = static_cast(obj); + if (stream->IsFlushPending()) { // Queue flush message - ServerQueueFlush(); + if (!flush_timer.IsActive()) + if (flush_count < NORM_ROBUST_FACTOR) + { + ServerQueueFlush(); + } + else if (NORM_ROBUST_FACTOR == flush_count) + { + DMSG(6, "NormSession::Serve() node>%lu server flush complete ...\n", + LocalNodeId()); + flush_count++; + } } if (!posted_tx_queue_empty) { posted_tx_queue_empty = true; Notify(NormController::TX_QUEUE_EMPTY, (NormServerNode*)NULL, obj); // (TBD) Was session deleted? + //Serve(); return; } } else { DMSG(0, "NormSession::Serve() pending non-stream obj, no message?.\n"); - //ASSERT(0); + ASSERT(repair_timer.IsActive()); } } } @@ -361,18 +486,146 @@ void NormSession::Serve() LocalNodeId()); } } - else if (flush_count < NORM_ROBUST_FACTOR) + else { - // Queue flush message - ServerQueueFlush(); - } - else if (flush_count == NORM_ROBUST_FACTOR) - { - DMSG(6, "NormSession::Serve() node>%lu server flush complete ...\n", - LocalNodeId()); - flush_count++; + // No pending objects or positive acknowledgement request + if (!posted_tx_queue_empty) + { + posted_tx_queue_empty = true; + Notify(NormController::TX_QUEUE_EMPTY, + (NormServerNode*)NULL, + (NormObject*)NULL); + // (TBD) Was session deleted? + //Serve(); + return; + } + if (flush_count < NORM_ROBUST_FACTOR) + { + // Queue flush message + ServerQueueFlush(); + } + else if (flush_count == NORM_ROBUST_FACTOR) + { + DMSG(6, "NormSession::Serve() node>%lu server flush complete ...\n", + LocalNodeId()); + flush_count++; + } } } // end NormSession::Serve() + +void NormSession::ServerSetWatermark(NormObjectId objectId, + NormBlockId blockId, + NormSegmentId segmentId) +{ + TRACE("NormSession::ServerSetWatermark(%hu:%lu:%hu) ...\n", + (UINT16)objectId, (UINT32)blockId, (UINT16)segmentId); + watermark_pending = true; + watermark_object_id = objectId; + watermark_block_id = blockId; + watermark_segment_id = segmentId; + acks_collected = 0; + // Reset acking_node_list + NormNodeTreeIterator iterator(acking_node_tree); + NormNode* next; + while ((next = iterator.GetNextNode())) + static_cast(next)->Reset(NORM_ROBUST_FACTOR); + PromptServer(); +} // end Norm::ServerSetWatermark() + +bool NormSession::ServerAddAckingNode(NormNodeId nodeId) +{ + NormAckingNode* theNode = static_cast(acking_node_tree.FindNodeById(nodeId)); + if (NULL == theNode) + { + theNode = new NormAckingNode(*this, nodeId); + if (NULL != theNode) + { + theNode->Reset(NORM_ROBUST_FACTOR); + acking_node_tree.AttachNode(theNode); + acking_node_count++; + return true; + } + else + { + DMSG(0, "NormSession::AddAckingNode() new NormAckingNode error: %s\n", GetErrorString()); + } + } + else + { + DMSG(0, "NormSession::AddAckingNode() warning: node already in list!?\n"); + } + return true; +} // end NormSession::AddAckingNode(NormNodeId nodeId) + +void NormSession::ServerRemoveAckingNode(NormNodeId nodeId) +{ + NormAckingNode* theNode = static_cast(acking_node_tree.FindNodeById(nodeId)); + if (theNode) + { + if (watermark_pending && theNode->AckReceived()) + acks_collected--; + acking_node_tree.DeleteNode(theNode); + acking_node_count--; + } +} // end NormSession::RemoveAckingNode() + +bool NormSession::ServerQueueWatermarkFlush() +{ + if (flush_timer.IsActive()) return false; + NormCmdFlushMsg* flush = static_cast(GetMessageFromPool()); + if (flush) + { + flush->Init(); + flush->SetDestination(address); + flush->SetGrtt(grtt_quantized); + flush->SetBackoffFactor((unsigned char)backoff_factor); + flush->SetGroupSize(gsize_quantized); + flush->SetObjectId(watermark_object_id); + flush->SetFecBlockId(watermark_block_id); + flush->SetFecSymbolId(watermark_segment_id); + NormNodeTreeIterator iterator(acking_node_tree); + NormAckingNode* next; + watermark_pending = false; + while ((next = static_cast(iterator.GetNextNode()))) + { + if (next->IsPending()) + { + // Add node to list + if (flush->AppendAckingNode(next->GetId(), segment_size)) + { + next->DecrementReqCount(); + watermark_pending = true; + } + else + { + DMSG(8, "NormSession::ServeQueueWatermarkFlush() full cmd ...\n"); + break; + } + } + } + if (watermark_pending) + { + flush_count++; + QueueMessage(flush); + DMSG(8, "NormSession::ServeQueueWatermarkFlush() node>%lu cmd queued ...\n", + LocalNodeId()); + } + else + { + DMSG(4, "NormSession::ServeQueueWatermarkFlush() node>%lu watermark ack finished incomplete\n"); + // (TBD) notify app + return false; + } + } + else + { + DMSG(0, "NormSession::ServerQueueWatermarkRequest() node>%lu message_pool exhausted! (couldn't req)\n", + LocalNodeId()); + } + flush_timer.SetInterval(2*grtt_advertised); + ActivateTimer(flush_timer); + return true; +} // end NormSession::ServerQueueWatermarkFlush() void NormSession::ServerQueueFlush() { @@ -400,6 +653,7 @@ void NormSession::ServerQueueFlush() } else { + // Why did I do this? - Brian // (TBD) send NORM_CMD(EOT) instead? if (ServerQueueSquelch(next_tx_object_id)) { @@ -424,22 +678,22 @@ void NormSession::ServerQueueFlush() flush->SetFecSymbolId(segmentId); QueueMessage(flush); flush_count++; - flush_timer.SetInterval(2*grtt_advertised); - ActivateTimer(flush_timer); - DMSG(8, "NormSession::ServerQueueFlush() node>%lu, flush queued (flush_count:%u)...\n", + DMSG(0, "NormSession::ServerQueueFlush() node>%lu, flush queued (flush_count:%u)...\n", LocalNodeId(), flush_count); } else { - DMSG(0, " NormSession::ServerQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n", + DMSG(0, "NormSession::ServerQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n", LocalNodeId()); - } + } + flush_timer.SetInterval(2*grtt_advertised); + ActivateTimer(flush_timer); } // end NormSession::ServerQueueFlush() bool NormSession::OnFlushTimeout(ProtoTimer& /*theTimer*/) { flush_timer.Deactivate(); - Serve(); // (TBD) Change this to PromptServer() ?? + PromptServer();//Serve(); // (TBD) Change this to PromptServer() ?? return false; } // NormSession::OnFlushTimeout() @@ -459,12 +713,12 @@ void NormSession::QueueMessage(NormMsg* msg) } lastTime = currentTime; */ - if (!tx_timer.IsActive()) + if (!tx_timer.IsActive() && (tx_rate > 0.0)) { tx_timer.SetInterval(0.0); ActivateTimer(tx_timer); } - message_queue.Append(msg); + if (msg) message_queue.Append(msg); } // end NormSesssion::QueueMessage(NormMsg& msg) @@ -478,7 +732,7 @@ NormFileObject* NormSession::QueueTxFile(const char* path, DMSG(0, "NormSession::QueueTxFile() Error: server is closed\n"); return NULL; } - NormFileObject* file = new NormFileObject(this, (NormServerNode*)NULL, next_tx_object_id); + NormFileObject* file = new NormFileObject(*this, (NormServerNode*)NULL, next_tx_object_id); if (!file) { DMSG(0, "NormSession::QueueTxFile() new file object error: %s\n", @@ -491,7 +745,7 @@ NormFileObject* NormSession::QueueTxFile(const char* path, delete file; return NULL; } - if (QueueTxObject(file, false)) + if (QueueTxObject(file)) { return file; } @@ -503,6 +757,42 @@ NormFileObject* NormSession::QueueTxFile(const char* path, } } // end NormSession::QueueTxFile() +NormDataObject* NormSession::QueueTxData(const char* dataPtr, + UINT32 dataLen, + const char* infoPtr, + UINT16 infoLen) +{ + if (!IsServer()) + { + DMSG(0, "NormSession::QueueTxData() Error: server is closed\n"); + return NULL; + } + NormDataObject* obj = new NormDataObject(*this, (NormServerNode*)NULL, next_tx_object_id); + if (!obj) + { + DMSG(0, "NormSession::QueueTxData() new data object error: %s\n", + strerror(errno)); + return NULL; + } + if (!obj->Open((char*)dataPtr, dataLen, infoPtr, infoLen)) + { + DMSG(0, "NormSession::QueueTxData() object open error\n"); + delete obj; + return NULL; + } + if (QueueTxObject(obj)) + { + return obj; + } + else + { + obj->Close(); + delete obj; + return NULL; + } +} // end NormSession::QueueTxData() + + NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, const char* infoPtr, UINT16 infoLen) @@ -512,7 +802,7 @@ NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, DMSG(0, "NormSession::QueueTxStream() Error: server is closed\n"); return NULL; } - NormStreamObject* stream = new NormStreamObject(this, (NormServerNode*)NULL, next_tx_object_id); + NormStreamObject* stream = new NormStreamObject(*this, (NormServerNode*)NULL, next_tx_object_id); if (!stream) { DMSG(0, "NormSession::QueueTxStream() new stream object error: %s\n", @@ -525,7 +815,7 @@ NormStreamObject* NormSession::QueueTxStream(UINT32 bufferSize, delete stream; return NULL; } - if (QueueTxObject(stream, true)) + if (QueueTxObject(stream)) { // (???: stream has nothing pending until user writes to it???) //stream->Reset(); @@ -547,7 +837,7 @@ NormSimObject* NormSession::QueueTxSim(unsigned long objectSize) DMSG(0, "NormSession::QueueTxSim() Error: server is closed\n"); return NULL; } - NormSimObject* simObject = new NormSimObject(this, NULL, next_tx_object_id); + NormSimObject* simObject = new NormSimObject(*this, NULL, next_tx_object_id); if (!simObject) { DMSG(0, "NormSession::QueueTxSim() new sim object error: %s\n", @@ -561,7 +851,7 @@ NormSimObject* NormSession::QueueTxSim(unsigned long objectSize) delete simObject; return NULL; } - if (QueueTxObject(simObject, false)) + if (QueueTxObject(simObject)) { return simObject; } @@ -573,7 +863,7 @@ NormSimObject* NormSession::QueueTxSim(unsigned long objectSize) } // end NormSession::QueueTxSim() #endif // SIMULATE -bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) +bool NormSession::QueueTxObject(NormObject* obj) { if (!IsServer()) { @@ -599,9 +889,8 @@ bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) } else { - tx_table.Remove(oldest); - oldest->Close(); - delete oldest; + Notify(NormController::TX_OBJECT_PURGED, (NormServerNode*)NULL, oldest); + DeleteTxObject(oldest); } count = tx_table.Count(); } @@ -613,6 +902,7 @@ bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) ASSERT(0); return false; } + obj->Retain(); tx_pending_mask.Set(obj->GetId()); ASSERT(tx_pending_mask.Test(obj->GetId())); next_tx_object_id++; @@ -624,13 +914,14 @@ bool NormSession::QueueTxObject(NormObject* obj, bool touchServer) void NormSession::DeleteTxObject(NormObject* obj) { - NormObjectId objectId = obj->GetId(); - ASSERT(obj == tx_table.Find(objectId)); - tx_table.Remove(obj); + if (tx_table.Remove(obj)) + { + NormObjectId objectId = obj->GetId(); + tx_pending_mask.Unset(objectId); + tx_repair_mask.Unset(objectId); + } obj->Close(); - tx_pending_mask.Unset(objectId); - tx_repair_mask.Unset(objectId); - delete obj; + obj->Release(); } // end NormSession::DeleteTxObject() NormBlock* NormSession::ServerGetFreeBlock(NormObjectId objectId, @@ -919,11 +1210,7 @@ void NormSession::HandleReceiveMessage(NormMsg& msg, bool wasUnicast) { // for suppression of unicast nack feedback advertise_repairs = true; - if (!tx_timer.IsActive()) - { - tx_timer.SetInterval(0.0); - ActivateTimer(tx_timer); - } + QueueMessage(NULL); // to prompt transmit timeout } } if (IsClient()) ClientHandleNackMessage((NormNackMsg&)msg); @@ -949,23 +1236,48 @@ void NormSession::ClientHandleObjectMessage(const struct timeval& currentTime, // Do common updates for servers we already know. NormNodeId sourceId = msg.GetSourceId(); NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(sourceId); - if (!theServer) + if (theServer) { - if ((theServer = new NormServerNode(this, msg.GetSourceId()))) + if (msg.GetSessionId() != theServer->GetSessionId()) { - server_tree.AttachNode(theServer); - DMSG(4, "NormSession::ClientHandleObjectMessage() node>%lu new remote server:%lu ...\n", - LocalNodeId(), msg.GetSourceId()); + DMSG(2, "NormSession::ClientHandleObjectMessage() node>%lu server>%lu sessionId change - resyncing.\n", + LocalNodeId(), theServer->GetId()); + theServer->Close(); + if (!theServer->Open(msg.GetSessionId())) + { + DMSG(0, "NormSession::ClientHandleObjectMessage() node>%lu error re-opening NormServerNode\n"); + // (TBD) notify application of error + return; + } + } + } + else + { + if ((theServer = new NormServerNode(*this, msg.GetSourceId()))) + { + if (theServer->Open(msg.GetSessionId())) + { + server_tree.AttachNode(theServer); + theServer->Retain(); + DMSG(4, "NormSession::ClientHandleObjectMessage() node>%lu new remote server:%lu ...\n", + LocalNodeId(), msg.GetSourceId()); + } + else + { + DMSG(0, "NormSession::ClientHandleObjectMessage() node>%lu error opening NormServerNode\n"); + // (TBD) notify application of error + return; + } } else { - DMSG(0, "NormSession::ClientHandleObjectMessage() new server node error: %s\n", + DMSG(0, "NormSession::ClientHandleObjectMessage() new NormServerNode error: %s\n", strerror(errno)); // (TBD) notify application of error return; } } - if (theServer->IsOpen()) theServer->Activate(); + theServer->Activate(); theServer->UpdateLossEstimate(currentTime, msg.GetSequence()); theServer->SetAddress(msg.GetSource()); theServer->IncrementRecvTotal(msg.GetLength()); // for statistics only (TBD) #ifdef NORM_DEBUG @@ -979,26 +1291,50 @@ void NormSession::ClientHandleCommand(const struct timeval& currentTime, // Do common updates for servers we already know. NormNodeId sourceId = cmd.GetSourceId(); NormServerNode* theServer = (NormServerNode*)server_tree.FindNodeById(sourceId); - if (!theServer) + if (theServer) + { + if (cmd.GetSessionId() != theServer->GetSessionId()) + { + DMSG(2, "NormSession::ClientHandleCommand() node>%lu server>%lu sessionId change - resyncing.\n", + LocalNodeId(), theServer->GetId()); + theServer->Close(); + if (!theServer->Open(cmd.GetSessionId())) + { + DMSG(0, "NormSession::ClientHandleCommand() node>%lu error re-opening NormServerNode\n"); + // (TBD) notify application of error + return; + } + } + } + else { //DMSG(0, "NormSession::ClientHandleCommand() node>%lu recvd command from unknown server ...\n", // LocalNodeId()); - if ((theServer = new NormServerNode(this, cmd.GetSourceId()))) + if ((theServer = new NormServerNode(*this, cmd.GetSourceId()))) { - - server_tree.AttachNode(theServer); - DMSG(4, "NormSession::ClientHandleCommand() node>%lu new remote server:%lu ...\n", - LocalNodeId(), cmd.GetSourceId()); + if (theServer->Open(cmd.GetSessionId())) + { + server_tree.AttachNode(theServer); + theServer->Retain(); + DMSG(4, "NormSession::ClientHandleCommand() node>%lu new remote server:%lu ...\n", + LocalNodeId(), cmd.GetSourceId()); + } + else + { + DMSG(0, "NormSession::ClientHandleCommand() node>%lu error opening NormServerNode\n"); + // (TBD) notify application of error + return; + } } else { - DMSG(0, "NormSession::ClientHandleCommand() new server node error: %s\n", + DMSG(0, "NormSession::ClientHandleCommand() new NormServerNode error: %s\n", strerror(errno)); // (TBD) notify application of error return; } } - if (theServer->IsOpen()) theServer->Activate(); + theServer->Activate(); theServer->UpdateLossEstimate(currentTime, cmd.GetSequence()); theServer->SetAddress(cmd.GetSource()); theServer->IncrementRecvTotal(cmd.GetLength()); // for statistics only (TBD) #ifdef NORM_DEBUG @@ -1159,7 +1495,7 @@ void NormSession::ServerHandleCCFeedback(NormNodeId nodeId, // There was no active CLR if (!next) { - if ((next = new NormCCNode(this, nodeId))) + if ((next = new NormCCNode(*this, nodeId))) { cc_node_list.Append(next); } @@ -1186,7 +1522,7 @@ void NormSession::ServerHandleCCFeedback(NormNodeId nodeId, NormCCNode* candidate = NULL; if (cc_node_list.GetCount() < 5) { - if ((candidate = new NormCCNode(this, nodeId))) + if ((candidate = new NormCCNode(*this, nodeId))) { cc_node_list.Append(candidate); } @@ -1288,11 +1624,7 @@ void NormSession::ServerHandleAckMessage(const struct timeval& currentTime, cons { // for suppression of unicast feedback advertise_repairs = true; - if (!tx_timer.IsActive()) - { - tx_timer.SetInterval(0.0); - ActivateTimer(tx_timer); - } + QueueMessage(NULL); } switch (ack.GetAckType()) @@ -1301,8 +1633,51 @@ void NormSession::ServerHandleAckMessage(const struct timeval& currentTime, cons // Everything is in the ACK header for this one break; + case NormAck::FLUSH: + if (watermark_pending) + { + NormAckingNode* acker = + static_cast(acking_node_tree.FindNodeById(ack.GetSourceId())); + if (acker) + { + if (!acker->AckReceived()) + { + const NormAckFlushMsg& flushAck = static_cast(ack); + if ((watermark_object_id == flushAck.GetObjectId()) && + (watermark_block_id == flushAck.GetFecBlockId()) && + (watermark_segment_id == flushAck.GetFecSymbolId())) + { + acker->MarkAckReceived(); + acks_collected++; + if (acks_collected >= acking_node_count) + { + watermark_pending = false; + DMSG(4, "NormSession::ServerHandleAckMessage() watermark acknowledgement complete\n"); + // (TBD) notify app + } + } + else + { + DMSG(0, "NormSession::ServerHandleAckMessage() received wrong watermark ACK?!\n"); + } + } + else + { + DMSG(0, "NormSession::ServerHandleAckMessage() received redundant watermark ACK?!\n"); + } + } + else + { + DMSG(0, "NormSession::ServerHandleAckMessage() received watermark ACK from unknown acker?!\n"); + } + } + else + { + DMSG(0, "NormSession::ServerHandleAckMessage() received unsolicited watermark ACK?!\n"); + } + break; + // (TBD) Handle other acknowledgement types - default: DMSG(0, "NormSession::ServerHandleAckMessage() node>%lu received " "unsupported ack type:%d\n", LocalNodeId(), ack.GetAckType()); @@ -1357,15 +1732,19 @@ void NormSession::ServerHandleNackMessage(const struct timeval& currentTime, Nor if (ServerGetFirstPending(txObjectIndex)) { NormObject* obj = tx_table.Find(txObjectIndex); - ASSERT(obj && obj->IsPending()); + ASSERT(obj); if (obj->IsPendingInfo()) { txBlockIndex = 0; } + else if (obj->GetFirstPending(txBlockIndex)) + { + txBlockIndex++; + } else { - obj->GetFirstPending(txBlockIndex); - txBlockIndex++; + txObjectIndex = next_tx_object_id; + txBlockIndex = 0; } } else @@ -1584,30 +1963,41 @@ void NormSession::ServerHandleNackMessage(const struct timeval& currentTime, Nor } if (!(block = object->FindBlock(nextBlockId))) { - // Try to recover block including parity calculation - if (!(block = object->ServerRecoverBlock(nextBlockId))) + // Is this entire block already tx pending? + if (!object->IsPendingSet(nextBlockId)) { - if (NormObject::STREAM == object->GetType()) + // Try to recover block including parity calculation + if (!(block = object->ServerRecoverBlock(nextBlockId))) { - DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu " - "recvd repair request for old stream block(%lu) ...\n", - LocalNodeId(), (UINT32)nextBlockId); - inRange = false; - if (!squelchQueued) + if (NormObject::STREAM == object->GetType()) { - ServerQueueSquelch(nextObjectId); - squelchQueued = true; + DMSG(4, "NormSession::ServerHandleNackMessage() node>%lu " + "recvd repair request for old stream block(%lu) ...\n", + LocalNodeId(), (UINT32)nextBlockId); + inRange = false; + if (!squelchQueued) + { + ServerQueueSquelch(nextObjectId); + squelchQueued = true; + } + continue; } - continue; + else + { + // Resource constrained, move on. + DMSG(2, "NormSession::ServerHandleNackMessage() node>%lu " + "Warning - server is resource contrained ...\n"); + inRange = false; + continue; + } } - else - { - // Resource constrained, move on. - DMSG(2, "NormSession::ServerHandleNackMessage() node>%lu " - "Warning - server is resource contrained ...\n"); - inRange = false; - continue; - } + } + else + { + // Entire block already tx pending, don't recover + DMSG(0, "NormSession::ServerHandleNackMessage() node>%lu " + "recvd SEGMENT repair request for pending block.\n"); + continue; } } freshBlock = false; @@ -1902,7 +2292,13 @@ bool NormSession::ServerBuildRepairAdv(NormCmdRepairAdvMsg& cmd) if (form != prevForm) { if (NormRepairRequest::INVALID != prevForm) - cmd.PackRepairRequest(req); // (TBD) error check; + { + if (0 == cmd.PackRepairRequest(req)) + { + DMSG(0, "NormSession::ServerBuildRepairAdv() warning: full msg\n"); + break; + } + } req.SetForm(form); cmd.AttachRepairRequest(req, segment_size); prevForm = form; @@ -1945,8 +2341,8 @@ bool NormSession::ServerBuildRepairAdv(NormCmdRepairAdvMsg& cmd) } // end while (nextObject) if (NormRepairRequest::INVALID != prevForm) { - cmd.PackRepairRequest(req); // (TBD) error check; - prevForm = NormRepairRequest::INVALID; + if (0 == cmd.PackRepairRequest(req)) + DMSG(0, "NormSession::ServerBuildRepairAdv() warning: full msg\n"); } return true; } // end NormSession::ServerBuildRepairAdv() @@ -2011,7 +2407,6 @@ bool NormSession::OnRepairTimeout(ProtoTimer& /*theTimer*/) return true; } // end NormSession::OnRepairTimeout() - // (TBD) Should pass current system time to ProtoTimer timeout handlers // for more efficiency ... bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) @@ -2069,6 +2464,8 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) advertise_repairs = false; else ReturnMessageToPool(msg); + // Pre-serve to allow pre-prompt for empty tx queue + if (message_queue.IsEmpty() && IsServer()) Serve(); } else { @@ -2089,7 +2486,7 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) // Check that any possible notifications posted in // the previous call to Serve() may have caused a // change in server state making it ready to send - if (IsServer()) Serve(); + //if (IsServer()) Serve(); return false; } else @@ -2100,6 +2497,7 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) } } // end NormSession::OnTxTimeout() + bool NormSession::SendMessage(NormMsg& msg) { struct timeval currentTime; @@ -2163,7 +2561,7 @@ bool NormSession::SendMessage(NormMsg& msg) msg.SetSourceId(local_node_id); UINT16 msgSize = msg.GetLength(); bool result = true; - // Drop some tx messages for testing purposes + // Possibly drop some tx messages for testing purposes bool drop = (UniformRand(100.0) < tx_loss_rate); if (drop || (clientMsg && client_silent)) { @@ -2236,6 +2634,11 @@ bool NormSession::OnProbeTimeout(ProtoTimer& /*theTimer*/) probe_timer.Deactivate(); return false; } + else if (0.0 == tx_rate) + { + // Sender paused, just idle probing until transmission is resumed + return true; + } // 2) Update grtt_estimate _if_ sufficient time elapsed. // This new code allows more liberal downward adjustment of @@ -2447,12 +2850,12 @@ bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/) struct timeval currentTime; ProtoSystemTime(currentTime); struct tm* ct = gmtime((time_t*)¤tTime.tv_sec); - DMSG(2, "Report time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", + DMSG(2, "REPORT time>%02d:%02d:%02d.%06lu node>%lu ***************************************\n", ct->tm_hour, ct->tm_min, ct->tm_sec, currentTime.tv_usec, LocalNodeId()); if (IsServer()) { - DMSG(2, "Local server:\n"); - DMSG(2, " tx_rate>%9.3lf kbps grtt>%lf\n", + DMSG(2, "Local status:\n"); + DMSG(2, " txRate>%9.3lf kbps grtt>%lf\n", ((double)tx_rate)*8.0/1000.0, grtt_advertised); } if (IsClient()) @@ -2461,18 +2864,19 @@ bool NormSession::OnReportTimeout(ProtoTimer& /*theTimer*/) NormServerNode* next; while ((next = (NormServerNode*)iterator.GetNextNode())) { - DMSG(2, "Remote server>%lu\n", next->GetId()); + DMSG(2, "Remote sender>%lu\n", next->GetId()); double rxRate = 8.0e-03*((double)next->RecvTotal()) / report_timer.GetInterval(); // kbps double rxGoodput = 8.0e-03*((double)next->RecvGoodput()) / report_timer.GetInterval(); // kbps next->ResetRecvStats(); - DMSG(2, " rx_rate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput); - DMSG(2, " objects completed>%lu pending>%lu failed:%lu\n", + DMSG(2, " rxRate>%9.3lf kbps rx_goodput>%9.3lf kbps\n", rxRate, rxGoodput); + DMSG(2, " rxObjects> completed>%lu pending>%lu failed:%lu\n", next->CompletionCount(), next->PendingCount(), next->FailureCount()); - DMSG(2, " buffer usage> current:%lu peak:%lu (overuns:%lu)\n", next->CurrentBufferUsage(), - next->PeakBufferUsage(), - next->BufferOverunCount()); + DMSG(2, " bufferUsage> current>%lu peak>%lu (overuns>%lu)\n", next->CurrentBufferUsage(), + next->PeakBufferUsage(), + next->BufferOverunCount()); DMSG(2, " resyncs>%lu nacks>%lu suppressed>%lu\n", next->ResyncCount(), - next->NackCount(), next->SuppressCount()); + next->NackCount(), + next->SuppressCount()); } } // end if (IsClient()) @@ -2506,7 +2910,7 @@ NormSession* NormSessionMgr::NewSession(const char* sessionAddress, UINT16 sessionPort, NormNodeId localNodeId) { - if (NORM_NODE_ANY == localNodeId) + if ((NORM_NODE_ANY == localNodeId) || (NORM_NODE_NONE == localNodeId)) { // Use local ip address to assign default localNodeId ProtoAddress localAddr; diff --git a/common/normSession.h b/common/normSession.h index ac9d866..88140d9 100644 --- a/common/normSession.h +++ b/common/normSession.h @@ -13,11 +13,20 @@ class NormController public: enum Event { + EVENT_INVALID = 0, + TX_QUEUE_VACANCY, TX_QUEUE_EMPTY, + TX_OBJECT_SENT, + TX_OBJECT_PURGED, + LOCAL_SERVER_CLOSED, + REMOTE_SERVER_NEW, + REMOTE_SERVER_INACTIVE, + REMOTE_SERVER_ACTIVE, RX_OBJECT_NEW, RX_OBJECT_INFO, RX_OBJECT_UPDATE, - RX_OBJECT_COMPLETE, + RX_OBJECT_COMPLETED, + RX_OBJECT_ABORTED }; virtual void Notify(NormController::Event event, @@ -54,9 +63,10 @@ class NormSessionMgr } void ActivateTimer(ProtoTimer& timer) {timer_mgr.ActivateTimer(timer);} - ProtoTimerMgr& GetTimerMgr() {return timer_mgr;} - ProtoSocket::Notifier& GetSocketNotifier() {return socket_notifier;} + ProtoTimerMgr& GetTimerMgr() const {return timer_mgr;} + ProtoSocket::Notifier& GetSocketNotifier() const {return socket_notifier;} + NormController* GetController() const {return controller;} private: ProtoTimerMgr& timer_mgr; ProtoSocket::Notifier& socket_notifier; @@ -86,26 +96,34 @@ class NormSession static const UINT16 DEFAULT_NPARITY; // General methods - const NormNodeId& LocalNodeId() {return local_node_id;} + const NormNodeId& LocalNodeId() const {return local_node_id;} bool Open(const char* interfaceName = NULL); void Close(); bool IsOpen() {return (rx_socket.IsOpen() || tx_socket.IsOpen());} const ProtoAddress& Address() {return address;} void SetAddress(const ProtoAddress& addr) {address = addr;} + bool SetTTL(UINT8 theTTL) + { + bool result = tx_socket.IsOpen() ? tx_socket.SetTTL(theTTL) : true; + ttl = result ? theTTL : ttl; + return result; + } + bool SetLoopback(bool state) + { + bool result = tx_socket.IsOpen() ? tx_socket.SetLoopback(state) : true; + loopback = result ? state : loopback; + return result; + } static double CalculateRate(double size, double rtt, double loss); + NormSessionMgr& GetSessionMgr() {return session_mgr;} // Session parameters double TxRate() {return (tx_rate * 8.0);} // (TBD) watch timer scheduling and min/max bounds - void SetTxRate(double txRate) {tx_rate = txRate / 8.0;} + void SetTxRate(double txRate); double BackoffFactor() {return backoff_factor;} void SetBackoffFactor(double value) {backoff_factor = value;} - void SetLoopback(bool state) - { - rx_socket.SetLoopback(state); - tx_socket.SetLoopback(state); - } bool CongestionControl() {return cc_enable;} void SetCongestionControl(bool state) {cc_enable = state;} @@ -130,11 +148,12 @@ class NormSession next_tx_object_id = IsServer() ? next_tx_object_id : baseId; session_id = IsServer() ? session_id : (UINT16)baseId; } - bool StartServer(unsigned long bufferSpace, - UINT16 segmentSize, - UINT16 numData, - UINT16 numParity, - const char* interfaceName = NULL); + bool IsServer() {return is_server;} + bool StartServer(UINT32 bufferSpace, + UINT16 segmentSize, + UINT16 numData, + UINT16 numParity, + const char* interfaceName = NULL); void StopServer(); NormStreamObject* QueueTxStream(UINT32 bufferSize, const char* infoPtr = NULL, @@ -142,15 +161,27 @@ class NormSession NormFileObject* QueueTxFile(const char* path, const char* infoPtr = NULL, UINT16 infoLen = 0); - - bool IsServer() {return is_server;} - UINT16 ServerSegmentSize() {return segment_size;} - UINT16 ServerBlockSize() {return ndata;} - UINT16 ServerNumParity() {return nparity;} - UINT16 ServerAutoParity() {return auto_parity;} + NormDataObject* QueueTxData(const char* dataPtr, + UINT32 dataLen, + const char* infoPtr = NULL, + UINT16 infoLen = 0); + void DeleteTxObject(NormObject* obj); + + // postive ack mgmnt + void ServerSetWatermark(NormObjectId objectId, + NormBlockId blockId, + NormSegmentId segmentId); + bool ServerAddAckingNode(NormNodeId nodeId); + void ServerRemoveAckingNode(NormNodeId nodeId); + + + UINT16 ServerSegmentSize() const {return segment_size;} + UINT16 ServerBlockSize() const {return ndata;} + UINT16 ServerNumParity() const {return nparity;} + UINT16 ServerAutoParity() const {return auto_parity;} void ServerSetAutoParity(UINT16 autoParity) {ASSERT(autoParity <= nparity); auto_parity = autoParity;} - UINT16 ServerExtraParity() {return extra_parity;} + UINT16 ServerExtraParity() const {return extra_parity;} void ServerSetExtraParity(UINT16 extraParity) {extra_parity = extraParity;} @@ -161,6 +192,13 @@ class NormSession objectId = (UINT16)index; return result; } + bool ServerGetFirstRepairPending(NormObjectId& objectId) + { + UINT32 index; + bool result = tx_repair_mask.GetFirstSet(index); + objectId = (UINT16)index; + return result; + } double ServerGrtt() {return grtt_advertised;} void ServerSetGrtt(double grttValue) @@ -192,15 +230,7 @@ class NormSession void ServerPutFreeSegment(char* segment) {segment_pool.Put(segment);} - void PromptServer() - { - if (!tx_timer.IsActive()) - { - tx_timer.SetInterval(0.0); - ActivateTimer(tx_timer); - } - } - + void PromptServer() {QueueMessage(NULL);} void TouchServer() { @@ -213,13 +243,23 @@ class NormSession bool StartClient(unsigned long bufferSpace, const char* interfaceName = NULL); void StopClient(); - bool IsClient() {return is_client;} - unsigned long RemoteServerBufferSize() + bool IsClient() const {return is_client;} + unsigned long RemoteServerBufferSize() const {return remote_server_buffer_size;} void SetUnicastNacks(bool state) {unicast_nacks = state;} - bool UnicastNacks() {return unicast_nacks;} + bool UnicastNacks() const {return unicast_nacks;} void ClientSetSilent(bool state) {client_silent = state;} - bool ClientIsSilent() {return client_silent;} + bool ClientIsSilent() const {return client_silent;} + + NormObject::NackingMode ClientGetDefaultNackingMode() const + {return default_nacking_mode;} + void ClientSetDefaultNackingMode(NormObject::NackingMode nackingMode) + {default_nacking_mode = nackingMode;} + + NormServerNode::RepairBoundary ClientGetDefaultRepairBoundary() const + {return default_repair_boundary;} + void ClientSetDefaultRepairBoundary(NormServerNode::RepairBoundary repairBoundary) + {default_repair_boundary = repairBoundary;} // Debug settings void SetTrace(bool state) {trace = state;} @@ -239,13 +279,12 @@ class NormSession ~NormSession(); void Serve(); - bool QueueTxObject(NormObject* obj, bool touchServer = true); - void DeleteTxObject(NormObject* obj); + bool QueueTxObject(NormObject* obj); + bool OnTxTimeout(ProtoTimer& theTimer); bool OnRepairTimeout(ProtoTimer& theTimer); bool OnFlushTimeout(ProtoTimer& theTimer); - bool OnWatermarkTimeout(ProtoTimer& theTimer); bool OnProbeTimeout(ProtoTimer& theTimer); bool OnReportTimeout(ProtoTimer& theTimer); @@ -271,6 +310,7 @@ class NormSession void AdjustRate(bool onResponse); bool ServerQueueSquelch(NormObjectId objectId); void ServerQueueFlush(); + bool ServerQueueWatermarkFlush(); bool ServerBuildRepairAdv(NormCmdRepairAdvMsg& cmd); void ServerUpdateGroupSize(); @@ -282,102 +322,111 @@ class NormSession void ClientHandleNackMessage(const NormNackMsg& nack); void ClientHandleAckMessage(const NormAckMsg& ack); - NormSessionMgr& session_mgr; - bool notify_pending; - ProtoTimer tx_timer; - ProtoSocket tx_socket; - ProtoSocket rx_socket; - NormMessageQueue message_queue; - NormMessageQueue message_pool; - ProtoTimer report_timer; - UINT16 tx_sequence; + NormSessionMgr& session_mgr; + bool notify_pending; + ProtoTimer tx_timer; + ProtoSocket tx_socket; + ProtoSocket rx_socket; + NormMessageQueue message_queue; + NormMessageQueue message_pool; + ProtoTimer report_timer; + UINT16 tx_sequence; // General session parameters - NormNodeId local_node_id; - ProtoAddress address; // session destination address & port - UINT8 ttl; // session multicast ttl - double tx_rate; // bytes per second - double backoff_factor; + NormNodeId local_node_id; + ProtoAddress address; // session destination address & port + UINT8 ttl; // session multicast ttl + bool loopback; // to receive own traffic + double tx_rate; // bytes per second + double backoff_factor; // Server parameters and state - bool is_server; - UINT16 session_id; - UINT16 segment_size; - UINT16 ndata; - UINT16 nparity; - UINT16 auto_parity; - UINT16 extra_parity; + bool is_server; + UINT16 session_id; + UINT16 segment_size; + UINT16 ndata; + UINT16 nparity; + UINT16 auto_parity; + UINT16 extra_parity; - NormObjectTable tx_table; - NormSlidingMask tx_pending_mask; - NormSlidingMask tx_repair_mask; - ProtoTimer repair_timer; - NormBlockPool block_pool; - NormSegmentPool segment_pool; - NormEncoder encoder; + NormObjectTable tx_table; + NormSlidingMask tx_pending_mask; + NormSlidingMask tx_repair_mask; + ProtoTimer repair_timer; + NormBlockPool block_pool; + NormSegmentPool segment_pool; + NormEncoder encoder; - NormObjectId next_tx_object_id; - unsigned int tx_cache_count_min; - unsigned int tx_cache_count_max; - NormObjectSize tx_cache_size_max; - ProtoTimer flush_timer; - int flush_count; - bool posted_tx_queue_empty; - ProtoTimer watermark_timer; - int watermark_count; - // (TBD) watermark_object_id, watermark_block_id, watermark_symbol_id + NormObjectId next_tx_object_id; + unsigned int tx_cache_count_min; + unsigned int tx_cache_count_max; + NormObjectSize tx_cache_size_max; + ProtoTimer flush_timer; + int flush_count; + bool posted_tx_queue_empty; + + // For postive acknowledgement collection + NormNodeTree acking_node_tree; + unsigned int acking_node_count; + bool watermark_pending; + NormObjectId watermark_object_id; + NormBlockId watermark_block_id; + NormSegmentId watermark_segment_id; + unsigned int acks_collected; // for unicast nack/cc feedback suppression - bool advertise_repairs; - bool suppress_nonconfirmed; - double suppress_rate; - double suppress_rtt; + bool advertise_repairs; + bool suppress_nonconfirmed; + double suppress_rate; + double suppress_rtt; - ProtoTimer probe_timer; // GRTT/congestion control probes - bool probe_proactive; - bool probe_pending; // true while CMD(CC) enqueued - bool probe_reset; + ProtoTimer probe_timer; // GRTT/congestion control probes + bool probe_proactive; + bool probe_pending; // true while CMD(CC) enqueued + bool probe_reset; - double grtt_interval; // current GRTT update interval - double grtt_interval_min; // minimum GRTT update interval - double grtt_interval_max; // maximum GRTT update interval + double grtt_interval; // current GRTT update interval + double grtt_interval_min; // minimum GRTT update interval + double grtt_interval_max; // maximum GRTT update interval - double grtt_max; - unsigned int grtt_decrease_delay_count; - bool grtt_response; - double grtt_current_peak; - double grtt_measured; - double grtt_age; - double grtt_advertised; - UINT8 grtt_quantized; - double gsize_measured; - double gsize_advertised; - UINT8 gsize_quantized; + double grtt_max; + unsigned int grtt_decrease_delay_count; + bool grtt_response; + double grtt_current_peak; + double grtt_measured; + double grtt_age; + double grtt_advertised; + UINT8 grtt_quantized; + double gsize_measured; + double gsize_advertised; + UINT8 gsize_quantized; // Server congestion control parameters - bool cc_enable; - UINT8 cc_sequence; - NormNodeList cc_node_list; - bool cc_slow_start; - double sent_rate; // measured sent rate - struct timeval prev_update_time; // for sent_rate measurement - unsigned long sent_accumulator; // for sent_rate measurement - double nominal_packet_size; + bool cc_enable; + UINT8 cc_sequence; + NormNodeList cc_node_list; + bool cc_slow_start; + double sent_rate; // measured sent rate + struct timeval prev_update_time; // for sent_rate measurement + unsigned long sent_accumulator; // for sent_rate measurement + double nominal_packet_size; // Client parameters - bool is_client; - NormNodeTree server_tree; - unsigned long remote_server_buffer_size; - bool unicast_nacks; - bool client_silent; + bool is_client; + NormNodeTree server_tree; + unsigned long remote_server_buffer_size; + bool unicast_nacks; + bool client_silent; + NormServerNode::RepairBoundary default_repair_boundary; + NormObject::NackingMode default_nacking_mode; // Protocol test/debug parameters - bool trace; - double tx_loss_rate; // for correlated loss - double rx_loss_rate; // for uncorrelated loss + bool trace; + double tx_loss_rate; // for correlated loss + double rx_loss_rate; // for uncorrelated loss // Linkers - NormSession* next; + NormSession* next; }; // end class NormSession diff --git a/common/normSimAgent.cpp b/common/normSimAgent.cpp index 7e81900..804b0e4 100644 --- a/common/normSimAgent.cpp +++ b/common/normSimAgent.cpp @@ -831,7 +831,7 @@ void NormSimAgent::Notify(NormController::Event event, break; } // end switch (object->GetType()) break; - case RX_OBJECT_COMPLETE: + case RX_OBJECT_COMPLETED: { switch(object->GetType()) { diff --git a/common/normTest.cpp b/common/normTest.cpp new file mode 100644 index 0000000..1c66c1f --- /dev/null +++ b/common/normTest.cpp @@ -0,0 +1,231 @@ +//////////////////////////////////////////////////// +// This is a test application for experimenting +// with the NORM API implementation during its +// development. A better-documented and complete +// example of the NORM API usage will be provided +// when the NORM API is more complete. + +#include "normApi.h" +#include "protokit.h" // for protolib debug, stuff, etc + +#include +#ifdef UNIX +#include // for "sleep()" +#endif // UNIX +int main(int argc, char* argv[]) +{ + printf("normTest starting ...\n"); + + SetDebugLevel(4); + + NormInstanceHandle instance = NormCreateInstance(); + +#ifdef WIN32 + const char* cachePath = "C:\\Adamson\\Temp\\cache\\"; +#else + const char* cachePath = "/tmp/"; +#endif // if/else WIN32/UNIX + + NormSetCacheDirectory(instance, cachePath); + + NormSessionHandle session = NormCreateSession(instance, + "224.1.2.3", + 6003, + NORM_NODE_ANY); + + NormSetMessageTrace(session, true); + + NormSetTxLoss(session, 10.0); // 1% packet loss + + NormSetGrttEstimate(session, 0.1);//0.001); // 1 msec initial grtt + + NormSetTransmitRate(session, 1.0e+05); // in bits/second + + NormSetDefaultRepairBoundary(session, NORM_BOUNDARY_BLOCK); + + // Uncomment to receive own traffic + NormSetLoopback(session, true); + + // Uncomment this line to participate as a receiver + NormStartReceiver(session, 1024*1024); + + // Uncomment the following line to start sender + NormStartSender(session, 1024*1024, 1024, 64, 0); + + NormAddAckingNode(session, NormGetLocalNodeId(session)); + + NormObjectHandle stream = NORM_OBJECT_INVALID; + const char* filePath = "/home/adamson/images/art/giger/giger205.jpg"; + const char* fileName = "ferrari.jpg"; + + // Uncomment this line to send a stream instead of the file + stream = NormOpenStream(session, 1024*1024); + NormSetStreamFlushMode(stream, NORM_FLUSH_NONE); + + int index = -1; // used to monitor reliable stream reception + int sendCount = 0; + int sendMax = 200; + NormEvent theEvent; + while (NormGetNextEvent(instance, &theEvent)) + { + switch (theEvent.type) + { + case NORM_TX_QUEUE_EMPTY: + if (NORM_OBJECT_INVALID != stream) + { + // Write a message to the "stream" + char buffer[1024]; + sprintf(buffer, "normTest says hello %d ...\n", sendCount++); + unsigned int len = strlen(buffer); + TRACE("writing to stream ...\n"); + if (len != NormWriteStream(stream, buffer, len)) + TRACE("incomplete write:%u\n", len); + else + { + NormFlushStream(stream); + NormSetWatermark(session, stream); + } + } + else + { + NormObjectHandle txFile = + NormQueueFile(session, + filePath, + fileName, + strlen(fileName)); + // Repeatedly queue our file for sending + if (NORM_OBJECT_INVALID == txFile) + { + DMSG(0, "normTest: error queuing file: %s\n", filePath); + break; + } + else + { + TRACE("QUEUED FILE ...\n"); + NormSetWatermark(session, txFile); + } + sendCount++; + } + break; + + case NORM_TX_OBJECT_PURGED: + DMSG(3, "normTest: NORM_TX_OBJECT_PURGED event ...\n"); + break; + + case NORM_RX_OBJECT_NEW: + DMSG(3, "normTest: NORM_RX_OBJECT_NEW event ...\n"); + break; + + case NORM_RX_OBJECT_INFO: + // Assume info contains '/' delimited string + if (NORM_OBJECT_FILE == NormGetObjectType(theEvent.object)) + { + NormObjectTransportId id = NormGetObjectTransportId(theEvent.object); + if (0 != (id & 0x01)) + { + //NormCancelObject(theEvent.object); + //break; + } + + char fileName[PATH_MAX]; + strcpy(fileName, cachePath); + int pathLen = strlen(fileName); + unsigned short nameLen = PATH_MAX - pathLen; + NormGetObjectInfo(theEvent.object, fileName+pathLen, &nameLen); + fileName[nameLen + pathLen] = '\0'; + char* ptr = fileName + 5; + while ('\0' != *ptr) + { + if ('/' == *ptr) *ptr = PROTO_PATH_DELIMITER; + ptr++; + } + if (!NormSetFileName(theEvent.object, fileName)) + TRACE("normTest: NormSetFileName(%s) error\n", fileName); + DMSG(3, "normTest: recv'd info for file: %s\n", fileName); + + + } + break; + + case NORM_RX_OBJECT_UPDATE: + { + //TRACE("normTest: NORM_RX_OBJECT_UPDATE event ...\n"); + if (NORM_OBJECT_STREAM != NormGetObjectType(theEvent.object)) + break; + char buffer[1024]; + unsigned int len = 1023; + do + { + len = 1023; + if (NormReadStream(theEvent.object, buffer, &len)) + { + buffer[len] = '\0'; + if (len) + { + TRACE("normTest: recvd(%u):\n\"%s\"\n", len, buffer); + // This while() loop is cheesy test, looking for broken stream + // + char* ptr = buffer; + while ((ptr = strstr(ptr, "hello"))) + { + int value; + if (1 == sscanf(ptr, "hello %d", &value)) + { + if (index >= 0) + { + if (1 != (value - index)) + TRACE("WARNING! possible break? value:%d index:%d\n", + value, index); + } + index = value; + } + else + { + TRACE("couldn't find index\n"); + } + ptr += 5; + } + } + } + else + { + TRACE("normTest: error reading stream\n"); + break; + } + } while (0 != len); + break; + } + + case NORM_RX_OBJECT_COMPLETED: + TRACE("normTest: NORM_RX_OBJECT_COMPLETED event ...\n"); + break; + + case NORM_RX_OBJECT_ABORTED: + TRACE("normTest: NORM_RX_OBJECT_ABORTED event ...\n"); + break; + + default: + TRACE("Got event type: %d\n", theEvent.type); + } // end switch(theEvent.type) + + // Break after sending "sendMax" messages or files + if (sendCount >= sendMax) break; + + } // end while (NormGetNextEvent()) + + fprintf(stderr, "normTest shutting down in 30 sec ...\n"); +#ifdef WIN32 + Sleep(30000); +#else + sleep(30); // allows time for cleanup if we're sending to someone else +#endif // if/else WIN32/UNIX + + NormCloseStream(stream); + NormStopReceiver(session); + NormStopSender(session); + NormDestroySession(session); + NormDestroyInstance(instance); + + fprintf(stderr, "normTest: Done.\n"); + return 0; +} // end main() diff --git a/common/normVersion.h b/common/normVersion.h index ff49c60..804dcc6 100644 --- a/common/normVersion.h +++ b/common/normVersion.h @@ -32,6 +32,6 @@ #ifndef _NORM_VERSION #define _NORM_VERSION -#define VERSION "1.2b1" +#define VERSION "1.2b2" #endif // _NORM_VERSION diff --git a/common/raft.cpp b/common/raft.cpp index 64f9cdf..27b419c 100644 --- a/common/raft.cpp +++ b/common/raft.cpp @@ -61,7 +61,7 @@ class RaftApp : public ProtoApp const char* const RaftApp::CMD_LIST[] = { "+debug", // debug - "+listen", // recv [/] + "+listen", // recv [/] "+dest", // send / "+rtspProxy", // rtsp NULL @@ -467,7 +467,7 @@ void RaftApp::OnClientSocketEvent(ProtoSocket& /*theSocket*/, while (rtsp_client_socket.Recv(buffer, buflen)) { buffer[buflen] = '\0'; - TRACE("%s"); + TRACE("%s", buffer); } TRACE("\n"); break; diff --git a/ns/example.tcl b/ns/example.tcl index d5c7579..87c867c 100644 --- a/ns/example.tcl +++ b/ns/example.tcl @@ -82,7 +82,7 @@ for {set i 2} {$i <= $numNodes} {incr i} { } $ns_ at 0.0 "$norm(1) sendFile 64000" -$ns_ at 3000.0 "finish $ns_ $f $nf" +$ns_ at 300.0 "finish $ns_ $f $nf" proc finish {ns_ f nf} { $ns_ flush-trace diff --git a/unix/Makefile.common b/unix/Makefile.common index 43523ec..0d4a474 100644 --- a/unix/Makefile.common +++ b/unix/Makefile.common @@ -13,7 +13,7 @@ NS = ../ns INCLUDES = $(TCL_INCL_PATH) $(SYSTEM_INCLUDES) -I$(UNIX) -I$(COMMON) -I$(PROTOLIB)/common -CFLAGS = -DPROTO_DEBUG -DUNIX -O -fPIC $(SYSTEM_HAVES) $(INCLUDES) +CFLAGS = -g -DPROTO_DEBUG -DUNIX -D_FILE_OFFSET_BITS=64 -O -fPIC $(SYSTEM_HAVES) $(INCLUDES) LDFLAGS = $(SYSTEM_LDFLAGS) @@ -83,7 +83,15 @@ APP_OBJ = $(APP_SRC:.cpp=.o) norm: $(APP_OBJ) libnorm.a $(LIBPROTO) $(CC) $(CFLAGS) -o $@ $(APP_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS) - + + +# (normTest) test of NORM API +TEST_SRC = $(COMMON)/normApi.cpp $(COMMON)/normTest.cpp +TEST_OBJ = $(TEST_SRC:.cpp=.o) + +normTest: $(TEST_OBJ) libnorm.a $(LIBPROTO) + $(CC) $(CFLAGS) -o $@ $(TEST_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS) + # (raft) command-line reliable tunnel helper RAFT_SRC = $(COMMON)/raft.cpp RAFT_OBJ = $(RAFT_SRC:.cpp=.o) diff --git a/unix/Makefile.linux b/unix/Makefile.linux index 00f862f..01a0c43 100644 --- a/unix/Makefile.linux +++ b/unix/Makefile.linux @@ -62,8 +62,8 @@ SYSTEM_LIBS = -ldl # (We export these for other Makefiles as needed) # -SYSTEM_HAVES = -DLINUX -DHAVE_IPV6 -DHAVE_GETLOGIN -DHAVE_LOCKF \ --DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT $(NETSEC) +SYSTEM_HAVES = -DLINUX -DHAVE_IPV6 -DHAVE_GETLOGIN -D_FILE_OFFSET_BITS=64 -DHAVE_LOCKF \ +-DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT SYSTEM_SRC = diff --git a/unix/Makefile.macosx b/unix/Makefile.macosx index 09e6b73..aed799e 100644 --- a/unix/Makefile.macosx +++ b/unix/Makefile.macosx @@ -6,9 +6,9 @@ # (Where to find X11 libraries, etc) # -SYSTEM_INCLUDES = -I/usr/X11R6/include -SYSTEM_LDFLAGS = -L/usr/X11R6/lib -SYSTEM_LIBS = +SYSTEM_INCLUDES = +SYSTEM_LDFLAGS = +SYSTEM_LIBS = -lresolv # 2) System specific capabilities # Must choose appropriate for the following: @@ -36,15 +36,16 @@ SYSTEM_LIBS = # (We export these for other Makefiles as needed) # -export SYSTEM_HAVES = -DMACOSX -DHAVE_IPV6 -DHAVE_ASSERT -DHAVE_GETLOGIN -DHAVE_FLOCK -DHAVE_DIRFD $(DNETSEC) -DSOCKLEN_T=int +SYSTEM_HAVES = -DMACOSX -DHAVE_IPV6 -DHAVE_ASSERT -DHAVE_GETLOGIN -DHAVE_FLOCK \ +-D_FILE_OFFSET_BITS=64 -DHAVE_DIRFD -DSOCKLEN_T=int SYSTEM_SRC = bsdRouteMgr.cpp # The "SYSTEM" keyword can be used for dependent makes SYSTEM = macosx -export CC = g++ -export RANLIB = ranlib -export AR = ar +CC = g++ +RANLIB = ranlib +AR = ar include Makefile.common diff --git a/unix/Makefile.solaris b/unix/Makefile.solaris index 532f76f..f26edf1 100644 --- a/unix/Makefile.solaris +++ b/unix/Makefile.solaris @@ -5,8 +5,8 @@ # 1) System specific additional libraries, include paths, etc # (Where to find X11 libraries, etc) # -SYSTEM_INCLUDES = -I/usr/openwin/include -SYSTEM_LDFLAGS = -L/usr/openwin/lib -R/usr/openwin/lib +SYSTEM_INCLUDES = +SYSTEM_LDFLAGS = SYSTEM_LIBS = -ldl -lnsl -lsocket -lresolv # 2) System specific capabilities @@ -35,7 +35,8 @@ SYSTEM_LIBS = -ldl -lnsl -lsocket -lresolv # (We export these for other Makefiles as needed) # -SYSTEM_HAVES = -DHAVE_IPV6 -DHAVE_ASSERT -DHAVE_CUSERID -DHAVE_LOCKF $(NETSEC) -DSOLARIS +SYSTEM_HAVES = -DHAVE_IPV6 -DHAVE_ASSERT -DHAVE_CUSERID -DHAVE_LOCKF \ +-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -DSOLARIS SYSTEM = solaris diff --git a/unix/unixPostProcess.cpp b/unix/unixPostProcess.cpp index 710e843..166bff4 100644 --- a/unix/unixPostProcess.cpp +++ b/unix/unixPostProcess.cpp @@ -234,6 +234,10 @@ bool UnixPostProcessor::ProcessFile(const char* path) // Restore signal handlers for parent signal(SIGTERM, sigtermHandler); signal(SIGINT, sigintHandler); + // The use of "waitpid()" here is a work-around + // for an IRIX SIGCHLD issue + int status; + while (waitpid(-1, &status, WNOHANG) > 0); signal(SIGCHLD, sigchldHandler); break; } diff --git a/win32/Norm.dsw b/win32/Norm.dsw index ed0c822..ef867b8 100644 --- a/win32/Norm.dsw +++ b/win32/Norm.dsw @@ -45,6 +45,24 @@ Package=<4> ############################################################################### +Project: "normTest"=..\normTest.dsp - Package Owner=<4> + +Package=<5> +{{{ +}}} + +Package=<4> +{{{ + Begin Project Dependency + Project_Dep_Name NormLib + End Project Dependency + Begin Project Dependency + Project_Dep_Name Protokit + End Project Dependency +}}} + +############################################################################### + Global: Package=<5> diff --git a/win32/Norm.opt b/win32/Norm.opt index 8b3b48e..f2abf06 100644 Binary files a/win32/Norm.opt and b/win32/Norm.opt differ diff --git a/win32/normTest.dsp b/win32/normTest.dsp new file mode 100644 index 0000000..5a747bb --- /dev/null +++ b/win32/normTest.dsp @@ -0,0 +1,106 @@ +# Microsoft Developer Studio Project File - Name="normTest" - Package Owner=<4> +# Microsoft Developer Studio Generated Build File, Format Version 6.00 +# ** DO NOT EDIT ** + +# TARGTYPE "Win32 (x86) Console Application" 0x0103 + +CFG=normTest - Win32 Debug +!MESSAGE This is not a valid makefile. To build this project using NMAKE, +!MESSAGE use the Export Makefile command and run +!MESSAGE +!MESSAGE NMAKE /f "normTest.mak". +!MESSAGE +!MESSAGE You can specify a configuration when running NMAKE +!MESSAGE by defining the macro CFG on the command line. For example: +!MESSAGE +!MESSAGE NMAKE /f "normTest.mak" CFG="normTest - Win32 Debug" +!MESSAGE +!MESSAGE Possible choices for configuration are: +!MESSAGE +!MESSAGE "normTest - Win32 Release" (based on "Win32 (x86) Console Application") +!MESSAGE "normTest - Win32 Debug" (based on "Win32 (x86) Console Application") +!MESSAGE + +# Begin Project +# PROP AllowPerConfigDependencies 0 +# PROP Scc_ProjName "" +# PROP Scc_LocalPath "" +CPP=cl.exe +RSC=rc.exe + +!IF "$(CFG)" == "normTest - Win32 Release" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 0 +# PROP BASE Output_Dir "Release" +# PROP BASE Intermediate_Dir "Release" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 0 +# PROP Output_Dir "Release" +# PROP Intermediate_Dir "Release" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /c +# ADD CPP /nologo /MD /W3 /GX /O2 /I "..\common" /I "..\protolib\common" /I "..\protolib\win32" /D "NDEBUG" /D "PROTO_DEBUG" /D "HAVE_IPV6" /D "HAVE_ASSERT" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /YX /FD /c +# ADD BASE RSC /l 0x409 /d "NDEBUG" +# ADD RSC /l 0x409 /d "NDEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 +# ADD LINK32 iphlpapi.lib ws2_32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /machine:I386 + +!ELSEIF "$(CFG)" == "normTest - Win32 Debug" + +# PROP BASE Use_MFC 0 +# PROP BASE Use_Debug_Libraries 1 +# PROP BASE Output_Dir "Debug" +# PROP BASE Intermediate_Dir "Debug" +# PROP BASE Target_Dir "" +# PROP Use_MFC 0 +# PROP Use_Debug_Libraries 1 +# PROP Output_Dir "Debug" +# PROP Intermediate_Dir "Debug" +# PROP Ignore_Export_Lib 0 +# PROP Target_Dir "" +# ADD BASE CPP /nologo /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c +# ADD CPP /nologo /MDd /W3 /GX /Od /I "..\common" /I "..\protolib\common" /I "..\protolib\win32" /D "_DEBUG" /D "PROTO_DEBUG" /D "HAVE_IPV6" /D "HAVE_ASSERT" /D "WIN32" /D "_CONSOLE" /D "_MBCS" /YX /FD /GZ /c +# ADD BASE RSC /l 0x409 /d "_DEBUG" +# ADD RSC /l 0x409 /d "_DEBUG" +BSC32=bscmake.exe +# ADD BASE BSC32 /nologo +# ADD BSC32 /nologo +LINK32=link.exe +# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept +# ADD LINK32 iphlpapi.lib ws2_32.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /subsystem:console /debug /machine:I386 /pdbtype:sept + +!ENDIF + +# Begin Target + +# Name "normTest - Win32 Release" +# Name "normTest - Win32 Debug" +# Begin Group "Source Files" + +# PROP Default_Filter "cpp;c;cxx;rc;def;r;odl;idl;hpj;bat" +# Begin Source File + +SOURCE=..\common\normApi.cpp +# End Source File +# Begin Source File + +SOURCE=..\common\normTest.cpp +# End Source File +# End Group +# Begin Group "Header Files" + +# PROP Default_Filter "h;hpp;hxx;hm;inl" +# End Group +# Begin Group "Resource Files" + +# PROP Default_Filter "ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe" +# End Group +# End Target +# End Project