diff --git a/README-PyNorm.txt b/README-PyNorm.txt index fd7d5c7..114b178 100644 --- a/README-PyNorm.txt +++ b/README-PyNorm.txt @@ -18,7 +18,7 @@ scripts. Requirements ------------ -PyNORM has been tested on Python versions 2.5 and 2.6. The code may work on +PyNORM has been tested on Python versions 2.5, 2.6, and 2.7. The code may work on earlier versions, but was not tested. PyNORM requires NORM to be built as a shared library (.so, .dylib, .dll), so @@ -29,6 +29,41 @@ PATH environment variable. On Windows, PyNORM requires the "PyWin32" module, available at: http://sf.net/projects/pywin32 + +------------- +Installation +------------- +A "setup.py" script is included that installs the 'pynorm' packages. Note, as +mentioned above, the NORM shared library must be installed on your system. The +'waf' install script will do this or it can be built with the NORM 'makefiles' +and manually emplaced. If you use the 'setup.py' script, you do _not_ need +to use the '--build-python' option of the 'waf' configuration script even if you +use 'waf' to build the NORM shared library. The 'waf' build tool can also be used +to install the 'pynorm' package. + +For example, to build and install "libnorm.so" and "pynorm" on Linux, use the +following steps. + +cd norm/makefiles +make -f Makefile.linux libnorm.so +sudo cp libnorm.so /usr/local/lib/ +cd ../ +sudo python setup.py install + +On MacOS, the same steps can be used, but using "libnorm.dylib" instead. + +----------------------- +Examples +----------------------- +Some 'pynorm' *.py examples are in the "norm/examples" directory. + +The 'normMsgr.py' example is functionally equivalent to the 'normMsgr.cpp' +example and the 'java/NormMsgr.java' example and illustrates multi-threaded +use of the 'pynorm' package for NORM-based data transmission and reception. +Note the "NORM_OBJECT_DATA" model is used and may provide out-of-order delivery +of the transmitted messages. A similar 'normStreamer.py' example will be +provided in the future to illustrate the NORM ordered, message stream delivery +mode of operation. ----------------------- "Extra" Package Modules diff --git a/VERSION.TXT b/VERSION.TXT index 34eaa7d..d420bd4 100644 --- a/VERSION.TXT +++ b/VERSION.TXT @@ -1,6 +1,32 @@ NORM Version History +Version 1.5b2 +============= + - Fixed bug with "graceful" stream closure when the sender application + did not first flush the stream before calling NormStreamClose() + - Added NORM_TX_FLUSH_COMPLETED notification for active stream + flushes, including the end of transmission flush when a stream + has been gracefully closed. + - Added NORM_SYNC_STREAM sync policy that is equivalent to NORM_SYNC_CURRENT + but attempts to sync to _start_ of stream (byte offset 0) + - Fixed issue with ackowledged, graceful stream closure to allow application + to choose to reset watermark before closing stream. The app needs to close + streams that remain unacknowledged at closure (i.e. catch TX_WATERMARK_COMPLETED + notification check acking status). Acknowledged, graceful shutdown streams, + are automatically closed. + - Fixed issue with compilation for NormObjectSize constructor, MSB(), and LSB() + methods on 32-bit systems. Thanks to Bill Skiba. + Version 1.5b1 +============= + - Rolled up all of the features described in Version 1.4b4 below + as a new "stable release" even though some minor documentation + issues are pending. + - Also includes the beginnings of code for "auto population" of + the sender acking node list for easier (less precoordination) + invocation of ACK-based flow control, etc + +Version 1.4b4 ============= - Added option to set NORM receiver "rx cache count max", i.e. the maximum number of pending object for which the receiver will keep diff --git a/examples/java/NormMsgr.java b/examples/java/NormMsgr.java new file mode 100644 index 0000000..6ed9680 --- /dev/null +++ b/examples/java/NormMsgr.java @@ -0,0 +1,789 @@ + +// This is a Java implementation of the same +// NORM "messenger" approach as normMsgr.cpp + +import java.util.concurrent.Semaphore; +import java.nio.ByteBuffer; +import java.util.Random; +import java.util.HashMap; +import java.util.LinkedList; + +import mil.navy.nrl.norm.NormEvent; +import mil.navy.nrl.norm.NormInstance; +import mil.navy.nrl.norm.NormNode; +import mil.navy.nrl.norm.NormSession; +import mil.navy.nrl.norm.NormObject; +import mil.navy.nrl.norm.NormData; +import mil.navy.nrl.norm.enums.NormEventType; +import mil.navy.nrl.norm.enums.NormObjectType; +import mil.navy.nrl.norm.enums.NormAckingStatus; +import mil.navy.nrl.norm.enums.NormSyncPolicy; + +public class NormMsgr +{ + static final int MSG_SIZE_MAX = 65536; + static final int MSG_HDR_SIZE = 2; + + private static Random randGen = new Random(System.currentTimeMillis()); + + // These members track NORM's "tx ready" status + private Semaphore normTxLock; // this acts as a mutex to manage "tx ready" status in a thread safe way + private Semaphore normTxReady; // this blocks data enqueuing until tx ready + private boolean norm_tx_vacancy; + private int norm_tx_queue_count; + private int norm_tx_queue_max; + private boolean norm_tx_watermark_pending; + + public NormInstance normInstance; + private NormSession normSession; + private boolean norm_acking; + + // This HashMap is where we cache enqueued Messages until NORM_TX_OBJECT_PURGED + // (In the C++ version we used the NormObjectSetUserData() which isn't available in NORM Java API) + private HashMap input_msg_list = new HashMap(); + + // Messages received from NORM are enqueued here until + // retrieved by the OutputWriter thread + private MessageQueue output_msg_queue; + private Semaphore normRxLock; // protects inter-thread access to output_msg_queue + private Semaphore normRxReady; // blocks OutputWriter when output_msg_queue is empty + + // Porbably will build these congestion control modes into the NORM API + public enum NormCCMode + { + NORM_FIXED, + NORM_CC, + NORM_CCE, + NORM_CCL; + } + + // Constructor + public NormMsgr() throws java.io.IOException, InterruptedException + { + normTxLock = new Semaphore(1); + normTxReady = new Semaphore(1); + try { + normInstance = new NormInstance(); + normInstance.setDebugLevel(3); + } + catch (java.io.IOException ex) { + System.err.println(ex); + throw(ex); + } + // Default parameter values + norm_tx_vacancy = true; + norm_tx_queue_max = 2048; // this is set largish, since I'm testing with _small_ messages + norm_tx_queue_count = 0; + norm_tx_watermark_pending = false; + norm_acking = false; + + normRxLock = new Semaphore(1); + normRxReady = new Semaphore(1); + output_msg_queue = new MessageQueue(); + normRxReady.acquire(); // nothing received yet + } + + public boolean openNormSession(String addr, short port, long nodeId) + { + try + { + normSession = normInstance.createSession(addr, port, nodeId); + if (null == normSession) + { + System.err.println("normMsgr error: unable to create NORM session"); + return false; + } + // Set some default parameters (maybe we should put parameter setting in Start()) + normSession.setRxCacheLimit(2*norm_tx_queue_max); // we let the receiver track some extra objects + normSession.setDefaultSyncPolicy(NormSyncPolicy.NORM_SYNC_ALL); + normSession.setDefaultUnicastNack(true); + normSession.setTxCacheBounds(10*1024*1024, norm_tx_queue_max, norm_tx_queue_max); + + normSession.setCongestionControl(true, true); + + //normSession.setMessageTrace(true); + } + catch (java.io.IOException ex) + { + System.err.println(ex); + } + return true; + } // end NormMsgr::openNormSession() + + public void addAckingNode(long nodeId) + { + try { + normSession.addAckingNode(nodeId); + } + catch (java.io.IOException ex) { + System.err.println(ex); + return; + } + norm_acking = true; + } // end NormMsgr::addAckingNode() + + public boolean setNormMulticastInterface(String ifaceName) + { + try { + normSession.setMulticastInterface(ifaceName); + } + catch (java.io.IOException ex) { + System.err.println(ex); + return false; + } + return true; + } // end NormMsgr::setMulticastInterface() + + public void setNormCCMode(NormCCMode ccMode) + { + switch (ccMode) + { + case NORM_CC: // default TCP-friendly congestion control + normSession.setEcnSupport(false, false, false); + break; + case NORM_CCE: // "wireless-ready" ECN-only congestion control + normSession.setEcnSupport(true, true); + break; + case NORM_CCL: // "loss tolerant", non-ECN congestion control + normSession.setEcnSupport(false, false, true); + break; + case NORM_FIXED: + normSession.setEcnSupport(false, false, false); + break; + } + if (NormCCMode.NORM_FIXED != ccMode) + normSession.setCongestionControl(true); + else + normSession.setCongestionControl(false); + } // end NormMsgr::setNormCCMode() + + public void setNormTxRate(double bitsPerSecond) + { + normSession.setTxRate(bitsPerSecond); + } + + public void setNormDebugLevel(int level) + {normInstance.setDebugLevel(level);} + + public void setNormMessageTrace(boolean state) + {normSession.setMessageTrace(state);} + + public boolean start(boolean send, boolean recv) + { + // Start NORM sender and/or receiver operation + boolean recvStarted = false; + try + { + if (recv) + { + normSession.startReceiver(10*1024*1024); + recvStarted = true; + } + if (send) + { + if (norm_acking) + { + // ack-based flow control enabled on command-line, + // so disable timer-based flow control + normSession.setFlowControl(0.0); + } + // Pick a random instance id for now + int instanceId = randGen.nextInt(); + normSession.startSender(instanceId, 10*1024*1024, 1400, (short)16, (short)4); + } + } + catch (java.io.IOException ex) + { + System.err.println(ex); + if (recvStarted) normSession.stopReceiver(); + return false; + } + return true; + } // end NormMsgr::start() + + public boolean sendMessage(Message msg) + { + // Future version will support NORM_OBJECT_STREAM as an option + while (!enqueueMessageObject(msg)); // keep trying until success (we're blocked by "normTxReady" semaphore) + return true; + } // end NormMsgr::sendMessage() + + public boolean enqueueMessageObject(Message msg) + { + try + { + normTxReady.acquire(); // caller will be blocked if NORM is not "tx ready" + normTxLock.acquire(); // this guarantees protected access to "tx ready" state variables + } + catch (InterruptedException ex) + { + System.err.println(ex); + if (!normTxReady.tryAcquire()) + normTxReady.release(); + return false; + } + NormData obj = null; + try { + obj = normSession.dataEnqueue(msg.getBuffer(), 0, msg.getSize()); + } + catch (java.io.IOException ex) { + //System.err.println(ex); + obj = null; + } + if (null == obj) + { + // Note we don't call normTxReady.release() here, it's up to the + // NormEventHandler to do that upon NORM_TX_QUEUE_EMPTY (or VACANCY for streams) + //System.err.println("NormMsgr::SendMessage() warning: data enqueue was blocked."); + norm_tx_vacancy = false; // there was no room at the inn + normTxLock.release(); + return false; + } + //System.err.println("caching msg for object " + obj + "\n"); + // Cache the msg associated with the resultant tx object. We use the + // input_msg_list HashMap so we can remove the msg upon NORM_TX_OBJECT_PURGED + input_msg_list.put(obj, msg); + + if (norm_acking) + { + norm_tx_queue_count++; + if (!norm_tx_watermark_pending && (norm_tx_queue_count >= (norm_tx_queue_max / 2))) + { + try { + normSession.setWatermark(obj); + norm_tx_watermark_pending = true; + } + catch (java.io.IOException ex) { + System.err.println(ex); + } + } + if (norm_tx_queue_count >= norm_tx_queue_max) + { + // We've filled our tx cache, so don't call normTxReady.release() + // NormEventHandler will do this upon watermark completion + normTxLock.release(); + return true; + } + } + normTxReady.release(); + normTxLock.release(); + return true; + } + + public void onNormTxObjectPurged(NormObject normObject) + { + try { + normTxLock.acquire(); + } + catch (InterruptedException ex) { + System.err.println(ex); + return; // Should kill program here (or maybe we should acquire uninterruptably)? + } + if (NormObjectType.NORM_OBJECT_DATA == normObject.getType()) + { + // removed "msg" will get garbage-collected + //System.err.println("purging msg for object " + (NormData)normObject + "\n"); + Message msg = input_msg_list.remove((NormData)normObject); + if (null == msg) // Shouldn't happen (and it doesn't after adding normTxLock.acquire()) + System.err.println("normMsgr warning: purged invalid object?!"); + } + normTxLock.release(); + } // end NormMsgr::onNormTxObjectPurged() + + // These next two methods are used by the NormEventHandler to update the + // NORM "tx ready" status variables in a thread-safe manner. Release of + // the "normTxReady" semaphore will unblock the InputThread if it was blocked + // due filling up the NORM tx queue. + // (The "!wasTxReady" check in these avoids any race condition with the normTxReady semaphore + public void onNormTxQueueVacancy() + { + try { + normTxLock.acquire(); + } + catch (InterruptedException ex) { + System.err.println(ex); + return; // Should kill program here (maybe we should acquire uninterruptably) + } + boolean wasTxReady = norm_tx_vacancy && (norm_acking ? (norm_tx_queue_count < norm_tx_queue_max) : true); + norm_tx_vacancy = true; + boolean isTxReady = norm_acking ? (norm_tx_queue_count < norm_tx_queue_max) : true; + if (!wasTxReady && isTxReady) + { + if (normTxReady.tryAcquire()) + System.err.println("NormMsgr::setNormTxVacancy() warning: normTxReady wasn't locked?!"); + normTxReady.release(); + } + normTxLock.release(); + } // end NormMsgr::onNormTxQueueVacancy() + + public void onNormTxWatermarkCompleted() + { + try { + normTxLock.acquire(); + } + catch (InterruptedException ex) { + System.err.println(ex); + return; // Should kill program here (or maybe we should acquire uninterruptably)? + } + boolean wasTxReady = norm_tx_vacancy && (norm_acking ? (norm_tx_queue_count < norm_tx_queue_max) : true); + norm_tx_queue_count -= (norm_tx_queue_max / 2); + norm_tx_watermark_pending = false; + boolean isTxReady = norm_tx_vacancy && (norm_acking ? (norm_tx_queue_count < norm_tx_queue_max) : true); + if (!wasTxReady && isTxReady) + { + if (normTxReady.tryAcquire()) + System.err.println("NormMsgr::decrementNormTxQueueCount() warning: normTxReady wasn't locked?!"); + normTxReady.release(); + } + normTxLock.release(); + } // end NormMsgr::onNormTxQueueVacancy() + + public void onNormRxObjectCompleted(NormObject obj) + { + try { + normRxLock.acquire(); + } + catch (InterruptedException ex) { + System.err.println(ex); + return; // Should kill program here (or maybe we should acquire uninterruptably)? + } + if (NormObjectType.NORM_OBJECT_DATA == obj.getType()) + { + // It's a message, so put it in the output_msg_queue for the OutputWriter thread + boolean wasEmpty = output_msg_queue.isEmpty(); + Message msg = new Message(((NormData)obj).getData(), (int)obj.getSize()); + output_msg_queue.append(msg); + if (wasEmpty) normRxReady.release(); // this will unblock the waiting OutputWriter + } + normRxLock.release(); + } // end NormMsgr::onNormRxObjectCompleted() + + // Called by OutputWriter thread to fetch received messages + public Message getRxMsg() + { + try { + normRxReady.acquire(); // will block if output_msg_queue is empty + normRxLock.acquire(); + } + catch (InterruptedException ex) { + System.err.println(ex); + return null; + } + Message msg = output_msg_queue.removeHead(); + if (!output_msg_queue.isEmpty()) + normRxReady.release(); + normRxLock.release(); + if (null == msg) // shouldn't happen + System.err.println("NormMsgr warning: output_msg_queue unexpectedly empty?!"); + return msg; + } + + public void RunThreads() + { + InputReader inputReader = new InputReader(this); + System.err.println("main thread starting input thread ..."); + inputReader.start(); + + NormEventHandler normEventHandler = new NormEventHandler(this); + System.err.println("main thread starting norm event handler thread ..."); + normEventHandler.start(); + + OutputWriter outputWriter = new OutputWriter(this); + System.err.println("main thread starting output thread ..."); + outputWriter.start(); + + // For now, we put the "inputReader" thread in the driver's seat. + // In the future, we'll be acquiring a semaphore shared among the + // threads and determine what thread(s) are signaling the parent + // and their status(es) + System.err.println("main thread waiting on input thread ..."); + inputReader.acquireLock(); // when acquired, indicates thread is done + + try { + inputReader.join(); + normInstance.stopInstance(); + normInstance.destroyInstance(); + normEventHandler.join(); + } + catch (InterruptedException ex) { + System.err.println(ex); + } + System.err.println("main thread exiting ..."); + } + + public class Message + { + private ByteBuffer msgBuffer; + + // Constructors + public Message(int size) + { + msgBuffer = ByteBuffer.allocateDirect(size); + } + public Message(byte[] buffer, int size) + { + msgBuffer = ByteBuffer.wrap(buffer, 0, size); + } + + public int getSize() + {return msgBuffer.capacity();} + public ByteBuffer getBuffer() + {return msgBuffer;} + + } // end class NormMsgr::Message + + public class MessageQueue extends LinkedList + { + public void append(Message msg) + {addLast(msg);} + public void prepend(Message msg) + {addFirst(msg);} + public Message removeHead() + {return removeFirst();} + public Message removeTail() + {return removeLast();} + public boolean isEmpty() + { + if (null == peekFirst()) + return true; + else + return false; + } + } // end class NormMsgr::MessageQueue + + private class InputReader extends Thread + { + private NormMsgr parentMsgr; + private boolean inputDone; + private Semaphore threadLock; + + public InputReader(NormMsgr parent) + { + parentMsgr = parent; + inputDone = false; + threadLock = new Semaphore(1); + } + + public boolean acquireLock() + { + try + { + threadLock.acquire(); + return true; + } + catch (InterruptedException ex) + { + System.err.println(ex); + return false; + } + } + + @Override + public void run() + { + System.err.println("input thread acquiring its lock ..."); + try + { + threadLock.acquire(); + } + catch (InterruptedException ex) + { + System.err.println(ex); + return; + } + System.err.println("input thread entering its main loop ..."); + byte[] msgHeader = new byte[MSG_HDR_SIZE]; + byte[] tmpBuffer = new byte[MSG_SIZE_MAX]; + while (!inputDone) + { + // Read in one message at time. + int want = MSG_HDR_SIZE; + int got = 0; + while (got < want) + { + try + { + int result = System.in.read(msgHeader, got, want - got); + if (-1 == result) + { + System.err.println("normMsgr: input end-of-file!"); + inputDone = true; + threadLock.release(); + System.err.println("input thread exiting 1 ..."); + return; + } + //System.err.println("input thread read " + result + " bytes of msg header\n"); + got += result; + } + catch (java.io.IOException ex) + { + System.err.println(ex); + } + } + int msgSize = 256*msgHeader[0] + msgHeader[1]; + //System.err.println("msg header is (256 * " + msgHeader[0] + ") + " + msgHeader[1] + " = " + msgSize); + got = 0; + msgSize -= 2; // already read header + while (got < msgSize) + { + try + { + int result = System.in.read(tmpBuffer, got, msgSize - got); + if (-1 == result) + { + System.err.println("normMsgr: input end-of-file!"); + inputDone = true; + threadLock.release(); + System.err.println("input thread exiting 2 ..."); + return; + } + //System.err.println("input thread read " + result + "bytes of message data\n"); + got += result; + } + catch (java.io.IOException ex) + { + System.err.println(ex); + } + } + int totalSize = msgSize + 2; + //System.err.println("normMsgr: read a " + totalSize + " byte message ...\n"); + + Message msg = new Message(msgSize); + msg.getBuffer().put(tmpBuffer, 0, msgSize); + parentMsgr.sendMessage(msg); + + } + System.err.println("input thread releasing its lock ..."); + threadLock.release(); + System.err.println("input thread exiting 3 ..."); + + } // end InputReader::run() + } // end class NormMsgr::InputReader + + + private class NormEventHandler extends Thread + { + private NormMsgr parentMsgr; + + public NormEventHandler(NormMsgr parent) + { + parentMsgr = parent; + } + + @Override + public void run() + { + try + { + System.err.println("entering NormEventHandler loop ..."); + NormEvent event; + while (null != (event = normInstance.getNextEvent())) + { + //System.err.println(event); + NormSession session = event.getSession(); + switch (event.getType()) + { + case NORM_TX_QUEUE_EMPTY: + case NORM_TX_QUEUE_VACANCY: + // This will unblock the InputThread if it was blocked + parentMsgr.onNormTxQueueVacancy(); + break; + + case NORM_TX_WATERMARK_COMPLETED: + if (NormAckingStatus.NORM_ACK_SUCCESS == session.getAckingStatus(NormNode.NORM_NODE_ANY)) + { + // This will unblock the InputThread if it was blocked + //System.err.println("sender tx watermark completed ..."); + parentMsgr.onNormTxWatermarkCompleted(); + } + else + { + // TBD - we could see who didn't ACK and possibly remove them + // from our acking list. For now, we are infinitely + // persistent by resetting watermark ack request + session.resetWatermark(); + } + break; + + case NORM_TX_OBJECT_PURGED: + parentMsgr.onNormTxObjectPurged(event.getObject()); + break; + + case NORM_RX_OBJECT_COMPLETED: + parentMsgr.onNormRxObjectCompleted(event.getObject()); + break; + + case NORM_REMOTE_SENDER_INACTIVE: + // optionally free memory resources that were in + // use by this remote sender + break; + + default: + break; + } + } + System.err.println("NormEventHandler got null event ..."); + } + catch (java.io.IOException ex) + { + System.err.println("NormGetNextEvent IOException:"); + System.err.println(ex); + } + System.err.println("exiting NormEventHandler ..."); + } // end NormEventHandler::run() + } // end class NormMsgr::NormEventHandler + + private class OutputWriter extends Thread + { + private NormMsgr parentMsgr; + boolean outputDone = false; + + // Constructors + public OutputWriter(NormMsgr parent) + { + parentMsgr = parent; + } + + @Override + public void run() + { + while (!outputDone) + { + Message msg = parentMsgr.getRxMsg(); // this will block if none available + // Make and write 2-byte length header (adding to the length to account for header, too) + int msgSize = msg.getSize() + MSG_HDR_SIZE; + byte[] msgHeader = new byte[MSG_HDR_SIZE]; + // This currently assumes 2-byte MSG_HDR_SIZE + msgHeader[0] = (byte)((msgSize >> 8) & 0xff); + msgHeader[1] = (byte)(msgSize & 0xff); + System.out.write(msgHeader, 0, MSG_HDR_SIZE); + // Write message content + System.out.write(msg.getBuffer().array(), 0, msg.getSize()); + } + System.err.println("NormMsgr output thread exiting ..."); + } + } // end class NormMsgr::OutputWriter() + + public static void usage() + { + System.err.println("Usage: normMsgr id {send &| recv} [addr [/]][ack [,,...]\n" + + " [cc|cce|ccl|rate ][interface ][debug ][trace]\n"); + } + + public static void main(String[] args) throws Throwable + { + // Default parameters + String sessionAddr = "224.1.2.3"; + short sessionPort = 6003; + long nodeId = 0; + boolean send = false; + boolean recv = false; + String[] ackerList = null; + String mcastIface = null; + int debugLevel = 0; + boolean normTrace = false; + NormCCMode ccMode = NormCCMode.NORM_CC; + double txRate = 1.0e+06; // only applies for NORM_FIXED ccMode + + int i = 0; + while (i < args.length) + { + String cmd = args[i++]; + if (cmd.equals("id")) + { + nodeId = Integer.parseInt(args[i++]); + } + else if (cmd.equals("send")) + { + send = true; + } + else if (cmd.equals("recv")) + { + recv = true; + } + else if (cmd.equals("addr")) + { + String[] items = args[i++].split("/"); + sessionAddr = items[0]; + if (items.length > 1) + sessionPort = (short)Integer.parseInt(items[1]); + } + else if (cmd.equals("interface")) + { + mcastIface = args[i++]; + } + else if (cmd.equals("cce")) + { + ccMode = NormCCMode.NORM_CCE; + } + else if (cmd.equals("cce")) + { + ccMode = NormCCMode.NORM_CCL; + } + else if (cmd.equals("rate")) + { + ccMode = NormCCMode.NORM_FIXED; + txRate = Double.parseDouble(args[i++]); + } + else if (cmd.equals("ack")) + { + ackerList = args[i++].split(","); + } + else if (cmd.equals("debug")) + { + debugLevel = Integer.parseInt(args[i++]); + } + else if (cmd.equals("trace")) + { + normTrace = true; + } + else + { + System.err.println("normMsgr error: invalid command '" + cmd + "'"); + usage(); + return; + } + } + if (!send && !recv) + { + System.err.println("normMsgr error: not configured to send or recv!"); + usage(); + return; + } + if (0 == nodeId) + { + System.err.println("normMsgr error: no local 'id' provided!"); + usage(); + return; + } + + NormMsgr msgr = new NormMsgr(); + + msgr.setNormDebugLevel(debugLevel); + + msgr.openNormSession(sessionAddr, sessionPort, nodeId); + + if (null != mcastIface) + msgr.setNormMulticastInterface(mcastIface); + + if (null != ackerList) + { + for (i = 0; i < ackerList.length; i++) + msgr.addAckingNode(Integer.parseInt(ackerList[i])); + } + + msgr.setNormCCMode(ccMode); + if (NormCCMode.NORM_FIXED == ccMode) + msgr.setNormTxRate(txRate); + + msgr.setNormMessageTrace(normTrace); + + msgr.start(send, recv); + + msgr.RunThreads(); + + System.err.println("normMsgr: main() exiting ..."); + } + + +} // end class NormMsgr diff --git a/examples/java/normMsgr.sh b/examples/java/normMsgr.sh new file mode 100755 index 0000000..a492eab --- /dev/null +++ b/examples/java/normMsgr.sh @@ -0,0 +1,2 @@ +#!/bin/bash +java -Djava.library.path=../../lib -cp .:../../lib/norm-1.0.0.jar NormMsgr $@ diff --git a/examples/normDataRecv.cpp b/examples/normDataRecv.cpp index 3a3626a..58764f5 100644 --- a/examples/normDataRecv.cpp +++ b/examples/normDataRecv.cpp @@ -87,10 +87,11 @@ int main(int argc, char* argv[]) NormInstanceHandle instance = NormCreateInstance(); // 2) Create a NormSession using default "automatic" local node id + fprintf(stderr, "joining session at addr/port %s/%hu\n", sessionAddr, sessionPort); NormSessionHandle session = NormCreateSession(instance, sessionAddr, sessionPort, - NORM_NODE_ANY); + 2);//NORM_NODE_ANY); // NOTE: These are debugging routines available @@ -124,8 +125,13 @@ int main(int argc, char* argv[]) { // Uncomment to allow multiple NORM processes on same session port number //NormSetRxPortReuse(session, true, sessionAddr); + + // NormSetDefaultUnicastNack(session, true); } + // Set a big rx cache for our current testing + NormSetRxCacheLimit(session, 4096); + // 3) Start the receiver with 1 Mbyte buffer per sender NormStartReceiver(session, 1024*1024); @@ -141,7 +147,7 @@ int main(int argc, char* argv[]) { case NORM_RX_OBJECT_NEW: gettimeofday(&startTime, NULL); - fprintf(stderr, "normDataRecv: NORM_RX_OBJECT_NEW event ...\n"); + //fprintf(stderr, "normDataRecv: NORM_RX_OBJECT_NEW event ...\n"); break; case NORM_RX_OBJECT_INFO: @@ -174,9 +180,10 @@ int main(int argc, char* argv[]) case NORM_RX_OBJECT_COMPLETED: { gettimeofday(&endTime, NULL); - fprintf(stderr, "normDataRecv: NORM_RX_OBJECT_COMPLETED event ...\n"); + //fprintf(stderr, "normDataRecv: NORM_RX_OBJECT_COMPLETED event ...\n"); unsigned int objSize = NormObjectGetSize(theEvent.object); const char* dataPtr = NormDataAccessData(theEvent.object); + // next 3 lines are temp for normMsgr testing // Validate that the data is complete/accurate // a) compare data size against the size embedded in the "INFO" char dataInfo[8192]; diff --git a/examples/normMsgr.cpp b/examples/normMsgr.cpp new file mode 100644 index 0000000..1ba8017 --- /dev/null +++ b/examples/normMsgr.cpp @@ -0,0 +1,972 @@ + +#include "normApi.h" + +#include // for printf(), etc +#include // for srand() +#include // for strrchr(), memset(), etc +#include // for gettimeofday() +#include // for htons() +#include // for, well, fnctl() +#include // obvious child +#include // embarrassingly obvious + +class NormMsgr +{ + public: + NormMsgr(); + ~NormMsgr(); + + // some day build these directly into NORM API + enum CCMode {NORM_FIXED, NORM_CC, NORM_CCE, NORM_CCL}; + + enum + { + MSG_SIZE_MAX = 65535, + MSG_HEADER_SIZE = 2 + }; + + // helper class and methods for message management + class MessageQueue; + class Message + { + friend class MessageQueue; + public: + Message(); + Message(char* buffer, unsigned int size); + ~Message(); + + bool Init(unsigned int size); + void Destroy(); + + unsigned int GetSize() const + {return msg_size;} + const char* GetHeader() const + {return msg_header;} + char* AccessHeader() + {return msg_header;} + const char* GetBuffer() const + {return msg_buffer;} + char* AccessBuffer() + {return msg_buffer;} + + void ResetIndex() + {msg_index = 0;} + void IncrementIndex(unsigned int count) + {msg_index += count;} + unsigned int GetIndex() const + {return msg_index;} + + bool IsComplete() const + {return (msg_index == (MSG_HEADER_SIZE + msg_size));} + + private: + unsigned int msg_size; + unsigned int msg_index; + char msg_header[MSG_HEADER_SIZE]; // this could be externalized to NormMsgr::input_msg_header / output_msg_header members + char* msg_buffer; + Message* prev; + Message* next; + }; // end class NormMsgr::Message + + class MessageQueue + { + public: + MessageQueue(); + ~MessageQueue(); + + bool IsEmpty() const + {return (NULL == head);} + + void Prepend(Message& msg); + void Append(Message& msg); + void Remove(Message& msg); + Message* RemoveHead(); + Message* RemoveTail(); + + void Destroy(); + + private: + Message* head; + Message* tail; + }; // end class NormMsgr::MessageQueue + Message* NewMessage(unsigned int size); + + bool OpenNormSession(NormInstanceHandle instance, + const char* addr, + unsigned short port, + NormNodeId nodeId); + void CloseNormSession(); + + void SetNormCongestionControl(CCMode ccMode); + void SetNormTxRate(double bitsPerSecond) + { + assert(NORM_SESSION_INVALID != norm_session); + NormSetTxRate(norm_session, bitsPerSecond); + } + void SetNormMulticastInterface(const char* ifaceName) + { + assert(NORM_SESSION_INVALID != norm_session); + NormSetMulticastInterface(norm_session, ifaceName); + } + void SetNormMessageTrace(bool state) + { + assert(NORM_SESSION_INVALID != norm_session); + NormSetMessageTrace(norm_session, state); + } + void AddAckingNode(NormNodeId ackId) + { + assert(NORM_SESSION_INVALID != norm_session); + NormAddAckingNode(norm_session, ackId); + norm_acking = true; // invoke ack-based flow control + } + + bool Start(bool sender, bool receiver); + void Stop() + {is_running = false;} + bool IsRunning() const + {return is_running;} + void HandleNormEvent(const NormEvent& event); + + // Sender methods + FILE* GetInputFile() const + {return input_file;} + bool InputNeeded() const + {return input_needed;} + bool InputMessageReady() const + {return ((NULL != input_msg) && !input_needed);} + bool ReadInput(); + + bool NormTxReady() const; + bool SendMessage(); + bool EnqueueMessageObject(); + + // Receiver methods + FILE* GetOutputFile() const + {return output_file;} + void SetOutputReady() + {output_ready = true;} + bool OutputReady() const + {return output_ready;} + bool OutputPending() const + {return output_pending;} + bool WriteOutput(); + + private: + bool is_running; + FILE* input_file; // stdin by default + bool input_needed; // + Message* input_msg; // current input message being read/sent* + MessageQueue input_msg_list; // list of enqueued messages (in norm sender cache) + + NormSessionHandle norm_session; + unsigned int norm_tx_queue_max; // max number of objects that can be enqueued at once + unsigned int norm_tx_queue_count; // count of unacknowledged enqueued objects (TBD - optionally track size too) + bool norm_tx_watermark_pending; + bool norm_tx_vacancy; + bool norm_acking; + + FILE* output_file; + bool output_ready; + bool output_pending; + Message* output_msg; + MessageQueue output_msg_queue; + +}; // end class NormMsgr + +NormMsgr::NormMsgr() + : input_file(stdin), input_needed(true), input_msg(NULL), + norm_session(NORM_SESSION_INVALID), norm_tx_queue_max(2048), norm_tx_queue_count(0), + norm_tx_watermark_pending(false), norm_tx_vacancy(true), norm_acking(false), + output_file(stdout), output_ready(true), output_pending(false), output_msg(NULL) +{ +} + +NormMsgr::~NormMsgr() +{ + if (NULL != input_msg) delete input_msg; + input_msg_list.Destroy(); + if (NULL != output_msg) delete output_msg; + output_msg_queue.Destroy(); +} + +bool NormMsgr::OpenNormSession(NormInstanceHandle instance, const char* addr, unsigned short port, NormNodeId nodeId) +{ + norm_session = NormCreateSession(instance, addr, port, nodeId); + if (NORM_SESSION_INVALID == norm_session) + { + fprintf(stderr, "normMsgr error: unable to create NORM session\n"); + return false; + } + // Set some default parameters (maybe we should put parameter setting in Start()) + NormSetRxCacheLimit(norm_session, norm_tx_queue_max); + NormSetDefaultSyncPolicy(norm_session, NORM_SYNC_ALL); + + NormSetDefaultUnicastNack(norm_session, true); + NormSetTxCacheBounds(norm_session, 10*1024*1024, norm_tx_queue_max, norm_tx_queue_max); + + //NormSetMessageTrace(norm_session, true); + return true; +} // end NormMsgr::OpenNormSession() + +void NormMsgr::CloseNormSession() +{ + if (NORM_SESSION_INVALID == norm_session) return; + NormDestroySession(norm_session); + norm_session = NORM_SESSION_INVALID; +} // end NormMsgr::CloseNormSession() + +void NormMsgr::SetNormCongestionControl(CCMode ccMode) +{ + assert(NORM_SESSION_INVALID != norm_session); + switch (ccMode) + { + case NORM_CC: // default TCP-friendly congestion control + NormSetEcnSupport(norm_session, false, false, false); + break; + case NORM_CCE: // "wireless-ready" ECN-only congestion control + NormSetEcnSupport(norm_session, true, true); + break; + case NORM_CCL: // "loss tolerant", non-ECN congestion control + NormSetEcnSupport(norm_session, false, false, true); + break; + case NORM_FIXED: + NormSetEcnSupport(norm_session, false, false, false); + break; + } + if (NORM_FIXED != ccMode) + NormSetCongestionControl(norm_session, true); + else + NormSetCongestionControl(norm_session, false); +} // end NormMsgr::SetNormCongestionControl() + +bool NormMsgr::Start(bool sender, bool receiver) +{ + if (receiver) + { + if (!NormStartReceiver(norm_session, 10*1024*1024)) + { + fprintf(stderr, "normMsgr error: unable to start NORM receiver\n"); + return false; + } + } + if (sender) + { + if (norm_acking) + { + // ack-based flow control enabled on command-line, + // so disable timer-based flow control + NormSetFlowControl(norm_session, 0.0); + } + // Pick a random instance id for now + struct timeval currentTime; + gettimeofday(¤tTime, NULL); + srand(currentTime.tv_usec); // seed random number generator + NormSessionId instanceId = (NormSessionId)rand(); + if (!NormStartSender(norm_session, instanceId, 10*1024*1024, 1400, 16, 4)) + { + fprintf(stderr, "normMsgr error: unable to start NORM sender\n"); + if (receiver) NormStopReceiver(norm_session); + return false; + } + } + is_running = true; + return true; +} // end NormMsgr::Start(); + +// Returns "true" when a complete message has been read +bool NormMsgr::ReadInput() +{ + assert(input_needed); + while (input_needed) + { + size_t readLength; + char* bufferPtr; + if ((NULL == input_msg) || (input_msg->GetIndex() < MSG_HEADER_SIZE)) + { + if (NULL == input_msg) + { + // Allocate a new message/buffer for input + if (NULL == (input_msg = new Message())) + { + perror("normMsgr new Message error"); + Stop(); // fatal out of memory error + return false; + } + } + // need to read msg "length" header + readLength = MSG_HEADER_SIZE - input_msg->GetIndex(); + bufferPtr = input_msg->AccessHeader() + input_msg->GetIndex(); + } + else + { + unsigned int offset = input_msg->GetIndex() - MSG_HEADER_SIZE; + readLength = input_msg->GetSize() - offset; + bufferPtr = input_msg->AccessBuffer() + offset; + } + size_t result = fread(bufferPtr, 1, readLength, input_file); + if (result > 0) + { + input_msg->IncrementIndex(result); + if (result < readLength) + { + // Still need more input + // (wait for next input notification to read more) + return false; + } + else if (MSG_HEADER_SIZE == input_msg->GetIndex()) + { + // We have now read the message size header + // TBD - support other message header formats? + // (for now, assume 2-byte message length header) + uint16_t msgSize ; + memcpy(&msgSize, input_msg->GetHeader(), MSG_HEADER_SIZE); + msgSize = ntohs(msgSize) - MSG_HEADER_SIZE; + if (!input_msg->Init(msgSize)) + { + perror("normMsgr: input message initialization error"); + Stop(); // fatal out of memory error + return false; + } + } + if (input_msg->IsComplete()) // have read complete header and message + { + // We have now read in the complete message + input_needed = false; + } + } + else + { + if (feof(input_file)) + { + // end-of-file reached, TBD - trigger final flushing and wrap-up + fprintf(stderr, "normMsgr: input end-of-file detected ...\n"); + } + else if (ferror(input_file)) + { + switch (errno) + { + case EINTR: + continue; // interupted, try again + case EAGAIN: + // input starved, wait for next notification + break; + default: + perror("normMsgr error reading input"); + break; + } + } + return false; + } + } + return true; +} // end NormMsgr::ReadInput() + +bool NormMsgr::WriteOutput() +{ + while (output_pending) + { + size_t writeLength; + const char* bufferPtr; + if (output_msg->GetIndex() < MSG_HEADER_SIZE) + { + writeLength = MSG_HEADER_SIZE - output_msg->GetIndex(); + bufferPtr = output_msg->GetHeader() + output_msg->GetIndex(); + } + else + { + unsigned int offset = output_msg->GetIndex() - MSG_HEADER_SIZE; + writeLength = output_msg->GetSize() - offset; + bufferPtr = output_msg->GetBuffer() + offset; + } + size_t result = fwrite(bufferPtr, 1, writeLength, output_file); + output_msg->IncrementIndex(result); + if (result < writeLength) + { + if (feof(output_file)) + { + // end-of-file reached, TBD - stop acting as receiver, signal sender we're done? + fprintf(stderr, "normMsgr: output end-of-file detected ...\n"); + } + else if (ferror(input_file)) + { + switch (errno) + { + case EINTR: + continue; // interupted, try again + case EAGAIN: + // input starved, wait for next notification + output_ready = false; + break; + default: + perror("normMsgr error writing output"); + break; + } + } + return false; + } + if (output_msg->IsComplete()) + { + delete output_msg; + output_msg = output_msg_queue.RemoveHead(); + if (NULL != output_msg) + output_msg->ResetIndex(); + else + output_pending = false; + break; + } + } + return true; +} // end NormMsgr::WriteOutput() + +bool NormMsgr::NormTxReady() const +{ + // This returns true if new tx data can be enqueued to NORM + // This is based on the state with respect to prior successful data + // data enqueuing (or stream writing) and NORM_TX_QUEUE_EMPTY or + // NORM_TX_QUEUE_VACANCY notifications (tracked by the "norm_tx_vacancy" variable, + // _and_ (if ack-based flow control is enabled) the norm_tx_queue_count or + // norm_stream_buffer_count status. + if (norm_tx_vacancy) + { + if (norm_tx_queue_count >= norm_tx_queue_max) + return false; // still waiting for ACK + return true; + } + else + { + return false; + } +} // end NormMsgr::NormTxReady() + +bool NormMsgr::SendMessage() +{ + // TBD - call EnqueueMessageObject() or WriteMessageStream() + // depending upon on configured mode + if (EnqueueMessageObject()) + { + // Our buffered message was sent, so reset input indices + // and request next message from input + input_msg_list.Append(*input_msg); + input_msg = NULL; + input_needed = true; + return true; + } + // else will be prompted to retry by NORM event (queue vacancy, watermark completion) + return false; +} // end NormMsgr::SendMessage() + +bool NormMsgr::EnqueueMessageObject() +{ + if (norm_acking) + { + assert(norm_tx_queue_count < norm_tx_queue_max); + if (norm_tx_queue_count >= norm_tx_queue_max) + return false; + } + + // Enqueue the message data for transmission + NormObjectHandle object = NormDataEnqueue(norm_session, input_msg->AccessBuffer(), input_msg->GetSize()); + if (NORM_OBJECT_INVALID == object) + { + // This might happen if a non-acking receiver is present and + // has nacked for the oldest object in the queue even if all + // of our acking receivers have acknowledged it. + norm_tx_vacancy = false; + return false; + } + NormObjectSetUserData(object, input_msg); // so we can remove/delete upon purge + if (norm_acking) + { + // ack-based flow control has been enabled + norm_tx_queue_count++; + if (!norm_tx_watermark_pending && (norm_tx_queue_count >= (norm_tx_queue_max / 2))) + { + NormSetWatermark(norm_session, object); + norm_tx_watermark_pending = true; + } + } + return true; +} // end NormMsgr::EnqueueMessageObject() + +void NormMsgr::HandleNormEvent(const NormEvent& event) +{ + switch (event.type) + { + case NORM_TX_QUEUE_EMPTY: + case NORM_TX_QUEUE_VACANCY: + norm_tx_vacancy = true; + break; + + case NORM_GRTT_UPDATED: + //fprintf(stderr, "new GRTT = %lf\n", NormGetGrttEstimate(norm_session)); + break; + + case NORM_TX_WATERMARK_COMPLETED: + if (NORM_ACK_SUCCESS == NormGetAckingStatus(norm_session)) + { + //fprintf(stderr, "WATERMARK COMPLETED\n"); + norm_tx_queue_count -= (norm_tx_queue_max / 2); + norm_tx_watermark_pending = false; + } + else + { + // TBD - we could see who didn't ACK and possibly remove them + // from our acking list. For now, we are infinitely + // persistent by resetting watermark ack request + NormResetWatermark(norm_session); + } + break; + + case NORM_TX_OBJECT_PURGED: + { + NormDataDetachData(event.object); + Message* msg = (Message*)NormObjectGetUserData(event.object); + if (NULL != msg) + { + input_msg_list.Remove(*msg); + delete msg; + } + break; + } + case NORM_RX_OBJECT_COMPLETED: + { + char* data = NormDataDetachData(event.object); + if (NULL != data) + { + Message* msg = new Message(data, NormObjectGetSize(event.object)); + if (NULL == msg) + { + perror("normMsgr new receive Message error"); + delete[] data; // TBD - may need to finally implement NormDelete() function! + // TBD Stop() as a fatal out of memory error? + break; + } + if (NULL == output_msg) + { + output_msg = msg; + output_msg->ResetIndex(); + output_pending = true; + } + else + { + output_msg_queue.Append(*msg); + } + } + // else TBD - "termination" info-only object? + break; + } + + default: + break; + } +} // end NormMsgr::HandleNormEvent() + + +void Usage() +{ + fprintf(stderr, "Usage: normMsgr id {send &| recv} [addr [/]][ack [,,...]\n" + " [cc|cce|ccl|rate ][interface ][debug ][trace]\n"); +} +int main(int argc, char* argv[]) +{ + // REQUIRED parameters initiailization + NormNodeId nodeId = NORM_NODE_NONE; + bool send = false; + bool recv = false; + + char sessionAddr[64]; + strcpy(sessionAddr, "224.1.2.3"); + unsigned int sessionPort = 6003; + + NormNodeId ackingNodeList[256]; + unsigned int ackingNodeCount = 0; + + double txRate = 0.0; // used for non-default NORM_FIXED ccMode + NormMsgr::CCMode ccMode = NormMsgr::NORM_CC; + const char* mcastIface = NULL; + + int debugLevel = 0; + bool trace = false; + + // Parse command-line + int i = 1; + while (i < argc) + { + const char* cmd = argv[i++]; + size_t len = strlen(cmd); + if (0 == strncmp(cmd, "send", len)) + { + send = true; + } + else if (0 == strncmp(cmd, "recv", len)) + { + recv = true; + } + else if (0 == strncmp(cmd, "addr", len)) + { + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'addr[/port]' value!\n"); + Usage(); + return -1; + } + const char* addrPtr = argv[i++]; + const char* portPtr = strchr(addrPtr, '/'); + if (NULL == portPtr) + { + strncpy(sessionAddr, addrPtr, 63); + sessionAddr[63] = '\0'; + } + else + { + size_t addrLen = portPtr - addrPtr; + if (addrLen > 63) addrLen = 63; // should issue error message + strncpy(sessionAddr, addrPtr, addrLen); + sessionAddr[addrLen] = '\0'; + portPtr++; + sessionPort = atoi(portPtr); + } + } + else if (0 == strncmp(cmd, "id", len)) + { + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'id' value!\n"); + Usage(); + return -1; + } + nodeId = atoi(argv[i++]); + } + else if (0 == strncmp(cmd, "ack", len)) + { + // comma-delimited acking node id list + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'id' value!\n"); + Usage(); + return -1; + } + const char* alist = argv[i++]; + while ((NULL != alist) && (*alist != '\0')) + { + // TBD - Do we need to skip leading white space? + if (1 != sscanf(alist, "%d", ackingNodeList + ackingNodeCount)) + { + fprintf(stderr, "nodeMsgr error: invalid acking node list!\n"); + Usage(); + return -1; + } + ackingNodeCount++; + alist = strchr(alist, ','); + if (NULL != alist) alist++; // point past comma + } + } + else if (0 == strncmp(cmd, "rate", len)) + { + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'rate' value!\n"); + Usage(); + return -1; + } + if (1 != sscanf(argv[i++], "%lf", &txRate)) + { + fprintf(stderr, "nodeMsgr error: invalid transmit rate!\n"); + Usage(); + return -1; + } + // set fixed-rate operation + ccMode = NormMsgr::NORM_FIXED; + } + else if (0 == strcmp(cmd, "cc")) + { + ccMode = NormMsgr::NORM_CC; + } + else if (0 == strcmp(cmd, "cce")) + { + ccMode = NormMsgr::NORM_CCE; + } + else if (0 == strcmp(cmd, "ccl")) + { + ccMode = NormMsgr::NORM_CCL; + } + else if (0 == strncmp(cmd, "interface", len)) + { + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'interface' !\n"); + Usage(); + return -1; + } + mcastIface = argv[i++]; + } + else if (0 == strncmp(cmd, "debug", len)) + { + if (i >= argc) + { + fprintf(stderr, "nodeMsgr error: missing 'interface' !\n"); + Usage(); + return -1; + } + debugLevel = atoi(argv[i++]); + } + else if (0 == strncmp(cmd, "trace", len)) + { + trace = true; + } + else if (0 == strncmp(cmd, "help", len)) + { + Usage(); + return 0; + } + else + { + fprintf(stderr, "nodeMsgr error: invalid command \"%s\"!\n", cmd); + Usage(); + return -1; + } + } + + if (!send && !recv) + { + fprintf(stderr, "normMsgr error: not configured to send or recv!\n"); + Usage(); + return -1; + } + if (NORM_NODE_NONE == nodeId) + { + fprintf(stderr, "normMsgr error: no local 'id' provided!\n"); + Usage(); + return -1; + } + + // TBD - should provide more error checking of calls + NormInstanceHandle normInstance = NormCreateInstance(); + NormSetDebugLevel(debugLevel); + + NormMsgr normMsgr; + + if (!normMsgr.OpenNormSession(normInstance, sessionAddr, sessionPort, (NormNodeId)nodeId)) + { + fprintf(stderr, "normMsgr error: unable to open NORM session\n"); + NormDestroyInstance(normInstance); + return false; + } + + for (unsigned int i = 0; i < ackingNodeCount; i++) + normMsgr.AddAckingNode(ackingNodeList[i]); + + normMsgr.SetNormCongestionControl(ccMode); + if (NormMsgr::NORM_FIXED == ccMode) + normMsgr.SetNormTxRate(txRate); + if (NULL != mcastIface) + normMsgr.SetNormMulticastInterface(mcastIface); + + if (trace) normMsgr.SetNormMessageTrace(true); + + // TBD - set NORM session parameters + normMsgr.Start(send, recv); + + int normfd = NormGetDescriptor(normInstance); + // Get input/output descriptors and set to non-blocking i/o + int inputfd = fileno(normMsgr.GetInputFile()); + if (-1 == fcntl(inputfd, F_SETFL, fcntl(inputfd, F_GETFL, 0) | O_NONBLOCK)) + perror("normMsgr: fcntl(inputfd, O_NONBLOCK) error"); + int outputfd = fileno(normMsgr.GetOutputFile()); + if (-1 == fcntl(outputfd, F_SETFL, fcntl(outputfd, F_GETFL, 0) | O_NONBLOCK)) + perror("normMsgr: fcntl(outputfd, O_NONBLOCK) error"); + fd_set fdsetInput, fdsetOutput; + FD_ZERO(&fdsetInput); + FD_ZERO(&fdsetOutput); + while (normMsgr.IsRunning()) + { + int maxfd = normfd; + FD_SET(normfd, &fdsetInput); + if (normMsgr.InputNeeded()) + { + //fprintf(stderr, "NEED INPUT ...\n"); + FD_SET(inputfd, &fdsetInput); + if (inputfd > maxfd) + maxfd = inputfd; + } + else + { + FD_CLR(inputfd, &fdsetInput); + } + int result; + if (normMsgr.OutputPending() && !normMsgr.OutputReady()) + { + FD_SET(outputfd, &fdsetOutput); + if (outputfd > maxfd) maxfd = outputfd; + result = select(maxfd+1, &fdsetInput, &fdsetOutput, NULL, NULL); + } + else + { + FD_CLR(outputfd, &fdsetOutput); + result = select(maxfd+1, &fdsetInput, NULL, NULL, NULL); + } + switch (result) + { + case -1: + switch (errno) + { + case EINTR: + case EAGAIN: + continue; + default: + perror("normMsgr select() error"); + // TBD - stop NormMsgr + break; + } + break; + case 0: + // shouldn't occur for now (no timeout) + continue; + default: + if (FD_ISSET(inputfd, &fdsetInput)) + { + normMsgr.ReadInput(); + } + if (FD_ISSET(normfd, &fdsetInput)) + { + NormEvent event; + if (NormGetNextEvent(normInstance, &event)) + normMsgr.HandleNormEvent(event); + } + if (FD_ISSET(outputfd, &fdsetOutput)) + { + normMsgr.SetOutputReady(); + } + break; + } + // As a result of reading input or NORM notification events, + // we may be ready to send a message if it's been read + if (normMsgr.InputMessageReady() && normMsgr.NormTxReady()) + normMsgr.SendMessage(); + // and/or output a received message if we need + if (normMsgr.OutputPending() && normMsgr.OutputReady()) + normMsgr.WriteOutput(); // TBD - implement output async i/o notification as needed + + } // end while(normMsgr.IsRunning() + NormDestroyInstance(normInstance); +} // end main() + +NormMsgr::Message::Message() + : msg_size(0), msg_index(0), msg_buffer(NULL), + prev(NULL), next(NULL) +{ +} + +NormMsgr::Message::Message(char* buffer, unsigned int size) + : msg_size(size), msg_index(size+MSG_HEADER_SIZE), msg_buffer(buffer), + prev(NULL), next(NULL) +{ + assert(2 == MSG_HEADER_SIZE); + uint16_t msgSize = size + MSG_HEADER_SIZE; + msgSize = htons(msgSize); + memcpy(msg_header, &msgSize, MSG_HEADER_SIZE); +} + +NormMsgr::Message::~Message() +{ + // in future we may need to use NormDelete() + // to delete msg_buffer if it came from NORM + if (NULL != msg_buffer) + { + delete[] msg_buffer; + msg_buffer = NULL; + } +} + +bool NormMsgr::Message::Init(unsigned int size) +{ + if ((NULL != msg_buffer) && (size != msg_size)) + { + delete[] msg_buffer; + msg_buffer = NULL; + } + if (NULL == msg_buffer) + msg_buffer = new char[size]; + if (NULL != msg_buffer) + { + assert(2 == MSG_HEADER_SIZE); + uint16_t msgSize = size + MSG_HEADER_SIZE; + msgSize = htons(msgSize); + memcpy(msg_header, &msgSize, MSG_HEADER_SIZE); + msg_index = MSG_HEADER_SIZE; // empty message + msg_size = size; + return true; + } + else + { + msg_index = msg_size = 0; + return false; + } +} // end NormMsgr::Message::Init() + +NormMsgr::MessageQueue::MessageQueue() + : head(NULL), tail(NULL) +{ +} + +NormMsgr::MessageQueue::~MessageQueue() +{ +} + +void NormMsgr::MessageQueue::Destroy() +{ + while (NULL != head) + { + Message* msg = RemoveHead(); + delete msg; + } +} // end NormMsgr::MessageQueue::Destroy() + +void NormMsgr::MessageQueue::Prepend(Message& msg) +{ + msg.prev = NULL; + if (NULL != head) + head->prev = &msg; + else + tail = &msg; + msg.next = head; + head = &msg; +} // end NormMsgr::MessageQueue::Prepend() + +void NormMsgr::MessageQueue::Append(Message& msg) +{ + msg.next = NULL; + if (NULL != tail) + tail->next = &msg; + else + head = &msg; + msg.prev = tail; + tail = &msg; +} // end NormMsgr::MessageQueue::Append() + +void NormMsgr::MessageQueue::Remove(Message& msg) +{ + if (NULL == msg.prev) + head = msg.next; + else + msg.prev->next = msg.next; + if (NULL == msg.next) + tail = msg.prev; + else + msg.next->prev = msg.prev; + msg.prev = msg.next = NULL; +} // end NormMsgr::MessageQueue::Remove() + +NormMsgr::Message* NormMsgr::MessageQueue::RemoveHead() +{ + Message* msg = head; + if (NULL != head) Remove(*head); + return msg; +} // end NormMsgr::MessageQueue::RemoveHead() + +NormMsgr::Message* NormMsgr::MessageQueue::RemoveTail() +{ + Message* msg = tail; + if (NULL != tail) Remove(*tail); + return msg; +} // end NormMsgr::MessageQueue::RemoveTail() diff --git a/examples/normStreamRecv.cpp b/examples/normStreamRecv.cpp index 6995f89..8fe95ca 100644 --- a/examples/normStreamRecv.cpp +++ b/examples/normStreamRecv.cpp @@ -42,7 +42,7 @@ int main(int argc, char* argv[]) // 2) Create a NormSession using default "automatic" local node id NormSessionHandle session = NormCreateSession(instance, - "127.0.0.1", + "224.1.2.3", 6003, NORM_NODE_ANY); @@ -51,7 +51,7 @@ int main(int argc, char* argv[]) // (Need to include "protolib/common/protoDebug.h" for this NormSetDebugLevel(3); // Uncomment to turn on debug NORM message tracing - //NormSetMessageTrace(session, true); + NormSetMessageTrace(session, true); // Uncomment to write debug output to file "normLog.txt" //NormOpenDebugLog(instance, "normLog.txt"); @@ -129,7 +129,6 @@ int main(int argc, char* argv[]) } case NORM_RX_OBJECT_UPDATED: { - //fprintf(stderr, "normStreamRecv: NORM_RX_OBJECT_UPDATED event ...\n"); if (theEvent.object != stream) { @@ -176,11 +175,12 @@ int main(int argc, char* argv[]) msgSync = false; continue; // try to re-sync and read again } + fprintf(stderr, "read %u bytes from stream ...\n", numBytes); msgIndex += numBytes; if (msgIndex == msgLen) { // Complete message read - //fprintf(stderr, "normStreamRecv msg: %s\n", msgBuffer+2); + fprintf(stderr, "normStreamRecv msg: %s\n", msgBuffer+2); msgLen = msgIndex = 0; // reset state variables for next message } else diff --git a/examples/normStreamSend.cpp b/examples/normStreamSend.cpp index b385f15..73ab113 100644 --- a/examples/normStreamSend.cpp +++ b/examples/normStreamSend.cpp @@ -1,5 +1,5 @@ /****************************************************************************** - Simple NORM_OBJECT_DATA sender example app using the NORM API + Simple NORM_OBJECT_STREAM sender example app using the NORM API USAGE: @@ -17,9 +17,7 @@ // Notes: -// 1) A single file is sent. -// 2) The program exits upon NORM_TX_FLUSH_COMPLETED notification (or user ) -// 3) NORM receiver should be started first (before sender starts) +// 1) A series of text messages are sent over a stream #include "normApi.h" // for NORM API @@ -33,10 +31,11 @@ int main(int argc, char* argv[]) { // 0) Default parameter values - const unsigned int MSG_LENGTH_MIN = 1400; - const unsigned int MSG_LENGTH_MAX = 1400; + const int MSG_COUNT_MAX = 10; + const unsigned int MSG_LENGTH_MIN = 40; + const unsigned int MSG_LENGTH_MAX = 40; UINT32 streamBufferSize = 4*1024*1024; // 1 Mbyte stream buffer size - double normRate = 1.0e+07; // 1 Mbps default NORM tx rate for fixed rate operation (bits/sec units here) + double normRate = 1.0e+07; // 10 Mbps default NORM tx rate for fixed rate operation (bits/sec units here) double msgRate = -1.0; //1e+06; // 32 kbits/sec default message rate // 1) Create a NORM API "NormInstance" @@ -45,7 +44,7 @@ int main(int argc, char* argv[]) // 2) Create a NormSession using default "automatic" local node id (based on IP addr) // TBD - add an option to set a specific NormNodeId NormSessionHandle session = NormCreateSession(instance, - "loon", //"224.1.2.3", + "224.1.2.3", 6003, 1);//NORM_NODE_ANY); @@ -53,7 +52,7 @@ int main(int argc, char* argv[]) // (not necessary for normal app use) NormSetDebugLevel(3); // Uncomment to turn on debug NORM message tracing - //NormSetMessageTrace(session, true); + NormSetMessageTrace(session, true); // Uncomment to turn on some random packet loss //NormSetTxLoss(session, 25.0); // 25% packet loss for testing purposes @@ -177,6 +176,10 @@ int main(int argc, char* argv[]) int result = select(normfd+1, &fdset, NULL, NULL, timeoutPtr); + bool keepSending = true; + if ((MSG_COUNT_MAX > 0) && (msgCount >= (unsigned int)MSG_COUNT_MAX)) + keepSending = false; + if (result > 0) { // Get and handle NORM API event @@ -194,7 +197,7 @@ int main(int argc, char* argv[]) else fprintf(stderr, "normStreamSend: NORM_TX_QUEUE_EMPTY event ...\n"); */ - if (!vacancy) + if (keepSending && (bytesWritten < msgLen)) { // Finish writing remaining pending message content (as much as can be written) bytesWritten += NormStreamWrite(stream, msgData + bytesWritten, msgLen - bytesWritten); @@ -219,7 +222,7 @@ int main(int argc, char* argv[]) break; case NORM_GRTT_UPDATED: - fprintf(stderr, "normStreamRecv: NORM_GRTT_UPDATED event ...\n"); + fprintf(stderr, "normStreamSend: NORM_GRTT_UPDATED event ...\n"); break; default: @@ -234,6 +237,7 @@ int main(int argc, char* argv[]) break; } + // This code writes _new_ message(s) to the stream _if_ there is "vacancy" // and it is time based on "msgRate" and how much time has passed since "lastTime" struct timeval currentTime; @@ -244,7 +248,7 @@ int main(int argc, char* argv[]) else timeDelta -= 1.0e-06 * (lastTime.tv_usec - currentTime.tv_usec); timeAccumulator += timeDelta; - while (vacancy && (timeAccumulator > delayTime)) + while (keepSending && vacancy && (timeAccumulator > delayTime)) { timeAccumulator -= delayTime; // subtract last message tx duration from accumulator // Fill buffer with new message "data" text character (a-z) @@ -270,6 +274,12 @@ int main(int argc, char* argv[]) NormStreamMarkEom(stream); msgCount++; delayTime = (msgRate > 0.0) ? ((double)msgLen / msgRate) : 0.0; + if ((MSG_COUNT_MAX > 0) && ((unsigned int)msgCount >= MSG_COUNT_MAX)) + { + fprintf(stderr, "closing stream after %u messages ...\n", msgCount); + NormStreamClose(stream, true); // gracefully close stream + keepSending = false; + } } } if (timeAccumulator <= delayTime) @@ -279,6 +289,7 @@ int main(int argc, char* argv[]) } lastTime = currentTime; + } // end while (keepGoing) NormStopSender(session); diff --git a/examples/python/normMsgr.py b/examples/python/normMsgr.py new file mode 100644 index 0000000..4ee6305 --- /dev/null +++ b/examples/python/normMsgr.py @@ -0,0 +1,396 @@ + +from threading import Thread, Lock +import sys +import random +import pynorm +import signal +import time +from collections import deque + +MSG_HDR_SIZE = 2 + +class InputThread(Thread): + """This thread reads 'messages" from STDIN and send them""" + + def __init__(self, parent, *args, **kwargs): + super(InputThread, self).__init__(*args, **kwargs) + self.setDaemon(True) ;# this is "child" daemon thread + self.msgr = parent + + def run(self): + while True: + try: + msgHdr = bytearray(sys.stdin.read(MSG_HDR_SIZE)) + except: + sys.stderr.write("normMsgr: input thread end-of-file 1 ...\n") + return + try: + msgSize = 256*int(msgHdr[0]) + int(msgHdr[1]) + msgBuffer = sys.stdin.read(msgSize - 2) + except: + sys.stderr.write("normMsgr: input thread end-of-file 2 ...\n") + return + msgr.sendMessage(msgBuffer) ;# will block if NORM not "tx ready" + +class OutputThread(Thread): + """This thread writes received 'messages" to STDOUT.""" + + def __init__(self, parent, *args, **kwargs): + super(OutputThread, self).__init__(*args, **kwargs) + self.setDaemon(True) ;# this is "child" daemon thread + self.msgr = parent + + def run(self): + while True: + msg = msgr.getRxMsg() ;# will block if none ready + msgLen = len(msg) + MSG_HDR_SIZE + msgHeader = bytearray(MSG_HDR_SIZE) + msgHeader[0] = (msgLen >> 8) & 0x00ff + msgHeader[1] = msgLen & 0x00ff + sys.stdout.write(msgHeader) + sys.stdout.write(msg) + del msg + + +class NormMsgr: + """This class keeps state for NORM tx/rx operations""" + + def __init__(self): + self.normInstance = pynorm.Instance() + self.normSession = None + # Sender state members + self.normTxLock = Lock() ;# for thread-safe access to NORM tx state variables + self.normTxReady = Lock() + self.norm_tx_vacancy = True + self.norm_tx_queue_count = 0 + self.norm_tx_queue_max = 2048 + self.norm_tx_watermark_pending = False + self.norm_acking = False + self.tx_msg_cache = {} + # Receiver state members + self.normRxLock = Lock() + self.normRxReady = Lock() + self.normRxReady.acquire() ;# no rx messages yet + self.output_msg_queue = deque() + random.seed(None) ;# seeds with current time + + def openNormSession(self, addr, port, nodeId): + # Create a NormSession and set some default parameters + self.normSession = self.normInstance.createSession(addr, port, nodeId) + self.normSession.setRxCacheLimit(2*self.norm_tx_queue_max) ;# we let the receiver track some extra objects + self.normSession.setDefaultSyncPolicy(pynorm.NORM_SYNC_ALL); + self.normSession.setDefaultUnicastNack(True); + self.normSession.setTxCacheBounds(10*1024*1024, self.norm_tx_queue_max, self.norm_tx_queue_max); + self.normSession.setCongestionControl(True, True); + return self.normSession + + def addAckingNode(self, nodeId): + self.normSession.addAckingNode(nodeId); + self.norm_acking = True + + def setNormMulticastInterface(self, ifaceName): + self.normSession.setMulticastInterface(ifaceName) + + def setNormCCMode(self, ccMode): + if ccMode == "cc": + self.normSession.setEcnSupport(False, False, False) + elif ccMode == "cce": + self.normSession.setEcnSupport(True, True) + elif ccMode == "ccl": + self.normSession.setEcnSupport(False, False, True) + elif ccMode == "fixed": + self.normSession.setEcnSupport(False, False, False) + else: + raise Exception("normMsgr: invalid ccMode \"%s\"" % ccMode) + if ccMode != "fixed": + self.normSession.setCongestionControl(True) + else: + self.normSession.setCongetstionControl(False) + + def setNormTxRate(self, bitsPerSecond): + self.normSession.setTxRate(bitsPerSecond) + + def setNormDebugLevel(self, level): + self.normInstance.setDebugLevel(level) + + def setNormMessageTrace(self, state): + self.normSession.setMessageTrace(state) + + def start(self, send, recv): + if (recv): + self.normSession.startReceiver(10*1024*1024) + if (send): + if (self.norm_acking): + self.normSession.setFlowControl(0.0) + # We use a random sender instanceId in case of stop/restart + instanceId = random.randint(0, 0xffff) + self.normSession.startSender(instanceId, 10*1024*1024, 1400, 16, 4); + + def stop(self): + del self.normInstance + self.normInstance = None + + def sendMessage(self, msgBuf): + # caller will be blocked if NORM is (or becomes) not "tx ready" + while not self.enqueueMessageObject(msgBuf): + #sys.stderr.write("enqueue message was blocked\n") + continue + + def enqueueMessageObject(self, msgBuf): + self.normTxReady.acquire() ;# this blocks until NORM is "tx ready" + with self.normTxLock: + #sys.stderr.write("normMsgr: sending %d byte message payload ...\n" % len(msgBuf)) + obj = self.normSession.dataEnqueue(msgBuf) + if obj is None: + self.norm_tx_vacancy = False ;# will be cleared by NORM_TX_QUEUE_EMPTY, etc + return False + # cache the sent msgBuf until NORM_TX_OBJECT_PURGED + self.tx_msg_cache[obj] = msgBuf + if (self.norm_acking): + # Manage ack-based flow control state + self.norm_tx_queue_count += 1 + if not self.norm_tx_watermark_pending: + if self.norm_tx_queue_count >= self.norm_tx_queue_max/2: + #sys.stderr.write("setting watermark ...\n") + self.normSession.setWatermark(obj) + #sys.stderr.write("watermark set.\n"); + self.norm_tx_watermark_pending = True + if self.norm_tx_queue_count >= self.norm_tx_queue_max: + # Don't release "normTxReady" since cache is filled + # (Will be released upon NORM_TX_WATERMARK_COMPLETED) + return True + self.normTxReady.release() + return True + + def onNormTxObjectPurged(self, obj): + with self.normTxLock: + if pynorm.NORM_OBJECT_DATA == obj.getType(): + del self.tx_msg_cache[obj] + + def onNormTxQueueVacancy(self): + with self.normTxLock: + wasTxReady = self.norm_tx_vacancy + if wasTxReady and self.norm_acking: + wasTxReady = self.norm_tx_queue_count < self.norm_tx_queue_max + self.norm_tx_vacancy = True + if self.norm_acking: + isTxReady = self.norm_tx_queue_count < self.norm_tx_queue_max + else: + isTxReady = False + if isTxReady and not wasTxReady: + if self.normTxReady.acquire(False): + sys.stderr.write("normMsgr onNormTxQueueVacancy() warning: normTxReady wasn't locked?!\n") + #sys.stderr.write("tx vacancy releasing norm tx ready ...\n"); + self.normTxReady.release() + + def onNormTxWatermarkCompleted(self): + with self.normTxLock: + wasTxReady = self.norm_tx_vacancy + if wasTxReady and self.norm_acking: + wasTxReady = self.norm_tx_queue_count < self.norm_tx_queue_max + self.norm_tx_watermark_pending = False + self.norm_tx_queue_count -= self.norm_tx_queue_max / 2 + isTxReady = self.norm_tx_vacancy + if isTxReady and self.norm_acking: + isTxReady = self.norm_tx_queue_count < self.norm_tx_queue_max + else: + isTxReady = False + if isTxReady and not wasTxReady: + if self.normTxReady.acquire(False): + sys.stderr.write("normMsgr onNormTxWatermarkCompleted() warning: normTxReady wasn't locked?!\n") + #sys.stderr.write("watermark completion releasing norm tx ready ...\n") + self.normTxReady.release() + + def onNormRxObjectCompleted(self, obj): + with self.normRxLock: + if pynorm.NORM_OBJECT_DATA == obj.getType(): + if 0 != len(self.output_msg_queue): + wasEmpty = False + else: + wasEmpty = True + msg = obj.getData() + self.output_msg_queue.append(msg) + if wasEmpty: + #sys.stderr.write("releasing normRxReady ...\n") + self.normRxReady.release() ;# unblocks waiting OutputThread + + def getRxMsg(self): + self.normRxReady.acquire() ;# blocks if output_msg_queue is empty + with self.normRxLock: + msg = self.output_msg_queue.popleft() + if 0 != len(self.output_msg_queue): + self.normRxReady.release() ;# not empty yet + return msg + + def getNextNormEvent(self): + if self.normInstance is None: + return None + else: + return self.normInstance.getNextEvent() + +class NormEventHandler(Thread): + """This thread calls normInstance.getNextEvent() and handles the events""" + + def __init__(self, parentMsgr, *args, **kwargs): + super(NormEventHandler, self).__init__(*args, **kwargs) + self.setDaemon(True) ;# this is "child" daemon thread + self.lock = Lock() + self.msgr = parentMsgr + + def run(self): + self.lock.acquire() + while True: + try: + event = self.msgr.getNextNormEvent() + except: + sys.stderr.write("get next event exception\n"); + self.lock.release() + return + if event is None: + break + if pynorm.NORM_EVENT_INVALID == event.type: + continue + elif pynorm.NORM_TX_QUEUE_EMPTY == event.type or pynorm.NORM_TX_QUEUE_VACANCY == event.type: + msgr.onNormTxQueueVacancy() + elif pynorm.NORM_TX_WATERMARK_COMPLETED == event.type: + if pynorm.NORM_ACK_SUCCESS == event.session.getAckingStatus(): + # All receivers acknowledged + msgr.onNormTxWatermarkCompleted() + else: + # TBD - we could see who didn't ACK and possibly remove them + # from our acking list. For now, we are infinitely + # persistent by resetting watermark ack request + event.session.resetWatermark() + elif pynorm.NORM_TX_OBJECT_PURGED == event.type: + msgr.onNormTxObjectPurged(event.object) + elif pynorm.NORM_RX_OBJECT_COMPLETED == event.type: + msgr.onNormRxObjectCompleted(event.object) + #else: + # sys.stderr.write("normMsgr: NormEventHandler warning: unhandled event: %s\n" % str(event)) + sys.stderr.write("normMsgr: NormEventHandler thread exiting ...\n"); + self.lock.release() + +def usage(): + sys.stderr.write("Usage: normMsgr.py id {send &| recv} [addr [/]][ack [,,...]\n" + + " [cc|cce|ccl|rate ][interface ][debug ][trace]\n") + +# Default parameters +nodeId = None +sessionAddr = "224.1.2.3" +sessionPort = 6003 +send = False +recv = False +ccMode = "cc" +txRate = None +ackerList = [] +debugLevel = 3 +normTrace = False +mcastIface = None + +# Parse command-line +cmd = None +val = None +try: + i = 1 + while i < len(sys.argv): + cmd = sys.argv[i] + i += 1 + if "id" == cmd: + val = sys.argv[i] + nodeId = int(val) + i += 1 + elif "addr" == cmd: + val = sys.argv[i] + i += 1 + if "/" in val: + field = val.split('/') + sessionAddr = field[0] + sessionPort = int(field[1]) + else: + sessionAddr = val + elif "send" == cmd: + send = True + elif "recv" == cmd: + recv = True + elif "cc" == cmd: + ccMode = "cc" + elif "cce" == cmd: + ccMode = "cce" + elif "ccl" == cmd: + ccMode = "ccl" + elif "rate" == cmd: + val = sys.argv[i] + rxRate = float(val) + ccMode = "fixed" + i += 1 + elif "ack" == cmd: + alist = sys.argv[i].split(',') + for val in alist: + ackerList.append(int(val)) + elif "debug" == cmd: + val = sys.argv[i] + debugLevel = int(val) + i += 1 + elif "trace" == cmd: + normTrace = True + else: + sys.stderr.write("normMsgr error: invalid command \"%s\"\n" % cmd) +except Exception as e: + sys.stderr.write("normMsgr \"" + cmd + " " + val + "\" argument error: " + e.__str__() + "\n") + usage() + sys.exit(-1) + +if not send and not recv: + sys.stderr.write("normMsgr error: not configured to send or recv!\n") + usage() + sys.exit(-1) + +if nodeId is None: + sys.stderr.write("normMsgr error: no local 'id' provided!\n") + usage() + sys.exit(-1) + +# Instantiate a NormMsgr and set its parameters +msgr = NormMsgr() +msgr.setNormDebugLevel(debugLevel) +sys.stderr.write("normMsgr: opening norm session ...\n") +msgr.openNormSession(sessionAddr, sessionPort, nodeId) +if mcastIface: + msgr.setNormMulticastInterface(mcastIface) +for node in ackerList: + msgr.addAckingNode(node) +msgr.setNormCCMode(ccMode); +if "fixed" == ccMode: + msgr.setNormTxRate(txRate) +msgr.setNormMessageTrace(normTrace) +msgr.start(send, recv) + +sys.stderr.write("normMsgr: starting NormEventHandler ...\n") +normEventHandler = NormEventHandler(msgr) +normEventHandler.start() + +if send: + sys.stderr.write("normMsgr: starting input thread ...\n") + inputThread = InputThread(msgr) + inputThread.start() + +if recv: + sys.stderr.write("normMsgr: starting output thread ...\n") + outputThread = OutputThread(msgr) + outputThread.start() + +# The main thread just sits on a loop that sleeps and wakes up +# once in a while. We could have made any of the other threads +# the main loop if we had wanted. Since all the other threads +# were child "daemons", they will get killed when this main exits +# TBD - provide for a graceful/clean sender/receiver termination +try: + sys.stderr.write("normMsgr: running (use Crtl-C to exit) ...\n") + while True: + time.sleep(5) + #sys.stderr.write("woke up ...\n") +except KeyboardInterrupt: + #sys.stderr.write("exception while waiting on input thread ..\n"); + pass + +sys.stderr.write("normMsgr: Done.\n") diff --git a/include/normApi.h b/include/normApi.h index 81e79ff..7708f66 100644 --- a/include/normApi.h +++ b/include/normApi.h @@ -58,6 +58,10 @@ typedef uint32_t UINT32; extern "C" { #endif /* __cplusplus */ +#define NORM_VERSION_MAJOR 1 +#define NORM_VERSION_MINOR 0 +#define NORM_VERSION_PATCH 0 + /** NORM API Types */ typedef const void* NormInstanceHandle; extern NORM_API_LINKAGE @@ -143,7 +147,8 @@ typedef enum NormProbingMode NORM_API_LINKAGE typedef enum NormSyncPolicy { - NORM_SYNC_CURRENT, // attempt to receiver current/new objects only + NORM_SYNC_CURRENT, // attempt to receiver current/new objects only, join mid-stream + NORM_SYNC_STREAM, // sync to current stream, but to beginning of stream NORM_SYNC_ALL // attempt to receive old and new objects } NormSyncPolicy; @@ -170,7 +175,7 @@ typedef enum NormEventType NORM_REMOTE_SENDER_NEW, NORM_REMOTE_SENDER_ACTIVE, NORM_REMOTE_SENDER_INACTIVE, - NORM_REMOTE_SENDER_PURGED, + NORM_REMOTE_SENDER_PURGED, // not yet implemented NORM_RX_CMD_NEW, NORM_RX_OBJECT_NEW, NORM_RX_OBJECT_INFO, @@ -192,6 +197,12 @@ typedef struct /** NORM API General Initialization and Operation Functions */ + +NORM_API_LINKAGE +int NormGetVersion(int* major DEFAULT((int*)0), + int* minor DEFAULT((int*)0), + int* patch DEFAULT((int*)0)); + NORM_API_LINKAGE NormInstanceHandle NormCreateInstance(bool priorityBoost DEFAULT(false)); @@ -252,6 +263,9 @@ NormSessionHandle NormCreateSession(NormInstanceHandle instanceHandle, NORM_API_LINKAGE void NormDestroySession(NormSessionHandle sessionHandle); +NORM_API_LINKAGE +bool NormIsUnicastAddress(const char* address); + NORM_API_LINKAGE void NormSetUserData(NormSessionHandle sessionHandle, const void* userData); @@ -358,6 +372,9 @@ double NormGetReportInterval(NormSessionHandle sessionHandle); /** NORM Sender Functions */ +NORM_API_LINKAGE +NormSessionId NormGetRandomSessionId(); + NORM_API_LINKAGE bool NormStartSender(NormSessionHandle sessionHandle, NormSessionId instanceId, @@ -457,10 +474,19 @@ NormObjectHandle NormStreamOpen(NormSessionHandle sessionHandle, const char* infoPtr DEFAULT((const char*)0), unsigned int infoLen DEFAULT(0)); +NORM_API_LINKAGE +void NormObjectSetUserData(NormObjectHandle objectHandle, const void* userData); + +NORM_API_LINKAGE +const void* NormObjectGetUserData(NormObjectHandle objectHandle); + // TBD - we should add a "bool watermark" option to "graceful" stream closure??? NORM_API_LINKAGE void NormStreamClose(NormObjectHandle streamHandle, bool graceful DEFAULT(false)); +NORM_API_LINKAGE +unsigned int NormGetStreamBufferSegmentCount(unsigned int bufferBytes, UINT16 segmentSize, UINT16 blockSize); + NORM_API_LINKAGE unsigned int NormStreamWrite(NormObjectHandle streamHandle, const char* buffer, @@ -508,8 +534,8 @@ void NormSetAutoAckingNodes(NormSessionHandle sessionHandle, NormTrackingStatus trackingStatus); NORM_API_LINKAGE -NormAckingStatus NormGetAckingStatus(NormSessionHandle sessionHandle, - NormNodeId nodeId DEFAULT(NORM_NODE_ANY)); +NormAckingStatus NormGetAckingStatus(NormSessionHandle sessionHandle, + NormNodeId nodeId DEFAULT(NORM_NODE_ANY)); NORM_API_LINKAGE bool NormGetNextAckingNode(NormSessionHandle sessionHandle, diff --git a/include/normMessage.h b/include/normMessage.h index 67be134..26bcf5d 100644 --- a/include/normMessage.h +++ b/include/normMessage.h @@ -110,23 +110,27 @@ class NormObjectSize { public: #ifdef WIN32 +#define _FILE_OFFSET_BITS 64 typedef __int64 Offset; #else typedef off_t Offset; -#endif +#endif // if/else WIN32 NormObjectSize() : size(0) {} NormObjectSize(Offset theSize) : size(theSize) {} NormObjectSize(UINT16 msb, UINT32 lsb) { size = (Offset)lsb; - size |= (sizeof(Offset) > 4) ? (((Offset)msb) << 32): (Offset)0; +#if _FILE_OFFSET_BITS > 32 + size |= ((Offset)msb) << 32; +#endif } Offset GetOffset() const {return size;} - UINT16 MSB() const - {return ((sizeof(Offset) > 4) ? (UINT16)((size >> 32) & 0x0000ffff) : 0);} - UINT32 LSB() const - {return ((UINT32)(size & 0xffffffff));} - +#if _FILE_OFFSET_BITS > 32 + UINT16 MSB() const {return ((UINT16)((size >> 32) & 0x0000ffff));} +#else + UINT16 MSB() const {return 0;} +#endif + UINT32 LSB() const {return ((UINT32)(size & 0xffffffff));} // Operators bool operator==(const NormObjectSize& b) const {return (b.size == size);} diff --git a/include/normNode.h b/include/normNode.h index 7fa08f6..511d2f2 100644 --- a/include/normNode.h +++ b/include/normNode.h @@ -263,8 +263,9 @@ class NormSenderNode : public NormNode enum SyncPolicy { - SYNC_CURRENT, // sync to detect transmit point, iff NORM_DATA from first FEC block - SYNC_ALL // permiscuously sync as far back as possible + SYNC_CURRENT, // sync to detect transmit point, iff NORM_DATA from first FEC block unless stream + SYNC_STREAM, // same as SYNC_CURRENT, but attempts to recover stream block zero + SYNC_ALL // permiscuously sync as far back as possible given rx cache size }; NormSenderNode(class NormSession& theSession, NormNodeId nodeId); diff --git a/include/normObject.h b/include/normObject.h index 40e5e79..c1dbfbf 100644 --- a/include/normObject.h +++ b/include/normObject.h @@ -64,6 +64,12 @@ class NormObject class NormSenderNode* GetSender() const {return sender;} NormNodeId GetSenderNodeId() const; + // for NORM API usage only + void SetUserData(const void* userData) + {user_data = userData;} + const void* GetUserData() const + {return user_data;} + bool IsOpen() {return (0 != segment_size);} // Opens (inits) object for tx operation bool Open(const NormObjectSize& objectSize, @@ -291,6 +297,8 @@ class NormObject bool accepted; bool notify_on_update; + const void* user_data; // for NORM API usage only + NormObject* next; }; // end class NormObject @@ -616,7 +624,6 @@ class NormObjectTable const NormObjectTable& table; bool reset; NormObjectId index; - bool backwards; }; private: diff --git a/include/normSegment.h b/include/normSegment.h index 35f2e48..ff1a6f0 100644 --- a/include/normSegment.h +++ b/include/normSegment.h @@ -19,7 +19,7 @@ class NormSegmentPool void Put(char* segment) { ASSERT(seg_count < seg_total); - *((char**)segment) = seg_list; // this might make a warning on Solaris + *((char**)((void*)segment)) = seg_list; // this might make a warning on Solaris seg_list = segment; seg_count++; } diff --git a/include/normSession.h b/include/normSession.h index f0d0714..d031e5e 100644 --- a/include/normSession.h +++ b/include/normSession.h @@ -303,8 +303,10 @@ class NormSession bool SendMessage(NormMsg& msg); void ActivateTimer(ProtoTimer& timer) {session_mgr.ActivateTimer(timer);} - void SetUserData(const void* userData) {user_data = userData;} - const void* GetUserData() {return user_data;} + void SetUserData(const void* userData) + {user_data = userData;} + const void* GetUserData() const + {return user_data;} // Sender methods void SenderSetBaseObjectId(NormObjectId baseId) @@ -335,7 +337,7 @@ class NormSession bool RequeueTxObject(NormObject* obj); - void DeleteTxObject(NormObject* obj); + void DeleteTxObject(NormObject* obj, bool notify); NormObject* SenderFindTxObject(NormObjectId objectId) {return tx_table.Find(objectId);} diff --git a/include/normSimAgent.h b/include/normSimAgent.h index 173b5e6..a3d6869 100644 --- a/include/normSimAgent.h +++ b/include/normSimAgent.h @@ -31,6 +31,12 @@ class NormSimAgent : public NormController, public ProtoMessageSink NORM_CCE, // strict ECN-based congestion control NORM_CCL // "loss-tolerant" congestion control }; + + // These functions support ACK-based flow-controlled streaming using + // some additional state variables (stream_buffer_max, stream_buffer_count, etc) + static unsigned int ComputeStreamBufferSegmentCount(unsigned int bufferBytes, UINT16 segmentSize, UINT16 blockSize); + unsigned int WriteToStream(const char* buffer, unsigned int numBytes); + bool AddAckingNode(NormNodeId nodeId); protected: NormSimAgent(ProtoTimerMgr& timerMgr, @@ -49,7 +55,7 @@ class NormSimAgent : public NormController, public ProtoMessageSink private: void OnInputReady(); - bool FlushStream(); + bool FlushStream(bool eom);// = true); virtual void Notify(NormController::Event event, class NormSessionMgr* sessionMgr, class NormSession* session, @@ -107,6 +113,12 @@ class NormSimAgent : public NormController, public ProtoMessageSink NormStreamObject* stream; bool auto_stream; + unsigned int stream_buffer_max; + unsigned int stream_buffer_count; + unsigned int stream_bytes_remain; + bool watermark_pending; + bool flow_control; + bool push_mode; NormStreamObject::FlushMode flush_mode; char* tx_msg_buffer; diff --git a/include/normVersion.h b/include/normVersion.h index 923bce8..fa86254 100644 --- a/include/normVersion.h +++ b/include/normVersion.h @@ -30,8 +30,12 @@ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. ********************************************************************/ +// This defines the NORM source code version +// The real code library version is given by NORM_VERSION +// in "normApi.h" + #ifndef _NORM_VERSION #define _NORM_VERSION -#define VERSION "1.5b1" +#define VERSION "1.5b2" #endif // _NORM_VERSION diff --git a/makefiles/Makefile.common b/makefiles/Makefile.common index f9a6374..a53bffd 100644 --- a/makefiles/Makefile.common +++ b/makefiles/Makefile.common @@ -168,7 +168,14 @@ normThreadTest2: $(TTEST2_OBJ) libnorm.a $(LIBPROTO) mkdir -p ../bin cp $@ ../bin/$@ +# (normMsgr) message sender/receiver +MSGR_SRC = $(EXAMPLE)/normMsgr.cpp +MSGR_OBJ = $(MSGR_SRC:.cpp=.o) +normMsgr: $(MSGR_OBJ) libnorm.a $(LIBPROTO) + $(CC) $(CFLAGS) -o $@ $(MSGR_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS) + mkdir -p ../bin + cp $@ ../bin/$@ # (pcap2norm) - parses pcap (e.g. tcpdump) file and prints NORM trace PCAP_SRC = $(COMMON)/pcap2norm.cpp diff --git a/makefiles/android/jni/Android.mk b/makefiles/android/jni/Android.mk index 5895d8c..6332712 100644 --- a/makefiles/android/jni/Android.mk +++ b/makefiles/android/jni/Android.mk @@ -52,4 +52,12 @@ LOCAL_SRC_FILES := \ ../../../src/unix/unixPostProcess.cpp include $(BUILD_EXECUTABLE) + +include $(CLEAR_VARS) +LOCAL_MODULE := normMsgr +LOCAL_STATIC_LIBRARIES := norm +LOCAL_SRC_FILES := \ + ../../../examples/normMsgr.cpp +include $(BUILD_EXECUTABLE) + $(call import-module,protolib/makefiles/android/jni) diff --git a/makefiles/android/jni/Application.mk b/makefiles/android/jni/Application.mk index b448d58..1737164 100644 --- a/makefiles/android/jni/Application.mk +++ b/makefiles/android/jni/Application.mk @@ -1,2 +1,3 @@ APP_ABI := all APP_PLATFORM := android-9 +NDK_MODULE_PATH := ../..:../../protolib/ diff --git a/makefiles/android/project.properties b/makefiles/android/project.properties index 616f300..5ca7d62 100644 --- a/makefiles/android/project.properties +++ b/makefiles/android/project.properties @@ -9,4 +9,4 @@ android.library=true # Project target. -target=android-16 +target=android-14 diff --git a/makefiles/java/Makefile.common b/makefiles/java/Makefile.common index be0feaf..0d512d3 100644 --- a/makefiles/java/Makefile.common +++ b/makefiles/java/Makefile.common @@ -35,6 +35,7 @@ LIB_OBJ = $(LIB_SRC:.cpp=.o) LIB_DEP = $(LIB_SRC:.cpp=.d) all: libmil_navy_nrl_norm.$(SYSTEM_SOEXT) + # NORM JNI shared library libmil_navy_nrl_norm.$(SYSTEM_SOEXT): $(LIB_OBJ) $(LIBNORM) $(LIBPROTO) @@ -51,7 +52,7 @@ $(LIBPROTO): -include $(LIB_DEP) %.d: %.cpp - $(CC) -MM -MT $(@:.d=.o) -MF $@ $(CPPFLAGS) $< + $(CC) -MM -MT $(@:.d=.o) -MF $@ $(CFLAGS) $< clean: rm -f $(NORMJNI)/*.o \ diff --git a/makefiles/java/Makefile.macosx b/makefiles/java/Makefile.macosx new file mode 100644 index 0000000..9044e98 --- /dev/null +++ b/makefiles/java/Makefile.macosx @@ -0,0 +1,17 @@ +# +# MacOSX NORM-JNI Makefile definitions +# + +# System specific additional libraries, include paths, etc +SYSTEM_INCLUDES = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/darwin +SYSTEM_LIBS = +SYSTEM_SRC = + +SYSTEM = macosx + +CC = g++ +SYSTEM_CFLAGS = -Wall -Wcast-align -fPIC -arch x86_64 -arch i386 +SYSTEM_SOFLAGS = -dynamiclib +SYSTEM_SOEXT = dylib + +include Makefile.common diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6bc73d2 --- /dev/null +++ b/setup.py @@ -0,0 +1,13 @@ +# This Python script can be used to install the Python 'pynorm' package +import platform +from distutils.core import setup + +# Note to use 'pynorm", you will need to have the libnorm shared library +# (libnorm.so or libnorm.dylib, etc) installed where your Python installation +# will find it with dlopen() (e.g. /usr/local/lib or something) + + +setup(name='pynorm', + version = '1.0', + packages = ['pynorm', 'pynorm.extra'], + package_dir={'' : 'src'}) diff --git a/src/common/normApi.cpp b/src/common/normApi.cpp index 43e6b2d..ff53b56 100644 --- a/src/common/normApi.cpp +++ b/src/common/normApi.cpp @@ -640,7 +640,7 @@ void NormInstance::Shutdown() notify_event = NULL; } #else - if (notify_fd[0] > 0) + if (notify_fd[0] >= 0) { close(notify_fd[0]); // close read end of pipe close(notify_fd[1]); // close write end of pipe @@ -715,6 +715,15 @@ UINT32 NormInstance::CountCompletedObjects(NormSession* session) // NORM API FUNCTION IMPLEMENTATIONS // +NORM_API_LINKAGE +int NormGetVersion(int* major, int* minor, int* patch) +{ + if (NULL != major) *major = NORM_VERSION_MAJOR; + if (NULL != minor) *minor = NORM_VERSION_MINOR; + if (NULL != patch) *patch = NORM_VERSION_PATCH; + return NORM_VERSION_MAJOR; +} // end NormGetVersion() + /** NORM API Initialization */ NORM_API_LINKAGE @@ -818,6 +827,7 @@ bool NormGetNextEvent(NormInstanceHandle instanceHandle, NormEvent* theEvent, bo instance->dispatcher.ResumeThread(); if (!instance->WaitForEvent()) { + // Indication that NormInstance is dead return false; } // re-suspend thread after wait @@ -828,10 +838,23 @@ bool NormGetNextEvent(NormInstanceHandle instanceHandle, NormEvent* theEvent, bo instance->dispatcher.ResumeThread(); } } - return result; + // We do this so we only return "false" when NormInstance is dead + if (!result && (NULL != theEvent)) + theEvent->type = NORM_EVENT_INVALID; + return true; } // end NormGetNextEvent() +NORM_API_LINKAGE +bool NormIsUnicastAddress(const char* address) +{ + ProtoAddress addr; + if ((NULL != address) && addr.ResolveFromString(address)) + return addr.IsUnicast(); + else + return false; // not valid, so not unicast +} // end NormIsUnicast() + /** NORM Session Creation and Control Functions */ NORM_API_LINKAGE NormSessionHandle NormCreateSession(NormInstanceHandle instanceHandle, @@ -1179,6 +1202,7 @@ void NormCloseDebugPipe(NormInstanceHandle instanceHandle) NORM_API_LINKAGE void NormSetDebugLevel(unsigned int level) { + // Sets underlying Protolib debug leve SetDebugLevel(level); } @@ -1218,6 +1242,15 @@ double NormGetReportInterval(NormSessionHandle sessionHandle) /** NORM Sender Functions */ +NORM_API_LINKAGE +NormSessionId NormGetRandomSessionId() +{ + ProtoTime currentTime; + currentTime.GetCurrentTime(); + srand(currentTime.usec()); // seed random number generator + return (NormSessionId)rand(); +} // end NormGetRandomSessionId() + NORM_API_LINKAGE bool NormStartSender(NormSessionHandle sessionHandle, NormSessionId sessionId, @@ -1576,6 +1609,15 @@ void NormStreamClose(NormObjectHandle streamHandle, bool graceful) } } // end NormStreamClose() +NORM_API_LINKAGE +unsigned int NormGetStreamBufferSegmentCount(unsigned int bufferBytes, UINT16 segmentSize, UINT16 blockSize) +{ + // This same computation is performed in NormStreamObject::Open() in "normObject.cpp" + unsigned int numBlocks = bufferBytes / (blockSize * segmentSize); + if (numBlocks < 2) numBlocks = 2; // NORM enforces a 2-block minimum buffer size + return (numBlocks * blockSize); +} // end NormGetStreamBufferSegmentCount() + NORM_API_LINKAGE unsigned int NormStreamWrite(NormObjectHandle streamHandle, const char* buffer, @@ -1704,16 +1746,18 @@ bool NormSetWatermark(NormSessionHandle sessionHandle, NORM_API_LINKAGE bool NormResetWatermark(NormSessionHandle sessionHandle) { - bool result = false; NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); if (instance && instance->dispatcher.SuspendThread()) { NormSession* session = (NormSession*)sessionHandle; session->SenderResetWatermark(); - result = true; instance->dispatcher.ResumeThread(); + return true; + } + else + { + return false; } - return true; } // end NormResetWatermark() NORM_API_LINKAGE @@ -1874,7 +1918,6 @@ bool NormStartReceiver(NormSessionHandle sessionHandle, NORM_API_LINKAGE void NormStopReceiver(NormSessionHandle sessionHandle) { - NormInstance* instance = NormInstance::GetInstanceFromSession(sessionHandle); if (instance && instance->dispatcher.SuspendThread()) { @@ -2165,13 +2208,34 @@ void NormObjectCancel(NormObjectHandle objectHandle) if (sender) sender->DeleteObject(obj); else - obj->GetSession().DeleteTxObject(obj); + obj->GetSession().DeleteTxObject(obj, false); instance->PurgeObjectNotifications(objectHandle); instance->dispatcher.ResumeThread(); } } } // end NormObjectCancel() +NORM_API_LINKAGE +void NormObjectSetUserData(NormObjectHandle objectHandle, const void* userData) +{ + NormInstance* instance = NormInstance::GetInstanceFromObject(objectHandle); + if (instance) + { + if (instance->dispatcher.SuspendThread()) + { + ((NormObject*)objectHandle)->SetUserData(userData); + instance->dispatcher.ResumeThread(); + } + } +} // end NormObjectSetUserData() + +NORM_API_LINKAGE +const void* NormObjectGetUserData(NormObjectHandle objectHandle) +{ + return (((NormObject*)objectHandle)->GetUserData()); +} // end NormNormObjectGetUserData() + + NORM_API_LINKAGE void NormObjectRetain(NormObjectHandle objectHandle) { @@ -2409,6 +2473,10 @@ void NormNodeRelease(NormNodeHandle nodeHandle) // is done so that this current approach is not very costly) // + +// Not sure why this function exists. It just counts the number +// of RX_OBJECT_COMPLETED events currently in notification queue +// which is a temporary and transitory thing??? NORM_API_LINKAGE UINT32 NormCountCompletedObjects(NormSessionHandle sessionHandle) { diff --git a/src/common/normNode.cpp b/src/common/normNode.cpp index 0daf7d2..c15b2c4 100644 --- a/src/common/normNode.cpp +++ b/src/common/normNode.cpp @@ -334,7 +334,7 @@ bool NormSenderNode::AllocateBuffers(UINT8 fecId, { if (NULL == (decoder = new NormDecoderRS8)) { - PLOG(PL_FATAL, "NormSenderNode::AllocateBuffers() new NormDecoderMDP error: %s\n", GetErrorString()); + PLOG(PL_FATAL, "NormSenderNode::AllocateBuffers() new NormDecoderRS8 error: %s\n", GetErrorString()); Close(); return false; } @@ -959,7 +959,7 @@ void NormSenderNode::HandleRepairContent(const UINT32* buffer, UINT16 bufferLen) bool freshBlock = true; NormBlockId prevBlockId = 0; NormBlock* block = NULL; - while ((requestLength = req.Unpack(buffer, bufferLen))) + while (0 != (requestLength = req.Unpack(buffer, bufferLen))) { // Point "buffer" to next request and adjust "bufferLen" buffer += (requestLength/4); @@ -1081,8 +1081,7 @@ void NormSenderNode::CalculateGrttResponse(const struct timeval& currentTime, if (grttResponse.tv_usec < grtt_recv_time.tv_usec) { grttResponse.tv_sec = grttResponse.tv_sec - grtt_recv_time.tv_sec - 1; - grttResponse.tv_usec = 1000000 - (grtt_recv_time.tv_usec - - grttResponse.tv_usec); + grttResponse.tv_usec = 1000000 - (grtt_recv_time.tv_usec - grttResponse.tv_usec); } else { @@ -1511,9 +1510,15 @@ void NormSenderNode::HandleObjectMessage(const NormObjectMsg& msg) session.Notify(NormController::RX_OBJECT_NEW, this, obj); if (obj->Accepted()) { - // (TBD) Do I _need_ to call "StreamUpdateStatus()" here? if (obj->IsStream()) - (static_cast(obj))->StreamUpdateStatus(blockId); + { + // This initial "StreamUpdateStatus()" syncs the stream according to our sync policy + NormStreamObject* stream = static_cast(obj); + if (SYNC_CURRENT != sync_policy) + stream->StreamResync(NormBlockId(0)); + else + stream->StreamUpdateStatus(blockId); + } PLOG(PL_DETAIL, "NormSenderNode::HandleObjectMessage() node>%lu sender>%lu new obj>%hu\n", LocalNodeId(), GetId(), (UINT16)objectId); } @@ -1597,6 +1602,7 @@ bool NormSenderNode::SyncTest(const NormObjectMsg& msg) const switch (sync_policy) { case SYNC_CURRENT: // default, more conservative "sync policy" + case SYNC_STREAM: { // Allow sync on stream at any time bool result = msg.FlagIsSet(NormObjectMsg::FLAG_STREAM); @@ -1697,6 +1703,7 @@ void NormSenderNode::Sync(NormObjectId objectId) switch (sync_policy) { case SYNC_CURRENT: // this is the usual default + case SYNC_STREAM: sync_id = next_id = max_pending_object = objectId; break; case SYNC_ALL: // gratuitously sync for anything in our "range" @@ -2299,7 +2306,7 @@ void NormSenderNode::UpdateRecvRate(const struct timeval& currentTime, unsigned if (prev_update_time.tv_sec || prev_update_time.tv_usec) { double interval = (double)(currentTime.tv_sec - prev_update_time.tv_sec); - if (currentTime.tv_usec > prev_update_time.tv_sec) + if (currentTime.tv_sec > prev_update_time.tv_sec) interval += 1.0e-06*(double)(currentTime.tv_usec - prev_update_time.tv_usec); else interval -= 1.0e-06*(double)(prev_update_time.tv_usec - currentTime.tv_usec); @@ -3218,7 +3225,7 @@ bool NormLossEstimator2::Update(const struct timeval& currentTime, event_time = event_time_orig = currentTime; newLossEvent = true; - TRACE("time>%lf new loss event fraction = %lf\n", ProtoTime(currentTime).GetValue(), LossFraction()); + //TRACE("time>%lf new loss event fraction = %lf\n", ProtoTime(currentTime).GetValue(), LossFraction()); } } diff --git a/src/common/normObject.cpp b/src/common/normObject.cpp index fd2d85e..3c2141f 100644 --- a/src/common/normObject.cpp +++ b/src/common/normObject.cpp @@ -15,7 +15,8 @@ NormObject::NormObject(NormObject::Type theType, transport_id(transportId), segment_size(0), pending_info(false), repair_info(false), current_block_id(0), next_segment_id(0), max_pending_block(0), max_pending_segment(0), - info_ptr(NULL), info_len(0), first_pass(true), accepted(false), notify_on_update(true) + info_ptr(NULL), info_len(0), first_pass(true), accepted(false), notify_on_update(true), + user_data(NULL), next(NULL) { if (theSender) { @@ -158,11 +159,14 @@ bool NormObject::Open(const NormObjectSize& objectSize, { small_block_size = large_block_size = numData; small_block_count = large_block_count = numBlocks.LSB(); - final_segment_size = segmentSize; - NormStreamObject* stream = static_cast(this); - // The call here to "StreamResync()" make us attempt to repair back to object start. - // TBD - we may only want to do this for the SYNC_ALL sync policy? - stream->StreamResync(NormBlockId(0)); + final_segment_size = segmentSize; + if (NULL == sender) + { + // This inits sender stream state to set things tx pending + // (the receiver stream is sync'd according to sync policy) + NormStreamObject* stream = static_cast(this); + stream->StreamResync(NormBlockId(0)); + } } else { @@ -2594,7 +2598,7 @@ bool NormStreamObject::Open(UINT32 bufferSize, UINT16 segmentSize, numData; if (sender) { - // receive streams have already be pre-opened + // receive streams have already beeen pre-opened segmentSize = segment_size; numData = ndata; } @@ -2634,6 +2638,7 @@ bool NormStreamObject::Open(UINT32 bufferSize, // (TBD) we really only need one set of indexes & offset // since our objects are exclusively read _or_ write read_init = true; + read_index.block = read_index.segment = 0; write_index.block = write_index.segment = 0; tx_index.block = tx_index.segment = 0; @@ -2682,7 +2687,6 @@ void NormStreamObject::Close(bool graceful) { if (graceful && (NULL == sender)) { - stream_closing = true; Terminate(); //SetFlushMode(FLUSH_ACTIVE); //Flush(); @@ -2818,6 +2822,19 @@ bool NormStreamObject::StreamUpdateStatus(NormBlockId blockId) stream_sync = true; stream_sync_id = blockId; stream_next_id = blockId + pending_mask.GetSize(); + + if (read_init) + { + // This is a fresh rx stream, so init the read indices + // TBD - we may wish to do this here only for + // a "NORM_SYNC_STREAM" sync policy (beginning of current stream) + // This is a "fresh" stream + read_init = false; + read_index.block = blockId; + read_index.segment = 0; + read_offset = 0; + } + // Since we're doing a resync including "read_init", dump any buffered data // (TBD) this may not be necessary??? and is thus currently commented-out code /*read_init = true; @@ -3232,7 +3249,7 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek else if (session.RcvrIsLowDelay()) { // Has the sender moved forward to the next FEC blocks - long delta = max_pending_block - read_index.block; + INT32 delta = max_pending_block - read_index.block; if (delta > session.RcvrGetMaxDelay()) forceForward = true; } @@ -3281,7 +3298,7 @@ bool NormStreamObject::ReadPrivate(char* buffer, unsigned int* buflen, bool seek else if (session.RcvrIsLowDelay()) { // Has the sender moved forward to the next FEC blocks - long delta = max_pending_block - read_index.block; + INT32 delta = max_pending_block - read_index.block; if (delta > session.RcvrGetMaxDelay()) forceForward = true; } @@ -3504,8 +3521,9 @@ void NormStreamObject::Terminate() { // Flush stream and create a ZERO length segment to send Flush(); // should eom be set for this call? + stream_closing = true; NormBlock* block = stream_buffer.Find(write_index.block); - if (!block) + if (NULL == block) { if (NULL == (block = block_pool.Get())) { @@ -3572,9 +3590,9 @@ void NormStreamObject::Terminate() } else { - // Make sure the segment is not still referenced - ASSERT(0 == NormDataMsg::ReadStreamPayloadMsgStart(segment)); - ASSERT(0 == NormDataMsg::ReadStreamPayloadLength(segment)); + // Make sure the segment is not still referenced (TBD - why was I checking this???) + //ASSERT(0 == NormDataMsg::ReadStreamPayloadMsgStart(segment)); + //ASSERT(0 == NormDataMsg::ReadStreamPayloadLength(segment)); } NormDataMsg::WriteStreamPayloadOffset(segment, write_offset); @@ -3600,7 +3618,7 @@ UINT32 NormStreamObject::Write(const char* buffer, UINT32 len, bool eom) PLOG(PL_ERROR, "NormStreamObject::Write() error: stream is closing (len:%lu eom:%d)\n", len, eom); len = 0; } - break; + break; } // This old code detected buffer "fullness" by offset instead of segment index // but, the problem there was when apps wrote & flushed messages smaller than @@ -3884,8 +3902,7 @@ void NormObjectTable::SetRangeMax(UINT16 rangeMax) NormSession& session = obj->GetSession(); if (NULL == sender) { - session.Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, obj); - session.DeleteTxObject(obj); + session.DeleteTxObject(obj, true); } else { diff --git a/src/common/normSegment.cpp b/src/common/normSegment.cpp index 41b0a6b..b735444 100644 --- a/src/common/normSegment.cpp +++ b/src/common/normSegment.cpp @@ -66,7 +66,7 @@ char* NormSegmentPool::Get() if (ptr) { //memcpy(&seg_list, ptr, sizeof(char*)); - seg_list = *((char**)ptr); + seg_list = *((char**)((void*)ptr)); seg_count--; //#ifdef NORM_DEBUG overrun_flag = false; diff --git a/src/common/normSession.cpp b/src/common/normSession.cpp index 9ddff5a..8c0f723 100644 --- a/src/common/normSession.cpp +++ b/src/common/normSession.cpp @@ -257,7 +257,7 @@ bool NormSession::Open() result &= tx_socket->SetMulticastInterface(interface_name); if (!result) { - PLOG(PL_FATAL, "NormSession::Open() t_socket::SetMulticastInterface() error\n"); + PLOG(PL_FATAL, "NormSession::Open() tx_socket::SetMulticastInterface() error\n"); Close(); return false; } @@ -529,12 +529,12 @@ double NormSession::GetTxRate() } } // end NormSession::GetTxRate() -/* + static double PoissonRand(double mean) { return(-log(((double)rand())/((double)RAND_MAX))*mean); } -*/ + // This hack can give us a tx rate interval that is POISSON instead of PERIODIC static inline double GetTxInterval(unsigned int msgSize, double txRate) @@ -974,7 +974,8 @@ void NormSession::Serve() // If command is enqueued if (SenderQueueAppCmd()) return; } - + + bool watermarkJustCompleted = false; if (watermark_pending && !flush_timer.IsActive()) { // Determine next message (objectId::blockId::segmentId) to be sent @@ -1072,12 +1073,14 @@ void NormSession::Serve() if (watermark_flushes) flush_count = GetTxRobustFactor(); } + watermarkJustCompleted = true; } } else { // The sender tx position is < watermark // Reset non-acked acking nodes since sender has rewound + // TBD - notify application that watermark ack has been reset ??? if (watermark_active) { watermark_active = false; @@ -1089,7 +1092,7 @@ void NormSession::Serve() } } // end if (watermark_pending) - if (obj) + if (NULL != obj) { NormObjectMsg* msg = (NormObjectMsg*)GetMessageFromPool(); if (msg) @@ -1150,24 +1153,33 @@ void NormSession::Serve() { SenderQueueFlush(); } - else if (GetTxRobustFactor() == flush_count) + else if (GetTxRobustFactor() == flush_count) //xxx { - PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", + + PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender stream flush complete ...\n", LocalNodeId()); + Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode*)NULL, stream); flush_count++; data_active = false; if (stream->IsClosing()) { - stream->Close(); - Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, stream); - DeleteTxObject(stream); - obj = NULL; + // If the stream just failed end-of-stream watermark flush, we don't + // close the stream yet, but instead give app chance to reset watermark + bool watermarkFailed = watermarkJustCompleted && (obj->GetId() == watermark_object_id) && + (ACK_FAILURE == SenderGetAckingStatus(NORM_NODE_ANY)); + if (!watermarkFailed) + { + // end of stream was successfully acknowledged + stream->Close(); + DeleteTxObject(stream, true); + obj = NULL; + } } } } } //ASSERT(stream->IsPending() || stream->IsRepairPending() || stream->IsClosing()); - if (!posted_tx_queue_empty && !stream->IsClosing() && stream->IsPending()) + if (!posted_tx_queue_empty && !stream->IsClosing() && stream->IsPending()) // post if pending || !repair_timer.IsActive() || (repair_timer.GetRepeatCount() == 0) ??? { //data_active = false; @@ -1204,14 +1216,13 @@ void NormSession::Serve() } else if (GetTxRobustFactor() == flush_count) { - PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", - LocalNodeId()); + PLOG(PL_TRACE, "NormSession::Serve() node>%lu sender flush complete ...\n", LocalNodeId()); Notify(NormController::TX_FLUSH_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); flush_count++; data_active = false; - } + } } } // end NormSession::Serve() @@ -1401,7 +1412,7 @@ bool NormSession::SenderQueueWatermarkFlush() watermark_pending = false; NormAckingNode* nodeNone = NULL; acking_success_count = 0; - while ((next = static_cast(iterator.GetNextNode()))) + while (NULL != (next = static_cast(iterator.GetNextNode()))) { // Save NORM_NODE_NONE for last if (NORM_NODE_NONE == next->GetId()) @@ -1447,7 +1458,7 @@ bool NormSession::SenderQueueWatermarkFlush() if (watermark_pending) { - // (TBD) we should increment the "flush_count" here only iff the watemark + // (TBD) we should increment the "flush_count" here only iff the watermark // corresponds to our "last_tx_object_id", etc //if ((GetTxRobustFactor() < 0) || (flush_count < GetTxRobustFactor())) // flush_count++; @@ -1524,8 +1535,7 @@ void NormSession::SenderQueueFlush() { PLOG(PL_ERROR, "NormSession::SenderQueueFlush() node>%lu message_pool exhausted! (couldn't flush)\n", LocalNodeId()); - } - + } } else { @@ -1769,8 +1779,8 @@ bool NormSession::QueueTxObject(NormObject* obj) double delay = GetFlowControlDelay() - oldest->GetNackAge(); if (delay < 1.0e-06) { - Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, oldest); - DeleteTxObject(oldest); + if (FlowControlIsActive()) DeactivateFlowControl(); + DeleteTxObject(oldest, true); } else { @@ -1828,11 +1838,12 @@ bool NormSession::RequeueTxObject(NormObject* obj) } } // end NormSession::RequeueTxObject() -void NormSession::DeleteTxObject(NormObject* obj) +void NormSession::DeleteTxObject(NormObject* obj, bool notify) { ASSERT(NULL != obj); if (tx_table.Remove(obj)) { + Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, obj); NormObjectId objectId = obj->GetId(); tx_pending_mask.Unset(objectId); tx_repair_mask.Unset(objectId); @@ -1863,8 +1874,7 @@ bool NormSession::SetTxCacheBounds(NormObjectSize sizeMax, // Remove oldest (hopefully non-pending ) object NormObject* oldest = tx_table.Find(tx_table.RangeLo()); ASSERT(NULL != oldest); - Notify(NormController::TX_OBJECT_PURGED, (NormSenderNode*)NULL, oldest); - DeleteTxObject(oldest); + DeleteTxObject(oldest, true); count = tx_table.GetCount(); } if (tx_cache_count_max < DEFAULT_TX_CACHE_MAX) @@ -2110,7 +2120,7 @@ void NormSession::OnPktCapture(ProtoChannel& theChannel, if (numBytes == 0) break; // no more packets to receive // Map ProtoPktETH instance into buffer and init for processing - ProtoPktETH ethPkt((UINT32*)ethBuffer, BUFFER_MAX - 2); + ProtoPktETH ethPkt((UINT32*)((void*)ethBuffer), BUFFER_MAX - 2); if (!ethPkt.InitFromBuffer(numBytes)) { PLOG(PL_ERROR, "NormSession::OnPktCapture() error: bad Ether frame\n"); @@ -2973,9 +2983,13 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons ext.GetCCSequence()); if (wasUnicast && probe_proactive && Address().IsMulticast()) { - // for suppression of unicast cc feedback - advertise_repairs = true; - QueueMessage(NULL); + // if it's the CLR, it doesn't suppress anyone, don't advertise + if (!ext.CCFlagIsSet(NormCC::CLR)) + { + // for suppression of unicast cc feedback + advertise_repairs = true; + QueueMessage(NULL); + } } break; } @@ -3006,27 +3020,46 @@ void NormSession::SenderHandleAckMessage(const struct timeval& currentTime, cons (watermark_segment_id == flushAck.GetFecSymbolId(fec_m))) { acker->MarkAckReceived(); + /* This code was an attempt to expedite delivery of the TX_WATERMARK_COMPLETED + notification to the application, but breaks some other desired behavior. + watermark_pending = false; + acking_success_count = 0; + NormNodeTreeIterator iterator(acking_node_tree); + NormAckingNode* next; + while (NULL != (next = static_cast(iterator.GetNextNode()))) + { + if (next->IsPending()) + watermark_pending = true; + else if (next->AckReceived() || (NORM_NODE_NONE == next->GetId())) + acking_success_count++; + } + if (!watermark_pending) + { + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() node>%lu watermark ack finished.\n"); + Notify(NormController::TX_WATERMARK_COMPLETED, (NormSenderNode*)NULL, (NormObject*)NULL); + } + */ } else { // This can happen when new watermarks are set when an old watermark is still // pending (i.e. receivers may still be in the process of replying) - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received wrong watermark ACK?!\n"); + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received wrong watermark ACK?!\n"); } } else { - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received redundant watermark ACK?!\n"); + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received redundant watermark ACK?!\n"); } } else { - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received watermark ACK from unknown acker?!\n"); + PLOG(PL_WARN, "NormSession::SenderHandleAckMessage() received watermark ACK from unknown acker?!\n"); } } else { - PLOG(PL_ERROR, "NormSession::SenderHandleAckMessage() received unsolicited watermark ACK?!\n"); + PLOG(PL_DEBUG, "NormSession::SenderHandleAckMessage() received unsolicited watermark ACK?!\n"); } break; @@ -3953,7 +3986,7 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) { if (0 == cmd.PackRepairRequest(req)) { - PLOG(PL_FATAL, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); + PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case prevForm = NormRepairRequest::INVALID; break; @@ -3981,21 +4014,26 @@ bool NormSession::SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd) currentId, 0, ndata, 0); break; } - cmd.PackRepairRequest(req); + prevForm = NormRepairRequest::INVALID; + if (0 == cmd.PackRepairRequest(req)) + { + PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); + // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case + break; + } objectCount = 0; } if (!repairEntireObject) { if ((NormRepairRequest::INVALID != prevForm) && currentObject->IsRepairPending()) { + prevForm = NormRepairRequest::INVALID; if (0 == cmd.PackRepairRequest(req)) { - PLOG(PL_FATAL, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); + PLOG(PL_WARN, "NormSession::SenderBuildRepairAdv() warning: full msg\n"); // (TBD) set NORM_REPAIR_ADV_LIMIT flag in this case - prevForm = NormRepairRequest::INVALID; break; } - prevForm = NormRepairRequest::INVALID; currentObject->AppendRepairAdv(cmd); } else @@ -4167,7 +4205,6 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) tx_timer.Deactivate(); tx_socket->StartOutputNotification(); return false; - } } else @@ -4177,7 +4214,8 @@ bool NormSession::OnTxTimeout(ProtoTimer& /*theTimer*/) if (message_queue.IsEmpty()) { - tx_timer.Deactivate(); + if (tx_timer.IsActive()) + tx_timer.Deactivate(); // Check that any possible notifications posted in // the previous call to Serve() may have caused a // change in sender state making it ready to send diff --git a/src/common/normSimAgent.cpp b/src/common/normSimAgent.cpp index a726407..98976e1 100644 --- a/src/common/normSimAgent.cpp +++ b/src/common/normSimAgent.cpp @@ -25,8 +25,10 @@ NormSimAgent::NormSimAgent(ProtoTimerMgr& timerMgr, tx_object_size_min(0), tx_object_size_max(0), tx_repeat_count(0), tx_repeat_interval(0.0), tx_requeue(0), tx_requeue_count(0), - stream(NULL), auto_stream(false), push_mode(false), - flush_mode(NormStreamObject::FLUSH_PASSIVE), + stream(NULL), auto_stream(false), + stream_buffer_max(0), stream_buffer_count(0), + stream_bytes_remain(0), watermark_pending(false), + flow_control(false), push_mode(false), flush_mode(NormStreamObject::FLUSH_PASSIVE), msg_sync(false), mgen_bytes(0), mgen_pending_bytes(0), tracing(false), log_file_ptr(NULL), tx_loss(0.0), rx_loss(0.0) { @@ -90,6 +92,7 @@ const char* const NormSimAgent::cmd_list[] = "+push", // "on" means real-time push stream advancement (non-blocking) "+flush", // stream flush mode "-doFlush", // invoke flushing of stream + "+ackingNode", // ; add to acking node list "+unicastNacks", // receivers will unicast feedback "+silentReceiver", // receivers will not transmit NULL @@ -520,7 +523,7 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) { if (session) { - if (stream) + if (NULL != stream) { PLOG(PL_FATAL, "NormSimAgent::ProcessCommand(openStream) stream already open!\n"); return false; @@ -545,6 +548,14 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) stream->SetPushMode(push_mode); auto_stream = false; tx_msg_len = tx_msg_index = 0; + if (flow_control) + { + // ACK-based flow control has been enabled, so initialize + stream_buffer_max = ComputeStreamBufferSegmentCount(tx_object_size, segment_size, ndata); + stream_buffer_max -= ndata; // a little safety margin + stream_buffer_count = stream_bytes_remain = 0; + watermark_pending = false; + } } else { @@ -571,7 +582,7 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) { if (session) { - return FlushStream(); + return FlushStream(true); // TBD - provide control over EOM marking here? } else { @@ -592,6 +603,10 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) } return true; } + else if (!strncmp("ackingNode", cmd, len)) + { + return AddAckingNode(atoi(val)); + } else if (!strncmp("unicastNacks", cmd, len)) { if (!strcmp(val, "on")) @@ -623,6 +638,68 @@ bool NormSimAgent::ProcessCommand(const char* cmd, const char* val) return true; } // end NormSimAgent::ProcessCommand() +unsigned int NormSimAgent::ComputeStreamBufferSegmentCount(unsigned int bufferBytes, UINT16 segmentSize, UINT16 blockSize) +{ + // This same computation is performed in NormStreamObject::Open() in "normObject.cpp" + unsigned int numBlocks = bufferBytes / (blockSize * segmentSize); + if (numBlocks < 2) numBlocks = 2; // NORM enforces a 2-block minimum buffer size + return (numBlocks * blockSize); +} // end NormSimAgent::ComputeStreamBufferSegmentCount() + +unsigned int NormSimAgent::WriteToStream(const char* buffer, + unsigned int numBytes) +{ + ASSERT(NULL != stream); + if (!flow_control) + { + return stream->Write(buffer, numBytes, false); + } + else if (stream_buffer_count < stream_buffer_max) + { + // This method uses stream->Write(), but limits writes by explicit ACK-based flow control status + // 1) How many buffer bytes are available? + unsigned int bytesAvailable = segment_size * (stream_buffer_max - stream_buffer_count); + bytesAvailable -= stream_bytes_remain; // unflushed segment portiomn + if (numBytes <= bytesAvailable) + { + unsigned int totalBytes = numBytes + stream_bytes_remain; + unsigned int numSegments = totalBytes / segment_size; + stream_bytes_remain = totalBytes % segment_size; + stream_buffer_count += numSegments; + } + else + { + numBytes = bytesAvailable; + stream_buffer_count = stream_buffer_max; + } + // 2) Write to the stream + unsigned int bytesWritten = stream->Write(buffer, numBytes, false); + + ASSERT(bytesWritten == numBytes); // this could fail if timer-based flow control is left enabled + // We had a "stream is closing" case here somehow? Need to make sure we don't try + // to write to norm stream + + // 3) Do we need to issue a watermark ACK request? + if (!watermark_pending && (stream_buffer_count >= (stream_buffer_max >> 1))) + { + //TRACE("NormSimAgent::WriteToStream() initiating watermark ACK request (buffer count:%lu max:%lu usage:%u)...\n", + // stream_buffer_count, stream_buffer_max, NormStreamGetBufferUsage(tx_stream)); + session->SenderSetWatermark(stream->GetId(), + stream->FlushBlockId(), + stream->FlushSegmentId(), + false); + watermark_pending = true; + } + return bytesWritten; + } + else + { + PLOG(PL_DETAIL, "NormSimAgent::WriteToStream() is blocked pending acknowledgment from receiver\n"); + return 0; + } +} // end NormSimAgent::WriteToStream() + + void NormSimAgent::SetCCMode(NormCC ccMode) { cc_mode = ccMode; @@ -693,15 +770,15 @@ bool NormSimAgent::SendMessage(unsigned int len, const char* txBuffer) { if (session) { - if (!stream) + if (NULL == stream) { - if(!(tx_msg_buffer = new char[65536])) + if(NULL == (tx_msg_buffer = new char[65536])) { PLOG(PL_FATAL, "NormSimAgent::SendMessage() error allocating tx_msg_buffer: %s\n", strerror(errno)); return false; } - if (!(stream = session->QueueTxStream(tx_object_size))) + if (NULL == (stream = session->QueueTxStream(tx_object_size))) { PLOG(PL_FATAL, "NormSimAgent::SendMessage() error opening stream!\n"); return false; @@ -736,27 +813,64 @@ bool NormSimAgent::SendMessage(unsigned int len, const char* txBuffer) void NormSimAgent::OnInputReady() { - if (tx_msg_index < tx_msg_len) + if (auto_stream) { - unsigned int bytesWrote = stream->Write(tx_msg_buffer+tx_msg_index, - tx_msg_len - tx_msg_index, - false); + // sending a dummy byte stream (fill the stream buffer up) + char buffer[NormMsg::MAX_SIZE]; + bool inputReady = true; + while (inputReady) + { + unsigned int numBytes = WriteToStream(buffer, segment_size); + inputReady = (numBytes == segment_size); + } + } + else if (tx_msg_index < tx_msg_len) + { + unsigned int bytesWrote; + if (flow_control) + { + bytesWrote = WriteToStream(tx_msg_buffer+tx_msg_index, + tx_msg_len - tx_msg_index); + } + else + { + bytesWrote = WriteToStream(tx_msg_buffer+tx_msg_index, + tx_msg_len - tx_msg_index); + } tx_msg_index += bytesWrote; if (tx_msg_index == tx_msg_len) { - // Mark EOM _and_ flush (using set flush mode) - stream->Write(NULL, 0, true); + // Mark EOM _and_ flush (using preset flush mode) + FlushStream(true); tx_msg_index = tx_msg_len = 0; } } -} // end NormSimAgent::InputReady() +} // end NormSimAgent::OnInputReady() // Do an _active_ flush of the stream (for use at end-of-transmission) -bool NormSimAgent::FlushStream() +bool NormSimAgent::FlushStream(bool eom) { if (stream && session && session->IsSender()) { - stream->Flush(true); + stream->Flush(eom); + if (flow_control && (NormStreamObject::FLUSH_ACTIVE == flush_mode)) + { + if (0 != stream_bytes_remain) + { + stream_buffer_count++; + stream_bytes_remain = 0; + if (!watermark_pending && (stream_buffer_count >= (stream_buffer_max >> 1))) + { + //TRACE("NormSimAgent::FlushStream() initiating watermark ACK request (buffer count:%lu max:%lu usage:%u)...\n", + // stream_buffer_count, stream_buffer_max, NormStreamGetBufferUsage(tx_stream)); + session->SenderSetWatermark(stream->GetId(), + stream->FlushBlockId(), + stream->FlushSegmentId(), + false); + watermark_pending = true; + } + } + } return true; } else @@ -780,22 +894,8 @@ void NormSimAgent::Notify(NormController::Event event, // Can queue a new object or write to stream for transmission if ((NULL != object) && (object == stream)) { - if (auto_stream) - { - // sending a dummy byte stream (fill the stream buffer up) - char buffer[NormMsg::MAX_SIZE]; - bool inputReady = true; - while (inputReady) - { - unsigned int numBytes = stream->Write(buffer, segment_size, false); - inputReady = (numBytes == segment_size); - } - } - else - { - // Stream starved, ask for input from "source" ? - OnInputReady(); - } + // Stream starved, get input + OnInputReady(); } else { @@ -807,9 +907,29 @@ void NormSimAgent::Notify(NormController::Event event, } break; - case TX_OBJECT_SENT: - if ((0 != tx_requeue) && (object != stream)) - { + case TX_WATERMARK_COMPLETED: + if (flow_control) + { + if (NormSession::ACK_SUCCESS == session->SenderGetAckingStatus(NORM_NODE_ANY)) + { + // if we've been blocking normal input, unblock it and prompt for more + // (we could post a vacancy notification here? + watermark_pending = false; + bool wasBlocked = stream_buffer_count >= stream_buffer_max; + stream_buffer_count -= (stream_buffer_max >> 1); + if (wasBlocked) OnInputReady(); + } + else + { + // reset watermark request + session->SenderResetWatermark(); + } + } + break; + + case TX_OBJECT_SENT: + if ((0 != tx_requeue) && (object != stream)) + { if (0 != tx_requeue_count) { if (session->RequeueTxObject(object)) @@ -829,8 +949,8 @@ void NormSimAgent::Notify(NormController::Event event, // reset "tx_requeue_count" for next tx object tx_requeue_count = tx_requeue; } - } - break; + } + break; case RX_OBJECT_NEW: { @@ -890,147 +1010,147 @@ void NormSimAgent::Notify(NormController::Event event, case RX_OBJECT_INFO: switch(object->GetType()) { - case NormObject::FILE: - case NormObject::DATA: - case NormObject::STREAM: - default: - break; + case NormObject::FILE: + case NormObject::DATA: + case NormObject::STREAM: + default: + break; } // end switch(object->GetType()) break; case RX_OBJECT_UPDATED: switch (object->GetType()) { - case NormObject::FILE: - // (TBD) update progress - break; - - case NormObject::STREAM: - { - // Read the stream when it's updated - if (msg_sink) - { - bool dataReady = true; - while (dataReady) - { - if (0 == mgen_pending_bytes) - { - // Reading (at least part of) 2 byte MGEN msg len header - unsigned int want = 2 - mgen_bytes; - unsigned int got = want; - bool findMsgSync = msg_sync ? false : true; - if ((static_cast(object))->Read(mgen_buffer + mgen_bytes, - &got, findMsgSync)) + case NormObject::FILE: + // (TBD) update progress + break; + + case NormObject::STREAM: + { + // Read the stream when it's updated + if (msg_sink) + { + bool dataReady = true; + while (dataReady) + { + if (0 == mgen_pending_bytes) { - mgen_bytes += got; - msg_sync = true; - if (got != want) dataReady = false; + // Reading (at least part of) 2 byte MGEN msg len header + unsigned int want = 2 - mgen_bytes; + unsigned int got = want; + bool findMsgSync = msg_sync ? false : true; + if ((static_cast(object))->Read(mgen_buffer + mgen_bytes, + &got, findMsgSync)) + { + mgen_bytes += got; + msg_sync = true; + if (got != want) dataReady = false; + } + else + { + if (msg_sync) + { + PLOG(PL_WARN, "NormSimAgent::Notify(1) detected stream break\n"); + //ASSERT(0); + mgen_bytes = mgen_pending_bytes = 0; + msg_sync = false; + continue; + } + else + { + break; + } + } + if (2 == mgen_bytes) + { + UINT16 msgSize; + memcpy(&msgSize, mgen_buffer, sizeof(UINT16)); + mgen_pending_bytes = ntohs(msgSize) - 2; + } + } + if (mgen_pending_bytes) + { + // Save the first part for MGEN logging + if (mgen_bytes < 64) + { + unsigned int want = MIN(mgen_pending_bytes, 62); + unsigned int got = want; + if ((static_cast(object))->Read(mgen_buffer+mgen_bytes, + &got)) + { + mgen_pending_bytes -= got; + mgen_bytes += got; + if (got != want) dataReady = false; + } + else + { + PLOG(PL_WARN, "NormSimAgent::Notify(2) detected stream break\n"); + mgen_bytes = mgen_pending_bytes = 0; + msg_sync = false; + continue; + } + } + while (dataReady && mgen_pending_bytes) + { + char buffer[256]; + unsigned int want = MIN(256, mgen_pending_bytes); + unsigned int got = want; + if ((static_cast(object))->Read(buffer, &got)) + { + mgen_pending_bytes -= got; + mgen_bytes += got; + if (got != want) dataReady = false; + } + else + { + PLOG(PL_WARN, "NormSimAgent::Notify(3) detected stream break\n"); + mgen_bytes = mgen_pending_bytes = 0; + msg_sync = false; + break; + } + } + if (msg_sync && (0 == mgen_pending_bytes)) + { + ProtoAddress srcAddr; + srcAddr.ResolveFromString(sender->GetAddress().GetHostString()); + srcAddr.SetPort(sender->GetAddress().GetPort()); + msg_sink->HandleMessage(mgen_buffer,mgen_bytes,srcAddr); + mgen_bytes = 0; + } + } // end if (mgen_pending_bytes) + } // end while(dataReady) + } + else + { + // Simply read and discard the byte stream for sim purposes + char buffer[1024]; + unsigned int want = 1024; + unsigned int got = want; + while (1) + { + if ((static_cast(object))->Read(buffer, &got)) + { + // Break when data is no longer available + if (got != want) break; } else { - if (msg_sync) - { - PLOG(PL_WARN, "NormSimAgent::Notify(1) detected stream break\n"); - //ASSERT(0); - mgen_bytes = mgen_pending_bytes = 0; - msg_sync = false; - continue; - } - else - { - break; - } - } - if (2 == mgen_bytes) - { - UINT16 msgSize; - memcpy(&msgSize, mgen_buffer, sizeof(UINT16)); - mgen_pending_bytes = ntohs(msgSize) - 2; + PLOG(PL_WARN, "NormSimAgent::Notify() detected stream break\n"); } + got = want = 1024; } - if (mgen_pending_bytes) - { - // Save the first part for MGEN logging - if (mgen_bytes < 64) - { - unsigned int want = MIN(mgen_pending_bytes, 62); - unsigned int got = want; - if ((static_cast(object))->Read(mgen_buffer+mgen_bytes, - &got)) - { - mgen_pending_bytes -= got; - mgen_bytes += got; - if (got != want) dataReady = false; - } - else - { - PLOG(PL_WARN, "NormSimAgent::Notify(2) detected stream break\n"); - mgen_bytes = mgen_pending_bytes = 0; - msg_sync = false; - continue; - } - } - while (dataReady && mgen_pending_bytes) - { - char buffer[256]; - unsigned int want = MIN(256, mgen_pending_bytes); - unsigned int got = want; - if ((static_cast(object))->Read(buffer, &got)) - { - mgen_pending_bytes -= got; - mgen_bytes += got; - if (got != want) dataReady = false; - } - else - { - PLOG(PL_WARN, "NormSimAgent::Notify(3) detected stream break\n"); - mgen_bytes = mgen_pending_bytes = 0; - msg_sync = false; - break; - } - } - if (msg_sync && (0 == mgen_pending_bytes)) - { - ProtoAddress srcAddr; - srcAddr.ResolveFromString(sender->GetAddress().GetHostString()); - srcAddr.SetPort(sender->GetAddress().GetPort()); - msg_sink->HandleMessage(mgen_buffer,mgen_bytes,srcAddr); - mgen_bytes = 0; - } - } // end if (mgen_pending_bytes) - } // end while(dataReady) - } - else - { - // Simply read and discard the byte stream for sim purposes - char buffer[1024]; - unsigned int want = 1024; - unsigned int got = want; - while (1) - { - if ((static_cast(object))->Read(buffer, &got)) - { - // Break when data is no longer available - if (got != want) break; - } - else - { - PLOG(PL_WARN, "NormSimAgent::Notify() detected stream break\n"); - } - got = want = 1024; } + break; } + + case NormObject::DATA: + PLOG(PL_FATAL, "NormSimAgent::Notify() DATA objects not supported...\n"); + ASSERT(0); + break; + + default: + // should never occur break; - } - - case NormObject::DATA: - PLOG(PL_FATAL, "NormSimAgent::Notify() DATA objects not supported...\n"); - ASSERT(0); - break; - - default: - // should never occur - break; } // end switch (object->GetType()) break; case RX_OBJECT_COMPLETED: @@ -1064,7 +1184,6 @@ void NormSimAgent::Notify(NormController::Event event, break; } case RX_OBJECT_ABORTED: - { switch(object->GetType()) { case NormObject::FILE: @@ -1092,10 +1211,10 @@ void NormSimAgent::Notify(NormController::Event event, break; } break; - } - default: - PLOG(PL_DEBUG, "NormSimAgent::Notify() unhandled NormEvent type\n"); + default: + PLOG(PL_DEBUG, "NormSimAgent::Notify() unhandled NormEvent type\n"); + break; } // end switch(event) } // end NormSimAgent::Notify() @@ -1111,7 +1230,6 @@ bool NormSimAgent::OnIntervalTimeout(ProtoTimer& theTimer) else { // Queue a NORM_OBJECT_SIM as long as there are repeats - if (tx_repeat_count > 0) tx_repeat_count--; if (tx_object_size_min > 0) { tx_object_size = tx_object_size_max - tx_object_size_min; @@ -1121,6 +1239,7 @@ bool NormSimAgent::OnIntervalTimeout(ProtoTimer& theTimer) NormObject* object = session->QueueTxSim(tx_object_size); if (NULL != object) { + if (tx_repeat_count > 0) tx_repeat_count--; PLOG(PL_DEBUG, "NormSimAgent::OnIntervalTimeout(() Queued file size: %lu bytes\n", tx_object_size); if (NULL != log_file_ptr) { @@ -1136,8 +1255,10 @@ bool NormSimAgent::OnIntervalTimeout(ProtoTimer& theTimer) } else { - TRACE("NormSimAgent::OnIntervalTimeout() Error queueing tx object.\n"); - PLOG(PL_WARN, "NormSimAgent::OnIntervalTimeout() Error queueing tx object.\n"); + // Note, probably was flow controlled, so we don't decrement the repeat count here and the + // next QUEUE_VACANCY event will trigger another attempt to enqueue + TRACE("NormSimAgent::OnIntervalTimeout() warning: unable to enqueue tx object. (flow control?)\n"); + PLOG(PL_WARN, "NormSimAgent::OnIntervalTimeout() warning: unable to enqueue tx object. (flow control?)\n"); } } interval_timer.SetInterval(tx_object_interval); @@ -1150,17 +1271,27 @@ bool NormSimAgent::OnIntervalTimeout(ProtoTimer& theTimer) return false; } } // end NormSimAgent::OnIntervalTimeout() - +bool NormSimAgent::AddAckingNode(NormNodeId nodeId) +{ + if (NULL == session) + { + PLOG(PL_FATAL, "NormSimAgent::AddAckingNode() error: sender not started!\n"); + return false; + } + flow_control = true; // assume ACK flow control is desired + return session->SenderAddAckingNode(nodeId); +} // end NormSimAgent::AddAckingNode() + bool NormSimAgent::StartSender() { - if (session) + if (NULL != session) { PLOG(PL_FATAL, "NormSimAgent::StartSender() Error! sender or receiver already started!\n"); return false; } // Validate our session settings - if (!address) + if (NULL == address) { PLOG(PL_FATAL, "NormSimAgent::StartSender() Error! no session address given."); return false; @@ -1168,7 +1299,7 @@ bool NormSimAgent::StartSender() // Create a new session on multicast group/port session = session_mgr.NewSession(address, port, GetAgentId()); - if (session) + if (NULL != session) { // Common session parameters session->SetTxRate(tx_rate); @@ -1203,13 +1334,13 @@ bool NormSimAgent::StartSender() bool NormSimAgent::StartReceiver() { - if (session) + if (NULL != session) { PLOG(PL_FATAL, "NormSimAgent::StartReceiver() Error! sender or receiver already started!\n"); return false; } // Validate our session settings - if (!address) + if (NULL == address) { PLOG(PL_FATAL, "NormSimAgent::StartReceiver() Error! no session address given."); return false; diff --git a/src/common/pcap2norm.cpp b/src/common/pcap2norm.cpp index 538096b..fa2268d 100644 --- a/src/common/pcap2norm.cpp +++ b/src/common/pcap2norm.cpp @@ -317,6 +317,10 @@ void NormTrace2(const struct timeval& currentTime, break; } } + struct timeval sendTime; + cc.GetSendTime(sendTime); + double delay = ProtoTime::Delta(ProtoTime(currentTime), ProtoTime(sendTime)); + PLOG(PL_ALWAYS, "delay>%lf ", delay); break; } default: @@ -328,6 +332,7 @@ void NormTrace2(const struct timeval& currentTime, case NormMsg::ACK: case NormMsg::NACK: { + PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); // look for NormCCFeedback extension NormHeaderExtension ext; while (msg.GetNextExtension(ext)) @@ -335,10 +340,22 @@ void NormTrace2(const struct timeval& currentTime, if (NormHeaderExtension::CC_FEEDBACK == ext.GetType()) { clrFlag = ((NormCCFeedbackExtension&)ext).CCFlagIsSet(NormCC::CLR); + // Print ccRtt (only valid if pcap file is from sender node) + double ccRtt = NormUnquantizeRtt(((NormCCFeedbackExtension&)ext).GetCCRtt()); + PLOG(PL_ALWAYS, "ccRtt:%lf ", ccRtt); break; } } - PLOG(PL_ALWAYS, "%s ", MSG_NAME[msgType]); + // Print locally measured rtt (only valid if pcap file is from sender node) + struct timeval grttResponse; + if (NormMsg::NACK == msgType) + static_cast(msg).GetGrttResponse(grttResponse); + else + static_cast(msg).GetGrttResponse(grttResponse); + + double rtt = ProtoTime::Delta(ProtoTime(currentTime), ProtoTime(grttResponse)); + PLOG(PL_ALWAYS, "rtt:%lf ", rtt); + PLOG(PL_ALWAYS, "len>%hu %s\n", length, clrFlag ? "(CLR)" : ""); if (NormMsg::NACK == msgType) { diff --git a/src/java/jni/normSessionJni.cpp b/src/java/jni/normSessionJni.cpp index fc02bb8..1c7322c 100644 --- a/src/java/jni/normSessionJni.cpp +++ b/src/java/jni/normSessionJni.cpp @@ -69,12 +69,12 @@ JNIEXPORT void JNICALL PKGNAME(NormSession_setRxPortReuse) } JNIEXPORT void JNICALL PKGNAME(NormSession_setEcnSupport) - (JNIEnv *env, jobject obj, jboolean ecnEnable, jboolean ignoreLoss) { + (JNIEnv *env, jobject obj, jboolean ecnEnable, jboolean ignoreLoss, jboolean tolerateLoss) { NormSessionHandle session; session = (NormSessionHandle)env->GetLongField(obj, fid_NormSession_handle); - NormSetEcnSupport(session, ecnEnable, ignoreLoss); + NormSetEcnSupport(session, ecnEnable, ignoreLoss, tolerateLoss); } JNIEXPORT void JNICALL PKGNAME(NormSession_setMulticastInterface) @@ -509,6 +509,15 @@ JNIEXPORT void JNICALL PKGNAME(NormSession_cancelWatermark) NormCancelWatermark(session); } +JNIEXPORT void JNICALL PKGNAME(NormSession_resetWatermark) + (JNIEnv *env, jobject obj) { + NormSessionHandle session; + + session = (NormSessionHandle)env->GetLongField(obj, fid_NormSession_handle); + + NormResetWatermark(session); +} + JNIEXPORT void JNICALL PKGNAME(NormSession_addAckingNode) (JNIEnv *env, jobject obj, jlong nodeId) { NormSessionHandle session; diff --git a/src/java/jni/normSessionJni.h b/src/java/jni/normSessionJni.h index 0fe1610..e3ea161 100644 --- a/src/java/jni/normSessionJni.h +++ b/src/java/jni/normSessionJni.h @@ -45,7 +45,7 @@ JNIEXPORT void JNICALL Java_mil_navy_nrl_norm_NormSession_setRxPortReuse * Signature: (ZZ)V */ JNIEXPORT void JNICALL Java_mil_navy_nrl_norm_NormSession_setEcnSupport - (JNIEnv *, jobject, jboolean, jboolean); + (JNIEnv *, jobject, jboolean, jboolean, jboolean); /* * Class: mil_navy_nrl_norm_NormSession @@ -319,6 +319,14 @@ JNIEXPORT void JNICALL Java_mil_navy_nrl_norm_NormSession_setWatermark JNIEXPORT void JNICALL Java_mil_navy_nrl_norm_NormSession_cancelWatermark (JNIEnv *, jobject); +/* + * Class: mil_navy_nrl_norm_NormSession + * Method: cancelWatermark + * Signature: ()V + */ +JNIEXPORT void JNICALL Java_mil_navy_nrl_norm_NormSession_resetWatermark + (JNIEnv *, jobject); + /* * Class: mil_navy_nrl_norm_NormSession * Method: addAckingNode diff --git a/src/java/src/mil/navy/nrl/norm/NormObject.java b/src/java/src/mil/navy/nrl/norm/NormObject.java index 0bd9da8..154f842 100644 --- a/src/java/src/mil/navy/nrl/norm/NormObject.java +++ b/src/java/src/mil/navy/nrl/norm/NormObject.java @@ -49,9 +49,8 @@ public class NormObject { */ public boolean equals(Object obj) { if (obj instanceof NormObject) { - return handle == ((NormObject)obj).handle; + return (handle == ((NormObject)obj).handle); } - return false; } } diff --git a/src/java/src/mil/navy/nrl/norm/NormSession.java b/src/java/src/mil/navy/nrl/norm/NormSession.java index 0ac5f87..d6fdf11 100644 --- a/src/java/src/mil/navy/nrl/norm/NormSession.java +++ b/src/java/src/mil/navy/nrl/norm/NormSession.java @@ -49,16 +49,17 @@ public class NormSession { public void setTxPort(int port) { setTxPort(port, false, null); } - public native void setTxPort(int port, boolean enableReuse, String txAddress); public void setRxPortReuse(boolean enable) { setRxPortReuse(enable, null, null, 0); } - public native void setRxPortReuse(boolean enable, String rxBindAddress, String senderAddress, int senderPort); - public native void setEcnSupport(boolean ecnEnable, boolean ignoreLoss); + public void setEcnSupport(boolean ecnEnable, boolean ignoreLoss) { + setEcnSupport(ecnEnable, ignoreLoss, false); + } + public native void setEcnSupport(boolean ecnEnable, boolean ignoreLoss, boolean tolerateLoss); public native void setMulticastInterface(String interfaceName) throws IOException; @@ -97,6 +98,9 @@ public class NormSession { public native void setTxSocketBuffer(long bufferSize) throws IOException; + public void setCongestionControl(boolean enable) { + setCongestionControl(enable, true); + } public native void setCongestionControl(boolean enable, boolean adjustRate); public native void setTxRateBounds(double rateMin, double rateMax); @@ -156,6 +160,8 @@ public class NormSession { throws IOException; public native void cancelWatermark(); + + public native void resetWatermark(); public native void addAckingNode(long nodeId) throws IOException; diff --git a/src/java/src/mil/navy/nrl/norm/enums/NormSyncPolicy.java b/src/java/src/mil/navy/nrl/norm/enums/NormSyncPolicy.java index 62bf19c..20d6c46 100644 --- a/src/java/src/mil/navy/nrl/norm/enums/NormSyncPolicy.java +++ b/src/java/src/mil/navy/nrl/norm/enums/NormSyncPolicy.java @@ -13,5 +13,6 @@ package mil.navy.nrl.norm.enums; */ public enum NormSyncPolicy { NORM_SYNC_CURRENT, + NORM_SYNC_STREAM, NORM_SYNC_ALL; } diff --git a/src/pynorm/constants.py b/src/pynorm/constants.py index 9308cca..4f24bca 100644 --- a/src/pynorm/constants.py +++ b/src/pynorm/constants.py @@ -36,6 +36,11 @@ NORM_ACK_SUCCESS = 3 NORM_PROBE_NONE = 0 NORM_PROBE_PASSIVE = 1 NORM_PROBE_ACTIVE = 2 + +# enum NormSyncPolicy +NORM_SYNC_CURRENT = 0 +NORM_SYNC_STREAM = 1 +NORM_SYNC_ALL = 2 # enum NormRepairBoundary NORM_BOUNDARY_BLOCK = 0 @@ -50,20 +55,23 @@ NORM_TX_WATERMARK_COMPLETED = 4 NORM_TX_CMD_SENT = 5 NORM_TX_OBJECT_SENT = 6 NORM_TX_OBJECT_PURGED = 7 -NORM_LOCAL_SENDER_CLOSED = 8 -NORM_REMOTE_SENDER_NEW = 9 -NORM_REMOTE_SENDER_ACTIVE = 10 -NORM_REMOTE_SENDER_INACTIVE = 11 -NORM_REMOTE_SENDER_PURGED = 12 -NORM_RX_CMD_NEW = 13 -NORM_RX_OBJECT_NEW = 14 -NORM_RX_OBJECT_INFO = 15 -NORM_RX_OBJECT_UPDATED = 16 -NORM_RX_OBJECT_COMPLETED = 17 -NORM_RX_OBJECT_ABORTED = 18 -NORM_GRTT_UPDATED = 19 -NORM_CC_ACTIVE = 20 -NORM_CC_INACTIVE = 21 +NORM_TX_RATE_CHANGED = 8 +NORM_LOCAL_SENDER_CLOSED = 9 +NORM_REMOTE_SENDER_NEW = 10 +NORM_REMOTE_SENDER_ACTIVE = 11 +NORM_REMOTE_SENDER_INACTIVE = 12 +NORM_REMOTE_SENDER_PURGED = 13 +NORM_RX_CMD_NEW = 14 +NORM_RX_OBJECT_NEW = 15 +NORM_RX_OBJECT_INFO = 16 +NORM_RX_OBJECT_UPDATED = 17 +NORM_RX_OBJECT_COMPLETED = 18 +NORM_RX_OBJECT_ABORTED = 19 +NORM_GRTT_UPDATED = 20 +NORM_CC_ACTIVE = 21 +NORM_CC_INACTIVE = 22 + + NORM_INSTANCE_INVALID = ctypes.c_void_p.in_dll(libnorm, "NORM_INSTANCE_INVALID").value NORM_SESSION_INVALID = ctypes.c_void_p.in_dll(libnorm, "NORM_SESSION_INVALID").value diff --git a/src/pynorm/core.py b/src/pynorm/core.py index cf2f62f..1d0b74c 100644 --- a/src/pynorm/core.py +++ b/src/pynorm/core.py @@ -232,8 +232,13 @@ def get_libnorm(): libnorm.NormSetFlowControl.argtypes = [ctypes.c_void_p, ctypes.c_double] libnorm.NormSetCongestionControl.restype = None - libnorm.NormSetCongestionControl.argtypes = [ctypes.c_void_p, - ctypes.c_bool, ctypes.c_bool] + libnorm.NormSetCongestionControl.argtypes = [ctypes.c_void_p, ctypes.c_bool, ctypes.c_bool] + + libnorm.NormSetEcnSupport.restype = None + libnorm.NormSetEcnSupport.argtypes = [ctypes.c_void_p, ctypes.c_bool, ctypes.c_bool, ctypes.c_bool] + + libnorm.NormSetFlowControl.restype = None + libnorm.NormSetFlowControl.argtypes = [ctypes.c_void_p, ctypes.c_double] libnorm.NormSetTxRateBounds.restype = None libnorm.NormSetTxRateBounds.argtypes = [ @@ -313,8 +318,7 @@ def get_libnorm(): libnorm.NormStreamMarkEom.restype = None libnorm.NormStreamMarkEom.argtypes = [ctypes.c_void_p] - libnorm.NormSetWatermark.argtypes = [ctypes.c_void_p, ctypes.c_void_p, - ctypes.c_bool] + libnorm.NormSetWatermark.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_bool] libnorm.NormSetWatermark.errcheck = errcheck_bool libnorm.NormResetWatermark.restype = ctypes.c_bool @@ -348,6 +352,9 @@ def get_libnorm(): libnorm.NormStopReceiver.restype = None libnorm.NormStopReceiver.argtypes = [ctypes.c_void_p] + + libnorm.NormSetDefaultSyncPolicy.restype = None + libnorm.NormSetDefaultSyncPolicy.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormSetRxCacheLimit.restype = None libnorm.NormSetRxCacheLimit.argtypes = [ctypes.c_void_p, ctypes.c_ushort] @@ -357,19 +364,19 @@ def get_libnorm(): libnorm.NormSetRxSocketBuffer.errcheck = errcheck_bool libnorm.NormSetSilentReceiver.restype = None - libnorm.NormSetSilentReceiver.argtypes = [ctypes.c_void_p, ctypes.c_bool, - ctypes.c_int] - + libnorm.NormSetSilentReceiver.argtypes = [ctypes.c_void_p, ctypes.c_bool, ctypes.c_int] + + libnorm.NormSetMessageTrace.restype = None + libnorm.NormSetMessageTrace.argtypes = [ctypes.c_void_p, ctypes.c_bool] + libnorm.NormSetDefaultUnicastNack.restype = None - libnorm.NormSetDefaultUnicastNack.argtypes = [ctypes.c_void_p, - ctypes.c_bool] + libnorm.NormSetDefaultUnicastNack.argtypes = [ctypes.c_void_p, ctypes.c_bool] libnorm.NormNodeSetUnicastNack.restype = None libnorm.NormNodeSetUnicastNack.argtypes = [ctypes.c_void_p, ctypes.c_bool] libnorm.NormSetDefaultNackingMode.restype = None - libnorm.NormSetDefaultNackingMode.argtypes = [ctypes.c_void_p, - ctypes.c_int] + libnorm.NormSetDefaultNackingMode.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormNodeSetNackingMode.restype = None libnorm.NormNodeSetNackingMode.argtypes = [ctypes.c_void_p, ctypes.c_int] @@ -378,24 +385,19 @@ def get_libnorm(): libnorm.NormObjectSetNackingMode.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormSetDefaultRepairBoundary.restype = None - libnorm.NormSetDefaultRepairBoundary.argtypes = [ctypes.c_void_p, - ctypes.c_int] + libnorm.NormSetDefaultRepairBoundary.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormNodeSetRepairBoundary.restype = None - libnorm.NormNodeSetRepairBoundary.argtypes = [ctypes.c_void_p, - ctypes.c_int] + libnorm.NormNodeSetRepairBoundary.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormSetDefaultRxRobustFactor.restype = None - libnorm.NormSetDefaultRxRobustFactor.argtypes = [ctypes.c_void_p, - ctypes.c_int] + libnorm.NormSetDefaultRxRobustFactor.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormNodeSetRxRobustFactor.restype = None - libnorm.NormNodeSetRxRobustFactor.argtypes = [ctypes.c_void_p, - ctypes.c_int] + libnorm.NormNodeSetRxRobustFactor.argtypes = [ctypes.c_void_p, ctypes.c_int] libnorm.NormStreamRead.restype = ctypes.c_bool - libnorm.NormStreamRead.argtypes = [ctypes.c_void_p, ctypes.c_char_p, - ctypes.POINTER(ctypes.c_uint)] + libnorm.NormStreamRead.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.POINTER(ctypes.c_uint)] libnorm.NormStreamRead.errcheck = return_bool libnorm.NormStreamSeekMsgStart.restype = ctypes.c_bool @@ -446,10 +448,10 @@ def get_libnorm(): libnorm.NormFileRename.argtypes = [ctypes.c_void_p, ctypes.c_char_p] libnorm.NormFileRename.errcheck = errcheck_bool - libnorm.NormDataAccessData.restype = ctypes.c_char_p + libnorm.NormDataAccessData.restype = ctypes.c_void_p libnorm.NormDataAccessData.argtypes = [ctypes.c_void_p] - libnorm.NormDataDetachData.restype = ctypes.c_char_p + libnorm.NormDataDetachData.restype = ctypes.c_void_p libnorm.NormDataDetachData.argtypes = [ctypes.c_void_p] libnorm.NormObjectGetSender.restype = ctypes.c_void_p diff --git a/src/pynorm/event.py b/src/pynorm/event.py index 0def624..8439932 100644 --- a/src/pynorm/event.py +++ b/src/pynorm/event.py @@ -13,11 +13,11 @@ from pynorm.node import Node from pynorm.object import Object class Event(object): - def __init__(self, type, session, sender, object): - self._type = type + def __init__(self, eventType, session, sender, normObject): + self._type = eventType self._session = session self._sender = sender - self._object = object + self._object = normObject # Properties type = property(lambda self: self._type) @@ -57,6 +57,8 @@ class Event(object): return "NORM_TX_OBJECT_SENT" elif self.type == c.NORM_TX_OBJECT_PURGED: return "NORM_TX_OBJECT_PURGED" + elif self.type == c.NORM_TX_RATE_CHANGED: + return "NORM_TX_RATE_CHANGED" elif self.type == c.NORM_LOCAL_SENDER_CLOSED: return "NORM_LOCAL_SENDER_CLOSED" elif self.type == c.NORM_REMOTE_SENDER_NEW: diff --git a/src/pynorm/instance.py b/src/pynorm/instance.py index 86499ed..237603c 100644 --- a/src/pynorm/instance.py +++ b/src/pynorm/instance.py @@ -9,6 +9,7 @@ import ctypes from select import select from weakref import WeakValueDictionary from platform import system +import sys if system() == 'Windows': import win32event @@ -48,6 +49,9 @@ class Instance(object): def setCacheDirectory(self, path): libnorm.NormSetCacheDirectory(self, path) + + def setDebugLevel(self, level): + libnorm.NormSetDebugLevel(level) def openDebugLog(self, path): libnorm.NormOpenDebugLog(self, path) @@ -64,23 +68,26 @@ class Instance(object): if not self._select(timeout): return None - libnorm.NormGetNextEvent(self, ctypes.byref(self._estruct), True) + result = libnorm.NormGetNextEvent(self, ctypes.byref(self._estruct), True) + if not result: + sys.stderr.write("NormInstance.getNextEvent() warning: no more NORM events\n") + return False + # Note a NORM_EVENT_INVALID can be OK (with NORM_SESSION_INVALID) + if self._estruct.type == c.NORM_EVENT_INVALID: + return Event(c.NORM_EVENT_INVALID, None, None, None) + if self._estruct.session == c.NORM_SESSION_INVALID: raise NormError("No new event") - try: sender = self._senders[self._estruct.sender] except KeyError: - sender = self._senders[self._estruct.sender] =\ - Node(self._estruct.sender) + sender = self._senders[self._estruct.sender] = Node(self._estruct.sender) try: object = self._objects[self._estruct.object] except KeyError: - object = self._objects[self._estruct.object] =\ - Object(self._estruct.object) - return Event(self._estruct.type, - self._sessions[self._estruct.session], sender, object) + object = self._objects[self._estruct.object] = Object(self._estruct.object) + return Event(self._estruct.type, self._sessions[self._estruct.session], sender, object) def getDescriptor(self): return libnorm.NormGetDescriptor(self) diff --git a/src/pynorm/object.py b/src/pynorm/object.py index 7bd56ee..2889066 100644 --- a/src/pynorm/object.py +++ b/src/pynorm/object.py @@ -18,7 +18,7 @@ class Object(object): def __init__(self, object): libnorm.NormObjectRetain(object) self._object = object - + def getType(self): return libnorm.NormObjectGetType(self) @@ -56,12 +56,19 @@ class Object(object): def renameFile(self, name): libnorm.NormFileRename(self, name) - def accessData(self): + # Because 'ctypes.string_at()' makes a _copy_ of the data, we don't + # support the usual NORM accessData / detachData options. We use + # NormDataAccessData() to get a pointer to the data we copy and + # let the underlying NORM release the received object and free the + # memory. + def getData(self): return ctypes.string_at(libnorm.NormDataAccessData(self), self.size) - def detachData(self): - return ctypes.string_at(libnorm.NormDataDetachData(self), self.size) - + #def accessData(self): + # return ctypes.string_at(libnorm.NormDataAccessData(self), self.size) + #def detachData(self): + # return ctypes.string_at(libnorm.NormDataDetachData(self), self.size) + def getSender(self): return Node(libnorm.NormObjectGetSender(self)) @@ -133,4 +140,7 @@ class Object(object): return cmp(self._as_parameter_, other._as_parameter_) def __hash__(self): - return self._as_parameter_ + return hash(self._as_parameter_) + + def __equ__(self, other): + return self._object == other._object diff --git a/src/pynorm/session.py b/src/pynorm/session.py index c11067a..7655f29 100644 --- a/src/pynorm/session.py +++ b/src/pynorm/session.py @@ -23,13 +23,13 @@ class Session(object): localId - NormNodeId """ self._instance = instance - self._session = libnorm.NormCreateSession(instance, address, port, - localId) + self._session = libnorm.NormCreateSession(instance, address, port, localId) self.sendGracefulStop = False self.gracePeriod = 0 def destroy(self): libnorm.NormDestroySession(self) + del self._instance._sessions[self] def setUserData(self, data): """data should be a string""" @@ -66,13 +66,12 @@ class Session(object): libnorm.NormSetReportInterval(self, interval) ## Sender functions - def startSender(self, sessionId, bufferSpace, segmentSize, blockSize, - numParity): + def startSender(self, sessionId, bufferSpace, segmentSize, blockSize, numParity): libnorm.NormStartSender(self, sessionId, bufferSpace, segmentSize, blockSize, numParity) - def stopSender(self, graceful=False): - libnorm.NormStopSender(self, graceful) + def stopSender(self): + libnorm.NormStopSender(self) def setTxRate(self, rate): libnorm.NormSetTxRate(self, rate) @@ -80,11 +79,17 @@ class Session(object): def setTxSocketBuffer(self, size): libnorm.NormSetTxSocketBuffer(self, size) - def setCongestionControl(self, cc): - libnorm.NormSetCongestionControl(self, cc) + def setCongestionControl(self, ccEnable, adjustRate=True): + libnorm.NormSetCongestionControl(self, ccEnable, adjustRate) + + def setEcnSupport(self, ecnEnable, ignoreLoss=False, tolerateLoss=False): + libnorm.NormSetEcnSupport(self, ecnEnable, ignoreLoss, tolerateLoss) + + def setFlowControl(self, flowControlFactor): + libnorm.NormSetFlowControl(self, flowControlFactor) - def setTxRateBounds(self, min, max): - libnorm.NormSetTxRateBounds(self, min, max) + def setTxRateBounds(self, rateMin, rateMax): + libnorm.NormSetTxRateBounds(self, rateMin, rateMax) def setTxCacheBounds(self, sizeMax, countMin, countMax): libnorm.NormSetTxCacheBounds(self, sizeMax, countMin, countMax) @@ -95,17 +100,17 @@ class Session(object): def getGrttEstimate(self): return libnorm.NormGetGrttEstimate(self) - def setGrttEstimate(self, grtt): + def setGrttEstimate(self, grttMax): libnorm.NormSetGrttEstimate(self, grtt) def setGrttMax(self, max): - libnorm.NormSetGrttMax(self, max) + libnorm.NormSetGrttMax(self, grttMax) def setGrttProbingMode(self, mode): libnorm.NormSetGrttProbingMode(self, mode) - def setGrttProbingInterval(self, min, max): - libnorm.NormSetGrttProbingInterval(self, min, max) + def setGrttProbingInterval(self, intervalMin, intervalMax): + libnorm.NormSetGrttProbingInterval(self, intervalMin, intervalMax) def setBackoffFactor(self, factor): libnorm.NormSetBackoffFactor(self, factor) @@ -114,22 +119,25 @@ class Session(object): libnorm.NormSetGroupSize(self, size) def fileEnqueue(self, filename, info=""): - return Object(libnorm.NormFileEnqueue(self, filename, info, - len(info))) + return Object(libnorm.NormFileEnqueue(self, filename, info, len(info))) def dataEnqueue(self, data, info=""): - return Object(libnorm.NormDataEnqueue(self, data, - len(data), info, len(info))) + return Object(libnorm.NormDataEnqueue(self, data, len(data), info, len(info))) - def requeueObject(self, object): - libnorm.NormRequeueObject(self, object) + def requeueObject(self, normObject): + libnorm.NormRequeueObject(self, normObject) def streamOpen(self, bufferSize, info=""): - return Object(libnorm.NormStreamOpen(self, bufferSize, info, - len(info))) + return Object(libnorm.NormStreamOpen(self, bufferSize, info, len(info))) - def setWatermark(self, object): - libnorm.NormSetWatermark(self, object) + def setWatermark(self, normObject, overrideFlush=False): + libnorm.NormSetWatermark(self, normObject, overrideFlush) + + def resetWatermark(self): + libnorm.NormResetWatermark(self) + + def cancelWatermark(self): + libnorm.NormCancelWatermark(self) def addAckingNode(self, nodeId): libnorm.NormAddAckingNode(self, nodeId) @@ -148,6 +156,9 @@ class Session(object): """This will be called automatically if the receiver is active""" libnorm.NormStopReceiver(self, gracePeriod) + def setRxCacheLimit(self, count): + libnorm.NormSetRxCacheLimit(self, count) + def setRxSocketBuffer(self, size): libnorm.NormSetRxSocketBuffer(self, size) @@ -158,12 +169,18 @@ class Session(object): def setDefaultUnicastNack(self, mode): libnorm.NormSetDefaultUnicastNack(self, mode) + + def setDefaultSyncPolicy(self, policy): + libnorm.NormSetDefaultSyncPolicy(self, policy) def setDefaultNackingMode(self, mode): libnorm.NormSetDefaultNackingMode(self, mode) def setDefaultRepairBoundary(self, boundary): libnorm.NormSetDefaultRepairBoundary(self, boundary) + + def setMessageTrace(self, state): + libnorm.NormSetMessageTrace(self, state) ## Properties nodeId = property(getNodeId) diff --git a/waf b/waf index e1e34d4..4e68fed 100755 Binary files a/waf and b/waf differ diff --git a/wscript b/wscript index 29fc37b..019941a 100644 --- a/wscript +++ b/wscript @@ -51,6 +51,7 @@ def build(ctx): target = 'norm', includes = ['include'], export_includes = ['include'], + vnum = '1.0.0', use = ctx.env.USE_BUILD_NORM, source = ['src/common/{0}.cpp'.format(x) for x in [ 'galois',