Merge branch 'USNavalResearchLaboratory:master' into master

BUILD.TXT

@@ -6,11 +6,26 @@ To see a full list of options, run:
 
     ./waf -h
 
+Note: if attempting to run waf as an executable as shown above results in a
+message like this: 
+
+/usr/bin/env: ‘python’: No such file or directory
+
+then try calling python directly:
+
+python waf <args>
+
+or
+
+python3 waf <args>
+
+For more background information about waf, visit the gitlab page at
+https://gitlab.com/ita1024/./waf/
 
 Configuring
 -----------
 
-To perform the configure checks, run:
+To perform the configuration checks, run:
 
     ./waf configure
 
@@ -51,6 +66,9 @@ To install, run:
 This will install the compiled library and headers to wherever your prefix was
 specified.  (See configure flags)
 
+This command may have to be run with root priviledges for certain prefix
+directories depending on your system setup.
+
 Uninstalling
 ------------
 

README-PyNorm.txt

@@ -28,7 +28,7 @@ in similar places as your system's compiler.  On UNIX systems, this is usually
 PATH environment variable.
 
 On Windows, PyNORM requires the "PyWin32" module, available at:
-    http://sf.net/projects/pywin32
+    https://sourceforge.net/projects/pywin32
     
 -------------
 Installation

android/README.md

@@ -1,4 +1,4 @@
-This directoy contains Android Studio project files for building the NORM
+This directory contains Android Studio project files for building the NORM
 library for Android.
 
 REQUIREMENTS:

android/lib/CMakeLists.txt

@@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.4.1)
 
 project("norm")
 
-add_definitions("-DANDROID=1" "-DANDROID_API_VERSION=15" "-DPUSH_NOTIFICATIONS=1" "-DANDROID" "-DLINUX" "-DUNIX" "-DHAVE_IPV6" "-DHAVE_DIRFD" "-DPROTO_DEBUG" "-DHAVE_ASSERT" "-DHAVE_GETLOGIN" "-DUSE_SELECT" "-D_FILE_OFFSET_BITS=64" "-DHAVE_OLD_SIGNALHANDLER" "-DHAVE_SCHED" "-DNO_SCM_RIGHTS" "-Wno-attributes" "-DAPP_VERSION=1.0.0" "-DAPP_VERSION_HEX=0x10000")
+add_definitions("-DANDROID=1" "-DANDROID_API_VERSION=24" "-DPUSH_NOTIFICATIONS=1" "-DANDROID" "-DLINUX" "-DUNIX" "-DHAVE_IPV6" "-DHAVE_DIRFD" "-DPROTO_DEBUG" "-DHAVE_ASSERT" "-DHAVE_GETLOGIN" "-DUSE_SELECT" "-D_FILE_OFFSET_BITS=64" "-DHAVE_OLD_SIGNALHANDLER" "-DHAVE_SCHED" "-DNO_SCM_RIGHTS" "-Wno-attributes" "-DAPP_VERSION=1.0.0" "-DAPP_VERSION_HEX=0x10000")
 
 include_directories( AFTER
     "../../OID_NDK}/sources/android/cpufeatures"

android/lib/build.gradle

@@ -1,8 +1,7 @@
 apply plugin: 'com.android.library'
 
 android {
-    compileSdkVersion 31
+    compileSdkVersion 34
-    buildToolsVersion "28.0.3"
     externalNativeBuild {
         cmake {
             path "CMakeLists.txt"
@@ -21,7 +20,7 @@ android {
     defaultConfig {
         externalNativeBuild {
             cmake {
-                arguments "-DANDROID_TOOLCHAIN=clang", "-DANDROID_PLATFORM=android-15", "-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja", "-DANDROID_STL=c++_static", "-DANDROID_CPP_FEATURES=exceptions rtti", "-DANDROID_ARM_MODE=arm", "-DANDROID_ARM_NEON=TRUE"
+                arguments "-DANDROID_TOOLCHAIN=clang", "-DANDROID_PLATFORM=android-24", "-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja", "-DANDROID_STL=c++_static", "-DANDROID_CPP_FEATURES=exceptions rtti", "-DANDROID_ARM_MODE=arm", "-DANDROID_ARM_NEON=TRUE"
                 cFlags "-fsigned-char"
                 cppFlags "-fsigned-char", "-std=c++14"
             }

doc/NormDeveloperGuide.xml

@@ -13,19 +13,19 @@
 
     <abstract>
       <para>This document describes an application programming interface (API)
-      for the <ulink url="http://norm.pf.itd.nrl.navy.mil/">Nack-Oriented
+      for the <ulink url="https://www.nrl.navy.mil/Our-Work/Areas-of-Research/Information-Technology/NCS/NORM/">Nack-Oriented
       Reliable Multicast (NORM)</ulink> protocol implementation developed by
       the Protocol Engineering and Advance Networking (<ulink
-      url="http://cs.itd.nrl.navy.mil/">PROTEAN</ulink>) Research Group of the
+      url="https://www.nrl.navy.mil/itd/ncs/">PROTEAN</ulink>) Research Group of the
-      United States <ulink url="http://www.nrl.navy.mil/">Naval Research
+      United States <ulink url="https://www.nrl.navy.mil/">Naval Research
       Laboratory</ulink> (NRL). The NORM protocol provides general purpose
       reliable data transport for applications wishing to use Internet
       Protocol (IP) Multicast services for group data delivery. NORM can also
       support unicast (point-to-point) data communication and may be used for
       such when deemed appropriate. The current NORM protocol specification is
-      given in the <ulink url="http://www.ietf.org/">Internet Engineering Task
+      given in the <ulink url="https://www.ietf.org/">Internet Engineering Task
       Force</ulink> (IETF) <ulink
-      url="http://norm.pf.itd.nrl.navy.mil/rfc3940.pdf">RFC 3940</ulink>. This
+      url="https://datatracker.ietf.org/doc/html/rfc5740">RFC 5740</ulink>. This
       document is currently a reference guide to the NORM API of the NRL
       reference implementation. More tutorial material may be include in a
       future version of this document or a separate developer's tutorial may
@@ -37,19 +37,19 @@
     <title>Background</title>
 
     <para>This document describes an application programming interface (API)
-    for the <ulink url="http://norm.pf.itd.nrl.navy.mil/">Nack-Oriented
+    for the <ulink url="https://www.nrl.navy.mil/Our-Work/Areas-of-Research/Information-Technology/NCS/NORM/">Nack-Oriented
     Reliable Multicast (NORM)</ulink> protocol implementation developed by the
     Protocol Engineering and Advance Networking (<ulink
-    url="http://cs.itd.nrl.navy.mil/">PROTEAN</ulink>) Research Group of the
+    url="https://www.nrl.navy.mil/itd/ncs/">PROTEAN</ulink>) Research Group of the
-    United States <ulink url="http://www.nrl.navy.mil/">Naval Research
+    United States <ulink url="https://www.nrl.navy.mil/">Naval Research
     Laboratory</ulink> (NRL). The NORM protocol provides general purpose
     reliable data transport for applications wishing to use Internet Protocol
     (IP) Multicast services for group data delivery. NORM can also support
     unicast (point-to-point) data communication and may be used for such when
     deemed appropriate. The current NORM protocol specification is given in
-    the <ulink url="http://www.ietf.org/">Internet Engineering Task
+    the <ulink url="https://www.ietf.org/">Internet Engineering Task
     Force</ulink> (IETF) <ulink
-    url="http://norm.pf.itd.nrl.navy.mil/rfc5740.pdf">RFC 5740</ulink>.</para>
+    url="https://datatracker.ietf.org/doc/html/rfc5740">RFC 5740</ulink>.</para>
 
     <para>The NORM protocol is designed to provide end-to-end reliable
     transport of bulk data objects or streams over generic IP multicast
@@ -134,7 +134,7 @@
     may participate as senders and/or receivers within a NORM session. NORM
     senders transmit data to the session destination address (usually an IP
     multicast group) while receivers are notified of incoming data. The NORM
-    API provides and event notification scheme to notify the application of
+    API provides an event notification scheme to notify the application of
     significant sender and receiver events. There are also a number support
     functions provided for the application to control and monitor its
     participation within a NORM transport session.</para>
@@ -159,7 +159,7 @@
       linkend="NormCreateInstance"><literal>NormCreateInstance()</literal></link>)
       as needed for applications with specific requirements for accessing and
       controlling participation in multiple <emphasis>NormSessions</emphasis>
-      from separate operating system multiple threads. Or, alternatively, a
+      from separate operating system threads. Or, alternatively, a
       single <emphasis>NormInstance</emphasis> could be used, with a "master
       thread" serving as an intermediary between the <link
       linkend="NormGetNextEvent"><literal>NormGetNextEvent()</literal></link>
@@ -194,7 +194,7 @@
       number of concurrently active senders, this may translate to significant
       memory allocation on receiver nodes. Currently, the API allows the
       application to control how much buffer space is allocated for each
-      active sender (NOTE: In the future, API functions may be provided limit
+      active sender (NOTE: In the future, API functions may be provided to limit
       the number of active senders monitored and/or provide the application
       with finer control over receive buffer allocation, perhaps on a per
       sender basis).</para>
@@ -227,7 +227,7 @@
         <title>Data Transmission</title>
 
         <para>The behavior of data transport operation is largely placed in
-        the control of the NORM sender(s). NORM senders controls their data
+        the control of the NORM sender(s). NORM senders control their data
         transmission rate, forward error correction (FEC) encoding settings,
         and parameters controlling feedback from the receiver group. Multiple
         senders may operate in a session, each with independent transmission
@@ -253,7 +253,7 @@
         be called before any calls to <link
         linkend="NormStartReceiver"><literal>NormStartReceiver()</literal></link>
         are made. However, note that the cache directory may be changed even
-        during active NORM reception. In this case, the new specified
+        during active NORM reception. In this case, the newly specified
         directory path will be used for subsequently-received files. Any files
         received before a directory path change will remain in the previous
         cache location. Note that the <link
@@ -477,15 +477,14 @@
     Protokit. These are described below.</para>
 
     <para>The "makefiles" directory contains Unix Makefiles for various
-    platforms the "win32" and "wince" sub-directories there contain Microsoft
+    platforms. The "win32" and "wince" sub-directories there contain Microsoft
     Visual C++ (VC++) and Embedded VC++ project files for building the NORM
     implementation. Additionally, a "waf" (Python-based build tool) build
     option is supported that can be used to build and install the NORM library
     code on the supported platforms. Finally, Python and Java bindings to the
-    NORM API are included and "src/python" and "src/java" directories contain
+    NORM API are included in the "src/python" and "src/java" directories and the
-    the code for these and the "makefiles/java" directory contains Makefiles
+    "makefiles/java" directory contains Makefiles to build the NORM Java JNI bindings.
-    to build the NORM Java JNI bindings. Note the "waf" tool can also be used
+    Note the "waf" tool can also be used to build the Java and Python bindings.</para>
-    to build the Java and Python bindings.</para>
 
     <sect2>
       <title>Unix Platforms</title>

doc/NormUserGuide.xml

@@ -96,11 +96,13 @@
 
       <orderedlist>
         <listitem>
-          <para>Download and unpack the NORM source code tarball.</para>
+            <para>Download and unpack the NORM source code tarball or clone the github
+            repository using 
+            <literal>git clone --recurse-submodules https://github.com/USNavalResearchLaboratory/norm.git</literal></para>
         </listitem>
 
         <listitem>
-          <para><literal>cd norm/unix</literal></para>
+          <para><literal>cd norm/makefiles</literal></para>
         </listitem>
 
         <listitem>
@@ -118,7 +120,8 @@
 
       <orderedlist>
         <listitem>
-          <para>Download and unpack the NORM source code tarball</para>
+            <para>Download and unpack the NORM source code tarball or clone the
+        repository as described in the section above.</para>
         </listitem>
 
         <listitem>
@@ -128,7 +131,7 @@
         </listitem>
 
         <listitem>
-          <para>Open the "<filename>norm.sln</filename>" (VC++ .Net),
+            <para>Inside the "<filename>norm/makefiles/win32</filename>" directory, open the "<filename>norm.sln</filename>" (VC++ .Net),
           "<filename>norm.dsw</filename>" (VC++ 6.0), or
           "<filename>norm.vcw</filename>" (Embedded VC++) workspace
           file.</para>
@@ -137,9 +140,14 @@
         <listitem>
           <para>The "<emphasis>norm</emphasis>" project can be selected and
           built.  The "<filename>norm.exe</filename>" file will be found in
-          the "<filename>norm/win32/norm/Release</filename>" directory.</para>
+          the "<filename>norm/Win32/norm/Release</filename>" directory.</para>
         </listitem>
       </orderedlist>
+      <para>The easiest way to run these builds is with Visual Studio. If you do not already have Visual Studio installed, then it can
+      be downloaded for free <ulink url="https://visualstudio.microsoft.com/downloads/?cid=learn-onpage-download-cta">here</ulink>.
+      When you launch Visual Studio, make sure to select "Visual C++" and open the "<filename>norm.sln</filename>" file.
+      This should load all of the project files and allow you to right-click on any of them in the Solution Explorer on the
+      left side and select "build".</para>
     </sect2>
   </sect1>
 
@@ -380,7 +388,7 @@
     <para>A minimal example <emphasis>norm</emphasis> sender command-line
     syntax is:</para>
 
-    <para><literal>norm addr &lt;addr/port&gt; sendFile
+    <para><literal>norm addr &lt;addr/port&gt; sendfile
     &lt;fileName&gt;</literal></para>
 
     <para>The corresponding minimal <emphasis>norm</emphasis> receiver
@@ -1160,4 +1168,4 @@
       use with norm</emphasis>
     </para>
   </appendix>
-</article>
+</article>

examples/python/normCast.py

@@ -301,14 +301,16 @@ class NormCaster():
                 self.is_running = False
                 return 
         
-        obj:Optional[pynorm.Object] = session.fileEnqueue(self.pendingSendFilePath, info= self.pendingSendFilePath.replace("\\","/").encode() )
+        lastSlash = self.pendingSendFilePath.rfind('/')
+        filename = self.pendingSendFilePath[lastSlash+1:]
+        obj:Optional[pynorm.Object] = session.fileEnqueue(self.pendingSendFilePath, info= filename.encode() )
         if obj:
             if self.opts.ack:
                 session.setWatermark(obj,True)
             logging.info(f"add file:{self.pendingSendFilePath}  {obj._object}")
             self.pendingSendFilePath = None # succeed enqued
         else:
-            logging.warning(f"fileEnqueue: {sendFilePath} failure!")
+            logging.warning(f"fileEnqueue: {self.pendingSendFilePath} failure!")
 
     def handle_norm_event(self, event:pynorm.Event):
         session:pynorm.Session = event.session

examples/python/normFileSend.py

@@ -44,7 +44,7 @@ def main(argv):
     session.startSender(randint(0, 1000), 1024**2, 1400, 64, 16)
 
     print(('Sending file %s' % filename))
-    session.fileEnqueue(filepath, filename)
+    session.fileEnqueue(filepath, info=filename.encode())
 
     try:
         for event in instance:

protolib

@@ -1 +1 @@
-Subproject commit 7323fd3455ea6c16137ad3b84c950d74cc21055b
+Subproject commit dc92c6b8ccc75a01ecd2d7850f6bc7d8a7fe671d

setup.py

@@ -1,6 +1,6 @@
 # This Python script can be used to install the Python 'pynorm' package
 import platform
-from distutils.core import setup, Extension
+from setuptools import setup, Extension
 
 # Note to use 'pynorm", you will need to have the libnorm shared library
 # (libnorm.so or libnorm.dylib, etc) installed where your Python  installation

src/common/normApp.cpp

@@ -96,6 +96,8 @@ class NormApp : public NormController, public ProtoApp
         
         enum EcnMode {ECN_OFF, ECN_ON, ECN_ONLY};
         
+        enum {MSG_BUFFER_SIZE = 65535};
+        
 #ifdef UNIX
         static void SignalHandler(int sigNum);
 #endif // UNIX
@@ -115,7 +117,7 @@ class NormApp : public NormController, public ProtoApp
         FILE*               output; // output stream
         char*               output_io_buffer;
         unsigned int        output_io_bufsize;
-        char                input_buffer[65535];
+        char                input_buffer[MSG_BUFFER_SIZE];
         unsigned int        input_index;
         unsigned int        input_length;
         bool                input_active;
@@ -127,7 +129,7 @@ class NormApp : public NormController, public ProtoApp
         bool                msg_test;
         unsigned int        msg_test_length;
         UINT32              msg_test_seq;
-        char                output_buffer[65535];
+        char                output_buffer[MSG_BUFFER_SIZE];
         UINT16              output_index;
         bool                output_messaging;
         UINT16              output_msg_length;
@@ -161,7 +163,7 @@ class NormApp : public NormController, public ProtoApp
         NormSession::ProbingMode grtt_probing_mode;
         double              group_size;
         unsigned long       tx_buffer_size; // bytes
-	    unsigned int	    tx_sock_buffer_size;
+        unsigned int        tx_sock_buffer_size;
         unsigned long       tx_cache_min;
         unsigned long       tx_cache_max;
         NormObjectSize      tx_cache_size;        
@@ -213,7 +215,6 @@ class NormApp : public NormController, public ProtoApp
     
 }; // end class NormApp
 
-
 NormApp::FileCacheItem::FileCacheItem(const char* thePath)
 {
     strncpy(file_path, thePath, PATH_MAX);
@@ -603,7 +604,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val)
     else if (!strncmp("address", cmd, len))
     {
         size_t len = strlen(val);
-        if (address) delete address;
+        if (address) delete[] address;
         if (!(address = new char[len+1]))
         {
             PLOG(PL_FATAL, "NormApp::OnCommand(address) allocation error:%s\n",
@@ -614,7 +615,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val)
         char* ptr = strchr(address, '/');
         if (!ptr)
         {
-            delete address;
+            delete[] address;
             address = NULL;
             PLOG(PL_FATAL, "NormApp::OnCommand(address) missing port number!\n");   
             return false;
@@ -623,7 +624,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val)
         int portNum = atoi(ptr);
         if ((portNum < 1) || (portNum > 65535))
         {
-            delete address;
+            delete[] address;
             address = NULL;
             PLOG(PL_FATAL, "NormApp::OnCommand(address) invalid port number!\n");   
             return false;
@@ -1405,7 +1406,7 @@ void NormApp::OnInputReady()
             {
                 if (input_msg_length)
                 {
-                    UINT16 bufferSpace = 65535 - input_index;
+                    UINT16 bufferSpace = MSG_BUFFER_SIZE - input_index;
                     UINT16 msgRemainder = input_msg_length - input_msg_index;
                     readLength = MIN(bufferSpace, msgRemainder);
                 }
@@ -1418,7 +1419,7 @@ void NormApp::OnInputReady()
                     memcpy(&input_msg_length, input_buffer, 2);
                     input_msg_length = ntohs(input_msg_length);
                     ASSERT(input_msg_length >= 2);
-                    UINT16 bufferSpace = 65535 - input_index;
+                    UINT16 bufferSpace = MSG_BUFFER_SIZE - input_index;
                     UINT16 msgRemainder = input_msg_length - 2;
                     readLength = MIN(bufferSpace, msgRemainder);  
                 }
@@ -1443,7 +1444,7 @@ void NormApp::OnInputReady()
                 {
                     hdrOffset = 0;
                 }
-                int hdrLen = sprintf(input_buffer + hdrOffset, "NORM Test Msg %08u ", msg_test_seq++);
+                int hdrLen = snprintf(input_buffer + hdrOffset, MSG_BUFFER_SIZE - hdrOffset, "NORM Test Msg %08u ", msg_test_seq++);
                 char testChar = 'a' + msg_test_seq % 26;
                 memset(input_buffer + hdrLen + hdrOffset, testChar, msg_test_length - (hdrLen + hdrOffset));
                 input_buffer[msg_test_length - 1] = '\n';

src/common/normFile.cpp

@@ -136,6 +136,7 @@ bool NormFile::Open(const char* thePath, int theFlags)
     if((fd = open(thePath, theFlags, 0640)) >= 0)
     {
         offset = 0;
+        flags = theFlags;
         return true;  // no error
     }
     else
@@ -246,9 +247,11 @@ bool NormFile::Rename(const char* oldName, const char* newName)
 #endif  // WIN32
     // In Win32, the old file can't be open and on Unix (Linux at least), it's better
     // performance to close/reopen a file being renamed
+    bool wasOpen = false;
 	int oldFlags = 0;
 	if (IsOpen())
 	{
+        wasOpen = true;
 		oldFlags = flags;
 		oldFlags &= ~(O_CREAT | O_TRUNC);  // unset these
 		Close();
@@ -321,22 +324,19 @@ bool NormFile::Rename(const char* oldName, const char* newName)
     {
         PLOG(PL_ERROR, "NormFile::Rename() rename() error: %s\n", GetErrorString());	
 #endif // if/else _WIN32_WCE / WIN32|UNIX
-        if (oldFlags) 
+        if (wasOpen && !Open(oldName, oldFlags))
-        {
+            PLOG(PL_ERROR, "NormFile::Rename() error re-opening file w/ old name\n");
-            if (!Open(oldName, oldFlags))
-                PLOG(PL_ERROR, "NormFile::Rename() error re-opening file w/ old name\n");
-        }  
         return false;
     }
     else
     {
-        if (oldFlags) 
+        if (wasOpen && !Open(newName, oldFlags))
         {
-            if (!Open(newName, oldFlags))
+           PLOG(PL_ERROR, "NormFile::Rename() error opening file w/ new name\n");
-                PLOG(PL_ERROR, "NormFile::Rename() error opening file w/ new name\n");
+           return false;
         }
-        return true;
     }
+    return true;
 }  // end NormFile::Rename()
 
 size_t NormFile::Read(char* buffer, size_t len)
@@ -388,10 +388,11 @@ size_t NormFile::Write(const char* buffer, size_t len)
 #endif // if/else _WIN32_WCE
 #else
         size_t result = write(fd, buffer+put, len-put);
+        if (result <= 0) perror("NormFile::Write() write() error");
 #endif // if/else WIN32
         if (result <= 0)
         {
- #ifndef _WIN32_WCE
+#ifndef _WIN32_WCE
             if (EINTR != errno)
 #endif // !_WIN32_WCE
             {

src/common/normPostProcess.cpp

@@ -15,6 +15,7 @@ NormPostProcessor::~NormPostProcessor()
     SetCommand(NULL);
 }
 
+/*  This was defined/implemented, but unused so commented out (for now)
 void NormPostProcessor::GetCommand(char* buffer, unsigned int buflen)
 {
     if (process_argv && buffer)
@@ -52,6 +53,7 @@ void NormPostProcessor::GetCommand(char* buffer, unsigned int buflen)
         if (buflen > 4) buffer[4] = '\0';
     }
 }  // end NormPostProcessor::GetCommand()
+*/
 
 bool NormPostProcessor::SetCommand(const char* cmd)
 {

src/pynorm/session.py

@@ -1,4 +1,4 @@
-"""
+o"""
 pynorm - Python wrapper for NRL's libnorm
 By: Tom Wambold <wambold@itd.nrl.navy.mil>
 """