From 8ae9df5607b259ea1614ff4eabc023b088d32eb2 Mon Sep 17 00:00:00 2001 From: bebopagogo Date: Sat, 2 Sep 2023 22:15:23 -0400 Subject: [PATCH 1/6] updated gradle and tweaked build.gradle file to API version 24 --- android/lib/CMakeLists.txt | 2 +- android/lib/build.gradle | 5 ++--- protolib | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/android/lib/CMakeLists.txt b/android/lib/CMakeLists.txt index 9af9a43..f92cd84 100755 --- a/android/lib/CMakeLists.txt +++ b/android/lib/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.4.1) project("norm") -add_definitions("-DANDROID=1" "-DANDROID_API_VERSION=15" "-DPUSH_NOTIFICATIONS=1" "-DANDROID" "-DLINUX" "-DUNIX" "-DHAVE_IPV6" "-DHAVE_DIRFD" "-DPROTO_DEBUG" "-DHAVE_ASSERT" "-DHAVE_GETLOGIN" "-DUSE_SELECT" "-D_FILE_OFFSET_BITS=64" "-DHAVE_OLD_SIGNALHANDLER" "-DHAVE_SCHED" "-DNO_SCM_RIGHTS" "-Wno-attributes" "-DAPP_VERSION=1.0.0" "-DAPP_VERSION_HEX=0x10000") +add_definitions("-DANDROID=1" "-DANDROID_API_VERSION=24" "-DPUSH_NOTIFICATIONS=1" "-DANDROID" "-DLINUX" "-DUNIX" "-DHAVE_IPV6" "-DHAVE_DIRFD" "-DPROTO_DEBUG" "-DHAVE_ASSERT" "-DHAVE_GETLOGIN" "-DUSE_SELECT" "-D_FILE_OFFSET_BITS=64" "-DHAVE_OLD_SIGNALHANDLER" "-DHAVE_SCHED" "-DNO_SCM_RIGHTS" "-Wno-attributes" "-DAPP_VERSION=1.0.0" "-DAPP_VERSION_HEX=0x10000") include_directories( AFTER "../../OID_NDK}/sources/android/cpufeatures" diff --git a/android/lib/build.gradle b/android/lib/build.gradle index 40cc8cb..d6e768a 100755 --- a/android/lib/build.gradle +++ b/android/lib/build.gradle @@ -1,8 +1,7 @@ apply plugin: 'com.android.library' android { - compileSdkVersion 31 - buildToolsVersion "28.0.3" + compileSdkVersion 34 externalNativeBuild { cmake { path "CMakeLists.txt" @@ -21,7 +20,7 @@ android { defaultConfig { externalNativeBuild { cmake { - arguments "-DANDROID_TOOLCHAIN=clang", "-DANDROID_PLATFORM=android-15", "-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja", "-DANDROID_STL=c++_static", "-DANDROID_CPP_FEATURES=exceptions rtti", "-DANDROID_ARM_MODE=arm", "-DANDROID_ARM_NEON=TRUE" + arguments "-DANDROID_TOOLCHAIN=clang", "-DANDROID_PLATFORM=android-24", "-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja", "-DANDROID_STL=c++_static", "-DANDROID_CPP_FEATURES=exceptions rtti", "-DANDROID_ARM_MODE=arm", "-DANDROID_ARM_NEON=TRUE" cFlags "-fsigned-char" cppFlags "-fsigned-char", "-std=c++14" } diff --git a/protolib b/protolib index 7323fd3..191df8d 160000 --- a/protolib +++ b/protolib @@ -1 +1 @@ -Subproject commit 7323fd3455ea6c16137ad3b84c950d74cc21055b +Subproject commit 191df8d69be6665298c207c1ba55222a1f4d5fc9 From 4b34fffa8e137e0ab96c34a174a5576307524330 Mon Sep 17 00:00:00 2001 From: bebopagogo Date: Sat, 2 Sep 2023 22:16:16 -0400 Subject: [PATCH 2/6] pulled updated submodule code --- protolib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protolib b/protolib index 191df8d..4aae218 160000 --- a/protolib +++ b/protolib @@ -1 +1 @@ -Subproject commit 191df8d69be6665298c207c1ba55222a1f4d5fc9 +Subproject commit 4aae218f692014e4fe180e4bd4d92b8ad8b1f2c2 From af9a238a70fbd75233a842258ac99146fb7e4daa Mon Sep 17 00:00:00 2001 From: bebopagogo Date: Sun, 3 Sep 2023 15:01:44 -0400 Subject: [PATCH 3/6] pulled updated submodule code --- android/README.md | 2 +- protolib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/android/README.md b/android/README.md index 40dc7b0..b293924 100755 --- a/android/README.md +++ b/android/README.md @@ -1,4 +1,4 @@ -This directoy contains Android Studio project files for building the NORM +This directory contains Android Studio project files for building the NORM library for Android. REQUIREMENTS: diff --git a/protolib b/protolib index 4aae218..dc92c6b 160000 --- a/protolib +++ b/protolib @@ -1 +1 @@ -Subproject commit 4aae218f692014e4fe180e4bd4d92b8ad8b1f2c2 +Subproject commit dc92c6b8ccc75a01ecd2d7850f6bc7d8a7fe671d From a992176873c3f3cef93a38809ba06b8ae23d6adf Mon Sep 17 00:00:00 2001 From: LVancina <95012459+LVancina@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:15:06 -0800 Subject: [PATCH 4/6] Documentation updates and pynorm example bug fixes (#82) * Added notes about waf execution in BUILD.TXT * Updated instructions for cloning and running make * Added more detailed instructions for win32 builds * Fixed typo on line 20 (capitalized 'f') that caused the command to fail * Generic editing and replaced dead URLs * Substituted setuptools package for the deprecated distutils in setup.py * Fixed typo in 'norm.exe' file path in Win32 build instructions * Fixed a bugs in normCast.py and normFileSend.py --------- Co-authored-by: Lucas --- BUILD.TXT | 20 ++++++++++++++++- README-PyNorm.txt | 2 +- doc/NormDeveloperGuide.xml | 39 ++++++++++++++++----------------- doc/NormUserGuide.xml | 22 +++++++++++++------ examples/python/normCast.py | 6 +++-- examples/python/normFileSend.py | 2 +- setup.py | 2 +- 7 files changed, 60 insertions(+), 33 deletions(-) diff --git a/BUILD.TXT b/BUILD.TXT index 3b3ebc2..4c6e360 100755 --- a/BUILD.TXT +++ b/BUILD.TXT @@ -6,11 +6,26 @@ To see a full list of options, run: ./waf -h +Note: if attempting to run waf as an executable as shown above results in a +message like this: + +/usr/bin/env: ‘python’: No such file or directory + +then try calling python directly: + +python waf + +or + +python3 waf + +For more background information about waf, visit the gitlab page at +https://gitlab.com/ita1024/./waf/ Configuring ----------- -To perform the configure checks, run: +To perform the configuration checks, run: ./waf configure @@ -51,6 +66,9 @@ To install, run: This will install the compiled library and headers to wherever your prefix was specified. (See configure flags) +This command may have to be run with root priviledges for certain prefix +directories depending on your system setup. + Uninstalling ------------ diff --git a/README-PyNorm.txt b/README-PyNorm.txt index 114b178..5e9160e 100755 --- a/README-PyNorm.txt +++ b/README-PyNorm.txt @@ -28,7 +28,7 @@ in similar places as your system's compiler. On UNIX systems, this is usually PATH environment variable. On Windows, PyNORM requires the "PyWin32" module, available at: - http://sf.net/projects/pywin32 + https://sourceforge.net/projects/pywin32 ------------- Installation diff --git a/doc/NormDeveloperGuide.xml b/doc/NormDeveloperGuide.xml index 6900095..4408de5 100755 --- a/doc/NormDeveloperGuide.xml +++ b/doc/NormDeveloperGuide.xml @@ -13,19 +13,19 @@ This document describes an application programming interface (API) - for the Nack-Oriented + for the Nack-Oriented Reliable Multicast (NORM) protocol implementation developed by the Protocol Engineering and Advance Networking (PROTEAN) Research Group of the - United States Naval Research + url="https://www.nrl.navy.mil/itd/ncs/">PROTEAN) Research Group of the + United States Naval Research Laboratory (NRL). The NORM protocol provides general purpose reliable data transport for applications wishing to use Internet Protocol (IP) Multicast services for group data delivery. NORM can also support unicast (point-to-point) data communication and may be used for such when deemed appropriate. The current NORM protocol specification is - given in the Internet Engineering Task + given in the Internet Engineering Task Force (IETF) RFC 3940. This + url="https://datatracker.ietf.org/doc/html/rfc5740">RFC 5740. This document is currently a reference guide to the NORM API of the NRL reference implementation. More tutorial material may be include in a future version of this document or a separate developer's tutorial may @@ -37,19 +37,19 @@ Background This document describes an application programming interface (API) - for the Nack-Oriented + for the Nack-Oriented Reliable Multicast (NORM) protocol implementation developed by the Protocol Engineering and Advance Networking (PROTEAN) Research Group of the - United States Naval Research + url="https://www.nrl.navy.mil/itd/ncs/">PROTEAN) Research Group of the + United States Naval Research Laboratory (NRL). The NORM protocol provides general purpose reliable data transport for applications wishing to use Internet Protocol (IP) Multicast services for group data delivery. NORM can also support unicast (point-to-point) data communication and may be used for such when deemed appropriate. The current NORM protocol specification is given in - the Internet Engineering Task + the Internet Engineering Task Force (IETF) RFC 5740. + url="https://datatracker.ietf.org/doc/html/rfc5740">RFC 5740. The NORM protocol is designed to provide end-to-end reliable transport of bulk data objects or streams over generic IP multicast @@ -134,7 +134,7 @@ may participate as senders and/or receivers within a NORM session. NORM senders transmit data to the session destination address (usually an IP multicast group) while receivers are notified of incoming data. The NORM - API provides and event notification scheme to notify the application of + API provides an event notification scheme to notify the application of significant sender and receiver events. There are also a number support functions provided for the application to control and monitor its participation within a NORM transport session. @@ -159,7 +159,7 @@ linkend="NormCreateInstance">NormCreateInstance()) as needed for applications with specific requirements for accessing and controlling participation in multiple NormSessions - from separate operating system multiple threads. Or, alternatively, a + from separate operating system threads. Or, alternatively, a single NormInstance could be used, with a "master thread" serving as an intermediary between the NormGetNextEvent() @@ -194,7 +194,7 @@ number of concurrently active senders, this may translate to significant memory allocation on receiver nodes. Currently, the API allows the application to control how much buffer space is allocated for each - active sender (NOTE: In the future, API functions may be provided limit + active sender (NOTE: In the future, API functions may be provided to limit the number of active senders monitored and/or provide the application with finer control over receive buffer allocation, perhaps on a per sender basis). @@ -227,7 +227,7 @@ Data Transmission The behavior of data transport operation is largely placed in - the control of the NORM sender(s). NORM senders controls their data + the control of the NORM sender(s). NORM senders control their data transmission rate, forward error correction (FEC) encoding settings, and parameters controlling feedback from the receiver group. Multiple senders may operate in a session, each with independent transmission @@ -253,7 +253,7 @@ be called before any calls to NormStartReceiver() are made. However, note that the cache directory may be changed even - during active NORM reception. In this case, the new specified + during active NORM reception. In this case, the newly specified directory path will be used for subsequently-received files. Any files received before a directory path change will remain in the previous cache location. Note that the The "makefiles" directory contains Unix Makefiles for various - platforms the "win32" and "wince" sub-directories there contain Microsoft + platforms. The "win32" and "wince" sub-directories there contain Microsoft Visual C++ (VC++) and Embedded VC++ project files for building the NORM implementation. Additionally, a "waf" (Python-based build tool) build option is supported that can be used to build and install the NORM library code on the supported platforms. Finally, Python and Java bindings to the - NORM API are included and "src/python" and "src/java" directories contain - the code for these and the "makefiles/java" directory contains Makefiles - to build the NORM Java JNI bindings. Note the "waf" tool can also be used - to build the Java and Python bindings. + NORM API are included in the "src/python" and "src/java" directories and the + "makefiles/java" directory contains Makefiles to build the NORM Java JNI bindings. + Note the "waf" tool can also be used to build the Java and Python bindings. Unix Platforms diff --git a/doc/NormUserGuide.xml b/doc/NormUserGuide.xml index cfb07fd..3e396b5 100755 --- a/doc/NormUserGuide.xml +++ b/doc/NormUserGuide.xml @@ -96,11 +96,13 @@ - Download and unpack the NORM source code tarball. + Download and unpack the NORM source code tarball or clone the github + repository using + git clone --recurse-submodules https://github.com/USNavalResearchLaboratory/norm.git - cd norm/unix + cd norm/makefiles @@ -118,7 +120,8 @@ - Download and unpack the NORM source code tarball + Download and unpack the NORM source code tarball or clone the + repository as described in the section above. @@ -128,7 +131,7 @@ - Open the "norm.sln" (VC++ .Net), + Inside the "norm/makefiles/win32" directory, open the "norm.sln" (VC++ .Net), "norm.dsw" (VC++ 6.0), or "norm.vcw" (Embedded VC++) workspace file. @@ -137,9 +140,14 @@ The "norm" project can be selected and built.  The "norm.exe" file will be found in - the "norm/win32/norm/Release" directory. + the "norm/Win32/norm/Release" directory. + The easiest way to run these builds is with Visual Studio. If you do not already have Visual Studio installed, then it can + be downloaded for free here. + When you launch Visual Studio, make sure to select "Visual C++" and open the "norm.sln" file. + This should load all of the project files and allow you to right-click on any of them in the Solution Explorer on the + left side and select "build". @@ -380,7 +388,7 @@ A minimal example norm sender command-line syntax is: - norm addr <addr/port> sendFile + norm addr <addr/port> sendfile <fileName> The corresponding minimal norm receiver @@ -1160,4 +1168,4 @@ use with norm - \ No newline at end of file + diff --git a/examples/python/normCast.py b/examples/python/normCast.py index 7dd61de..68ba319 100644 --- a/examples/python/normCast.py +++ b/examples/python/normCast.py @@ -301,14 +301,16 @@ class NormCaster(): self.is_running = False return - obj:Optional[pynorm.Object] = session.fileEnqueue(self.pendingSendFilePath, info= self.pendingSendFilePath.replace("\\","/").encode() ) + lastSlash = self.pendingSendFilePath.rfind('/') + filename = self.pendingSendFilePath[lastSlash+1:] + obj:Optional[pynorm.Object] = session.fileEnqueue(self.pendingSendFilePath, info= filename.encode() ) if obj: if self.opts.ack: session.setWatermark(obj,True) logging.info(f"add file:{self.pendingSendFilePath} {obj._object}") self.pendingSendFilePath = None # succeed enqued else: - logging.warning(f"fileEnqueue: {sendFilePath} failure!") + logging.warning(f"fileEnqueue: {self.pendingSendFilePath} failure!") def handle_norm_event(self, event:pynorm.Event): session:pynorm.Session = event.session diff --git a/examples/python/normFileSend.py b/examples/python/normFileSend.py index 49968f9..7f6c3d9 100755 --- a/examples/python/normFileSend.py +++ b/examples/python/normFileSend.py @@ -44,7 +44,7 @@ def main(argv): session.startSender(randint(0, 1000), 1024**2, 1400, 64, 16) print(('Sending file %s' % filename)) - session.fileEnqueue(filepath, filename) + session.fileEnqueue(filepath, info=filename.encode()) try: for event in instance: diff --git a/setup.py b/setup.py index 0684a18..556841e 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ # This Python script can be used to install the Python 'pynorm' package import platform -from distutils.core import setup, Extension +from setuptools import setup, Extension # Note to use 'pynorm", you will need to have the libnorm shared library # (libnorm.so or libnorm.dylib, etc) installed where your Python installation From 6188965887743be7371f37d2822b15383ee54bf2 Mon Sep 17 00:00:00 2001 From: bebopagogo Date: Wed, 24 Jan 2024 13:43:38 -0500 Subject: [PATCH 5/6] fixed improper delete operator usage in normApp.cpp and minor tweaks --- src/common/normApp.cpp | 21 +++++++++++---------- src/common/normPostProcess.cpp | 2 ++ src/pynorm/session.py | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/common/normApp.cpp b/src/common/normApp.cpp index 12333f0..3771e8f 100755 --- a/src/common/normApp.cpp +++ b/src/common/normApp.cpp @@ -96,6 +96,8 @@ class NormApp : public NormController, public ProtoApp enum EcnMode {ECN_OFF, ECN_ON, ECN_ONLY}; + enum {MSG_BUFFER_SIZE = 65535}; + #ifdef UNIX static void SignalHandler(int sigNum); #endif // UNIX @@ -115,7 +117,7 @@ class NormApp : public NormController, public ProtoApp FILE* output; // output stream char* output_io_buffer; unsigned int output_io_bufsize; - char input_buffer[65535]; + char input_buffer[MSG_BUFFER_SIZE]; unsigned int input_index; unsigned int input_length; bool input_active; @@ -127,7 +129,7 @@ class NormApp : public NormController, public ProtoApp bool msg_test; unsigned int msg_test_length; UINT32 msg_test_seq; - char output_buffer[65535]; + char output_buffer[MSG_BUFFER_SIZE]; UINT16 output_index; bool output_messaging; UINT16 output_msg_length; @@ -161,7 +163,7 @@ class NormApp : public NormController, public ProtoApp NormSession::ProbingMode grtt_probing_mode; double group_size; unsigned long tx_buffer_size; // bytes - unsigned int tx_sock_buffer_size; + unsigned int tx_sock_buffer_size; unsigned long tx_cache_min; unsigned long tx_cache_max; NormObjectSize tx_cache_size; @@ -213,7 +215,6 @@ class NormApp : public NormController, public ProtoApp }; // end class NormApp - NormApp::FileCacheItem::FileCacheItem(const char* thePath) { strncpy(file_path, thePath, PATH_MAX); @@ -603,7 +604,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val) else if (!strncmp("address", cmd, len)) { size_t len = strlen(val); - if (address) delete address; + if (address) delete[] address; if (!(address = new char[len+1])) { PLOG(PL_FATAL, "NormApp::OnCommand(address) allocation error:%s\n", @@ -614,7 +615,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val) char* ptr = strchr(address, '/'); if (!ptr) { - delete address; + delete[] address; address = NULL; PLOG(PL_FATAL, "NormApp::OnCommand(address) missing port number!\n"); return false; @@ -623,7 +624,7 @@ bool NormApp::OnCommand(const char* cmd, const char* val) int portNum = atoi(ptr); if ((portNum < 1) || (portNum > 65535)) { - delete address; + delete[] address; address = NULL; PLOG(PL_FATAL, "NormApp::OnCommand(address) invalid port number!\n"); return false; @@ -1405,7 +1406,7 @@ void NormApp::OnInputReady() { if (input_msg_length) { - UINT16 bufferSpace = 65535 - input_index; + UINT16 bufferSpace = MSG_BUFFER_SIZE - input_index; UINT16 msgRemainder = input_msg_length - input_msg_index; readLength = MIN(bufferSpace, msgRemainder); } @@ -1418,7 +1419,7 @@ void NormApp::OnInputReady() memcpy(&input_msg_length, input_buffer, 2); input_msg_length = ntohs(input_msg_length); ASSERT(input_msg_length >= 2); - UINT16 bufferSpace = 65535 - input_index; + UINT16 bufferSpace = MSG_BUFFER_SIZE - input_index; UINT16 msgRemainder = input_msg_length - 2; readLength = MIN(bufferSpace, msgRemainder); } @@ -1443,7 +1444,7 @@ void NormApp::OnInputReady() { hdrOffset = 0; } - int hdrLen = sprintf(input_buffer + hdrOffset, "NORM Test Msg %08u ", msg_test_seq++); + int hdrLen = snprintf(input_buffer + hdrOffset, MSG_BUFFER_SIZE - hdrOffset, "NORM Test Msg %08u ", msg_test_seq++); char testChar = 'a' + msg_test_seq % 26; memset(input_buffer + hdrLen + hdrOffset, testChar, msg_test_length - (hdrLen + hdrOffset)); input_buffer[msg_test_length - 1] = '\n'; diff --git a/src/common/normPostProcess.cpp b/src/common/normPostProcess.cpp index 3c07724..f318ca2 100755 --- a/src/common/normPostProcess.cpp +++ b/src/common/normPostProcess.cpp @@ -15,6 +15,7 @@ NormPostProcessor::~NormPostProcessor() SetCommand(NULL); } +/* This was defined/implemented, but unused so commented out (for now) void NormPostProcessor::GetCommand(char* buffer, unsigned int buflen) { if (process_argv && buffer) @@ -52,6 +53,7 @@ void NormPostProcessor::GetCommand(char* buffer, unsigned int buflen) if (buflen > 4) buffer[4] = '\0'; } } // end NormPostProcessor::GetCommand() +*/ bool NormPostProcessor::SetCommand(const char* cmd) { diff --git a/src/pynorm/session.py b/src/pynorm/session.py index 1b3463b..df0e8f0 100755 --- a/src/pynorm/session.py +++ b/src/pynorm/session.py @@ -1,4 +1,4 @@ -""" +o""" pynorm - Python wrapper for NRL's libnorm By: Tom Wambold """ From 2ae69e6e96d1ea5dab9d8353d2b8ca80b3a7683b Mon Sep 17 00:00:00 2001 From: bebopagogo Date: Sun, 28 Jan 2024 20:26:15 -0500 Subject: [PATCH 6/6] fix to NormFile::Open() where 'flags' were not properly saved and affected NormFile::Rename() --- src/common/normFile.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/common/normFile.cpp b/src/common/normFile.cpp index a723586..84faaf4 100755 --- a/src/common/normFile.cpp +++ b/src/common/normFile.cpp @@ -136,6 +136,7 @@ bool NormFile::Open(const char* thePath, int theFlags) if((fd = open(thePath, theFlags, 0640)) >= 0) { offset = 0; + flags = theFlags; return true; // no error } else @@ -246,9 +247,11 @@ bool NormFile::Rename(const char* oldName, const char* newName) #endif // WIN32 // In Win32, the old file can't be open and on Unix (Linux at least), it's better // performance to close/reopen a file being renamed + bool wasOpen = false; int oldFlags = 0; if (IsOpen()) { + wasOpen = true; oldFlags = flags; oldFlags &= ~(O_CREAT | O_TRUNC); // unset these Close(); @@ -321,22 +324,19 @@ bool NormFile::Rename(const char* oldName, const char* newName) { PLOG(PL_ERROR, "NormFile::Rename() rename() error: %s\n", GetErrorString()); #endif // if/else _WIN32_WCE / WIN32|UNIX - if (oldFlags) - { - if (!Open(oldName, oldFlags)) - PLOG(PL_ERROR, "NormFile::Rename() error re-opening file w/ old name\n"); - } + if (wasOpen && !Open(oldName, oldFlags)) + PLOG(PL_ERROR, "NormFile::Rename() error re-opening file w/ old name\n"); return false; } else { - if (oldFlags) + if (wasOpen && !Open(newName, oldFlags)) { - if (!Open(newName, oldFlags)) - PLOG(PL_ERROR, "NormFile::Rename() error opening file w/ new name\n"); + PLOG(PL_ERROR, "NormFile::Rename() error opening file w/ new name\n"); + return false; } - return true; } + return true; } // end NormFile::Rename() size_t NormFile::Read(char* buffer, size_t len) @@ -388,10 +388,11 @@ size_t NormFile::Write(const char* buffer, size_t len) #endif // if/else _WIN32_WCE #else size_t result = write(fd, buffer+put, len-put); + if (result <= 0) perror("NormFile::Write() write() error"); #endif // if/else WIN32 if (result <= 0) { - #ifndef _WIN32_WCE +#ifndef _WIN32_WCE if (EINTR != errno) #endif // !_WIN32_WCE {