fix ci builds by pinning msvc to windows-2022 and configuring waf on linux (#103)

* Fix uninitialized memory read vulnerability in packet validation

* Fix buffer overflows in temporary file generation

.devcontainer/devcontainer.json

@@ -0,0 +1,39 @@
+{
+  "image": "mcr.microsoft.com/devcontainers/cpp:1.0.0-buster",
+  "features": {
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.11"
+    },
+    "ghcr.io/devcontainers/features/dotnet:1": {
+      "version": "6"
+    },
+    "ghcr.io/devcontainers/features/java:1": {
+      "version": "17",
+      "installAnt": true,
+      "antVersion": "1.10.12"
+    },
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "20.10"
+    },
+    "ghcr.io/devcontainers/features/powershell:1": {
+      "version": "7.3"
+    }
+},
+"customizations": {
+	"vscode": {
+		"extensions": [
+			"DavidAnson.vscode-markdownlint",
+			"eamodio.gitlens",
+			"github.vscode-github-actions",
+			"GitHub.codespaces",
+			"jebbs.plantuml",
+			"ms-python.isort",
+			"ms-vscode-remote.remote-containers",
+			"ms-vscode.cpptools-themes",
+      "ms-vscode.PowerShell",
+			"tomoki1207.pdf",
+			"yzhang.markdown-all-in-one"
+		]
+	}
+}
+}

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.github/workflows/cmake.yml

@@ -0,0 +1,183 @@
+name: CMake build
+
+on: [push, pull_request]
+
+env:
+  CMAKE_VERSION: 3.16.2
+  NINJA_VERSION: 1.9.0
+  BUILD_TYPE: Release
+  CCACHE_VERSION: 3.7.7
+
+jobs:
+  build:
+    name: ${{ matrix.config.name }}
+    runs-on: ${{ matrix.config.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+        - {
+            name: "Windows 2022 MSVC Shared", artifact: "Windows-MSVC-shared.tar.xz",
+            os: windows-2022,
+            cc: "cl", cxx: "cl",
+            shared: "ON",
+            environment_script: "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
+          }
+        - {
+            name: "Windows 2022 MSVC", artifact: "Windows-MSVC.tar.xz",
+            os: windows-2022,
+            cc: "cl", cxx: "cl",
+            shared: "OFF",
+            environment_script: "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Auxiliary/Build/vcvars64.bat"
+            
+          }
+        - {
+            name: "Windows Latest MinGW", artifact: "Windows-MinGW.tar.xz",
+            os: windows-latest,
+            cc: "gcc", cxx: "g++",
+            shared: "OFF"
+          }
+        - {
+            name: "Ubuntu Latest GCC", artifact: "Linux.tar.xz",
+            os: ubuntu-latest,
+            cc: "gcc", cxx: "g++",
+            shared: "OFF"
+          }
+        - {
+            name: "macOS Latest Clang", artifact: "macOS.tar.xz",
+            os: macos-latest,
+            cc: "clang", cxx: "clang++",
+            shared: "OFF"
+          }
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download Ninja and CMake
+      id: cmake_and_ninja
+      shell: cmake -P {0}
+      run: |
+        set(cmake_version $ENV{CMAKE_VERSION})
+        set(ninja_version $ENV{NINJA_VERSION})
+        message(STATUS "Using host CMake version: ${CMAKE_VERSION}")
+        if ("${{ runner.os }}" STREQUAL "Windows")
+          set(ninja_suffix "win.zip")
+          set(cmake_suffix "win64-x64.zip")
+          set(cmake_dir "cmake-${cmake_version}-win64-x64/bin")
+        elseif ("${{ runner.os }}" STREQUAL "Linux")
+          set(ninja_suffix "linux.zip")
+          set(cmake_suffix "Linux-x86_64.tar.gz")
+          set(cmake_dir "cmake-${cmake_version}-Linux-x86_64/bin")
+        elseif ("${{ runner.os }}" STREQUAL "macOS")
+          set(ninja_suffix "mac.zip")
+          set(cmake_suffix "Darwin-x86_64.tar.gz")
+          set(cmake_dir "cmake-${cmake_version}-Darwin-x86_64/CMake.app/Contents/bin")
+        endif()
+        set(ninja_url "https://github.com/ninja-build/ninja/releases/download/v${ninja_version}/ninja-${ninja_suffix}")
+        file(DOWNLOAD "${ninja_url}" ./ninja.zip SHOW_PROGRESS)
+        execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvf ./ninja.zip)
+        set(cmake_url "https://github.com/Kitware/CMake/releases/download/v${cmake_version}/cmake-${cmake_version}-${cmake_suffix}")
+        file(DOWNLOAD "${cmake_url}" ./cmake.zip SHOW_PROGRESS)
+        execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvf ./cmake.zip)
+        # Save the path for other steps
+        file(TO_CMAKE_PATH "$ENV{GITHUB_WORKSPACE}/${cmake_dir}" cmake_dir)
+        message("::set-output name=cmake_dir::${cmake_dir}")
+        if (NOT "${{ runner.os }}" STREQUAL "Windows")
+          execute_process(
+            COMMAND chmod +x ninja
+            COMMAND chmod +x ${cmake_dir}/cmake
+          )
+        endif()
+    - name: Download ccache
+      id: ccache
+      shell: cmake -P {0}
+      run: |
+        set(ccache_url "https://github.com/cristianadam/ccache/releases/download/v$ENV{CCACHE_VERSION}/${{ runner.os }}.tar.xz")
+        file(DOWNLOAD "${ccache_url}" ./ccache.tar.xz SHOW_PROGRESS)
+        execute_process(COMMAND ${CMAKE_COMMAND} -E tar xvf ./ccache.tar.xz)
+    - name: Prepare ccache timestamp
+      id: ccache_cache_timestamp
+      shell: cmake -P {0}
+      run: |
+        string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC)
+        message("::set-output name=timestamp::${current_date}")
+    - name: ccache cache files
+      uses: actions/cache@v4
+      with:
+        path: .ccache
+        key: ${{ matrix.config.name }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+        restore-keys: |
+          ${{ matrix.config.name }}-ccache-
+    - name: Configure
+      shell: cmake -P {0}
+      run: |
+        set(ENV{CC} ${{ matrix.config.cc }})
+        set(ENV{CXX} ${{ matrix.config.cxx }})
+        if ("${{ runner.os }}" STREQUAL "Windows" AND NOT "x${{ matrix.config.environment_script }}" STREQUAL "x")
+          execute_process(
+            COMMAND "${{ matrix.config.environment_script }}" && set
+            OUTPUT_FILE environment_script_output.txt
+          )
+          file(STRINGS environment_script_output.txt output_lines)
+          foreach(line IN LISTS output_lines)
+            if (line MATCHES "^([a-zA-Z0-9_-]+)=(.*)$")
+              set(ENV{${CMAKE_MATCH_1}} "${CMAKE_MATCH_2}")
+            endif()
+          endforeach()
+        endif()
+        set(path_separator ":")
+        if ("${{ runner.os }}" STREQUAL "Windows")
+          set(path_separator ";")
+        endif()
+        set(ENV{PATH} "$ENV{GITHUB_WORKSPACE}${path_separator}$ENV{PATH}")
+        execute_process(
+          COMMAND ${{ steps.cmake_and_ninja.outputs.cmake_dir }}/cmake
+            -S .
+            -B build
+            -D CMAKE_BUILD_TYPE=$ENV{BUILD_TYPE}
+            -G Ninja
+            -D CMAKE_MAKE_PROGRAM=ninja
+            -D CMAKE_C_COMPILER_LAUNCHER=ccache
+            -D CMAKE_CXX_COMPILER_LAUNCHER=ccache
+            -D BUILD_SHARED_LIBS=${{ matrix.config.shared }}
+          RESULT_VARIABLE result
+        )
+        if (NOT result EQUAL 0)
+          message(FATAL_ERROR "Bad exit status")
+        endif()
+    - name: Build
+      shell: cmake -P {0}
+      run: |
+        set(ENV{NINJA_STATUS} "[%f/%t %o/sec] ")
+        if ("${{ runner.os }}" STREQUAL "Windows" AND NOT "x${{ matrix.config.environment_script }}" STREQUAL "x")
+          file(STRINGS environment_script_output.txt output_lines)
+          foreach(line IN LISTS output_lines)
+            if (line MATCHES "^([a-zA-Z0-9_-]+)=(.*)$")
+              set(ENV{${CMAKE_MATCH_1}} "${CMAKE_MATCH_2}")
+            endif()
+          endforeach()
+        endif()
+        set(path_separator ":")
+        if ("${{ runner.os }}" STREQUAL "Windows")
+          set(path_separator ";")
+        endif()
+        set(ENV{PATH} "$ENV{GITHUB_WORKSPACE}${path_separator}$ENV{PATH}")
+        file(TO_CMAKE_PATH "$ENV{GITHUB_WORKSPACE}" ccache_basedir)
+        set(ENV{CCACHE_BASEDIR} "${ccache_basedir}")
+        set(ENV{CCACHE_DIR} "${ccache_basedir}/.ccache")
+        set(ENV{CCACHE_COMPRESS} "true")
+        set(ENV{CCACHE_COMPRESSLEVEL} "6")
+        set(ENV{CCACHE_MAXSIZE} "400M")
+        if ("${{ matrix.config.cxx }}" STREQUAL "cl")
+          set(ENV{CCACHE_MAXSIZE} "600M")
+        endif()
+        execute_process(COMMAND ccache -p)
+        execute_process(COMMAND ccache -z)
+        execute_process(
+          COMMAND ${{ steps.cmake_and_ninja.outputs.cmake_dir }}/cmake --build build
+          RESULT_VARIABLE result
+        )
+        if (NOT result EQUAL 0)
+          message(FATAL_ERROR "Bad exit status")
+        endif()
+        execute_process(COMMAND ccache -s)

.github/workflows/dotnet-publish.yml

@@ -0,0 +1,142 @@
+name: ".NET Publish"
+
+on:
+  workflow_run:
+    workflows: ['.NET']
+    types:
+    - completed
+    branches:
+      - master
+      - main
+  workflow_dispatch:
+
+jobs:
+  publish:
+
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    defaults:
+      run:
+        working-directory: ./src/dotnet
+
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: 6.0.x
+        source-url: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
+      env:
+        NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+    - name: Package .NET
+      run: dotnet pack . -c Release
+    - name: Publish .NET
+      run: |
+        package=$(find . -type f -name "*.nupkg")
+        dotnet nuget push "$package" --skip-duplicate
+
+  publish-linux-x64:
+
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    
+    - name: Package .NET
+      uses: devcontainers/ci@v0.3
+      with:
+        push: never
+        runCmd: | 
+          cd ./src/dotnet
+          chmod 755 *.sh
+          ./pack-linux-x64.sh
+    - name: Set up .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: '6.0.x'
+        source-url: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
+      env:
+        NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+    - name: Publish .NET
+      run: |
+        package=$(find . -type f -name "*linux-x64*.nupkg")
+        dotnet nuget push "$package" --skip-duplicate
+
+  publish-win-x64:
+
+    runs-on: windows-latest
+    permissions:
+      packages: write
+    defaults:
+      run:
+        working-directory: ./src/dotnet
+
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Setup Python
+      uses: actions/setup-python@v5.1.0
+      with:
+        python-version: 3.x
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: 6.0.x
+        source-url: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
+      env:
+        NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+    - name: Package .NET
+      run: .\pack-win-x64.bat
+    - name: Publish .NET
+      run: |
+        $package=Get-ChildItem -Path .\ -Filter *win-x64*.nupkg -Recurse -File | ForEach-Object { $_.FullName }
+        dotnet nuget push "$package" --skip-duplicate
+
+  publish-win-x86:
+
+    runs-on: windows-latest
+    permissions:
+      packages: write
+    defaults:
+      run:
+        working-directory: ./src/dotnet
+
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Setup Python
+      uses: actions/setup-python@v5.1.0
+      with:
+        python-version: 3.x
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: 6.0.x
+        source-url: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json
+      env:
+        NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+    - name: Package .NET
+      run: .\pack-win-x86.bat
+    - name: Publish .NET
+      run: |
+        $package=Get-ChildItem -Path .\ -Filter *win-x86*.nupkg -Recurse -File | ForEach-Object { $_.FullName }
+        dotnet nuget push "$package" --skip-duplicate

.github/workflows/dotnet.yml

@@ -0,0 +1,49 @@
+name: .NET
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: ./src/dotnet
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: recursive
+    
+    - name: Setup Python
+      uses: actions/setup-python@v5.1.0
+      with:
+        python-version: 3.x
+    - name: Build NORM
+      run: ./waf
+      working-directory: .
+        
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: 6.0.x
+    - name: Restore dependencies
+      run: dotnet restore /property:Configuration=Release
+    - name: Build
+      run: dotnet build --no-restore --configuration Release
+    - name: Test
+      run: dotnet test --no-build --verbosity normal --configuration Release --logger trx --results-directory TestResults
+    - name: Upload test results
+      uses: actions/upload-artifact@v4
+      with:
+        name: dotnet-results
+        path: ./src/dotnet/TestResults
+      if: ${{ always() }}
+    - name: Publish test results
+      uses: EnricoMi/publish-unit-test-result-action@v2
+      with:
+        files: ./src/dotnet/TestResults/*.trx
+      if: ${{ always() }}

.gitignore

@@ -1,14 +1,37 @@
-# protolib
-protolib/
 
 # build executables
 build/
-lib/
+bin/
 makefiles/norm
 makefiles/normMsgr
 makefiles/normStreamer
+makefiles/normCast
+makefiles/normCastApp
+makefiles/normClient
+makefiles/normServer
 makefiles/npc
 makefiles/raft
+makefiles/pcap2norm
+norp/makefiles/norp
+
+# build object files
+*.o
+*.a
+
+# python build files
+src/pynorm.egg-info
+**/__pycache__/
+
+# build java files
+*.class
+
+#build android files
+android/.gradle
+
+# waf cruft
+.waf*
+.lock-waf*
+waf*/*
 
 # OS generated files #
 .DS_Store
@@ -18,3 +41,275 @@ makefiles/raft
 .Trashes
 ehthumbs.db
 Thumbs.db
+
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+
+# User-specific files
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+bld/
+[Bb]in/
+[Oo]bj/
+[Ll]og/
+
+# Visual Studio 2015 cache/options directory
+.vs/
+# Uncomment if you have tasks that create the project's static files in wwwroot
+#wwwroot/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUNIT
+*.VisualState.xml
+TestResult.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# DNX
+project.lock.json
+artifacts/
+
+*_i.c
+*_p.c
+*_i.h
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opendb
+*.opensdf
+*.sdf
+*.cachefile
+*.VC.db
+*.VC.VC.opendb
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+*.sap
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# JustCode is a .NET coding add-in
+.JustCode
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# Visual Studio code coverage results
+*.coverage
+*.coveragexml
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+nCrunchTemp_*
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# TODO: Comment the next line if you want to checkin your web deploy settings
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# Microsoft Azure Web App publish settings. Comment the next line if you want to
+# checkin your Azure Web App publish settings, but sensitive information contained
+# in these scripts will be unencrypted
+PublishScripts/
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/packages/*
+# except build/, which is used as an MSBuild target.
+!**/packages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/packages/repositories.config
+# NuGet v3's project.json files produces more ignoreable files
+*.nuget.props
+*.nuget.targets
+
+# Microsoft Azure Build Output
+csx/
+*.build.csdef
+
+# Microsoft Azure Emulator
+ecf/
+rcf/
+
+# Windows Store app package directories and files
+AppPackages/
+BundleArtifacts/
+Package.StoreAssociation.xml
+_pkginfo.txt
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!*.[Cc]ache/
+
+# Others
+ClientBin/
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.pfx
+*.publishsettings
+node_modules/
+orleans.codegen.cs
+
+# Since there are multiple workflows, uncomment next line to ignore bower_components
+# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
+#bower_components/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+
+# SQL Server files
+*.mdf
+*.ldf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# GhostDoc plugin setting file
+*.GhostDoc.xml
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# Visual Studio LightSwitch build output
+**/*.HTMLClient/GeneratedArtifacts
+**/*.DesktopClient/GeneratedArtifacts
+**/*.DesktopClient/ModelManifest.xml
+**/*.Server/GeneratedArtifacts
+**/*.Server/ModelManifest.xml
+_Pvt_Extensions
+
+# Paket dependency manager
+.paket/paket.exe
+paket-files/
+
+# FAKE - F# Make
+.fake/
+
+# JetBrains Rider
+.idea/
+*.sln.iml
+*.sqlite
+*.sqlite-shm
+*.sqlite-wal
+
+# Nuget files
+DeploymentSettings.props
+Directory.Build.props
+Directory.Build.targets
+Packages.props
+
+#Local Log Files
+log*.txt
+
+#.NET Extension Library Files
+/src/dotnet/lib

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "protolib"]
+	path = protolib
+	url = https://github.com/USNavalResearchLaboratory/protolib.git

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+    "recommendations": [
+        "github.vscode-pull-request-github"
+    ]
+}

BUILD.TXT

@@ -6,11 +6,26 @@ To see a full list of options, run:
 
     ./waf -h
 
+Note: if attempting to run waf as an executable as shown above results in a
+message like this: 
+
+/usr/bin/env: ‘python’: No such file or directory
+
+then try calling python directly:
+
+python waf <args>
+
+or
+
+python3 waf <args>
+
+For more background information about waf, visit the gitlab page at
+https://gitlab.com/ita1024/./waf/
 
 Configuring
 -----------
 
-To perform the configure checks, run:
+To perform the configuration checks, run:
 
     ./waf configure
 
@@ -51,6 +66,9 @@ To install, run:
 This will install the compiled library and headers to wherever your prefix was
 specified.  (See configure flags)
 
+This command may have to be run with root priviledges for certain prefix
+directories depending on your system setup.
+
 Uninstalling
 ------------
 
@@ -70,3 +88,8 @@ will delete all compiled files and configuration settings.
     ./waf distclean
 
 will do the same as clean, and additionally delete the waf cache files.
+
+
+ANDROID Build
+-------------
+See the "android" directory README for instructions.

CMakeLists.txt

@@ -0,0 +1,226 @@
+cmake_minimum_required(VERSION 3.14)
+cmake_policy(SET CMP0077 NEW)
+# set the project name
+project(norm VERSION 1.5.10)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+set(COMMON src/common)
+
+option(NORM_BUILD_EXAMPLES "Enables building of the examples in /examples." OFF)
+set(NORM_CUSTOM_PROTOLIB_VERSION OFF CACHE STRING "Set a custom protolib version to use, ./protolib to use the local version")
+
+include(CheckCXXSymbolExists)
+check_cxx_symbol_exists(dirfd "dirent.h" HAVE_DIRFD)
+if(HAVE_DIRFD)
+	list(APPEND PLATFORM_DEFINITIONS HAVE_DIRFD)
+endif()
+
+check_cxx_symbol_exists(lockf "unistd.h" HAVE_LOCKF)
+if(HAVE_LOCKF)
+	list(APPEND PLATFORM_DEFINITIONS HAVE_LOCKF)
+endif()
+
+check_cxx_symbol_exists(flock "sys/file.h" HAVE_FLOCK)
+if(HAVE_FLOCK)
+	list(APPEND PLATFORM_DEFINITIONS HAVE_FLOCK)
+endif()
+
+if(NOT NORM_CUSTOM_PROTOLIB_VERSION)
+	find_package(Git)
+	
+	if(Git_FOUND)
+		message("Git found: ${GIT_EXECUTABLE}")
+		execute_process(
+			COMMAND
+				${GIT_EXECUTABLE} ls-tree HEAD
+			RESULT_VARIABLE
+				GIT_SUBMODULE_STATUS
+			OUTPUT_VARIABLE
+	            GIT_SUBMODULE
+			WORKING_DIRECTORY 
+				${CMAKE_CURRENT_LIST_DIR}
+		)
+		string(REGEX MATCH "commit[ \t\r\n]*([a-z0-9]+)[ \t\r\n]*protolib" GIT_SUBMODULE_HASH_MATCH ${GIT_SUBMODULE})
+		if(GIT_SUBMODULE_HASH_MATCH)
+			set(NORM_PROTOKIT_GIT_TAG ${CMAKE_MATCH_1})
+		endif()
+	endif()
+elseif(NOT NORM_CUSTOM_PROTOLIB_VERSION STREQUAL "./protolib")
+	set(NORM_PROTOKIT_GIT_TAG ${NORM_CUSTOM_PROTOLIB_VERSION})
+endif()
+
+if(NORM_PROTOKIT_GIT_TAG)
+	message(STATUS "Building protokit from ${NORM_PROTOKIT_GIT_TAG}")
+	include(FetchContent)
+	FetchContent_Declare(
+	  protokit
+	  GIT_REPOSITORY https://github.com/USNavalResearchLaboratory/protolib.git
+	  GIT_TAG        ${NORM_PROTOKIT_GIT_TAG}
+	)
+	FetchContent_MakeAvailable(protokit)
+else()
+	message(STATUS "Building protokit from ${NORM_CUSTOM_PROTOLIB_VERSION}")
+	add_subdirectory(protolib)
+endif()
+
+
+# List header files
+list(APPEND PUBLIC_HEADER_FILES
+            include/galois.h
+            include/normApi.h
+            include/normEncoder.h
+            include/normEncoderMDP.h
+            include/normEncoderRS16.h
+            include/normEncoderRS8.h
+            include/normFile.h
+            include/normMessage.h
+            include/normNode.h
+            include/normObject.h
+            include/normPostProcess.h
+            include/normSegment.h
+            include/normSession.h
+            include/normSimAgent.h
+            include/normVersion.h
+)
+
+# List platform-independent source files
+list(APPEND COMMON_SOURCE_FILES 
+            ${COMMON}/galois.cpp
+            ${COMMON}/normApi.cpp
+            ${COMMON}/normEncoder.cpp
+            ${COMMON}/normEncoderMDP.cpp
+            ${COMMON}/normEncoderRS16.cpp
+            ${COMMON}/normEncoderRS8.cpp
+            ${COMMON}/normFile.cpp
+            ${COMMON}/normMessage.cpp
+            ${COMMON}/normNode.cpp
+            ${COMMON}/normObject.cpp
+            ${COMMON}/normSegment.cpp
+            ${COMMON}/normSession.cpp )
+
+# Setup platform independent include directory
+list(APPEND INCLUDE_DIRS ${CMAKE_CURRENT_LIST_DIR}/include )
+
+# Setup platform dependent libraries, defines, source file and compiler flags
+# (The "post processing" helper source is only needed for normApp build)
+#if(MSVC)
+#    list(APPEND PLATFORM_LIBS Shell32)
+#    list(APPEND PLATFORM_DEFINITIONS _CONSOLE)
+#    list(APPEND PLATFORM_SOURCE_FILES src/win32/win32PostProcess.cpp)
+#elseif(UNIX)
+#    list(APPEND PLATFORM_SOURCE_FILES   src/unix/unixPostProcess.cpp)
+#endif()
+
+include(GNUInstallDirs)
+
+# Setup target
+add_library(norm ${PLATFORM_SOURCE_FILES} ${COMMON_SOURCE_FILES} ${PUBLIC_HEADER_FILES})
+target_link_libraries(norm PRIVATE protokit::protokit)
+target_link_libraries(norm PUBLIC ${PLATFORM_LIBS})
+target_compile_definitions(norm PUBLIC ${PLATFORM_DEFINITIONS})
+target_compile_options(norm PUBLIC ${PLATFORM_FLAGS})
+target_include_directories(norm PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(norm PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> $<INSTALL_INTERFACE:include>)
+
+if(BUILD_SHARED_LIBS)
+	set_target_properties(norm PROPERTIES SOVERSION 1.5.10)
+	if(WIN32)
+		target_compile_definitions(norm PUBLIC NORM_USE_DLL)
+	endif()
+endif()
+
+if(BUILD_SHARED_LIBS AND BUILD_STATIC_LIBS)
+	add_library(norm-static STATIC ${PLATFORM_SOURCE_FILES} ${COMMON_SOURCE_FILES} ${PUBLIC_HEADER_FILES})
+	target_link_libraries(norm-static PRIVATE protokit::protokit)
+	target_compile_definitions(norm-static PUBLIC ${PLATFORM_DEFINITIONS})
+	target_compile_options(norm-static PUBLIC ${PLATFORM_FLAGS})
+	target_include_directories(norm-static PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+	target_include_directories(norm-static PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include> $<INSTALL_INTERFACE:include>)
+	set_target_properties(norm-static PROPERTIES OUTPUT_NAME "norm" PREFIX "lib")
+	set(NORM_STATIC "norm-static")
+endif()
+
+# Install target
+install( TARGETS norm ${NORM_STATIC} protokit EXPORT normTargets
+        RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
+        ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+        INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} )
+
+set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/norm)
+
+install( EXPORT normTargets
+    FILE normTargets.cmake
+    NAMESPACE norm::
+    DESTINATION ${INSTALL_CONFIGDIR}
+)
+
+install(FILES include/normApi.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+# Create pkg-config file norm.pc
+# TODO: once waf is removed, norm.pc.in can be edited to use the variables CMake sets directly, and
+# remove these re-definitions
+set(VERSION ${PROJECT_VERSION})
+set(PREFIX ${CMAKE_INSTALL_PREFIX})
+set(LIBDIR ${CMAKE_INSTALL_LIBDIR})
+set(STATIC_LIBS "libprotokit.a")
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/norm.pc.in ${CMAKE_CURRENT_BINARY_DIR}/norm.pc @ONLY)
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/norm.pc DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
+
+# Create a ConfigVersion.cmake file
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+    ${CMAKE_CURRENT_BINARY_DIR}/normConfigVersion.cmake
+    VERSION ${PROJECT_VERSION}
+    COMPATIBILITY AnyNewerVersion
+)
+
+configure_package_config_file(${CMAKE_CURRENT_LIST_DIR}/cmake/normConfig.cmake.in
+    ${CMAKE_CURRENT_BINARY_DIR}/normConfig.cmake
+    INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
+)
+
+# Install the config, configversion and custom find modules
+install(FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/normConfig.cmake
+    ${CMAKE_CURRENT_BINARY_DIR}/normConfigVersion.cmake
+    DESTINATION ${INSTALL_CONFIGDIR}
+)
+
+##############################################
+# Exporting from the build tree
+export(EXPORT normTargets
+    FILE ${CMAKE_CURRENT_BINARY_DIR}/normTargets.cmake
+    NAMESPACE norm::)
+
+# Register package in user's package registry
+export(PACKAGE norm)
+
+if(NORM_BUILD_EXAMPLES)
+    # Setup examples
+    list(APPEND examples 
+        #normDataExample
+        normDataRecv
+        normDataSend
+        normFileRecv
+        normFileSend
+        normStreamRecv
+        normStreamSend
+        normMsgr
+        normStreamer
+        normCast
+        chant
+        normClient
+        normServer
+        #wintest
+        )
+
+    foreach(example ${examples})
+        add_executable(${example} examples/${example}.cpp examples/normSocket.cpp)
+        target_link_libraries(${example} PRIVATE norm protokit::protokit)
+    endforeach()
+endif()
+

Dockerfile

@@ -0,0 +1,27 @@
+ARG platformName=ubuntu
+ARG platformVersion=latest
+FROM ${platformName}:${platformVersion}
+
+ARG version
+ARG configure
+ARG build="--targets=*"
+
+ENV NORM_DIR=/norm-staging
+ENV NORM_BUILD=$NORM_DIR/build
+ENV WAF_CONFIGURE=${configure}
+ENV WAF_BUILD=${build}
+
+ADD ./ ${NORM_DIR}
+
+# Step 1: dependencies
+RUN VERSION=${version} ${NORM_DIR}/docker/setup.sh
+
+# Step 2: build
+RUN cd ${NORM_DIR} && ./waf ${WAF_BUILD}
+
+# Step 3: install
+RUN cd ${NORM_DIR} && ./waf install
+
+# Step 4: expose
+COPY ./docker/run.sh /usr/bin/run-norm-example
+ENTRYPOINT [ "run-norm-example" ]

README-DotNet.md

@@ -0,0 +1,72 @@
+.NET extension for NORM
+==========================
+
+By:  
+  Jeff Muller <jeffrey.muller3.ctr@us.navy.mil>  
+  Sylvester Freeman <sylvester.freeman18.civ@us.navy.mil>  
+  Sergiy Yermak <sergiy.yermak.ctr@us.navy.mil>  
+
+The .NET extension for NORM provides a .NET wrapper for the NORM C API.
+
+For documentation about the main NORM API calls, refer to the NORM Developers
+guide in the regular NORM distribution.
+
+The .NET extension can be built using the .NET CLI.
+
+------------
+Requirements
+------------
+
+The .NET extension for NORM requires at least .NET SDK 6.0.
+
+The NORM library should be built prior to building the .NET extension since it is invoked by the .NET extension.
+
+------------
+Building
+------------
+
+To build the .NET extension for NORM, execute the .NET CLI command in the src/dotnet directory:
+
+  ```
+  dotnet build .
+  ```
+
+------------
+Testing
+------------
+
+To test the .NET extension for NORM, execute the .NET CLI command in the src/dotnet directory:
+
+  ```
+  dotnet test .
+  ```
+
+The test command results should show that all tests have passed.
+Some tests might be skipped due to IO exception.
+
+------------
+Packaging
+------------
+To package the .NET extension for NORM, execute the .NET CLI command in the src/dotnet directory:
+
+```
+dotnet pack . -c Release
+```
+
+To package the .NET extension for NORM that targets 64-bit Windows, execute the command in the src/dotnet directory:
+
+```
+pack-win-x64.bat
+```
+
+To package the .NET extension for NORM that targets 32-bit Windows, execute the command in the src/dotnet directory:
+
+```
+pack-win-x86.bat
+```
+
+To package the .NET extension for NORM that targets 64-bit Linux, execute the command in the src/dotnet directory:
+
+```
+./pack-linux-x64.sh
+```

README-Java.txt

@@ -4,28 +4,42 @@ Java JNI bindings for NORM
 By:
   Jason Rush <jason.rush@gd-ais.com>
   Peter Griffin <peter.griffin@gd-ais.com>
+  updates covering waf build system by Duc Nguyen <dnguyen@aypeks.com>
+
+Updated: 2018-03-14
 
 The Java JNI bindings for NORM provide Java bindings for the NORM C API.
 
 For documentation about the main NORM API calls, refer to the NORM Developers
 guide in the regular NORM distribution.
 
+The JNI bindings for NORM can be built using two methods. The original
+method uses Make and an Ant build.xml script in makefiles/java. The
+newer method uses the waf build system and builds for all platforms with
+various configurations via configuration switches.
+
 ------------
 Requirements
 ------------
 
 Java JNI bindings for NORM requires at least Java 1.5; however, it has also
-been tested with Java 1.6.
+been tested with Java 1.6 and Java 1.8
 
 The NORM library should be built prior to building the Java JNI bindings since
 they link against it.
 
+### Using Ant and Make ###
 Apache Ant is required for building the class files and jar file.
 
+### Python waf ###
+Tested with Python 2 but Python 3 should be ok.
+
 ------------
 Building
 ------------
 
+### Using Ant and Make ###
+
 The build files for Java JNI bindings for NORM are located in the
 makefiles/java directory.
 
@@ -42,6 +56,23 @@ makefiles/java directory with the correct make file for your system:
 
 This will produce a libmil_navy_nrl_norm.so file in the lib/ directory.
 
+### Python waf ###
+
+Reconfigure waf to use java and choose the target on a windows system:
+
+  > cd <norm root dir>
+  > waf configure --build-java --msvc_target=x64
+
+The msvc_target flag allows you to choose the target Windows
+architecture. The default is "x86" (i.e. 32bit arch)
+
+Build the NORM libraries:
+
+  > waf build
+
+The output will be in the build/ directory. Copy both
+mil_navy_nrl_norm.* and norm-1.* to the location the JRE libraries.
+
 ------------
 Installation
 ------------
@@ -75,9 +106,23 @@ If you observe an "java.lang.UnsatisfiedLinkError" exception while running
 your application, you do not have the libmil_navy_nrl_norm.so installed in
 the correct location.
 
+If you're running on Windows, you also need to have the PATH environment
+variable set with along with the -Djava.library.path=<dir>
+
 ------------
 Examples
 ------------
 
 Examples using the Java JNI bindings for NORM are provided in the examples/java
 directory.
+
+Running on Windows (could be put in a .bat script):
+
+  set PATH=%PATH%;<path to mil_navy_nrl_norm.dll>
+  java -Djava.library.path=<path to mil_navy_nrl_norm.dll> ...
+
+Running on Linux or MacOS:
+
+  java -Djava.library.path=<path to mil_navy_nrl_norm.dll> ...
+
+

README-PyNorm.txt

@@ -28,7 +28,7 @@ in similar places as your system's compiler.  On UNIX systems, this is usually
 PATH environment variable.
 
 On Windows, PyNORM requires the "PyWin32" module, available at:
-    http://sf.net/projects/pywin32
+    https://sourceforge.net/projects/pywin32
     
 -------------
 Installation

README.md

@@ -1,50 +1,55 @@
+# norm
 
+```
                     NORM SOURCE CODE RELEASE
 
-AUTHORIZATION TO USE AND DISTRIBUTE
+/*********************************************************************
+ *
+ * AUTHORIZATION TO USE AND DISTRIBUTE
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that: 
+ *
+ * (1) source code distributions retain this paragraph in its entirety, 
+ *  
+ * (2) distributions including binary code include this paragraph in
+ *     its entirety in the documentation or other materials provided 
+ *     with the distribution.
+ * 
+ *      "This product includes software written and developed 
+ *       by Code 5520 of the Naval Research Laboratory (NRL)." 
+ *         
+ *  The name of NRL, the name(s) of NRL  employee(s), or any entity
+ *  of the United States Government may not be used to endorse or
+ *  promote  products derived from this software, nor does the 
+ *  inclusion of the NRL written and developed software  directly or
+ *  indirectly suggest NRL or United States  Government endorsement
+ *  of this product.
+ * 
+ * THIS SOFTWARE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ ********************************************************************/
+```
 
-By receiving this distribution, you have agreed to the following 
-terms governing the use and redistribution of the prototype NRL
-NORM software release written and developed by Brian Adamson and
-Joe Macker:
+This is the NRL NORM source code repository. 
 
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that: 
+The Github "Discussions" feature is available here for comment and question in addition 
+to the regular code issue reporting:
 
-(1) source code distributions retain this paragraph in its entirety, 
-
-(2) all advertising materials mentioning features or use of this 
-  software display the following acknowledgment:
-
-   "This product includes software written and developed 
-    by the Naval Research Laboratory (NRL)." 
-
-The name of NRL, the name(s) of NRL  employee(s), or any entity
-of the United States Government may not be used to endorse or
-promote  products derived from this software, nor does the 
-inclusion of the NRL written and developed software  directly or
-indirectly suggest NRL or United States  Government endorsement
-of this product.
-
-THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
-IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
-
----------------------------------------------------------------------
-
-
-This is a release of the NRL MDP source code.  For most
-purposes, the authors would prefer that the code not be
-re-distributed so that users will obtain the latest (and most
-debugged/improved) release from the official NORM web site:
-
-<http://cs.itd.nrl.navy.mil/work/norm>
+https://github.com/USNavalResearchLaboratory/norm/discussions
 
 
 SOURCE CODE
 ===========
 
-The following items can be build from this source code release:
+The "norm" source Git repository includes "protolib" as a git submodule. You can
+add the "--recurse-submodules" to your git clone command to automate inclusion: 
+
+git clone --recurse-submodules https://github.com/USNavalResearchLaboratory/norm.git
+
+The following items can be built from this source code release:
 
 1) libnorm.a - static NORM library for Unix platforms, or
 
@@ -87,6 +92,43 @@ B) An application for reliably "tunneling" real-time UDP packet
    streams (like RTP video or audio) using NORM's streaming
    capability.
 
+NACK-Oriented Proxy (NORP)
+==========================
+NORM Proxy code is also included in the 'norp' directory.
+More info is available in the norp/doc directory, as well as the
+NORP website:  https://www.nrl.navy.mil/itd/ncs/products/norp
+
+
+
+Docker
+======
+A `Dockerfile` is provided to facilitate containerized NORM environment testing. Notable
+build arguments passed to `docker build` will influence how the image is created.
+
+- `platformName` represents the base image name (defaults to `ubuntu`)
+- `platformVersion` represents the tagged version for the base image (defaults to `latest`)
+- `version` is a 1:1 mapping to this repo's tag (defaults to `v1.5.9`)
+- `configure` is piped to `./waf configure` (defaults to "")
+- `build` is piped to `./waf` (defaults to `--targets=*`)
+
+Example usage:
+
+CentOS7 build
+```
+docker build --build-arg platformName=centos \
+  --build-arg platformVersion=7 \
+  --build-arg="--build-java" \
+  -t norm:centos-1.5.9 .
+```
+
+Running the data send and recv examples:
+```
+# Sender in one terminal
+docker run -it --rm --name norm-send <norm:image> normDataSend
+# Receiver in another terminal
+docker run -it --rm --name norm-recv <norm:image> normDataRecv
+```
+
 
 OTHER FILES:
 ============
@@ -102,14 +144,33 @@ NormUserGuide.pdf       - Guide to "norm" demo app usage
 
 VERSION.TXT             - NORM version history notes
 
-README.TXT              - this file
+README.md               - this file
+
+docker                  - directory containing docker container support
 
 
 NOTES:
 ======
 
-The NORM code depends upon the current "Protolib" release.  The NORM
-source code tarballs contain an appropriate release of "Protolib" as
-part of the NORM source tree.  If the NORM code is checked out from
-our CVS server, it is necessary to also check out "protolib" separately
-and provide paths to it for NORM to build.
+The NORM code depends upon the current "Protolib" release:
+
+  https://github.com/USNavalResearchLaboratory/protolib 
+  
+It has been addded as a git submodule to the NORM git repository.  So, to 
+to build you will need to do the following steps to download the protolib code:
+
+git clone --recurse-submodules https://github.com/USNavalResearchLaboratory/norm.git
+
+Alternatively after a basic "git clone" you can do the folowing to pull in the protolib source:
+
+cd norm
+git submodule update --init
+
+To keep the 'protolib' submodule up to date, you will need to do the 
+following:
+
+cd norm/protolib
+git checkout master
+
+This will enable you to issue 'git pull', etc to treat the 'protolib' 
+sub-directory as its own (sub-) repository.

VERSION.TXT

@@ -1,5 +1,60 @@
 			           NORM Version History
 
+
+Version 1.5.10 (in progress)
+=============
+    - NormSocket API improvements
+
+Version 1.5.9
+=============
+    - Added "normCast" file send/receive example 
+    - Protolib ProtoTimer fix for "long" timeout intervals
+
+Version 1.5.8
+=============
+    - Modified waf build to build pkg-config file.cThanks to Luca Boccassi
+       for this and some other patches for Gnu/Hurd, etc.)
+
+Version 1.5.7
+=============
+    - Fixed issue with NormSession::SenderRemoveAckingNode() method where
+      the removed NormAckingNode instance was not being properly deleted.
+      (Thanks to Chad Gioia)
+    - Fixed bug in normPrecoder (npc) when building on 64-bit architectures
+      (Thanks to Leon Meyer)
+    - Fixed bug in JNI NormNode.getAddress() method where an "unsigned short" 
+      type was used instead of "UINT16" to receive the port portion of the
+      address. (Thanks to Jame Nguyen)
+    - Reduced warnings, added gnu and other build options to 'waf' wscripts
+      (Thanks to Luca Buccassi)
+    - Greatly improved high speed performance via enhancements to Protolib
+    - Fixed bugs related to coding block id wrapping for very long-lived
+      (or very high speed) NORM sessions.
+    - Added NormSocket API extension in "norm/examples" that provides a 
+      sockets-like API and client-server connection-oriented paradigm
+      for using NORM. This currently supports a byte-stream service 
+      for both multicast and unicast using NORM_OBJECT_STREAM.  This API
+      extension is still work in progress and will be expanded to include
+      message-stream service support as well as NORM's file/data object
+      modes.  The "norm/examples/normClient.cpp" (and "normServer.cpp)
+      illustrates use of the the API extension defined in
+      "norm/examples/normSocket.h"
+    - Added "normStreamer" example that provides the same function as the
+      "normMsgr" example except using NORM_OBJECT_STREAM.
+    - The bug fixes and performance enhancements make this a key update.
+      The NORM Developer's Guide is not yet updated to described some of
+      the new API features added.
+    - Added APIs to iterate through sender "Acking Node List" to determine
+      who has and who has not acknowledged a watermark acknowledgement
+      request.
+    - Added NormSetAutoAckingNodes() call with NormTrackingStatus to 
+      automate detection of new group members.  The Acking Node List
+      iterator function NormGetNextAckingNode() can be used to learn
+      who has joined and the NORM_ACKING_NODE_NEW provides notification
+      to the application of new members.
+    - The 'waf' build scripts assign the current version number to
+      the NORM shared library built for better version control/tracking.
+
 Version 1.5r6
 =============
     - Fixed Protolib ProtoBitmask bug for NDEBUG (release) builds.

android/README.md

@@ -0,0 +1,61 @@
+This directory contains Android Studio project files for building the NORM
+library for Android.
+
+REQUIREMENTS:
+
+Ninja is required in order to run the ant build for the java .jar file
+
+Download and install ndk and cmake using the SDK manager within Android Studio. To do this:
+
+    -> Tools -> SDK Manager -> System Settings -> Android SDK -> SDK Tools -> Select 'Show Package Details' at the bottom
+
+The latest compatible versions are ndk version 20.0.5594570 and cmake version 3.10.2.4988404
+
+* Important *
+  If you decide to install different version of the SDK Tools above, make sure to edit the path locations in the local.properties file
+
+TO BUILD:
+
+./gradlew build
+
+The lib/build/ directory will contain the binary libraries built.
+
+The "./gradlew clean" command can be used to delete the files built.
+
+This directory may also be opened as an Android Studio project and built from
+the IDE GUI.
+
+This will evolve as we update NORM and associated projects to the newer Android
+Studio tool chain.
+
+USAGE:
+
+Copy the .aar file into your ${project.rootDir}/app/libs/ folder
+
+Within your app level build.gradle include the following two tasks before your dependencies section:
+
+```
+task extractSo(type: Copy) {
+    println 'Extracting *.so file(s)....'
+
+    from zipTree("${project.rootDir}/app/libs/lib-release_-release.aar")
+    into "${project.rootDir}/app/src/main/jniLibs"
+    include "jni/**/*.so"
+
+    eachFile {
+        def segments = it.getRelativePath().getSegments() as List
+        println segments
+        it.setPath(segments.tail().join("/"))
+        return it
+    }
+    includeEmptyDirs = false
+}
+
+task extractJar(type: Copy, dependsOn: extractSo) {
+    println 'Extracting *.jar file(s)....'
+
+    from zipTree("${project.rootDir}/app/libs/lib-release_-release.aar")
+    into "${project.rootDir}/app/libs/"
+    include "norm-*.jar"
+}
+```

android/build.gradle

@@ -0,0 +1,16 @@
+buildscript {
+   repositories {
+       google()
+       jcenter()
+   }
+   dependencies {
+       classpath 'com.android.tools.build:gradle:4.2.1'
+   }
+}
+
+allprojects {
+   repositories {
+       google()
+       jcenter()
+   }
+}

android/gradle/wrapper/LICENSE-for-gradlewrapper.txt

@@ -0,0 +1,202 @@
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

android/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

android/gradlew

@@ -0,0 +1,234 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##############################################################################
+#
+#   Gradle start up script for POSIX generated by Gradle.
+#
+#   Important for running:
+#
+#   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+#       noncompliant, but you have some other compliant shell such as ksh or
+#       bash, then to run this script, type that shell name before the whole
+#       command line, like:
+#
+#           ksh Gradle
+#
+#       Busybox and similar reduced shells will NOT work, because this script
+#       requires all of these POSIX shell features:
+#         * functions;
+#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+#         * compound commands having a testable exit status, especially «case»;
+#         * various built-in commands including «command», «set», and «ulimit».
+#
+#   Important for patching:
+#
+#   (2) This script targets any POSIX shell, so it avoids extensions provided
+#       by Bash, Ksh, etc; in particular arrays are avoided.
+#
+#       The "traditional" practice of packing multiple parameters into a
+#       space-separated string is a well documented source of bugs and security
+#       problems, so this is (mostly) avoided, by progressively accumulating
+#       options in "$@", and eventually passing that to Java.
+#
+#       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+#       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+#       see the in-line comments for details.
+#
+#       There are tweaks for specific operating systems such as AIX, CygWin,
+#       Darwin, MinGW, and NonStop.
+#
+#   (3) This script is generated from the Groovy template
+#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       within the Gradle project.
+#
+#       You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+    APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
+    [ -h "$app_path" ]
+do
+    ls=$( ls -ld "$app_path" )
+    link=${ls#*' -> '}
+    case $link in             #(
+      /*)   app_path=$link ;; #(
+      *)    app_path=$APP_HOME$link ;;
+    esac
+done
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+
+APP_NAME="Gradle"
+APP_BASE_NAME=${0##*/}
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+    echo "$*"
+} >&2
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in                #(
+  CYGWIN* )         cygwin=true  ;; #(
+  Darwin* )         darwin=true  ;; #(
+  MSYS* | MINGW* )  msys=true    ;; #(
+  NONSTOP* )        nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD=$JAVA_HOME/jre/sh/java
+    else
+        JAVACMD=$JAVA_HOME/bin/java
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD=java
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+    case $MAX_FD in #(
+      max*)
+        MAX_FD=$( ulimit -H -n ) ||
+            warn "Could not query maximum file descriptor limit"
+    esac
+    case $MAX_FD in  #(
+      '' | soft) :;; #(
+      *)
+        ulimit -n "$MAX_FD" ||
+            warn "Could not set maximum file descriptor limit to $MAX_FD"
+    esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+#   * args from the command line
+#   * the main class name
+#   * -classpath
+#   * -D...appname settings
+#   * --module-path (only if needed)
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+    APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+    JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    for arg do
+        if
+            case $arg in                                #(
+              -*)   false ;;                            # don't mess with options #(
+              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
+                    [ -e "$t" ] ;;                      #(
+              *)    false ;;
+            esac
+        then
+            arg=$( cygpath --path --ignore --mixed "$arg" )
+        fi
+        # Roll the args list around exactly as many times as the number of
+        # args, so each arg winds up back in the position where it started, but
+        # possibly modified.
+        #
+        # NB: a `for` loop captures its iteration list before it begins, so
+        # changing the positional parameters here affects neither the number of
+        # iterations, nor the values presented in `arg`.
+        shift                   # remove old arg
+        set -- "$@" "$arg"      # push replacement arg
+    done
+fi
+
+# Collect all arguments for the java command;
+#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+#     shell script including quotes and variable substitutions, so put them in
+#     double quotes to make sure that they get re-expanded; and
+#   * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+        "-Dorg.gradle.appname=$APP_BASE_NAME" \
+        -classpath "$CLASSPATH" \
+        org.gradle.wrapper.GradleWrapperMain \
+        "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+#   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+#   set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+        printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+        xargs -n1 |
+        sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+        tr '\n' ' '
+    )" '"$@"'
+
+exec "$JAVACMD" "$@"

android/gradlew.bat

@@ -0,0 +1,89 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

android/lib/CMakeLists.txt

@@ -0,0 +1,80 @@
+
+cmake_minimum_required(VERSION 3.4.1)
+
+project("norm")
+
+add_definitions("-DANDROID=1" "-DANDROID_API_VERSION=24" "-DPUSH_NOTIFICATIONS=1" "-DANDROID" "-DLINUX" "-DUNIX" "-DHAVE_IPV6" "-DHAVE_DIRFD" "-DPROTO_DEBUG" "-DHAVE_ASSERT" "-DHAVE_GETLOGIN" "-DUSE_SELECT" "-D_FILE_OFFSET_BITS=64" "-DHAVE_OLD_SIGNALHANDLER" "-DHAVE_SCHED" "-DNO_SCM_RIGHTS" "-Wno-attributes" "-DAPP_VERSION=1.0.0" "-DAPP_VERSION_HEX=0x10000")
+
+include_directories( AFTER
+    "../../OID_NDK}/sources/android/cpufeatures"
+    "../../include"
+    "../../protolib/include"
+)
+
+find_program(ANTPATH ant)
+
+execute_process(
+    COMMAND ${ANTPATH}
+    WORKING_DIRECTORY "$ENV{HOME}/norm/makefiles/java/"
+)
+
+enable_language(ASM)
+
+IF(BUILD_CONFIGURATION MATCHES "DEBUG")
+    SET(BINARY_NAME "norm")
+    add_definitions("-DDEBUG=1" "-D_DEBUG=1")
+ELSEIF(BUILD_CONFIGURATION MATCHES "RELEASE")
+    SET(BINARY_NAME "norm")
+    add_definitions("-DNDEBUG=1")
+ELSE(BUILD_CONFIGURATION MATCHES "DEBUG")
+    MESSAGE( FATAL_ERROR "No matching build-configuration found." )
+ENDIF(BUILD_CONFIGURATION MATCHES "DEBUG")
+
+add_subdirectory("../../protolib/android/lib" ./protolib)
+
+add_library( ${BINARY_NAME}
+
+    STATIC
+    
+    "../../src/common/galois.cpp"
+    "../../src/common/normApi.cpp"
+    "../../src/common/normEncoder.cpp"
+    "../../src/common/normEncoderMDP.cpp"
+    "../../src/common/normEncoderRS16.cpp"
+    "../../src/common/normEncoderRS8.cpp"
+    "../../src/common/normFile.cpp"
+    "../../src/common/normMessage.cpp"
+    "../../src/common/normNode.cpp"
+    "../../src/common/normObject.cpp"
+    "../../src/common/normSegment.cpp"
+    "../../src/common/normSession.cpp"
+)
+
+add_library( mil_navy_nrl_norm
+
+    SHARED
+    
+    "../../src/java/jni/normDataJni.cpp"
+	"../../src/java/jni/normEventJni.cpp"
+	"../../src/java/jni/normFileJni.cpp"
+	"../../src/java/jni/normInstanceJni.cpp"
+	"../../src/java/jni/normJni.cpp"
+	"../../src/java/jni/normNodeJni.cpp"
+	"../../src/java/jni/normObjectJni.cpp"
+	"../../src/java/jni/normSessionJni.cpp"
+	"../../src/java/jni/normStreamJni.cpp"
+)
+
+find_library(log "log")
+find_library(android "android")
+find_library(glesv2 "GLESv2")
+find_library(egl "EGL")
+
+target_link_libraries(mil_navy_nrl_norm
+    norm
+    protokit
+    ${log}
+    ${android}
+    ${glesv2}
+    ${egl}
+)

android/lib/build.gradle

@@ -0,0 +1,142 @@
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion 34
+    externalNativeBuild {
+        cmake {
+            path "CMakeLists.txt"
+        }
+    }
+    signingConfigs {
+        normSigning {
+            storeFile     file("${System.properties['user.home']}${File.separator}.android${File.separator}debug.keystore")
+            storePassword "android"
+            keyAlias      "androiddebugkey"
+            keyPassword   "android"
+            storeType     "jks"
+        }
+    }
+
+    defaultConfig {
+        externalNativeBuild {
+            cmake {
+                arguments "-DANDROID_TOOLCHAIN=clang", "-DANDROID_PLATFORM=android-24", "-DCMAKE_MAKE_PROGRAM=/usr/local/bin/ninja", "-DANDROID_STL=c++_static", "-DANDROID_CPP_FEATURES=exceptions rtti", "-DANDROID_ARM_MODE=arm", "-DANDROID_ARM_NEON=TRUE"
+                cFlags "-fsigned-char"
+                cppFlags "-fsigned-char", "-std=c++14"
+            }
+        }
+    }
+
+    buildTypes {
+         debug {
+             initWith debug
+             debuggable    true
+             jniDebuggable true
+             signingConfig signingConfigs.normSigning
+         }
+         release {
+             initWith release
+             debuggable    false
+             jniDebuggable false
+             signingConfig signingConfigs.normSigning
+         }
+    }
+
+    flavorDimensions "default"
+    productFlavors {
+        debug_ {
+            ndk {
+                abiFilters "armeabi-v7a", "x86"
+            }
+            externalNativeBuild {
+                cmake {
+                    targets "norm", "mil_navy_nrl_norm"
+                    arguments "-DBUILD_CONFIGURATION=DEBUG", "-DCMAKE_CXX_FLAGS_DEBUG=-O0", "-DCMAKE_C_FLAGS_DEBUG=-O0"
+                }
+            }
+
+            dimension "default"
+       }
+        release_ {
+            externalNativeBuild {
+                cmake {
+                    targets "norm", "mil_navy_nrl_norm"
+                    arguments "-DBUILD_CONFIGURATION=RELEASE", "-DCMAKE_CXX_FLAGS_RELEASE=-O3", "-DCMAKE_C_FLAGS_RELEASE=-O3"
+                }
+            }
+
+            dimension "default"
+       }
+    }
+
+    variantFilter { variant ->
+        def names = variant.flavors*.name
+        if (names.contains ("debug_")
+              && variant.buildType.name != "debug") {
+            setIgnore(true)
+        }
+        if (names.contains ("release_")
+              && variant.buildType.name != "release") {
+            setIgnore(true)
+        }
+    }
+
+repositories {
+}
+
+dependencies {
+}
+
+
+}
+
+task copyReleaseJar(type: Copy) {
+    println 'Unzipping release .aar and including .jar....'
+
+    if (file("${project.rootDir}/lib/build/outputs/aar/lib-release_-release.aar").exists()) {
+        from zipTree("${project.rootDir}/lib/build/outputs/aar/lib-release_-release.aar")
+        into "${project.rootDir}/lib/build/outputs/aar/lib-release_-release"
+
+        from "${System.env.HOME}/norm/lib/norm-1.0.0.jar"
+        into "${project.rootDir}/lib/build/outputs/aar/lib-release_-release/"
+    }
+}
+
+task copyDebugJar(type: Copy, dependsOn: copyReleaseJar) {
+    println 'Unzipping debug .aar and including .jar....'
+
+    if (file("${project.rootDir}/lib/build/outputs/aar/lib-debug_-debug.aar").exists()){
+        from zipTree("${project.rootDir}/lib/build/outputs/aar/lib-debug_-debug.aar")
+        into "${project.rootDir}/lib/build/outputs/aar/lib-debug_-debug"
+
+        from "${System.env.HOME}/norm/lib/norm-1.0.0.jar"
+        into "${project.rootDir}/lib/build/outputs/aar/lib-debug_-debug/"
+    }
+}
+
+task zipReleaseAAR(type: Exec, dependsOn: copyDebugJar) {
+    println 'Creating new release .aar..'
+    ignoreExitValue true
+    workingDir "${project.rootDir}/lib/build/outputs/aar/"
+    executable 'jar'
+    def releaseArgsList = ['cvf', 'lib-release_-release.aar', '-C', 'lib-release_-release/', '.']
+    args releaseArgsList
+}
+
+task zipDebugAAR(type: Exec, dependsOn: zipReleaseAAR) {
+    println 'Creating new debug .aar..'
+    ignoreExitValue true
+    workingDir "${project.rootDir}/lib/build/outputs/aar/"
+    executable 'jar'
+    def debugArgsList = ['cvf', 'lib-debug_-debug.aar', '-C', 'lib-debug_-debug/', '.']
+    args debugArgsList
+}
+
+task deleteAARAssets(type: Delete, dependsOn: zipDebugAAR) {
+    delete "${project.rootDir}/lib/build/outputs/aar/lib-release_-release"
+    delete "${project.rootDir}/lib/build/outputs/aar/lib-debug_-debug"
+}
+
+
+this.build.finalizedBy(deleteAARAssets)
+

android/lib/src/main/AndroidManifest.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android" android:versionCode="1" android:versionName="1.0.0"
+          package="mil.navy.nrl.norm">
+  <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+  <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+  <uses-permission android:name="android.permission.INTERNET"/>
+</manifest>

android/settings.gradle

@@ -0,0 +1 @@
+include ':lib'

cmake/normConfig.cmake.in

@@ -0,0 +1,7 @@
+
+include(GNUInstallDirs)
+
+include(CMakeFindDependencyMacro)
+
+# Add the targets file
+include("${CMAKE_CURRENT_LIST_DIR}/normTargets.cmake")

doc/NormDeveloperGuide.xml

@@ -13,19 +13,19 @@
 
     <abstract>
       <para>This document describes an application programming interface (API)
-      for the <ulink url="http://norm.pf.itd.nrl.navy.mil/">Nack-Oriented
+      for the <ulink url="https://www.nrl.navy.mil/Our-Work/Areas-of-Research/Information-Technology/NCS/NORM/">Nack-Oriented
       Reliable Multicast (NORM)</ulink> protocol implementation developed by
       the Protocol Engineering and Advance Networking (<ulink
-      url="http://cs.itd.nrl.navy.mil/">PROTEAN</ulink>) Research Group of the
-      United States <ulink url="http://www.nrl.navy.mil/">Naval Research
+      url="https://www.nrl.navy.mil/itd/ncs/">PROTEAN</ulink>) Research Group of the
+      United States <ulink url="https://www.nrl.navy.mil/">Naval Research
       Laboratory</ulink> (NRL). The NORM protocol provides general purpose
       reliable data transport for applications wishing to use Internet
       Protocol (IP) Multicast services for group data delivery. NORM can also
       support unicast (point-to-point) data communication and may be used for
       such when deemed appropriate. The current NORM protocol specification is
-      given in the <ulink url="http://www.ietf.org/">Internet Engineering Task
+      given in the <ulink url="https://www.ietf.org/">Internet Engineering Task
       Force</ulink> (IETF) <ulink
-      url="http://norm.pf.itd.nrl.navy.mil/rfc3940.pdf">RFC 3940</ulink>. This
+      url="https://datatracker.ietf.org/doc/html/rfc5740">RFC 5740</ulink>. This
       document is currently a reference guide to the NORM API of the NRL
       reference implementation. More tutorial material may be include in a
       future version of this document or a separate developer's tutorial may
@@ -37,19 +37,19 @@
     <title>Background</title>
 
     <para>This document describes an application programming interface (API)
-    for the <ulink url="http://norm.pf.itd.nrl.navy.mil/">Nack-Oriented
+    for the <ulink url="https://www.nrl.navy.mil/Our-Work/Areas-of-Research/Information-Technology/NCS/NORM/">Nack-Oriented
     Reliable Multicast (NORM)</ulink> protocol implementation developed by the
     Protocol Engineering and Advance Networking (<ulink
-    url="http://cs.itd.nrl.navy.mil/">PROTEAN</ulink>) Research Group of the
-    United States <ulink url="http://www.nrl.navy.mil/">Naval Research
+    url="https://www.nrl.navy.mil/itd/ncs/">PROTEAN</ulink>) Research Group of the
+    United States <ulink url="https://www.nrl.navy.mil/">Naval Research
     Laboratory</ulink> (NRL). The NORM protocol provides general purpose
     reliable data transport for applications wishing to use Internet Protocol
     (IP) Multicast services for group data delivery. NORM can also support
     unicast (point-to-point) data communication and may be used for such when
     deemed appropriate. The current NORM protocol specification is given in
-    the <ulink url="http://www.ietf.org/">Internet Engineering Task
+    the <ulink url="https://www.ietf.org/">Internet Engineering Task
     Force</ulink> (IETF) <ulink
-    url="http://norm.pf.itd.nrl.navy.mil/rfc5740.pdf">RFC 5740</ulink>.</para>
+    url="https://datatracker.ietf.org/doc/html/rfc5740">RFC 5740</ulink>.</para>
 
     <para>The NORM protocol is designed to provide end-to-end reliable
     transport of bulk data objects or streams over generic IP multicast
@@ -134,7 +134,7 @@
     may participate as senders and/or receivers within a NORM session. NORM
     senders transmit data to the session destination address (usually an IP
     multicast group) while receivers are notified of incoming data. The NORM
-    API provides and event notification scheme to notify the application of
+    API provides an event notification scheme to notify the application of
     significant sender and receiver events. There are also a number support
     functions provided for the application to control and monitor its
     participation within a NORM transport session.</para>
@@ -159,7 +159,7 @@
       linkend="NormCreateInstance"><literal>NormCreateInstance()</literal></link>)
       as needed for applications with specific requirements for accessing and
       controlling participation in multiple <emphasis>NormSessions</emphasis>
-      from separate operating system multiple threads. Or, alternatively, a
+      from separate operating system threads. Or, alternatively, a
       single <emphasis>NormInstance</emphasis> could be used, with a "master
       thread" serving as an intermediary between the <link
       linkend="NormGetNextEvent"><literal>NormGetNextEvent()</literal></link>
@@ -194,7 +194,7 @@
       number of concurrently active senders, this may translate to significant
       memory allocation on receiver nodes. Currently, the API allows the
       application to control how much buffer space is allocated for each
-      active sender (NOTE: In the future, API functions may be provided limit
+      active sender (NOTE: In the future, API functions may be provided to limit
       the number of active senders monitored and/or provide the application
       with finer control over receive buffer allocation, perhaps on a per
       sender basis).</para>
@@ -227,7 +227,7 @@
         <title>Data Transmission</title>
 
         <para>The behavior of data transport operation is largely placed in
-        the control of the NORM sender(s). NORM senders controls their data
+        the control of the NORM sender(s). NORM senders control their data
         transmission rate, forward error correction (FEC) encoding settings,
         and parameters controlling feedback from the receiver group. Multiple
         senders may operate in a session, each with independent transmission
@@ -253,7 +253,7 @@
         be called before any calls to <link
         linkend="NormStartReceiver"><literal>NormStartReceiver()</literal></link>
         are made. However, note that the cache directory may be changed even
-        during active NORM reception. In this case, the new specified
+        during active NORM reception. In this case, the newly specified
         directory path will be used for subsequently-received files. Any files
         received before a directory path change will remain in the previous
         cache location. Note that the <link
@@ -477,15 +477,14 @@
     Protokit. These are described below.</para>
 
     <para>The "makefiles" directory contains Unix Makefiles for various
-    platforms the "win32" and "wince" sub-directories there contain Microsoft
+    platforms. The "win32" and "wince" sub-directories there contain Microsoft
     Visual C++ (VC++) and Embedded VC++ project files for building the NORM
     implementation. Additionally, a "waf" (Python-based build tool) build
     option is supported that can be used to build and install the NORM library
     code on the supported platforms. Finally, Python and Java bindings to the
-    NORM API are included and "src/python" and "src/java" directories contain
-    the code for these and the "makefiles/java" directory contains Makefiles
-    to build the NORM Java JNI bindings. Note the "waf" tool can also be used
-    to build the Java and Python bindings.</para>
+    NORM API are included in the "src/python" and "src/java" directories and the
+    "makefiles/java" directory contains Makefiles to build the NORM Java JNI bindings.
+    Note the "waf" tool can also be used to build the Java and Python bindings.</para>
 
     <sect2>
       <title>Unix Platforms</title>
@@ -791,7 +790,7 @@
 {
     NormEventType     type;
     <link linkend="NormSessionHandle"><literal>NormSessionHandle</literal></link> session; 
-    <link linkend="NormNodeHandle"><literal>NormNodeHandle</literal></link>    node;
+    <link linkend="NormNodeHandle"><literal>NormNodeHandle</literal></link>    sender;
     <link linkend="NormObjectHandle"><literal>NormObjectHandle</literal></link>  object;
 } <link linkend="NormEvent"><literal>NormEvent</literal></link>;</programlisting></para>
 
@@ -801,7 +800,7 @@
         all <link
         linkend="NormEventType"><literal>NormEventType</literal></link> fields
         are relevant to all events. The <parameter>session</parameter>,
-        <parameter>node</parameter>, and <parameter>object</parameter> fields
+        <parameter>sender</parameter>, and <parameter>object</parameter> fields
         indicate the applicable <link
         linkend="NormSessionHandle"><literal>NormSessionHandle</literal></link>,
         <link
@@ -1181,7 +1180,7 @@ bool <link linkend="NormSetCacheDirectory"><literal>NormSetCacheDirectory</liter
 
 bool <link linkend="NormGetNextEvent"><literal>NormGetNextEvent</literal></link>(<link
               linkend="NormInstanceHandle"><literal>NormInstanceHandle</literal></link> instanceHandle,
-                      <link linkend="NormEvent"><literal>NormEvent</literal></link>*         theEvent);</programlisting>
+                      <link linkend="NormEvent"><literal>NormEvent*</literal></link>         theEvent);</programlisting>
         </sect4>
 
         <sect4>
@@ -3763,7 +3762,7 @@ void <link linkend="NormSetTxRobustFactor"><literal>NormSetTxRobustFactor</liter
 
           <programlisting>#include &lt;normApi.h&gt;
 
-boolNormRequeueObject(<link linkend="NormSessionHandle"><literal>NormSessionHandle</literal></link> sessionHandle,
+bool NormRequeueObject(<link linkend="NormSessionHandle"><literal>NormSessionHandle</literal></link> sessionHandle,
                       <link linkend="NormObjectHandle"><literal>NormObjectHandle</literal></link>  objectHandle);</programlisting>
         </sect4>
 
@@ -6596,7 +6595,7 @@ bool <link linkend="NormNodeGetAddress"><literal>NormNodeGetAddress</literal></l
 
           <programlisting>#include &lt;normApi.h&gt;
 
-double <link linkend="NormNodeGetId"><literal>NormNodeGetId</literal></link>(<link
+double <link linkend="NormNodeGetId"><literal>NormNodeGetGrtt</literal></link>(<link
               linkend="NormNodeHandle"><literal>NormNodeHandle</literal></link> nodeHandle);</programlisting>
         </sect4>
 

doc/NormUserGuide.xml

@@ -96,11 +96,13 @@
 
       <orderedlist>
         <listitem>
-          <para>Download and unpack the NORM source code tarball.</para>
+            <para>Download and unpack the NORM source code tarball or clone the github
+            repository using 
+            <literal>git clone --recurse-submodules https://github.com/USNavalResearchLaboratory/norm.git</literal></para>
         </listitem>
 
         <listitem>
-          <para><literal>cd norm/unix</literal></para>
+          <para><literal>cd norm/makefiles</literal></para>
         </listitem>
 
         <listitem>
@@ -118,7 +120,8 @@
 
       <orderedlist>
         <listitem>
-          <para>Download and unpack the NORM source code tarball</para>
+            <para>Download and unpack the NORM source code tarball or clone the
+        repository as described in the section above.</para>
         </listitem>
 
         <listitem>
@@ -128,7 +131,7 @@
         </listitem>
 
         <listitem>
-          <para>Open the "<filename>norm.sln</filename>" (VC++ .Net),
+            <para>Inside the "<filename>norm/makefiles/win32</filename>" directory, open the "<filename>norm.sln</filename>" (VC++ .Net),
           "<filename>norm.dsw</filename>" (VC++ 6.0), or
           "<filename>norm.vcw</filename>" (Embedded VC++) workspace
           file.</para>
@@ -137,9 +140,14 @@
         <listitem>
           <para>The "<emphasis>norm</emphasis>" project can be selected and
           built.  The "<filename>norm.exe</filename>" file will be found in
-          the "<filename>norm/win32/norm/Release</filename>" directory.</para>
+          the "<filename>norm/Win32/norm/Release</filename>" directory.</para>
         </listitem>
       </orderedlist>
+      <para>The easiest way to run these builds is with Visual Studio. If you do not already have Visual Studio installed, then it can
+      be downloaded for free <ulink url="https://visualstudio.microsoft.com/downloads/?cid=learn-onpage-download-cta">here</ulink>.
+      When you launch Visual Studio, make sure to select "Visual C++" and open the "<filename>norm.sln</filename>" file.
+      This should load all of the project files and allow you to right-click on any of them in the Solution Explorer on the
+      left side and select "build".</para>
     </sect2>
   </sect1>
 
@@ -380,7 +388,7 @@
     <para>A minimal example <emphasis>norm</emphasis> sender command-line
     syntax is:</para>
 
-    <para><literal>norm addr &lt;addr/port&gt; sendFile
+    <para><literal>norm addr &lt;addr/port&gt; sendfile
     &lt;fileName&gt;</literal></para>
 
     <para>The corresponding minimal <emphasis>norm</emphasis> receiver

doc/npcUserGuide.xml

@@ -0,0 +1,413 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE article PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
+"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd">
+<article lang="">
+  <articleinfo>
+    <title><inlinemediaobject>
+        <imageobject>
+          <imagedata align="center" fileref="NormLogo.gif" scale="50"/>
+        </imageobject>
+      </inlinemediaobject> <command>npc</command> User Guide</title>
+
+    <subtitle>(NORM Precoder User Guide)</subtitle>
+
+    <titleabbrev><command>npc</command> User Guide</titleabbrev>
+  </articleinfo>
+
+  <sect1>
+    <title>Background</title>
+
+    <para>The NACK-Oriented Reliable Multicast (NORM) protocol is capable of
+    supporting robust transmission of content to "silent" receivers that are
+    required or only capable of operating in an emission-controlled (EMCON)
+    manner. This capability is enabled when the NORM sender is configured to
+    proactively transmit Forward Error Correction (FEC) erasure coding content
+    as part of its original data transmission. For NACK-based operation, the
+    FEC repair packets are usually sent only reactively, in response to repair
+    requests (NACKs) from the receiver group. However, hybrid operation with a
+    combination of proactive FEC content and additional reactive FEC repairs
+    as needed is also supported. Similarly, a mix of nacking and silent
+    receivers may be supported with silent receivers capitalizing on the FEC
+    repair information sent proactively and/or reactively. The purpose of the
+    NORM Pre-Coder (<command>npc</command>) software utility described here is
+    to support additional robustness for purely-proactive sessions, where the
+    receivers are unable to request repair or retransmission of
+    content.</para>
+
+    <para>The Naval Research Laboratory (NRL) reference implementation of the
+    NORM protocol includes support for 8-bit and 16-bit Reed-Solomon FEC
+    encoding with additional support for other coding algorithms (e.g.,
+    Low-Density Parity Check (LDPC)) planned for the future. The NORM
+    specification allows for different FEC algorithms to be applied within the
+    protocol. The current Reed-Solomon NORM FEC algorithms in the NRL
+    implementation are limited to modest code block sizes (With 16-bit
+    Reed-Solomon coding, larger block sizes will be allowed but very high data
+    rates may not be possible). For channels with random errors, the current
+    NORM FEC codecs are often adequate as there is flexibility in how the
+    encoded data can be partitioned into FEC blocks (a block consists of some
+    number of data segments (packets)) and the number of FEC parity packets
+    that can computed and possibly transmitted per source data block. For
+    channels with large bursts of packet loss (with respect to the configured
+    NORM FEC block size), it is quite possible that the number of lost packets
+    (erasures) that occur within a NORM FEC block may exceed the configured
+    erasure-filling capability. The <command>npc</command> utility was created
+    to "pre-encode" (and "post-decode") files for NORM transmission to silent
+    (non- NACKing) receivers by adding additional FEC encoding, and
+    importantly, interleaving of the FEC segments (packets) to re-distribute
+    bursts of packet losses as random losses over the entire file. It is thus
+    most applicable to very large files (with respect to FEC block
+    sizes).</para>
+
+    <para>The NORM protocol is described in Internet Engineering Task Force
+    (IETF) Request For Comments (RFC) RFC 5740 and RFC 5741. NRL provides a
+    NORM protocol library with a well-defined API that it is suitable for
+    application development. Refer to the NORM website &lt;<ulink
+    url="https://github.com/USNavalResearchLaboratory/norm">https://github.com/USNavalResearchLaboratory/norm</ulink>&gt;
+    for these other components as well as up-to-date versions of this
+    demonstration application and documentation.</para>
+
+    <para>The <command>npc</command> tool is designed to use in conjunction
+    with the NORM protocol and accompanying NORM file transfer examples that
+    are part of the NORM source code distribution. However, the encoded file
+    format that <command>npc</command> creates can be use with other
+    transports as well. The key is to align the segmentation parameters of the
+    <command>npc</command> configuration with that of the intended transport
+    mechanism.</para>
+  </sect1>
+
+  <sect1>
+    <title>Overview</title>
+
+    <para>The <command>npc</command> utility takes, as input", a file and
+    logically divides it into segments, adding cyclic-redundancy checksum
+    (CRC) to the segments, encoding the source segments with Reeed-Solomon
+    encoding (adding a configurable number of parity segments per FEC source
+    block), and interleaves the source and encoding segments to an output
+    file. The use of the CRC allows erasure to be detected and also provides
+    additional assurance of correct content delivery by possibly detecting bit
+    errors that may have been undetected during transport (i.e., link-layer
+    framing, Internet Protocol (IP), and/or User Datagram Protocol (UDP)
+    checksums). The interleaving by default is a block interleaver using the
+    entire file as a logical block, but a limit on the interleaving size can
+    be set to help increase the speed of the <command>npc</command>
+    encoding/decoding process. This may be useful for extremely large file
+    sizes.</para>
+  </sect1>
+
+  <sect1>
+    <title>Usage</title>
+
+    <para>The following is a synopsis of <command>npc</command> usage:</para>
+
+    <programlisting>npc {encode|decode} input &lt;inFile&gt; [output &lt;outFile&gt;][segment &lt;segmentSize&gt;] 
+    {[[block &lt;numData&gt;][parity &lt;numParity&gt;]] | 
+     [auto &lt;parityPercentage&gt;][bmax &lt;maxBlockSize&gt;]}
+    [imax &lt;widthLimit&gt;][ibuffer &lt;bytes&gt;][background][help][debug &lt;debugLevel&gt;]</programlisting>
+
+    <para>The <command>npc</command> utility may be instructed to either
+    "encode" a file (add FEC content and interleaving to the given
+    &lt;inFile&gt;) or "decode" a file that was previously encoded with
+    <command>npc</command>. The ".npc" file extension is suggested to
+    delineate files that are of the <command>npc</command> encoded format.
+    Note the "output" filename is optional. By default, <command>npc</command>
+    will use the filename of the &lt;inFile&gt; as the output filename,
+    replacing the '.' extension delimiter with a '_' (underscore) and adding
+    the ".npc" extension suffix. The <command>npc</command> format includes
+    some minimal "meta-data" in the first encoded &lt;segmentSize&gt; to
+    convey the file size and name of the original file. On decoding, if the
+    "output" file option is omitted, this "meta-data" is used to name the
+    decoded output file.</para>
+
+    <para>The optional FEC parameters,
+    <parameter>&lt;segmentSize&gt;</parameter>,
+    <parameter>&lt;numData&gt;</parameter>, and
+    <parameter>&lt;numParity&gt;</parameter> control the logical segmentation,
+    blocking, and amount of FEC parity content added to the file. For use with
+    NORM, it is recommended that the
+    <parameter>&lt;segmentSize&gt;</parameter> value correspond to the same
+    segmentation size used for NORM transmission. The
+    <parameter>&lt;numData&gt;</parameter> (source segments per FEC encoding
+    block) and <parameter>&lt;numParity&gt;</parameter> parameters should be
+    selected to provide erasure filling coverage for the expected transmission
+    packet loss characteristics. Note that when used with proactive NORM FEC
+    transmission, the <command>npc</command> encoding provides an "inner" FEC
+    code and interleaving and the NORM protocol provides an "outer" FEC
+    encoding. The "outer" NORM code might be configured to deal with typical
+    random packet loss due to channel BER, etc and the "inner"
+    <command>npc</command> interleaving and coding could be correspondingly
+    configured to handle expected burst losses (e.g. outages) that might
+    occur.</para>
+
+    <para>The <parameter>&lt;auto&gt;</parameter> option provides an
+    alternative means for setting the FEC encoding protection level instead of
+    using the <parameter>&lt;block&gt;</parameter> and
+    <parameter>&lt;parity&gt;</parameter> options. First the
+    <parameter>&lt;auto&gt;</parameter> option causes <command>npc</command>
+    to select a block size corresponding to the entire input file size (plus
+    the segment of meta data information that npc adds). Then, the
+    <parameter>&lt;auto&gt;</parameter> option
+    <parameter>&lt;parityPercentage&gt;</parameter> value is used to set the
+    number of parity packets per encoding block to the given percentage of the
+    automatically determined block size. For exanple, the command
+    "<option>auto 100"</option> causes <command>npc</command> to set an
+    encoding rate of 100%. I.e., the parity segments sent will equal the
+    number of segments in a bock. Note that the
+    <parameter>&lt;parityPercentage&gt;</parameter> can even exceed 100% if
+    desired for high levels of loss protection. Also note that the percentage
+    here is _not_ a loss protection percentage with a 100%
+    <parameter>&lt;parityPercentage&gt;</parameter> value being able to
+    correct up to 50% errored or lost packets within a coding block. With 50%
+    uniform random packet loss, this would result in successful file transfer
+    about 50% of the time as, per Gaussian distribution, burst error
+    probabilities would result in half of blocks arriving with greater than
+    50% packet loss and half with less than 50% packet loss.</para>
+
+    <para>When the <command>npc</command> encoder uses the
+    "<option>auto</option>" command the <command>npc</command> decoder MUST
+    also use the "<option>auto</option>" command and be configured with the
+    same <parameter>&lt;segmentSize&gt;</parameter> and
+    <parameter>&lt;parityPercentage&gt;</parameter> values. Similarly when the
+    "<option>block</option>" and "<option>parity</option>" commands are used
+    to explicitly set the <parameter>&lt;numdData&gt;</parameter> and
+    <parameter>&lt;numParity&gt;</parameter> at the encoder, the decoder MUST
+    be configured with the same corresponding options and values, again
+    including <parameter>&lt;segmentSize&gt;</parameter>. And for use with
+    NORM protocol transport, the <parameter>&lt;segmentSize&gt;</parameter>
+    parameter SHOULD be matched for best coding gain performance. The NORM
+    block size (numData) and parity (numParity) parameters may be set
+    indepedently. Basically, the NORM protocol proactive erasure coding can be
+    configured to deal with expected short term random packet loss while the
+    <command>npc</command> parameters can be configured to counter large burst
+    (or outage) losses. The inner/outer encoding approach that the combination
+    of <command>npc</command> and NORM provides, can allow for a sort of
+    multiplicative coding gain to deal well with both random packet loss and
+    bursts or outages with lower FEC overhead. However, when the
+    <command>npc</command> coding is configured (e.g., via the "auto" option)
+    to encapsulate an entire file into a single logical coding block, the
+    desired level of loss protection can be simply "dialed into" the
+    <command>npc</command> <parameter>&lt;parityPercentage&gt;</parameter>
+    option. The tradeoff is that the larger FEC block size increases the
+    computational requirement for file encoding and decoding. Future versions
+    of <command>npc</command> may provide additional FEC code types</para>
+
+    <para>As basic example usage, to encode a file name "originalFile.txt"
+    with the default <command>npc</command> naming convention, FEC, and
+    interleaving parameters, use the following syntax:</para>
+
+    <programlisting>npc encode input originalFile.txt</programlisting>
+
+    <para>This will produce and output file named
+    "<filename>originalFile_txt.npc</filename>" in the current working
+    directory. The default <command>npc</command> configuration is
+    "<command>auto 100.0</command>" providing 100% parity content which makes
+    the encoded file size roughly double the input file. The original file can
+    be recovered (decoded) using the syntax:</para>
+
+    <programlisting>npc decode input originalFile_txt.npc</programlisting>
+
+    <para>This will decode the ".npc" file, and in this case produce a file
+    named "<filename>originalFile.txt"</filename> in the current working
+    directory. (The file name information was stored in first "meta data"
+    segment of the ".npc" file). This default naming convention can be
+    overridden by using the <command>npc</command> "<option>output</option>"
+    command. For example, the syntax:</para>
+
+    <programlisting>npc decode input originalFile_txt.npc output file.txt</programlisting>
+
+    <para>will produce a file named "<filename>file.txt</filename>" that is
+    identical in content to "<filename>originalFile.txt</filename>".</para>
+
+    <sect2>
+      <title>Notes</title>
+
+      <para>The FEC and interleaving parameters that are used for
+      <command>npc</command> encoding MUST be exactly matched to successfully
+      decode the encoded file. I.e., if the defaults are used for encoding,
+      the defaults must be used for decoding. The parameters that must match
+      include <parameter>&lt;segmentSize&gt;</parameter>,
+      <parameter>&lt;parityPercentage&gt;</parameter> and
+      <parameter>&lt;maxBlockSize&gt;</parameter> (or
+      <parameter>&lt;numData&gt;</parameter> and
+      <parameter>&lt;numParity&gt;</parameter>), and
+      <parameter>&lt;widthMax&gt;</parameter>.</para>
+
+      <para>It is possible that in some cases it may be beneficial to apply
+      more proactive FEC content with the <command>npc</command> program
+      instead of with the NORM transport. The trade-offs are
+      scenario-specific.</para>
+
+      <para>The NRL "<command>norm</command>" demonstration application has
+      commands included to support transport of <command>npc</command> encoded
+      files. The distinction here is that a file that _fails_ NORM transport
+      might still be successfully decoded with <command>npc</command>. There
+      are two receiver-side <command>norm</command> demo application options
+      that apply here:</para>
+
+      <orderedlist>
+        <listitem>
+          <para>The "<option>saveAborts</option>" command causes
+          <command>norm</command> to not delete (and attempt to postprocess)
+          "aborted" files (files that failed reliable NORM transport).</para>
+        </listitem>
+
+        <listitem>
+          <para>The <command>norm</command> "<option>lowDelay</option>"
+          command should be applied for silent-receivers to more immediately
+          deliver "failed" files to the application for post-processing (i.e.,
+          attempted <command>npc</command> decoding)</para>
+        </listitem>
+      </orderedlist>
+    </sect2>
+  </sect1>
+
+  <sect1>
+    <title><command>npc</command> Command Reference</title>
+
+    <para>The following table describes each of the <command>npc</command>
+    commands available in the command-line syntax.</para>
+
+    <informaltable frame="all">
+      <tgroup cols="2">
+        <colspec colnum="1" colwidth="1*"/>
+
+        <colspec colname="2" colwidth="2*"/>
+
+        <tbody>
+          <row>
+            <entry><para><option>encode</option> |
+            <option>decode</option></para></entry>
+
+            <entry><para>Determine whether <command>npc</command> is to encode
+            or decode the given <parameter>&lt;inFile&gt;</parameter>. This
+            option is required and only one should be given.</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>input
+            </option><parameter>&lt;inFile&gt;</parameter></para></entry>
+
+            <entry><para>Specifies the file to be processed. Required
+            command.</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>output
+            </option><parameter>&lt;outFile&gt;</parameter></para></entry>
+
+            <entry><para>Specifies the name of the output file to be produced.
+            Overrides the default <command>npc</command> output file naming
+            convention. Optional.</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>segment
+            </option><parameter>&lt;segmentSize&gt;</parameter></para></entry>
+
+            <entry><para>Sets the segmentation size (e.g., packet payload
+            size) in bytes. Note four bytes of the
+            <parameter>&lt;segmentSize&gt;</parameter> are used for a 32-bit
+            CRC that <command>npc</command> applies to each segment. (Default
+            <parameter>&lt;segmentSize&gt;</parameter> is 1024
+            bytes)</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>block
+            </option><parameter>&lt;numData&gt;</parameter></para></entry>
+
+            <entry><para>Specify the number of source data segments (packets)
+            per <command>npc</command> FEC coding block. (Default block sizing
+            is auto)</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>parity
+            </option><parameter>&lt;numParity&gt;</parameter></para></entry>
+
+            <entry><para>Specify the number of FEC parity segments (packets)
+            added per <command>npc</command> FEC coding block. (Default is 2
+            segments).</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>auto
+            </option><parameter>&lt;parityPercentage&gt;</parameter></para></entry>
+
+            <entry><para>Specifies automatic FEC block sizing with
+            <parameter>&lt;parityPercentage&gt;</parameter> indicating the
+            percentage of FEC parity segments to include per block. The "auto"
+            block sizing sets the block size as large as possible to treat the
+            entire files as one logical FEC block to maximize FEC performance.
+            The maximum possible block size currently supported by
+            <command>npc</command> are blocks where (numData + numParity) is
+            less than or equal to 65536. The maximum buffer size can be
+            limited by using the <option>bmax</option> command.</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>bmax
+            </option><parameter>&lt;maxBlockSize&gt;</parameter></para></entry>
+
+            <entry><para>Limits the maximum block size when the
+            <option>auto</option> command is used for automatic block sizing
+            (Default is 65536)</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>imax
+            </option><parameter>&lt;widthMax&gt;</parameter></para></entry>
+
+            <entry><para>Limits interleaving of encoded file to a maximum
+            interleaver width of <parameter>&lt;widthMax&gt;</parameter>
+            segments. A value of ZERO (or less) defaults to
+            <command>npc</command> calculating a block interleaver that
+            encompasses the entire encoded file size. For extremely large
+            files, this option may be beneficial to limit file seeking
+            operations required to interleave the file. If the encoded file
+            size is less than
+            <parameter>&lt;widthMax&gt;</parameter>*<parameter>&lt;widthMax&gt;</parameter>
+            segments, <command>npc</command> will again calculate its own
+            maximum block size. (Default is 1000 segments interleaver depth
+            (i.e., about 1 Gbyte interleaver size with the default 1024 byte
+            <parameter>&lt;segmentSize&gt;</parameter> value))</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>ibuffer
+            </option><parameter>&lt;bufferSize&gt;</parameter></para></entry>
+
+            <entry><para>This sets the maximum memory (in bytes) that
+            <command>npc</command> allocates for encoding. A larger value
+            allows <command>npc</command> to perform file input/output with
+            less seeking and improved encoding/decoding times can be achieved.
+            (Default is 1.5 GByte)</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>debug
+            </option><parameter>&lt;debugLevel&gt;</parameter></para></entry>
+
+            <entry><para>Specifies debug output verbosity. Higher number is
+            more verbose debugging information. (Default is
+            ZERO).</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>background</option></para></entry>
+
+            <entry><para>Sets percentage of received messages that are
+            randomly dropped (for testing purposes). Default = 0.0
+            percent.</para></entry>
+          </row>
+
+          <row>
+            <entry><para><option>help</option></para></entry>
+
+            <entry><para>Displays <command>npc</command> usage
+            statement.</para></entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable>
+  </sect1>
+</article>

docker/run.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo Running example $@
+
+exec "$NORM_BUILD/$@"

docker/setup.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+set -eux
+
+install_debian()
+{
+    apt-get update
+    DEBIAN_FRONTEND=noninteractive apt-get install -yq \
+       python2.7 \
+       g++ \
+       libxml2-dev \
+       libnetfilter-queue-dev
+
+    ln -s /usr/bin/python2.7 /usr/bin/python
+
+    if [ "$VERSION" != "" ]; then
+        apt-get install -yq git-core
+        cd $NORM_DIR && git checkout $VERSION && git submodule update --checkout --init --force
+    fi
+
+    if [[ "$WAF_CONFIGURE" =~ "java" && -z `which java` ]]; then
+        apt-get install -yq default-jdk-headless
+        export JAVA_HOME=$(readlink -f `which java` | sed 's/\/bin\/java//')
+    fi
+}
+
+install_redhat()
+{
+    yum update -y
+    yum install -y \
+      which \
+      centos-release-scl
+    yum install -y \
+      gcc-c++ \
+      libxml2-devel \
+      libnetfilter_queue-devel
+
+    if [ "$VERSION" != "" ]; then
+        yum install -y git
+        cd $NORM_DIR && git checkout $VERSION && git submodule update --checkout --init --force
+    fi
+
+    if [[ "$WAF_CONFIGURE" =~ "java" && -z `which java` ]]; then
+        yum install -y java-1.8.0-openjdk-devel
+        export JAVA_HOME=$(readlink -f `which java` | sed 's/\/jre\/bin\/java//')
+    fi
+}
+
+configure_norm()
+{
+    cd $NORM_DIR && ./waf configure ${WAF_CONFIGURE}
+}
+
+if [ ! -z `which apt` ]; then
+    install_debian
+elif [ ! -z $(which yum) ] || [ "$?" = "1" ]; then
+    install_redhat
+fi
+configure_norm

examples/README.TXT

@@ -2,16 +2,45 @@
 
 This directory contains some purposely simplified examples of NORM
 API usage. See comments in the source code files on how to build
-these on Unix platforms.  Eventually, "Makefiles" (and hopefully
-"waf" configure/build scripts) will be provided as well as Visual
-C++ project files for Win32 builds.  
+these on Unix platforms.  Most of these can be built with either the
+Makefiles or Visual Studio projects, but the 'waf' build option is more
+straightforward to use, particularly for Java and Python builds.  
 
-The following example programs are included:
+The most complete examples are the following:
 
-(These two programs use a hard-coded multicast address and port,
-but take a file name and directory name, respectively, as a
-command-line argument to determine the file sent and the directory
-to which receive files are stored.)
+normMsgr.cpp    - Also java/NormMsgr.java and python/normMsgr.py
+                  implementations are available with same functionality.
+                  This sends messages piped to STDIN and receiver
+                  instances can pipe received messages to STDOUT.
+                  "Messages" are binary with a two-byte (Big Endian)
+                  message length header.  A normMsgr instnace can act
+                  as a sender and/or receiver.  This program uses
+                  NORM_OBJECT_DATA as its transport mode.  It has options
+                  to illustrate ACK-based flow control, and passive/active
+                  flushing of NORM transmission stream.
+                  
+normStreamer.cpp - Similar to normMsgr, but uses NORM_OBJECT_STREAM.
+                   It can support byte- and message-stream transport
+                   with the same 2-byte message length header.
+                   (TBD - implement Java and Python versions of this)
+                   
+normCast.cpp    - Similar to above two, but uses NORM_OBEJCT_FILE 
+                  to transmit files and/or or directories of files.
+                  This will eventually have the same file casting
+                  options as the "norm" demo app has such as
+                  repeat iterations through the file/directory list
+                  and ability to monitor one or more directories
+                  as live "outboxes" where files can be deposited
+                  for transmission.                  
+  
+normClient/normServer - illustrate use of the NormSocket API extension
+                        that is a work in progress, but fully functional
+                        for TCP-like byte-streaming applications.
+
+
+The following example programs are mainly "sketches" that illustrate
+the NORM API at a high level.  These programs primarily use a hard-coded 
+multicast address and port, but some have simple options:
 
 normFileSend.cpp - simple file transmission program.  Sends one
                    file and exits.
@@ -34,32 +63,15 @@ normDataRecv.cpp - simple NORM_OBJECT_DATA reception program.  This
 
 java -             There is a README.TXT in the java directory.
                    It explains the java examples, how to build
-				   them, and how to run them.
+				   them, and how to run them.  Other than 
+                   'NormMsgr.java', the examples there
+                   haven't been tested thoroughly/recently.
                    
-NOTES:
-
-Althought the normDataSend/Recv example use relatively small data
-object sizes, the _intended_ use of NORM_OBJECT_DATA is for bulkier
-content stored in application memory space.
-
-For "messaging" applications that use modestly small message
-sizes,  the NORM_OBJECT_STREAM transport option in NORM is likely
-to provide more efficient service than NORM_OBJECT_DATA for small
-objects due to the packet level Forward Error Correction (FEC)
-based packet  recovery mechanism that NORM uses.
-
-In the future, a similar "simple" example pair will be provided to
-illustrate the NORM_OBJECT_STREAM form of transport.  Meanwhile,
-the "normTest.cpp" file in the "norm/src/common" directory 
-provides an example of the NORM API calls related to this.
-  
-Other examples will be added later including examples using the
-"NormGetDescriptor()" function to allow NORM API events to be
-multiplexed with other possible application events (e.g. other I/O
-or Windows messages, etc).
+python -           Other than 'normMsgr.py', the examples there
+                   haven't been tested thoroughly/recently. 
 
 Brian Adamson  
-<mailto:adamson@itd.nrl.navy.mil>
-20 August 2010
+<mailto: badamson@gmail.com>
+27 December 2019
 
 

examples/java/NormMsgr.java

@@ -239,7 +239,7 @@ public class NormMsgr
             normTxLock.release();
             return false;
         }
-        //System.err.println("caching msg for object " + obj + "\n");
+        // System.err.println("caching msg for object " + obj + "\n");
         // Cache the msg associated with the resultant tx object.  We use the
         // input_msg_list HashMap so we can remove the msg upon NORM_TX_OBJECT_PURGED
         input_msg_list.put(obj, msg);

examples/java/NormStreamSend.java

@@ -10,8 +10,8 @@ public class NormStreamSend {
 	static final long REPAIR_WINDOW_SIZE = 1024 * 1024;
 	static final long SESSION_BUFFER_SIZE = 1024 * 1024;
 	static final int SEGMENT_SIZE = 1400;
-	static final int BLOCK_SIZE = 64;
-	static final int PARITY_SEGMENTS = 16;
+	static final short BLOCK_SIZE = 64;
+	static final short PARITY_SEGMENTS = 16;
 	static final String DEST_ADDRESS = "224.1.2.3";
 	static final int DEST_PORT = 6003;
 

examples/normClient.cpp

@@ -1,9 +1,15 @@
 #include "normSocket.h"
 #include <stdio.h>
 #include <string.h>
+
+#ifdef WIN32
+#include "win32InputHandler.cpp"  // brings in the entire implementation
+#else
 #include <sys/select.h>
 #include <fcntl.h>       // for, well, fnctl()
 #include <errno.h>       // obvious child
+#endif  // if/else WIN32/UNIX
+
 
 // BUILD: (assumes "normApi.h" in "include" ...
 // g++ -I../include -o normClient normClient.cpp  normSocket.cpp ../lib/libnorm.a ../protolib/lib/libprotokit.a -lresolv 
@@ -19,6 +25,11 @@
 // To deny a connection, the "server" can send a stream close???  Which means the server will have
 // to establish state just to deny a client a connection (yuck!)
 
+void Usage()
+{
+    fprintf(stderr, "Usage: normClient [connect <serverAddr>/<port>[,<groupAddr>]][debug <level>][trace]\n");
+}
+
 const unsigned int MSG_LENGTH_MAX = 64;
 
 int main(int argc, char* argv[])
@@ -33,18 +44,19 @@ int main(int argc, char* argv[])
     bool trace = false;
     unsigned int debugLevel = 0;
     
-    for (int i = 1; i < argc; i++)
+	for (int i = 1; i < argc; i++)
     {
         const char* cmd = argv[i];
         unsigned int len = strlen(cmd);
         if (0 == strncmp(cmd, "connect", len))
         {
-            // connect <serverAddr>/<port>[,<groupAddr>
+            // connect <serverAddr>/<port>[,<groupAddr>]
             const char* val = argv[++i];
             const char* portPtr = strchr(val, '/');
             if (NULL == portPtr)
             {
                 fprintf(stderr, "normClient error: missing <port> number\n");
+                Usage();
                 return -1;
             }
             portPtr++;
@@ -58,6 +70,7 @@ int main(int argc, char* argv[])
             else
             {
                 fprintf(stderr, "normClient error: missing <serverAddr>\n");
+                Usage();
                 return -1;
             }
             char portText[32];
@@ -76,6 +89,7 @@ int main(int argc, char* argv[])
             if (1 != sscanf(portText, "%hu", &serverPort))
             {
                 fprintf(stderr, "normClient error: invalid <port>\n");
+                Usage();
                 return -1;
             }
             if (NULL != groupPtr)
@@ -96,7 +110,8 @@ int main(int argc, char* argv[])
         {
             if (1 != sscanf(argv[++i], "%u", &debugLevel))
             {
-                fprintf(stderr, "normServer error: invalid debug level\n");
+                fprintf(stderr, "normClient error: invalid debug level\n");
+                Usage();
                 return -1;
             }
         }
@@ -107,38 +122,57 @@ int main(int argc, char* argv[])
             if (1 != sscanf(val, "%lu", &id))
             {
                 fprintf(stderr, "normClient error: invalid 'id' value\n");
+                Usage();
                 return -1;
             }
             clientId = id;
         }
         else
         {
-            fprintf(stderr, "normServer error: invalid command \"%s\"\n", cmd);
+            fprintf(stderr, "normClient error: invalid command \"%s\"\n", cmd);
+                Usage();
             return -1;
         }
     }  
     
     NormInstanceHandle instance = NormCreateInstance();
+	NormSocketHandle normSocket = NormOpen(instance);
+
+	if (trace)
+	{
+		NormSetMessageTrace(NormGetSocketSession(normSocket), true);
+		if (NULL != groupAddrPtr)
+			NormSetMessageTrace(NormGetSocketMulticastSession(normSocket), true);
+	}
+	if (0 != debugLevel) NormSetDebugLevel(debugLevel);
+
+	//NormSetDebugLevel(3);
+	//NormSetMessageTrace(NormGetSocketSession(normSocket), true);
     
     // Initate connection to server ...
     fprintf(stderr, "normClient: connecting to %s/%hu ...\n", serverAddr, serverPort);        
-    NormSocketHandle normSocket = NormConnect(instance, serverAddr, serverPort, groupAddrPtr, clientId);
+    // setting 'localPort' param here to zero lets an ephemeral port be picked
+    NormConnect(normSocket, serverAddr, serverPort, 0, groupAddrPtr, clientId);
+    /* // Optional code to test NormWrite() immediately after NormConnect() call
+    // (Note this is _not_ compatible with newer stale connection reject code)
+    const char* helloStr = "Hello\n";
+    unsigned int helloLen = strlen(helloStr) + 1;
+    NormWrite(normSocket, helloStr, helloLen);
+    NormFlush(normSocket);*/
     
-    if (trace)
-    {
-        NormSetMessageTrace(NormGetSession(normSocket), true);
-        if (NULL != groupAddrPtr)
-            NormSetMessageTrace(NormGetMulticastSession(normSocket), true);
-    }
-    if (0 != debugLevel) NormSetDebugLevel(debugLevel);
     
-    //NormSetDebugLevel(3);
-    //NormSetMessageTrace(NormGetSession(normSocket), true);
     
-    // We use a select() call to multiplex input reading and NormSocket handling
+#ifdef WIN32
+    HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
+    Win32InputHandler inputHandler;
+    inputHandler.Open();
+    HANDLE handleArray[2];
+    handleArray[0] = NormGetDescriptor(instance);
+    handleArray[1] = inputHandler.GetEventHandle();
+#else
+    // On Unix, use a select() call to multiplex input reading and NormSocket handling
     fd_set fdset;
     FD_ZERO(&fdset);
-    
     // Get our input (STDIN) descriptor and set non-blocking
     FILE* inputFile = stdin;
     int inputfd = fileno(inputFile);
@@ -146,15 +180,45 @@ int main(int argc, char* argv[])
         perror("normClient: fcntl(inputfd, O_NONBLOCK) error");
     // Get our NormInstance descriptor
     int normfd = NormGetDescriptor(instance);
+# endif // if/else WIN32
     bool keepGoing = true;
     bool writeReady = false;
-    unsigned int inputLength = 0;
+    int inputLength = 0;
     unsigned int bytesWritten = 0;
     const unsigned int BUFFER_LENGTH = 2048;
     char inputBuffer[BUFFER_LENGTH];
     bool inputNeeded = false;  // will be set to "true" upon CONNECT
     while (keepGoing)
     {
+        bool normEventPending = false;
+        bool inputEventPending = false;
+#ifdef WIN32
+        DWORD handleCount = inputNeeded ? 2 : 1;
+        DWORD waitStatus =  
+            MsgWaitForMultipleObjectsEx(handleCount,   // number of handles in array
+                                        handleArray,   // object-handle array
+                                        INFINITE,           // time-out interval
+                                        QS_ALLINPUT,   // input-event type
+                                        0);
+        if ((WAIT_OBJECT_0 <= waitStatus) && (waitStatus < (WAIT_OBJECT_0 + handleCount)))
+        {
+            if (0 == (waitStatus - WAIT_OBJECT_0))
+                normEventPending = true;
+            else
+                inputEventPending = true;
+        }
+        else if (-1 == waitStatus)
+        {
+            perror("normClient: MsgWaitForMultipleObjectsEx() error");
+            break;
+        }
+        else
+        {
+            // TBD - any other status we should handle?
+            // (e.g. WAIT_TIMEOUT, WAIT_ABANDONED or WAIT_IO_COMPLETION)
+            continue;  // ignore for now
+        }
+#else
         FD_SET(normfd, &fdset);
         int maxfd = normfd;
         if (inputNeeded)
@@ -173,8 +237,37 @@ int main(int argc, char* argv[])
             break;
         }
         if (FD_ISSET(inputfd, &fdset))
+            inputEventPending = true;
+        if (FD_ISSET(normfd, &fdset))
+            normEventPending = true;
+#endif  // if/else WIN32/UNIX
+        
+        if (inputEventPending)
         {
             // Read input into our txBuffer
+#ifdef WIN32
+			inputLength = inputHandler.ReadData(inputBuffer, BUFFER_LENGTH);
+			if (inputLength > 0)
+            {
+				// We got our input
+				bytesWritten = 0;
+                inputNeeded = false;
+            }
+            else if (inputLength < 0)
+            {
+				// Input stream has likely closed, initiate client shutown
+                // TBD - initiate client shutdown
+                //if (NULL == groupAddrPtr)
+                {
+                    fprintf(stderr, "normClient: CLOSING connection to %sserver ...\n",
+                                    (NULL != groupAddrPtr) ? : "multicast " : "");
+                    NormShutdown(normSocket);
+                }
+                inputNeeded = false;  // TBD -should we also fclose(inputFile)???
+                inputHandler.Close();
+            }
+			// else zero bytes read, still need input
+#else
             inputLength = fread(inputBuffer, 1, BUFFER_LENGTH, inputFile);
             if (inputLength > 0)
             {
@@ -185,13 +278,18 @@ int main(int argc, char* argv[])
             else if (feof(inputFile))
             {
                 // TBD - initiate client shutdown
-                if (NULL == groupAddrPtr)
+                //if (NULL == groupAddrPtr)
                 {
-                    fprintf(stderr, "normClient: CLOSING connection to server ...\n");
+                    fprintf(stderr, "normClient: CLOSING connection to %sserver ...\n",
+                                    (NULL != groupAddrPtr) ? "multicast " : "");
                     NormShutdown(normSocket);
                 }
                 inputNeeded = false;  // TBD -should we also fclose(inputFile)???
-                
+                if (stdin != inputFile)
+                {
+                    fclose(inputFile);
+                    inputFile = NULL;
+                }
                 // else stick around to receive stuff from the server
             }
             else if (ferror(inputFile))
@@ -205,14 +303,15 @@ int main(int argc, char* argv[])
                         // input starved, wait for next notification
                         break;
                     default:
-                        perror("normClient: error reading input");
+                        perror("normClient: error reading input?!");
                         break;
                 }
             }
-        }
-        if (FD_ISSET(normfd, &fdset))
-        {
+#endif // if/else WIN32/UNIX
+        }  // end if inputEventPending
         
+        if (normEventPending)
+        {
             // There's a NORM event pending
             NormSocketEvent event;
             if (NormGetSocketEvent(instance, &event))
@@ -225,9 +324,16 @@ int main(int argc, char* argv[])
 
                     case NORM_SOCKET_CONNECT:
                     {
-                        fprintf(stderr, "normClient: CONNECTED to server ...\n");
+						char remoteAddr[16];
+						unsigned int addrLen = 16;
+						UINT16 remotePort;
+						NormGetPeerName(normSocket, remoteAddr, &addrLen, &remotePort);
+						fprintf(stderr, "normClient: CONNECTED to %sserver %s/%hu\n",
+                                (NULL != groupAddrPtr) ? "multicast " : "", serverAddr, remotePort);
                         inputNeeded = true;
                         writeReady = true;
+                        if (trace && (NULL != groupAddrPtr))
+                            NormSetMessageTrace(NormGetSocketMulticastSession(normSocket), true);
                         break;   
                     }
                     case NORM_SOCKET_READ:
@@ -246,7 +352,17 @@ int main(int argc, char* argv[])
                                 continue;
                             }
                             if (bytesRead > 0)
+                            {
+#ifdef WIN32
+                                // Use Win32 WriteFile() so there is no buffer delay
+                                DWORD dwWritten;
+                                WriteFile(hStdout, buffer, bytesRead, &dwWritten, NULL);
+#else
                                 fwrite(buffer, sizeof(char), bytesRead, stdout);
+#endif // if/else WIN32
+                            }
+                            // If less bytes read than request then need
+                            // to wait for next NORM_SOCKET_READ event
                             if (bytesRead < 1024) rxReady = false;
                         }
 				        break;
@@ -255,14 +371,19 @@ int main(int argc, char* argv[])
                         writeReady = true;
 				        break;     
                     case NORM_SOCKET_CLOSING:
-                        fprintf(stderr, "normClient: server CLOSING connection ...\n");
+                        fprintf(stderr, "normClient: %sserver CLOSING connection ...\n",
+                                        (NULL != groupAddrPtr) ? "multicast " : "");
+                        writeReady = false;
                         inputNeeded = false;
 				        break;   
-                    case NORM_SOCKET_CLOSED:
+                    case NORM_SOCKET_CLOSE:
                     {
-                        fprintf(stderr, "normClient: connection to server CLOSED.\n");
+                        fprintf(stderr, "normClient: connection to %sserver CLOSED.\n",
+                                        (NULL != groupAddrPtr) ? "multicast " : "");
+                        writeReady = false;
                         inputNeeded = false;
                         keepGoing = false;
+                        
 				        break;   
                     }
                     case NORM_SOCKET_NONE:
@@ -273,7 +394,7 @@ int main(int argc, char* argv[])
             {
                 fprintf(stderr, "normClient: NormGetSocketEvent() returned false\n");
             }
-        }  // end if FD_ISSET(normfd)
+        }  // end if normEventPending
         
         // If we have data in our inputBuffer and the NormSocket is "writeReady", then send it
         if (writeReady && (inputLength > 0))
@@ -293,4 +414,16 @@ int main(int argc, char* argv[])
             }
         }
     }  // end while(keepGoing)
+    NormClose(normSocket);
+#ifdef WIN32
+    inputHandler.Close();
+#else
+    if ((stdin != inputFile) && (NULL != inputFile))
+    {
+        fclose(inputFile);
+        inputFile = NULL;
+    }
+#endif  // if/else WIN32
+	fprintf(stderr, "normClient: Done.\n");
+	return 0;
 } 

examples/normDataExample.cpp

@@ -3,7 +3,7 @@
 // The "data" sent and received are simple text messages
 // of randomly-varying length.
 
-XXXX - THIS CODE IS NOT YET COMPLETE !!!!!
+XXX - THIS CODE IS NOT YET A COMPLETE EXAMPLE !!!!!
 
 #include <normApi.h>
 
@@ -12,6 +12,8 @@ XXXX - THIS CODE IS NOT YET COMPLETE !!!!!
 #include <protoDebug.h>
 #include <protoTime.h>
 
+#include <stdlib.h>  // for rand(), srand()
+
 void Usage()
 {
     fprintf(stderr, "Usage: normDataExample [send][recv]\n");

examples/normFileRecv.cpp

@@ -58,9 +58,12 @@ int main(int argc, char* argv[])
                                                    6003,
                                                    NORM_NODE_ANY);
     
+    NormSetRxPortReuse(session, true);
+    NormSetMulticastLoopback(session, true);
+    
     // NOTE: These are debugging routines available 
     //       (not necessary for normal app use)
-    // (Need to include "protolib/common/protoDebug.h" for this
+    // (Need to include "common/protoDebug.h" for this
     //SetDebugLevel(2);
     // Uncomment to turn on debug NORM message tracing
     //NormSetMessageTrace(session, true);

examples/normFileSend.cpp

@@ -64,11 +64,14 @@ int main(int argc, char* argv[])
     NormSessionHandle session = NormCreateSession(instance,
                                                   "224.1.2.3", 
                                                    6003,
-                                                   NORM_NODE_ANY);
+                                                   1);
+    
+    NormSetRxPortReuse(session, true);
+    NormSetMulticastLoopback(session, true);
     
     // NOTE: These are some debugging routines available 
     //       (not necessary for normal app use)
-    // (Need to include "protolib/common/protoDebug.h" for this
+    // (Need to include "common/protoDebug.h" for this
     //SetDebugLevel(2);
     // Uncomment to turn on debug NORM message tracing
     NormSetMessageTrace(session, true);

examples/normServer.cpp

@@ -36,14 +36,27 @@
 //    socket" approach could be supported for unicast and SSM streams without too much
 //    difficulty.
 
-#include "normSocket.h"
-#include <arpa/inet.h>  // for inet_ntoa
 #include <stdio.h>      // for fprintf()
 #include <string.h>     // for memcmp()
 #include <map>          // for std::map<>
+#include <assert.h>     // more obvious
+
+#include "normSocket.h"
+#ifdef WIN32
+#include "win32InputHandler.cpp"  // to include full implementation
+#include <Winsock2.h>             // for inet_ntoa()
+#include <Ws2tcpip.h>             // for inet_ntop()
+#else
+#include <arpa/inet.h>  // for inet_ntoa
 #include <sys/select.h>
 #include <fcntl.h>       // for, well, fnctl()
 #include <errno.h>       // obvious child
+#endif // if/else WIN32/UNIX
+
+void Usage()
+{
+    fprintf(stderr, "Usage: normServer [listen [<groupAddr>/]<port>][debug <level>][trace]\n");
+}
 
 // Our "server" indexes clients by their source addr/port
 class ClientInfo
@@ -58,11 +71,11 @@ class ClientInfo
         UINT16 GetPort() const
             {return client_port;}
         
-        const char* GetAddressString() const;
-        void Print(FILE* filePtr) const;
+        const char* GetAddressString();
+        void Print(FILE* filePtr);
         
     private:
-        UINT8               addr_version;  // 4 or 6
+        UINT8               addr_version;    // 4 or 6
         char                client_addr[16]; // big enough for IPv6
         UINT16              client_port;
         
@@ -110,7 +123,7 @@ int ClientInfo::GetAddressFamily() const
 }  // end ClientInfo::GetAddressFamily()
 
 
-const char* ClientInfo::GetAddressString() const
+const char* ClientInfo::GetAddressString() 
 {
     static char text[64];
     text[63] = '\0';    
@@ -123,7 +136,7 @@ const char* ClientInfo::GetAddressString() const
     return text;
 }  // end ClientInfo::GetAddressString() 
 
-void ClientInfo::Print(FILE* filePtr) const
+void ClientInfo::Print(FILE* filePtr)
 {
     char text[64];
     text[63] = '\0';    
@@ -136,9 +149,45 @@ void ClientInfo::Print(FILE* filePtr) const
     fprintf(filePtr, "%s/%hu", text, client_port);
 }  // end ClientInfo::Print()
 
+class Client
+{
+    public:
+        Client(NormSocketHandle clientSocket);
+        ~Client();
+        
+        NormSocketHandle GetSocket() const
+            {return client_socket;}
+        
+        bool GetWriteReady() const
+            {return write_ready;}
+        void SetWriteReady(bool state)
+            {write_ready = state;}
+        
+        unsigned int GetBytesWritten() const
+            {return bytes_written;}
+        void SetBytesWritten(unsigned long numBytes)
+            {bytes_written = numBytes;}
+               
+    private:
+        NormSocketHandle   client_socket;
+        // These are state variables for unicast server -> client communication
+        bool                write_ready;
+        unsigned int        bytes_written;
+        
+};  // end class Client
+
+Client::Client(NormSocketHandle clientSocket)
+ : client_socket(clientSocket), 
+   write_ready(true), bytes_written(0)
+{
+}
+
+Client::~Client()
+{
+}
 
 // C++ map used to index client sessions by the client source addr/port
-typedef std::map<ClientInfo, NormSocketHandle> ClientMap;
+typedef std::map<ClientInfo, Client*> ClientMap;
 
 ClientInfo NormGetClientInfo(NormNodeHandle client)
 {
@@ -146,32 +195,46 @@ ClientInfo NormGetClientInfo(NormNodeHandle client)
     unsigned int addrLen = 16;
     UINT16 port;
     NormNodeGetAddress(client, addr, &addrLen, &port);
-    int addrFamily;
     UINT8 version;
     if (4 == addrLen)
-    {
-        addrFamily = AF_INET;
         version = 4;
-    }
     else
-    {
-        addrFamily = AF_INET6;
         version = 6;
-    }
     return ClientInfo(version, addr, port);
-}  // end NormGetClientInfo()
+}  // end NormGetClientInfo(NormNodeHandle)
 
-NormSocketHandle FindClientSocket(ClientMap& clientMap, const ClientInfo& clientInfo)
+static ClientInfo NormGetSocketInfo(NormSocketHandle socket)
+{
+    char addr[16]; // big enough for IPv6
+    unsigned int addrLen = 16;
+    UINT16 port;
+    NormGetPeerName(socket, addr, &addrLen, &port);
+    UINT8 version;
+    if (4 == addrLen)
+        version = 4;
+    else
+        version = 6;
+    return ClientInfo(version, addr, port);
+}  // end NormGetSocketInfo(NormSocketHandle)
+
+Client* FindClient(ClientMap& clientMap, const ClientInfo& clientInfo)
 {
     ClientMap::iterator it = clientMap.find(clientInfo);
     if (clientMap.end() != it) 
-        return &(it->second);
+        return it->second;
     else
+        return NULL;
+}  // end FindClient()
+
+NormSocketHandle FindClientSocket(ClientMap& clientMap, const ClientInfo& clientInfo)
+{
+    Client* client = FindClient(clientMap, clientInfo);
+    if (NULL == client)
         return NORM_SOCKET_INVALID;
+    else
+        return client->GetSocket();
 }  // end FindClientSocket()
 
-
-
 int main(int argc, char* argv[])
 {
     ClientMap clientMap;
@@ -185,7 +248,6 @@ int main(int argc, char* argv[])
     bool trace = false;
     unsigned int debugLevel = 0;
     
-    
     for (int i = 1; i < argc; i++)
     {
         const char* cmd = argv[i];
@@ -210,11 +272,13 @@ int main(int argc, char* argv[])
             if (1 != sscanf(portPtr, "%hu", &serverPort))
             {
                 fprintf(stderr, "normServer error: invalid <port> \"%s\"\n", portPtr);
+                Usage();
                 return -1;
             }
         }
         else if (0 == strncmp(cmd, "interface", len))
         {
+            // Note the NormSocket code does not yet expose mcast interface
             mcastInterface = argv[++i];
         }
         else if (0 == strncmp(cmd, "trace", len))
@@ -226,37 +290,42 @@ int main(int argc, char* argv[])
             if (1 != sscanf(argv[++i], "%u", &debugLevel))
             {
                 fprintf(stderr, "normServer error: invalid debug level\n");
+                Usage();
                 return -1;
             }
         }
         else
         {
             fprintf(stderr, "normServer error: invalid command \"%s\"\n", cmd);
+            Usage();
             return -1;
         }
     }    
     
-    // For unicast operation in this demo app, the server only "talks back"
-    // to one client on a first come, first serve basis. (Multiple clients
-    // can connect and send data _to_ the server, but in this simple example,
-    // the server only sends to one at a time. For the multicast server case,
-    // the server multicasts to the entire group.
-    NormSocketHandle firstClientSocket = NORM_SOCKET_INVALID;
-    
     NormInstanceHandle instance = NormCreateInstance();
     
-    NormSocketHandle serverSocket = NormListen(instance, serverPort, groupAddrPtr);
+    NormSocketHandle serverSocket = NormOpen(instance);
     
-    if (trace) NormSetMessageTrace(NormGetSession(serverSocket), true);
+    NormListen(serverSocket, serverPort, groupAddrPtr);
+    
+    if (trace) NormSetMessageTrace(NormGetSocketSession(serverSocket), true);
     if (0 != debugLevel) NormSetDebugLevel(debugLevel);
     
     //NormSetDebugLevel(8);
-    //NormSetMessageTrace(NormGetSession(serverSocket), true);
+    //NormSetMessageTrace(NormGetSocketSession(serverSocket), true);
     
+    
+#ifdef WIN32
+    HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
+    Win32InputHandler inputHandler;
+    inputHandler.Open();
+    HANDLE handleArray[2];
+    handleArray[0] = NormGetDescriptor(instance);
+    handleArray[1] = inputHandler.GetEventHandle();
+#else    
     // We use a select() call to multiplex input reading and NormSocket handling
     fd_set fdset;
     FD_ZERO(&fdset);
-    
     // Get our input (STDIN) descriptor and set non-blocking
     FILE* inputFile = stdin;
     int inputfd = fileno(inputFile);
@@ -264,15 +333,48 @@ int main(int argc, char* argv[])
         perror("normClient: fcntl(inputfd, O_NONBLOCK) error");
     // Get our NormInstance descriptor
     int normfd = NormGetDescriptor(instance);
+#endif // if/else WIN32/UNIX
+    
     bool keepGoing = true;
     bool writeReady = false;
-    unsigned int inputLength = 0;
+    int inputLength = 0;
     unsigned int bytesWritten = 0;
     const unsigned int BUFFER_LENGTH = 2048;
     char inputBuffer[BUFFER_LENGTH];
     bool inputNeeded = false;  // will be set to "true" upon CONNECT
+    bool inputClosed = false;
+    unsigned int clientCount = 0;
     while (keepGoing)
     {
+        bool normEventPending = false;
+        bool inputEventPending = false;
+#ifdef WIN32
+        DWORD handleCount = inputNeeded ? 2 : 1;
+        DWORD waitStatus =  
+            MsgWaitForMultipleObjectsEx(handleCount,   // number of handles in array
+                                        handleArray,   // object-handle array
+                                        INFINITE,           // time-out interval
+                                        QS_ALLINPUT,   // input-event type
+                                        0);
+        if ((WAIT_OBJECT_0 <= waitStatus) && (waitStatus < (WAIT_OBJECT_0 + handleCount)))
+        {
+            if (0 == (waitStatus - WAIT_OBJECT_0))
+                normEventPending = true;
+            else
+                inputEventPending = true;
+        }
+        else if (-1 == waitStatus)
+        {
+            perror("normServer: MsgWaitForMultipleObjectsEx() error");
+            break;
+        }
+        else
+        {
+            // TBD - any other status we should handle?
+            // (e.g. WAIT_TIMEOUT, WAIT_ABANDONED or WAIT_IO_COMPLETION)
+            continue;  // ignore for now
+        }
+#else
         FD_SET(normfd, &fdset);
         int maxfd = normfd;
         if (inputNeeded)
@@ -287,12 +389,33 @@ int main(int argc, char* argv[])
         int result = select(maxfd+1, &fdset, NULL, NULL, NULL);
         if (result <= 0)
         {
-            perror("normClient: select() error");
+            perror("normServer: select() error");
             break;
         }
         if (FD_ISSET(inputfd, &fdset))
+            inputEventPending = true;
+        if (FD_ISSET(normfd, &fdset))
+            normEventPending = true;
+#endif  // if/else WIN32/UNIX
+        
+        if (inputEventPending)
         {
             // Read input into our txBuffer
+#ifdef WIN32
+            inputLength = inputHandler.ReadData(inputBuffer, BUFFER_LENGTH);
+            if (inputLength > 0)
+            {
+				// We got our input
+				bytesWritten = 0;
+                inputNeeded = false;
+            }
+            else if (inputLength < 0)
+            {
+                inputHandler.Close();
+                inputClosed = true;
+            }
+            // else zero bytes read, still need input
+#else
             inputLength = fread(inputBuffer, 1, BUFFER_LENGTH, inputFile);
             if (inputLength > 0)
             {
@@ -302,10 +425,13 @@ int main(int argc, char* argv[])
             }
             else if (feof(inputFile))
             {
-                // TBD - initiate server shutdown if it's job is
-                // only until the input is closed
-                // Meanwhile, we just stick around to receive stuff from the clients
-                inputNeeded = false;  // TBD - should also fclose(inputFile)??
+                if (stdin != inputFile)
+                {
+                    fclose(inputFile);
+                    inputFile = NULL;
+                }
+                inputClosed = true;
+                
             }
             else if (ferror(inputFile))
             {
@@ -318,12 +444,35 @@ int main(int argc, char* argv[])
                         // input starved, wait for next notification
                         break;
                     default:
-                        perror("normClient: error reading input");
+                        perror("normServer: error reading input?!");
                         break;
                 }
             }
-        }
-        if (FD_ISSET(normfd, &fdset))
+#endif // if/else WIN32/UNIX
+            if (inputClosed)
+            {
+                inputNeeded = false;
+                // Gracefully shutdown any connected clients
+                // TBD - set state variable to indicate exit upon all closed?
+                if (clientMap.empty())
+                {
+                    keepGoing = false;
+                    continue;
+                }
+                else
+                {
+                    ClientMap::iterator it;
+                    for (it = clientMap.begin(); it != clientMap.end(); it++)
+                    {
+                        Client* client = it->second;
+                        NormSocketHandle clientSocket = client->GetSocket();
+                        NormShutdown(clientSocket);
+                    }
+                }
+            }
+        }  // end if inputEventPending
+        
+        if (normEventPending)
         {
 
             // There's a NORM event pending
@@ -333,37 +482,62 @@ int main(int argc, char* argv[])
                 ClientInfo clientInfo;
                 if (NORM_NODE_INVALID != event.sender)
                     clientInfo = NormGetClientInfo(event.sender);
+                else
+                    clientInfo = NormGetSocketInfo(event.socket);
                 switch (event.type)
                 {           
                     case NORM_SOCKET_ACCEPT:
                     {
                         if (event.socket == serverSocket)
                         {
+                            
+                            // TBD - now that the NormSocket code manages its own client_table by remote addr/port
+                            // and should eliminate the 'duplicative' connect itself, we can just keep track
+                            // of client sockets by their NormSocketHandle
+                            
                             // Possibly a new "client" connecting to our "server"
                             // First confirm that this really is a new client.
                             if (NORM_SOCKET_INVALID != FindClientSocket(clientMap, clientInfo))
                             {
                                 // We think we're already connected to this client
-                                fprintf(stderr, "normServer: duplicative %s client ...\n",
-                                        (NORM_REMOTE_SENDER_NEW == event.event.type) ? "new" : "reset");
+                                fprintf(stderr, "normServer: duplicative %s from client %s/%hu...\n",
+                                        (NORM_REMOTE_SENDER_NEW == event.event.type) ? "new" : "reset",
+                                        clientInfo.GetAddressString(), clientInfo.GetPort());
                                 continue;
                             }
                             NormSocketHandle clientSocket = NormAccept(serverSocket, event.sender);
                             
-                            // TBD - For multicast, if we are sending a tx_stream, we could flush it here to 
+                            Client* client = new Client(clientSocket);
+                            if (NULL == client)
+                            {
+                                perror("normServer: new Client() error");
+                                NormClose(clientSocket);
+                                continue;
+                            }
+
+							// TBD - For multicast, if we are sending a tx_stream, we could flush it here to 
                             // bring the new receiver "up to date" ... probably would be best to
                             // do this on some sort of timer-basis in the case of a bunch of receivers
                             // joining in a short window of time ...
                             
                             if (trace)  // note we're already tracing the mcast session
-                                NormSetMessageTrace(NormGetSession(clientSocket), true);
-                            //NormSetMessageTrace(NormGetSession(clientSocket), true);
-                            clientMap[clientInfo] = clientSocket;
+                                NormSetMessageTrace(NormGetSocketSession(clientSocket), true);
+                            //NormSetMessageTrace(NormGetSocketSession(clientSocket), true);
+                            clientMap[clientInfo] = client;
+                            
+                            // ACCEPTED is good as CONNECTED, so enable writing right away
+                            client->SetWriteReady(true);
+                            if (0 == clientCount)
+                            {
+                                // We have at least one client, so lets serve up some juicy input
+                                inputNeeded = true;
+                                writeReady = true;
+                            }
+                            clientCount++;
                             fprintf(stderr, "normServer: ACCEPTED connection from %s/%hu\n",
                                             clientInfo.GetAddressString(), clientInfo.GetPort());
-                            // We have at least one client, so lets serve up some juicy input
-                            inputNeeded = true;
-                            writeReady = true;
+                            // Note that an ACCEPTED socket is essentially CONNECTED, so we could
+                            // go ahead and set writeReady to true, etc here
                         }
                         else
                         {
@@ -375,8 +549,8 @@ int main(int argc, char* argv[])
                     {
                         fprintf(stderr, "normServer: CONNECTED to %s/%hu ...\n",
                                         clientInfo.GetAddressString(), clientInfo.GetPort());
-                        if (NORM_SOCKET_INVALID == firstClientSocket)
-                                firstClientSocket = event.socket;
+                        Client* client = FindClient(clientMap, clientInfo);
+                        assert(NULL != client);
                         break;   
                     }
                     case NORM_SOCKET_READ:
@@ -395,36 +569,60 @@ int main(int argc, char* argv[])
                                     continue;
                                 }
                                 if (bytesRead > 0)
+                                
+                                {
+#ifdef WIN32
+                                    // Use WriteFile() so there is no buffer delay
+                                    DWORD dwWritten;
+                                    WriteFile(hStdout, buffer, bytesRead, &dwWritten, NULL);
+#else
                                     fwrite(buffer, sizeof(char), bytesRead, stdout);
+#endif // if/else WIN32
+                                }
                                 if (bytesRead < 1024) rxReady = false;
                             }
+                            // Following lines are test code (to immediately close connection after first reaad)
+                            // (tests server stale connection "reject" command)
+                            //clientMap.erase(clientInfo);
+                            //NormClose(event.socket);
 				        break;
                     }      
                     case NORM_SOCKET_WRITE:
-                        // We only demo server data transmission for the multicast server case
-                        // (see comment below)
-                        if ((NULL != groupAddrPtr) || (event.socket == firstClientSocket)) 
+                    {
+                        if (NULL != groupAddrPtr)
+                        {
+                            // We are a multicast server
                             writeReady = true;
-				        break;     
+                        }
+                        else
+                        {
+                            Client* client = FindClient(clientMap, clientInfo);
+                            assert(NULL != client);
+                            client->SetWriteReady(true);
+                        }
+                        break;     
+                    }
                     case NORM_SOCKET_CLOSING:
+                    {
                         fprintf(stderr, "normServer: client %s/%hu CLOSING connection ...\n",
                                             clientInfo.GetAddressString(), clientInfo.GetPort());
-                        if (event.socket == firstClientSocket)
-                            firstClientSocket = NORM_SOCKET_INVALID;
-                        
-				        break;   
-                    case NORM_SOCKET_CLOSED:
+                        Client* client = FindClient(clientMap, clientInfo);
+                        assert(NULL != client);
+                        client->SetWriteReady(false);
+                        break;   
+                    }
+                    case NORM_SOCKET_CLOSE:
                     {
                         fprintf(stderr, "normServer: connection to client %s/%hu CLOSED ...\n",
                                             clientInfo.GetAddressString(), clientInfo.GetPort());
-                        if (event.socket == firstClientSocket)
-                            firstClientSocket = NORM_SOCKET_INVALID;
                         clientMap.erase(clientInfo);
-				        break;   
+                        NormClose(event.socket);
+                        if (inputClosed && clientMap.empty())
+                            keepGoing = false;
+                        break;   
                     }
                     case NORM_SOCKET_NONE:
 				        break; 
-                        break;
                 }  // end switch(event.type)   
             }
             else
@@ -433,31 +631,96 @@ int main(int argc, char* argv[])
             }
         }  // end if FD_ISSET(normfd)
         
-        // For our _multicast_ "normServer" example, the server can send to the group                        
-        // (For a unicast "normServer", we would need to do something more complex
-        //  to manage sending data to each individual client that connects to us.
-        //  So, for the moment, the unicast "normServer" only sends to the "firstClientSocket"
-        NormSocketHandle sendSocket = (NULL != groupAddrPtr) ? serverSocket : firstClientSocket;
-        if (NORM_SOCKET_INVALID != sendSocket)
+        // If the normServer app has unsent data from STDIN, send it to the clients.
+        // Note that a _multicast_ server multicasts to all clients at once while a 
+        // unicast server sends to each connected client individually
+        if ((inputLength > 0) && !inputNeeded)
         {
-            if (writeReady && (inputLength > 0))
+            // There is inputBuffer data for the server to send to the client(s)
+            if (NULL == groupAddrPtr)
             {
-                // We have data in our inputBuffer and the NormSocket is "writeReady", so send it
-                bytesWritten += NormWrite(sendSocket, inputBuffer + bytesWritten, inputLength - bytesWritten);
-                if (bytesWritten < inputLength)
+                // Unicast the data to each connected client individually by iterating
+                // over the clientMap and sending data out to each pending client socket
+                // (inputNeeded is reset to "true" when _all_ clients are non-pending)
+                bool clientPending = false;
+                ClientMap::iterator it;
+                for (it = clientMap.begin(); it != clientMap.end(); it++)
                 {
-                    // Couldn't write whole inputBuffer, need to wait for NORM_SOCKET_WRITE event
-                    writeReady = false;
+                    Client* client = it->second;
+                    if (!client->GetWriteReady())
+                    {
+                        clientPending = true;
+                        continue;
+                    }
+                    unsigned int numBytes = client->GetBytesWritten();
+                    if (numBytes < inputLength)
+                    {
+                        NormSocketHandle clientSocket = client->GetSocket();
+                        bytesWritten += NormWrite(clientSocket, inputBuffer + numBytes, inputLength - numBytes);
+                        client->SetBytesWritten(numBytes);
+                        if (bytesWritten < inputLength)
+                        {
+                            // Couldn't write whole inputBuffer, need to wait for NORM_SOCKET_WRITE event
+                            // for this client socket
+                            client->SetWriteReady(false);
+                            clientPending = true;
+                        }
+                        else
+                        {
+                            // inputBuffer has been completely written to this client
+                            NormFlush(clientSocket);
+                        }
+                    }
                 }  
-                else
+                if (!clientPending)
                 {
-                    // inputBuffer has been completely written
+                    // inputBuffer was sent to _all_ clients, so reset 
                     inputLength = 0;
                     inputNeeded = true;
-                    NormFlush(sendSocket);
+                    // Reset all client "bytes_written" to zero for next chunk of data
+                    for (it = clientMap.begin(); it != clientMap.end(); it++)
+                        it->second->SetBytesWritten(0);
                 }
             }
-        }
+            else
+            {
+                // Multicast the data out the "serverSocket" to all clients
+                // (Use the 'writeReady' and 'bytesWritten' state variables)
+                NormSocketHandle sendSocket = serverSocket;
+                if (NORM_SOCKET_INVALID != sendSocket)
+                {
+                    if (writeReady && (inputLength > 0))
+                    {
+						// We have data in our inputBuffer and the NormSocket is "writeReady", so send it
+                        bytesWritten += NormWrite(sendSocket, inputBuffer + bytesWritten, inputLength - bytesWritten);
+                        if (bytesWritten < inputLength)
+                        {
+                            // Couldn't write whole inputBuffer, need to wait for NORM_SOCKET_WRITE event
+                            writeReady = false;
+                        }
+                        else
+                        {
+                            // inputBuffer has been completely written
+                            inputLength = 0;
+                            inputNeeded = true;
+							NormFlush(sendSocket);
+                        }
+                    }
+                }
+            }
+        }  // end if ((inputLength > 0) && !inputNeeded)
+        
         
     }  // end while (keepGoing)
+#ifdef WIN32
+    inputHandler.Close();
+#else
+    if ((stdin != inputFile) && (NULL != inputFile))
+    {
+        fclose(inputFile);
+        inputFile = NULL;
+    }
+#endif // if/else WIN32
+    NormClose(serverSocket);
+    serverSocket = NORM_SOCKET_INVALID;
 }  // end main()

examples/normSocket.h

@@ -19,7 +19,7 @@
 //
 // 0) Make "NormInstanceHandle" optional for Listen/Connect/Accept
 //    (for eventual blocking usage, it wouldn't be needed)
-// 1) Support asymmetric server->multicast, client->unicast model
+// 1) Support asymmetric server->multicast, client->unicast model (DONE)
 // 2) Provide non-blocking option for read/write calls
 // 3) leverage ICMP (e.g., port unreachable) feedback to detect connect() failure (DONE)
 // 4) provide means for connect() to time out??? (DONE)
@@ -41,6 +41,12 @@
 // f) Buffer size (sender, stream, receiver)
 // g) cache bounds / limits
 
+
+#ifdef WIN32
+#include <BaseTsd.h>    // for SSIZE_T
+typedef SSIZE_T ssize_t;
+#endif // WIN32
+
 #include "normApi.h"
 
 typedef const void* NormSocketHandle;
@@ -55,44 +61,108 @@ extern const double NORM_DEFAULT_CONNECT_TIMEOUT;
 
 // Main NormSocket API Functions
 
-NormSocketHandle NormOpen(NormInstanceHandle instance = NORM_INSTANCE_INVALID);
+NormSocketHandle NormOpen(NormInstanceHandle instance);
 
-NormSocketHandle NormListen(NormInstanceHandle instance, UINT16 serverPort, const char* groupAddr = NULL);
+bool NormListen(NormSocketHandle    normSocket,
+                UINT16              serverPort,
+                const char*         groupAddr = NULL,
+                const char*         serverAddr = NULL);
 
-NormSocketHandle NormConnect(NormInstanceHandle instance, 
-                             const char*        serverAddr, 
-                              UINT16            serverPort, 
-                             const char*        groupAddr = NULL, 
-                             NormNodeId         clientId = NORM_NODE_NONE);
+bool NormConnect(NormSocketHandle   normSocket,
+                 const char*        serverAddr,
+                 UINT16             serverPort,
+                 UINT16             localPort = 0,
+                 const char*        groupAddr = NULL, 
+                 NormNodeId         clientId = NORM_NODE_ANY);
 
-NormSocketHandle NormAccept(NormSocketHandle serverSocket, NormNodeHandle clientNode, NormInstanceHandle instance = NORM_INSTANCE_INVALID);
+NormSocketHandle NormAccept(NormSocketHandle    serverSocket,
+                            NormNodeHandle      clientNode,
+                            NormInstanceHandle  instance = NORM_INSTANCE_INVALID);
+
+void NormReject(NormSocketHandle    serverSocket,
+                NormNodeHandle      clientNode);
         
 void NormShutdown(NormSocketHandle normSocket);
 
 void NormClose(NormSocketHandle normSocket);
 
-ssize_t NormRead(NormSocketHandle normSocket, void *buf, size_t nbyte);
+ssize_t NormRead(NormSocketHandle normSocket, void* buf, size_t nbyte);
 
-ssize_t NormWrite(NormSocketHandle normSocket, const void *buf, size_t nbyte);
+ssize_t NormWrite(NormSocketHandle normSocket, const void* buf, size_t nbyte);
 
 int NormFlush(NormSocketHandle normSocket);
 
-// Helper API functions
+/* 
 
-NormSessionHandle NormGetSession(NormSocketHandle normSocket);
-NormSessionHandle NormGetMulticastSession(NormSocketHandle normSocket);
+// These calls _will_ be used for file/data object and message-stream delivery. (Sketching the calls out for now)
+
+NormObject NormSendFile(NormSocketHandle normSocket, 
+                        const char*      filePath, 
+                        const char*      infoPtr, 
+                        ssize_t          infoLen);
+
+NormObject NormSendData(NormSocketHandle normSocket, 
+                        const char*      data,       // NormSocket will copy the "data" unlike NORM low-level API
+                        ssize_t          nbyte, 
+                        const char*      infoPtr, 
+                        ssize_t          infoLen); 
+ 
+// Notification events will indicate receipt of file/data objects.  Are additional NormSocket API calls
+// needed to manage receipt?
+
+ssize_t NormSendMsg(NormSocketHandle normSocket, const char* data, ssize_t nbyte, bool flush);
+ 
+ssize_t NormRecvMsg(NormSocketHandle normSocket, const char* data, ssize_t nbyte); 
+
+*/
+
+// NormSocket helper functions
+
+void NormSetSocketUserData(NormSocketHandle normSocket, const void* userData);
+const void* NormGetSocketUserData(NormSocketHandle normSocket);
+        
+NormInstanceHandle NormGetSocketInstance(NormSocketHandle normSocket);
+NormSessionHandle NormGetSocketSession(NormSocketHandle normSocket);
+NormSessionHandle NormGetSocketMulticastSession(NormSocketHandle normSocket);
+void NormGetPeerName(NormSocketHandle normSocket, char* addr, unsigned int* addrLen, UINT16* port);
+NormObjectHandle NormGetSocketTxStream(NormSocketHandle normSocket);
+NormObjectHandle NormGetSocketRxStream(NormSocketHandle normSocket);
+
+void NormSetSocketFlowControl(NormSocketHandle normSocket, bool enable);
+void NormSetSocketTrace(NormSocketHandle normSocket, bool enable);
 
 typedef enum NormSocketEventType
 {
-    NORM_SOCKET_NONE = 0,
+    NORM_SOCKET_NONE = 0,       // applications should generally ignore these
     NORM_SOCKET_ACCEPT,         // only issued for listening server sockets
     NORM_SOCKET_CONNECT,        // notification confirming connection to server
     NORM_SOCKET_READ,           // edge-triggered notification that socket is ready for reading
     NORM_SOCKET_WRITE,          // edge-triggered notification that socket is ready for writing
     NORM_SOCKET_CLOSING,        // indicates remote endpoint is closing socket (only read data at this point)
-    NORM_SOCKET_CLOSED          // indicates socket is now closed (invalid for further operations)
+    NORM_SOCKET_CLOSE           // indicates socket is now closed (invalid for further operations)
 } NormSocketEventType;
     
+    
+// Right now, these options MUST be set _after_ NormOpen() 
+// but _before_ NormConnect() or NormListen()
+// (This will be expanded, perhaps to include parameters 
+//  that can be changed after connection setup)
+typedef struct 
+{
+    UINT16          num_data;
+    UINT16          num_parity;
+    UINT16          num_auto;
+    UINT16          segment_size;
+    unsigned int    buffer_size;      // used for both FEC and stream buffer sizing
+    bool            silent_receiver;  // not yet used  (maybe should be nack_mode instead)
+    int             max_delay;        // not yet used
+} NormSocketOptions;
+
+void NormGetSocketOptions(NormSocketHandle normSocket, NormSocketOptions* options);
+bool NormSetSocketOptions(NormSocketHandle normSocket, NormSocketOptions* options);
+    
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wpedantic"
 typedef struct
 {
     NormSocketEventType type;
@@ -112,9 +182,9 @@ typedef struct
         };
     };
 } NormSocketEvent;
+#pragma GCC diagnostic pop
     
 bool NormGetSocketEvent(NormInstanceHandle normInstance, NormSocketEvent* event, bool waitForEvent = true);
 
 
-
 #endif // !_NORM_SOCKET

examples/normSocketNotes.txt

@@ -0,0 +1,155 @@
+NormSocket API Extension Notes
+
+The NormSocket API (currently defined in examples/normSocket.h) is an in-development extension to the
+base (low-level) NORM API (defined in include/normApi.h) that provides a more familiar (socket-like),
+easier-to-use API for some specific NORM use patterns.  The use patterns supported here are those of a
+client-server paradigm where "clients" have individual, reliable connections to the server and the
+"server" can have either a one-to-many multicast connection or individual unicast connections to the set
+of clients.  The paradigm follows a connection-oriented socket model where the server "listens" for
+incoming connections initiated by clients.  The low-level NORM API is used to establish multiple NORM
+sessions as needed to support the necessary data exhange relationships. Since this is a
+connection-oriented model, explicit ACK-based flow control used for both client-to-server and
+server-to-client(s) data transfers.  This is a more controlled model of NORM use as compared to the
+looser "connection-free" model of operation NORM use has typically observed.
+
+The NormSocket API code makes careful use of operating system socket-binding to enable port reuse and
+simpler session establishment where clients can "connect" to the server listening on a well-known port
+as compared to typical base NORM API use where participants can exchange data on a a priori known common
+port number (and group address if multicast).  However, the clients do need to know the unicast address
+of the server and the paradigm is somewhat that of Single-Source Multicast (SSM).  Client application
+can, of course, using multiple NormSocket instances connect to multiple multicast (or unicast) servers.. 
+THere
+A server application can use the NormListen() call to establish a session that listens for incoming
+connection requests, Client applications can use the NormConnect() call to initiate a connection to a
+listening server.  In response, the server application can use NormAccept() to establish connections with
+the clients.  
+
+At the moment a NormGetSocketEvent() (analogous to the NormGetNextEvent() call in the base NORM API) can
+be used to provide cues to changes in NormSocket state (notification of incoming connections, connection
+establishment, read/write readiness, etc.).  A descriptor is provided for asynchronous I/O notification
+of pending events.  At the moment, the default behavior is a single NormInstance, and hence a single
+descriptor, can be used for multiple NormSockets, but it is possible to call NormCreateInstance() and
+associate a separate NormInstance (and hence separate descriptor) with each NormSocket that is
+listending, connected or accepted if desired.  
+
+As the NormSocket API extension matures, more examples and documentation will be provided as well as
+incorporating it (possibly directly) into the "normApi.h" header file and NORM library.  At the moment,
+"examples/normClient.cpp" and "examples/normServer.cpp" illustrate (and test) basic usage of the API. 
+The "normServer" can be run as a unicast server using the syntax:
+
+./normServer listen <serverPort>
+
+or a multicast server using the syntax:
+
+./normServer listen <groupAddr>/<serverPort>
+
+Corresponding "normClient" instances can connect to unicast server using the syntax:
+
+./normClient connect <serverAddr>/<serverPort>
+
+or to a multicast server using the syntax:
+
+./normClient connect <serverAddr>/<serverPort>,<groupAddr>
+
+There are some additional command-line options shared by these examples such as "trace" to enable NORM's
+message trace output and "debug <debugLevel>" to invoke different levels of debug output.  Note that an
+option to specify the multicast interface is planned but not yet implemented in these examples.
+Similarly, an option to set a specific unicast server binding will also be provided.
+
+TODO (not in priority order):
+
+1) When client initiates connection and server-listener receives NORM_CMD(CC), add code
+   to directly "inject" the NORM_CMD(CC) into the newly created server-side client
+   session such that NORM_ACK is immediately sent.  This will give client a more immediate
+   RTT measurement, etc.
+   
+2) Add options to set transmit rate, congestion control, etc options via NormSocketHandle
+   (current code works at default tx rate of 64 kbps)
+   
+3) The server-listener session should be modified so NORM_DATA messages are ignored.  Also,
+   NormSetServerListener() could also set the session a silent receiver, too?
+      
+4) Add support for additional transport modes in addition to current byte stream:
+`   1) Message stream
+    2) NORM_OBJECT_DATA (w/ data copy option?)
+    3) NORM_OBJECT_FILE
+    
+5) Add option to specify multicast "interface" and use in examples.
+
+6) Enable setting a specific unicast server binding address.    
+
+
+===========================================================================
+MISC. DEVELOPMENT NOTES
+
+// Notional  program outline
+// (optional mcastAddr iff asymm multicast/unicast (i.e. SSM))
+
+normSocket = NormListen(port, [mcastAddr])
+
+_or_
+
+normSocket = NormConnect(addr, port, [mcastAddr])
+
+while NormGetSocketEvent(normSocket)
+{
+
+    switch (event.GetType())
+        NORM_ACCEPT:
+            clientSocket = NormAccept(normSocket, client, [instance]);  // rx-only for asymm mcast case
+            
+        NORM_CONNECTED:
+            // in response to NormConnect when "server" is detected (NEW_REMOTE_SENDER)
+            
+        NORM_TX_READY:
+            upon QUEUE_VACANCY, etc ... socket _might_ be write() ready
+            
+        NORM_RX_READY:
+
+            upon RX_OBJECT_NEW, RX_OBJECT_UPDATE ... socket _might_ be read() ready
+            
+        NORM_CLOSING: ???
+            upon stream end received from remote sender
+            
+        NORM_CLOSED: ???
+            upon stream end _and_ ack to our stream end
+}
+
+
+
+ *** Server listen on a port (optional mcast address):
+
+- Create NormSession that receive-only on given port (if it's asymm mcast, then sender enabled, too)
+
+- For each client (detected via NEW_REMOTE_SENDER yielding a NormSocket::NORM_ACCEPT event)
+      Create tightly binded session for rx (and tx if unicast) ... this will fail if binding already exists
+      
+
+*** Client connect
+
+- Create single socket (common ephemeral tx/rx port) NormSession for server destination addr/port.
+- Call NormStartSender() to initiate probing of server
+- NORM_CONNECTED upon NEW_REMOTE_SENDER (from server) ???
+
+
+QUESTIONS:
+
+1) How does a server reject a connection request?  (for now, just ignore)
+
+2) How is a "connection" gracefully terminated? (NormShutdown() vs. NormClose()?)
+
+
+NormClose(graceful = false) ... 
+ a) stop sender/receiver, destroy everything
+
+NormShutdown(graceful = true) ... 
+ a) close stream (gracefully, w/ ack request)
+ b) upon ack receipt (or timeout) _and_ rx stream end, destroy everything
+ 
+
+Upon receipt of rx stream end, close tx stream gracefully ...
+
+ initiator -> stream end; upon ack set tx_stream = NULL and wait for rx_stream end
+ 
+ receptor -> upon rx_stream end, initiate tx_stream shutdown and wait for ack
+ 

examples/normStreamRecv.cpp

@@ -48,7 +48,7 @@ int main(int argc, char* argv[])
     
     // NOTE: These are debugging routines available 
     //       (not necessary for normal app use)
-    // (Need to include "protolib/common/protoDebug.h" for this
+    // (Need to include "common/protoDebug.h" for this
     NormSetDebugLevel(3);
     // Uncomment to turn on debug NORM message tracing
     NormSetMessageTrace(session, true);

examples/python/debugStats.py

@@ -29,7 +29,7 @@ def main(argv):
     (opts, args) = get_option_parser().parse_args(argv)
 
     if len(args) != 1:
-        print USAGE
+        print(USAGE)
         return 1
 
     pipep = PipeParser(opts.pipe)
@@ -53,7 +53,7 @@ def main(argv):
         except IndexError:
             continue
         pipep.reports.task_done()
-    print "Exiting..."
+    print("Exiting...")
 #    g.reset()
 #    g.close()
 #    del g

examples/python/normCast.py

@@ -0,0 +1,425 @@
+'''
+
+LD_LIBRARY_PATH=/home/honglei/norm_test/
+
+'''
+# sender/Win10: 
+#--id 34252 --send "C:\Users\Admin\AppData\Roaming\Wing Pro 9" --repeat 1 --addr 224.1.2.4/6003 --txaddr 10.65.39.191/8002 --cc rate 5000 --ack  auto --grttprobing active --debug 4
+# receiver/Debian10 
+#--id 172042016 --recv "~/recvFiles" --addr 224.1.2.4/6003  --interface ens33  --unicast_nack --debug 4
+import sys
+import os
+if os.name =='nt':
+    os.add_dll_directory(os.path.dirname(__file__))
+import argparse
+import traceback
+import logging
+from typing import Optional
+
+import enum
+
+class CCEnum(enum.Enum):
+    CC = "cc"
+    CCE = "cce"
+    CCL = "ccl"
+    Fixed = "rate"
+
+class FlushEnum(enum.Enum):
+    none='none'
+    passive='passive'
+    active='active'
+
+class GrttEnum(enum.Enum):
+    none='none'
+    passive='passive'
+    active='active'  
+
+
+class EnumAction(argparse.Action):
+    """
+    Argparse action for handling Enums
+    from https://stackoverflow.com/questions/43968006/support-for-enum-arguments-in-argparse
+    """
+    def __init__(self, **kwargs):
+        # Pop off the type value
+        enum_type = kwargs.pop("type", None)
+
+        # Ensure an Enum subclass is provided
+        if enum_type is None:
+            raise ValueError("type must be assigned an Enum when using EnumAction")
+        if not issubclass(enum_type, enum.Enum):
+            raise TypeError("type must be an Enum when using EnumAction")
+
+        # Generate choices from the Enum
+        #kwargs.setdefault("choices", tuple(e.value for e in enum_type)) #if e!=CCEnum.rate else "rate <bitsPerSecond>" 
+
+        super(EnumAction, self).__init__(**kwargs)
+
+        self._enum = enum_type
+
+    def __call__(self, parser, namespace, values:list|str, option_string=None):
+        # Convert value back into an Enum
+        if isinstance(values,list):
+            if len(values) ==2:
+                value = int(values[1])
+            elif len(values) ==1:
+                value = self._enum( values[0].lower())
+        else:
+            value = self._enum(values.lower() )
+        setattr(namespace, self.dest, value)
+
+'''
+    norm.cpp: 
+
+    fprintf(stderr, "Usage: normCast {send <file/dir list> &| recv <rxCacheDir>} [silent {on|off}]\n"
+                    "                [repeat <interval> [updatesOnly]] [id <nodeIdInteger>]\n"
+                    "                [addr <addr>[/<port>]][txaddr <addr>[/<port>]][txport <port>]\n"
+                    "                [interface <name>][reuse][loopback]\n"
+                    "                [ack auto|<node1>[,<node2>,...]] [segment <bytes>]\n"
+                    "                [block <count>] [parity <count>] [auto <count>]\n"
+                    "                [cc|cce|ccl|rate <bitsPerSecond>] [rxloss <lossFraction>]\n"
+                    "                [txloss <lossFraction>] [flush {none|passive|active}]\n"
+                    "                [grttprobing {none|passive|active}] [grtt <secs>]\n"
+                    "                [ptos <value>] [processor <processorCmdLine>] [saveaborts]\n"
+                    "                [sentprocessor <processorCmdLine>]\n"
+                    "                [purgeprocessor <processorCmdLine>] [buffer <bytes>]\n"
+                    "                [txsockbuffer <bytes>] [rxsockbuffer <bytes>]\n"
+                    "                [debug <level>] [trace] [log <logfile>]\n");
+
+'''
+# Instantiate the parser
+
+def get_arg_options():
+    parser = argparse.ArgumentParser(description='Optional app description')
+
+    ##    Required positional argument
+    parser.add_argument('--id', type=int, required=True,
+                        help='id <nodeIdInteger>')
+
+    parser.add_argument('--send', '-S',nargs='+',
+                        help='send <file/dir list>')
+
+    parser.add_argument('--recv', '-R',
+                        help='recv <rxCacheDir>')
+
+    parser.add_argument('--repeat','-r', type=int, default=60,
+                        help='[repeat <interval> [updatesOnly]]')    
+
+    parser.add_argument('--interface', '-i',
+                        help='addr <addr>[/<port>]')
+
+    parser.add_argument('--addr', '-a',
+                        help='addr <addr>[/<port>]')
+
+    parser.add_argument('--txaddr', '-t',
+                        help='txaddr <addr>[/<port>]')
+    parser.add_argument('--txport', type=int, 
+                        help='txport <port>')
+
+    parser.add_argument('--ack', nargs='+', #nargs='?',
+                        help='ack auto|<node1>[,<node2>,...')
+
+
+    #To create an option that needs no value, set the action [docs] of it to 'store_const', 'store_true' or 'store_false'
+    parser.add_argument('--loopback', action='store_true',help='')
+    parser.add_argument('--segment', type=int,
+                        help='segment <bytes>')
+    parser.add_argument('--block', type=int,
+                        help='[block <count>] ')
+    parser.add_argument('--parity', type=int,
+                        help='[parity <count>]')
+    parser.add_argument('--auto', type=int,
+                        help='[auto <count>]')
+
+
+    parser.add_argument('--grttprobing', type=GrttEnum,  action=EnumAction,
+                        help='[grttprobing {none|passive|active}]') 
+    parser.add_argument('--grtt', type=int, help='[grtt <secs>]')    
+
+    parser.add_argument('--ptos', type=int,
+                        help='[ptos <value>]')
+
+    parser.add_argument('--flush', type=FlushEnum, action=EnumAction,
+                        help='[flush {none|passive|active}]')
+
+    parser.add_argument('--silent', action='store_true',help='')
+    parser.add_argument('--unicast_nack', action='store_true', help='')
+
+    parser.add_argument('--txloss', type=float, help='[txloss <lossFraction>]')
+    parser.add_argument('--rxloss', type=float, help='[rxloss <lossFraction>]')
+
+    parser.add_argument('--buffer', type=int, help='[buffer <bytes>]')
+    parser.add_argument('--txsockbuffer', type=int, help=' [txsockbuffer <bytes>]')
+    parser.add_argument('--rxsockbuffer', type=int, help='[rxsockbuffer <bytes>]')
+    parser.add_argument('--debug', type=int, help='[debug <level>]')         
+    parser.add_argument('--cc',type=CCEnum, nargs='+', action=EnumAction,
+                        help='[cc|cce|ccl|rate <bitsPerSecond>] '
+                        )
+
+    args = parser.parse_args()
+    return args
+
+import pynorm
+
+def create_session(instance:pynorm.Instance, opts:argparse.Namespace):
+    '''
+       use opts to create a NORM session
+    '''
+    sessionAddr ='224.1.2.3'
+    sessionPort = 6003
+
+    if opts.addr:
+        args = opts.addr.split('/')
+        if len(args) >0:
+            sessionAddr = args[0]
+        if len(args) ==2:
+            sessionPort = int(args[1])
+    session = instance.createSession(sessionAddr, sessionPort, localId=opts.id)
+
+
+    sessionTxAddr:str = None
+    sessionTxPort = 8002
+    if opts.txaddr:
+        args = opts.txaddr.split('/')
+        if len(args) >0:
+            sessionTxAddr = args[0]
+        if len(args) ==2:
+            sessionTxPort = int(args[1])
+
+    if sessionTxAddr:        
+        session.setTxPort(txPort=sessionTxPort, txBindAddr=sessionTxAddr)
+
+    if opts.interface:
+        session.setMulticastInterface(opts.interface)    
+    if opts.loopback:
+        session.setLoopback(loopbackEnable=True)
+
+    if opts.silent:
+        session.setSilentReceiver(True)
+
+    if opts.unicast_nack:
+        session.setDefaultUnicastNack(enable=True)
+
+    autoAck:bool = False  
+    ackingNodeList:list[int] =[]
+    if opts.ack:
+        if 'auto' == opts.ack[0]:
+            autoAck = True
+            session.setAutoAckingNodes(pynorm.TrackingStatus.RECEIVERS)
+        else:
+            ackingNodeList = [ int(i) for i in opts.ack]
+
+            for ackingNondeID in ackingNodeList:
+                session.addAckingNode(ackingNondeID)
+
+    if opts.txloss and opts.txloss>0:
+        session.setTxLoss(opts.txloss)
+
+    if opts.rxloss and opts.rxloss>0:
+        session.setRxLoss(opts.rxloss)    
+
+    # Congestion Control
+    if opts.cc:
+        if isinstance(opts.cc,int):
+            session.setTxRate(opts.cc)
+            session.setEcnSupport(ecnEnable=False)
+        else:
+            if opts.cc == CCEnum.CC:  # default TCP-friendly congestion control
+                session.setEcnSupport(ecnEnable=False)
+            elif opts.cc == CCEnum.CCE: #"wireless-ready" ECN-only congestion control
+                session.setEcnSupport(ecnEnable=True, ignoreLoss=True)
+            elif opts.cc == CCEnum.CCL: # "loss tolerant", non-ECN congestion control
+                session.setEcnSupport(ecnEnable=False, ignoreLoss=False, tolerateLoss=True)            
+
+    defaultBufferSpace = 64*1024*1024
+    defaultTxSocketBufferSize = 4*1024*1024
+    defaultRxSocketBufferSize = 6*1024*1024
+
+    bufferSpace = opts.buffer if opts.buffer else defaultBufferSpace
+    if opts.send:
+        if opts.ack:
+            session.setFlowControl(flowControlFactor=0) #// ack-based flow control enabled on command-line, so disable timer-based flow control
+        session.setBackoffFactor(0)
+        #FEC
+        session.startSender(sessionId=opts.id, 
+                            bufferSpace=bufferSpace, 
+                            segmentSize=opts.segment if opts.segment else 1400, 
+                            blockSize=opts.block if opts.block else 64, 
+                            numParity=opts.parity if opts.parity else 0,
+                            fecId=0)
+
+        if opts.auto:
+            session.setAutoParity(opts.auto)
+        session.setTxSocketBuffer(opts.txsockbuffer if opts.txsockbuffer else defaultTxSocketBufferSize )
+
+    if opts.recv:
+        session.startReceiver(bufferSpace=bufferSpace)
+        session.setRxSocketBuffer(opts.rxsockbuffer if opts.rxsockbuffer else defaultRxSocketBufferSize )
+
+    if opts.grttprobing:
+        session.setGrttProbingMode(pynorm.ProbingMode[ opts.grttprobing.value.upper() ]  )    
+
+    return session
+
+from pynorm import EventType   
+
+from typing import Iterable
+def listFiles( dirFileList:list[str]) -> Iterable[str]:
+    for dirFile in dirFileList:
+        if os.path.isfile(dirFile):
+            yield dirFile
+        elif os.path.isdir(dirFile):
+            for filename in os.scandir(dirFile):
+                if filename.is_file():
+                    yield filename.path       
+        else:
+            raise FileExistsError(f"{dirFile} is not a valid file/dir")
+
+import logging
+import ipaddress
+
+class NormCaster():
+    def __init__(self,instance:pynorm.Instance, opts:argparse.Namespace ):
+        self.opts:argparse.Namespace  = opts
+        self.recvDir:Optional[str]  = os.path.expanduser(opts.recv) if opts.recv else None
+
+        self.instance:pynorm.Instance = instance
+
+        self.session:pynorm.Session = create_session(instance, opts)
+        self.fileIterator:Iterable[str] = listFiles(opts.send)
+        self.is_running:bool = True
+        self.pendingSendFilePath:Optional[str] = None #  
+
+    def addOneFile(self,session):
+        '''
+
+        '''
+        if self.pendingSendFilePath is None:
+            try:
+                self.pendingSendFilePath:str = self.fileIterator.__next__()
+            except StopIteration:
+                self.is_running = False
+                return 
+        
+        lastSlash = self.pendingSendFilePath.rfind('/')
+        filename = self.pendingSendFilePath[lastSlash+1:]
+        obj:Optional[pynorm.Object] = session.fileEnqueue(self.pendingSendFilePath, info= filename.encode() )
+        if obj:
+            if self.opts.ack:
+                session.setWatermark(obj,True)
+            logging.info(f"add file:{self.pendingSendFilePath}  {obj._object}")
+            self.pendingSendFilePath = None # succeed enqued
+        else:
+            logging.warning(f"fileEnqueue: {self.pendingSendFilePath} failure!")
+
+    def handle_norm_event(self, event:pynorm.Event):
+        session:pynorm.Session = event.session
+        
+        evtType:pynorm.EventType = event.type
+        logging.info(evtType )
+        if evtType in (EventType.TX_QUEUE_EMPTY, EventType.TX_QUEUE_VACANCY):
+            self.addOneFile(session)
+        elif evtType == EventType.GRTT_UPDATED:
+            pass
+        elif evtType  == EventType.TX_WATERMARK_COMPLETED:
+            if pynorm.AckingStatus.SUCCESS == session.getAckingStatus():
+                logging.warning( "normCast: NORM_TX_WATERMARK_COMPLETED, NORM_ACK_SUCCESS");
+                self.addOneFile(session)
+            else:
+                logging.warning( "normCast: NORM_TX_WATERMARK_COMPLETED, _NOT_ NORM_ACK_SUCCESS");
+            obj = event.object
+            if obj is None:
+                session.resetWatermark()
+        elif evtType  == EventType.TX_FLUSH_COMPLETED:
+            pass
+        elif evtType  == EventType.TX_OBJECT_PURGED:
+            obj = event.object
+            if obj and obj.type== pynorm.ObjectType.FILE:
+                logging.info(f"normCast: send file purged: {obj.info.decode()}")
+        elif evtType  == EventType.TX_OBJECT_SENT:
+            obj = event.object
+            if obj and obj.type== pynorm.ObjectType.FILE:
+                logging.info(f"initial send complete: {obj.info.decode()}")
+                self.addOneFile(session)
+        elif evtType  == EventType.ACKING_NODE_NEW:
+
+            sender = event.sender
+            logging.info(f"normCast: new acking node: {sender.id} IP address{sender.address}")
+        elif evtType  == EventType.REMOTE_SENDER_INACTIVE:
+            pass
+        elif evtType == EventType.RX_OBJECT_ABORTED:
+            obj = event.object
+            if obj and obj.type is not pynorm.ObjectType.FILE:
+                logging.error("normCast: received invalid object type?!")
+                return
+            filePath = event.object.filename
+            event.object.cancel()
+            #remove temparary file if recv aborted.
+            os.remove(filePath) 
+            logging.info("")
+
+        elif evtType == EventType.RX_OBJECT_INFO:
+            pass
+        elif evtType == EventType.RX_OBJECT_COMPLETED:
+            obj = event.object
+            if obj and obj.type is not pynorm.ObjectType.FILE:
+                logging.error("normCast: received invalid object type?!")
+                return 
+            file_path = event.object.info.decode()
+            fileName = os.path.split(file_path)[-1]
+            path = os.path.join( self.recvDir,  fileName )
+            oldPath = event.object.filename
+
+            logging.debug (f"{oldPath=}")
+            try:
+                if os.path.isfile(path):
+                    os.remove(path)
+                os.rename(src=oldPath, dst=path)    
+            except Exception as e:
+                logging.error ( traceback.format_exc() )           
+
+
+    def getAckNodesStatus(self,session):
+        isSuccess, nodeID, nodeAckStatus = session.getNextAckingNode()
+        while isSuccess:
+            ip=ipaddress.IPv4Address(nodeID)
+            if pynorm.AckingStatus.SUCCESS == nodeAckStatus:
+                logging.info( f"normCast: node {nodeID} (IP address: {str(ip)}) acnkowledged.")
+            else:
+                logging.info( f"normCast: node {nodeID} (IP address: {str(ip)}) failed to acnkowledge.")    
+            isSuccess, nodeID, nodeAckStatus = session.getNextAckingNode()    
+
+    def run(self):
+        while self.is_running:
+            event: Optional[pynorm.Event,bool,None] = self.instance.getNextEvent(timeout=self.opts.repeat)
+            while event:
+                self.handle_norm_event(event) 
+                event = self.instance.getNextEvent(timeout=self.opts.repeat)
+            if event is None and self.opts.send:
+                self.addOneFile(self.session) # means timeout 
+        logging.warning("normCast exits!")         
+
+
+def main(argv):
+    opts:argparse.Namespace = get_arg_options()
+    instance = pynorm.Instance()
+    if opts.debug:
+        instance.setDebugLevel(level=pynorm.DebugLevel(opts.debug) )
+    if opts.recv:
+        recvPath:str = os.path.expanduser( opts.recv  ) 
+        if not os.path.isdir(recvPath): #create  is not exists!
+            os.makedirs( recvPath )
+        instance.setCacheDirectory( recvPath )
+
+    normCast:NormCaster = NormCaster(instance, opts)
+    if opts.send:
+        normCast.addOneFile(normCast.session)
+
+    normCast.run()
+
+
+
+
+if __name__ == '__main__':
+    logging.getLogger().setLevel(logging.INFO)
+    sys.exit(main(sys.argv))

examples/python/normFileRecv.py

@@ -28,7 +28,7 @@ def main(argv):
     (opts, args) = get_option_parser().parse_args(argv)
 
     if len(args) != 2:
-        print get_option_parser().get_usage()
+        print(get_option_parser().get_usage())
         return 1
 
     path = os.path.abspath(args[1])
@@ -45,26 +45,26 @@ def main(argv):
         for event in instance:
             if event == 'NORM_RX_OBJECT_INFO':
                 event.object.filename = os.path.join(path, event.object.info)
-                print 'Downloading file %s' % event.object.filename
+                print('Downloading file %s' % event.object.filename)
 
             elif event == 'NORM_RX_OBJECT_UPDATED':
-                print 'File %s - %i bytes left to download' % (
-                        event.object.filename, event.object.bytesPending)
+                print('File %s - %i bytes left to download' % (
+                        event.object.filename, event.object.bytesPending))
 
             elif event == 'NORM_RX_OBJECT_COMPLETED':
-                print 'File %s completed' % event.object.filename
+                print('File %s completed' % event.object.filename)
                 return 0
 
             elif event == 'NORM_RX_OBJECT_ABORTED':
-                print 'File %s aborted' % event.object.filename
+                print('File %s aborted' % event.object.filename)
                 return 1
 
             else:
-                print event
+                print(event)
     except KeyboardInterrupt:
         pass
 
-    print 'Exiting.'
+    print('Exiting.')
     return 0
 
 if __name__ == '__main__':

examples/python/normFileSend.py

@@ -10,7 +10,7 @@ from random import randint
 import pynorm
 
 USAGE = 'usage: %s [options] <file>' % sys.argv[0]
-DEFAULT_ADDR = '224.1.2.3'
+DEFAULT_ADDR = u'224.1.2.3'
 DEFAULT_PORT = 6003
 
 def get_option_parser():
@@ -29,7 +29,7 @@ def main(argv):
     (opts, args) = get_option_parser().parse_args(argv)
 
     if len(args) != 2:
-        print get_option_parser().get_usage()
+        print(get_option_parser().get_usage())
         return 1
 
     filepath = os.path.abspath(args[1])
@@ -40,23 +40,23 @@ def main(argv):
     session = instance.createSession(opts.address, opts.port)
     if opts.iface:
         session.setMulticastInterface(opts.iface)
-    session.setTxRate(256e10)
+    session.setTxRate(1.0e+06)
     session.startSender(randint(0, 1000), 1024**2, 1400, 64, 16)
 
-    print 'Sending file %s' % filename
-    session.fileEnqueue(filepath, filename)
+    print(('Sending file %s' % filename))
+    session.fileEnqueue(filepath, info=filename.encode())
 
     try:
         for event in instance:
-            if event == 'NORM_TX_FLUSH_COMPLETED':
-                print 'Flush completed, exiting.'
+            print(event)
+            if str(event) == 'NORM_TX_FLUSH_COMPLETED':
+                print('Flush completed, exiting.')
                 return 0
-            else:
-                print event
+                
     except KeyboardInterrupt:
         pass
 
-    print 'Exiting.'
+    print('Exiting.')
     return 0
 # end main
 

examples/python/normMsgr.py

@@ -14,19 +14,19 @@ class InputThread(Thread):
     
     def __init__(self, parent, *args, **kwargs):
         super(InputThread, self).__init__(*args, **kwargs)
-        self.setDaemon(True)  ;# this is "child" daemon thread
+        self.daemon = True
         self.msgr = parent
         
     def run(self):
         while True:
             try:
-                msgHdr = bytearray(sys.stdin.read(MSG_HDR_SIZE))
+                msgHdr = sys.stdin.buffer.read(MSG_HDR_SIZE)
             except:
                 sys.stderr.write("normMsgr: input thread end-of-file 1 ...\n")
                 return
             try:
                 msgSize = 256*int(msgHdr[0]) + int(msgHdr[1])
-                msgBuffer = sys.stdin.read(msgSize - 2)
+                msgBuffer = sys.stdin.buffer.read(msgSize - 2)
             except:
                 sys.stderr.write("normMsgr: input thread end-of-file 2 ...\n")
                 return
@@ -37,7 +37,7 @@ class OutputThread(Thread):
     
     def __init__(self, parent, *args, **kwargs):
         super(OutputThread, self).__init__(*args, **kwargs)
-        self.setDaemon(True)  ;# this is "child" daemon thread
+        self.daemon = True
         self.msgr = parent
         
     def run(self):
@@ -47,11 +47,15 @@ class OutputThread(Thread):
             msgHeader = bytearray(MSG_HDR_SIZE)
             msgHeader[0] = (msgLen >> 8) & 0x00ff
             msgHeader[1] = msgLen & 0x00ff
-            sys.stdout.write(msgHeader)
-            sys.stdout.write(msg)
+            try:
+                sys.stdout.buffer.write(msgHeader)
+                sys.stdout.buffer.write(msg)
+                sys.stdout.flush()
+            except:
+                sys.stderr.write("normMsgr: output thread exiting ...\n")
+                return
             del msg
             
-
 class NormMsgr:
     """This class keeps state for NORM tx/rx operations"""
     
@@ -78,10 +82,12 @@ class NormMsgr:
         # Create a NormSession and set some default parameters
         self.normSession = self.normInstance.createSession(addr, port, nodeId)
         self.normSession.setRxCacheLimit(2*self.norm_tx_queue_max) ;# we let the receiver track some extra objects
-        self.normSession.setDefaultSyncPolicy(pynorm.NORM_SYNC_ALL);
+        self.normSession.setDefaultSyncPolicy(pynorm.SyncPolicy.ALL);
         self.normSession.setDefaultUnicastNack(True);
         self.normSession.setTxCacheBounds(10*1024*1024, self.norm_tx_queue_max, self.norm_tx_queue_max);
         self.normSession.setCongestionControl(True, True);
+        self.normSession.setRxPortReuse(True)
+        self.normSession.setMulticastLoopback(True)
         return self.normSession
         
     def addAckingNode(self, nodeId):
@@ -203,7 +209,7 @@ class NormMsgr:
    
     def onNormRxObjectCompleted(self, obj):
         with self.normRxLock:
-            if pynorm.NORM_OBJECT_DATA == obj.getType():
+            if pynorm.ObjectType.DATA == obj.getType():
                 if 0 != len(self.output_msg_queue):
                     wasEmpty = False
                 else:
@@ -233,7 +239,8 @@ class NormEventHandler(Thread):
     
     def __init__(self, parentMsgr, *args, **kwargs):
         super(NormEventHandler, self).__init__(*args, **kwargs)
-        self.setDaemon(True)  ;# this is "child" daemon thread
+        #self.setDaemon(True)  ;# this is "child" daemon thread (setDaemon is deprecated)
+        self.daemon = True
         self.lock = Lock()
         self.msgr = parentMsgr
         
@@ -248,12 +255,12 @@ class NormEventHandler(Thread):
                 return
             if event is None:
                 break
-            if pynorm.NORM_EVENT_INVALID == event.type:
+            if pynorm.EventType.EVENT_INVALID == event.type:
                 continue
-            elif pynorm.NORM_TX_QUEUE_EMPTY == event.type or pynorm.NORM_TX_QUEUE_VACANCY == event.type:
+            elif pynorm.EventType.TX_QUEUE_EMPTY == event.type or pynorm.EventType.TX_QUEUE_VACANCY == event.type:
                 msgr.onNormTxQueueVacancy()
-            elif pynorm.NORM_TX_WATERMARK_COMPLETED == event.type:
-                if pynorm.NORM_ACK_SUCCESS == event.session.getAckingStatus():
+            elif pynorm.EventType.TX_WATERMARK_COMPLETED == event.type:
+                if pynorm.EventType.ACK_SUCCESS == event.session.getAckingStatus():
                     # All receivers acknowledged
                     msgr.onNormTxWatermarkCompleted()
                 else:
@@ -261,12 +268,12 @@ class NormEventHandler(Thread):
                     #       from our acking list.  For now, we are infinitely
                     #       persistent by resetting watermark ack request
                     event.session.resetWatermark()
-            elif pynorm.NORM_TX_OBJECT_PURGED == event.type:
+            elif pynorm.EventType.TX_OBJECT_PURGED == event.type:
                 msgr.onNormTxObjectPurged(event.object)
-            elif pynorm.NORM_RX_OBJECT_COMPLETED == event.type:
+            elif pynorm.EventType.RX_OBJECT_COMPLETED == event.type:
                 msgr.onNormRxObjectCompleted(event.object)
             #else:
-            #    sys.stderr.write("normMsgr: NormEventHandler warning: unhandled event: %s\n" % str(event))
+                #sys.stderr.write("normMsgr: NormEventHandler warning: unhandled event: %s\n" % pynorm.EventType(event.type))
         sys.stderr.write("normMsgr: NormEventHandler thread exiting ...\n");
         self.lock.release()
   
@@ -393,4 +400,9 @@ except KeyboardInterrupt:
     #sys.stderr.write("exception while waiting on input thread ..\n");
     pass
     
+# TBD - shut the NormEventHandler thread down in a graceful way
+if send:
+    inputThread.join()
+if recv:
+    outputThread.join()
 sys.stderr.write("normMsgr: Done.\n")

examples/win32InputHandler.cpp

@@ -0,0 +1,243 @@
+ 
+
+#include <windows.h>
+#include <stdio.h>	 // for perror()
+#include <string.h>  // for memset()
+#include <process.h>  // for _beginthreadex(), _endthreadex()
+
+// This class creates a child thread to read STDIN into a buffer.
+// An event is set that can be monitored by the parent to know
+// when input is available.  Mutexes are used to regulate the
+// thread's reading (i.e., flow control).
+class Win32InputHandler
+{
+	public:
+		Win32InputHandler();
+		~Win32InputHandler();
+
+		bool Open(int buflen = 4096);
+		void Close();
+
+		// The event handle returned here can be used in
+		// a call like WaitForSignalObject() to get notified
+		// when there is data to be read.
+		HANDLE GetEventHandle() const
+			{return input_event;}
+
+		// This does not block and the input_event
+		// handle is a cue for when to call it
+		// Returns number of bytes copied into "buffer"
+		// (-1 is returned upon input error (input closed))
+		int ReadData(char* buffer, int numBytes);
+
+	private:
+		static unsigned int __stdcall DoThreadOpen(void* param);
+		unsigned int Run();
+
+		HANDLE				input_handle;
+		HANDLE				input_event;
+		HANDLE				thread_handle;
+		bool				input_ready;
+		char*				input_buffer;
+		int					input_buflen;
+		int					input_index;
+		int					input_outdex;
+		CONDITION_VARIABLE	buffer_vacancy;
+		CRITICAL_SECTION	buffer_lock;	
+		bool				is_running;
+		int					input_length;
+		bool				input_error;
+		
+};  // end class Win32InputHandler
+
+Win32InputHandler::Win32InputHandler()
+	: input_handle(NULL), input_event(NULL), thread_handle(NULL),
+	  input_ready(false), input_buffer(NULL), input_buflen(0),
+	  input_index(0), input_outdex(0), is_running(false),
+	  input_length(0), input_error(false)
+{
+
+}
+
+Win32InputHandler::~Win32InputHandler()
+{
+	if (is_running) Close();
+}
+
+bool Win32InputHandler::Open(int buflen)
+{
+	if (is_running) Close();
+	if (NULL != input_buffer) 
+    {
+        input_buffer = NULL;
+        delete[] input_buffer;
+    }
+	input_buflen = input_index = input_outdex = input_length = 0;
+	input_error = false;
+	if (NULL != input_event) CloseHandle(input_event);
+
+	// Get the STDIN handle whatever it may be
+	input_handle = GetStdHandle(STD_INPUT_HANDLE);
+	/*if (FILE_TYPE_CHAR == GetFileType(input_handle))
+	{
+		// The STDIN is a console
+		DWORD consoleMode;
+		bool result = GetConsoleMode(input_handle, &consoleMode);
+		consoleMode &= ~(ENABLE_LINE_INPUT | ENABLE_MOUSE_INPUT | ENABLE_WINDOW_INPUT);
+		SetConsoleMode(input_handle, consoleMode);
+	}*/
+
+	// Create an event handle, non-signaled initially
+	input_event = CreateEvent(NULL, TRUE, FALSE, NULL);
+	if (NULL == input_event)
+	{
+		perror("Win32InputHandler::Open() CreateEvent() error");
+		return false;
+	}
+	if (NULL == (input_buffer = new char[buflen]))
+	{
+		perror("Win32InputHandler::Open() new input_buffer error");
+		CloseHandle(input_event);
+		input_event = NULL;
+		return false;
+	}
+	input_buflen = buflen;
+	InitializeConditionVariable(&buffer_vacancy);
+	InitializeCriticalSection(&buffer_lock);
+	is_running = true;
+	if (0 == (thread_handle = (HANDLE)_beginthreadex(NULL, 0, DoThreadOpen, this, 0, NULL)))
+	{
+		perror("Win32InputHandler::Open() _beginthreadex() error");
+		DeleteCriticalSection(&buffer_lock);
+		delete[] input_buffer;
+		input_buffer = NULL;
+		CloseHandle(input_event);
+		input_event = NULL;
+		return false;
+	}
+	return true;
+}  // end Win32InputHandler::Open()
+
+void  Win32InputHandler::Close()
+{
+	EnterCriticalSection(&buffer_lock);
+	is_running = false;
+	WakeConditionVariable(&buffer_vacancy);  // in case it was full
+	LeaveCriticalSection(&buffer_lock);
+	WaitForSingleObject(thread_handle, INFINITE);
+	DeleteCriticalSection(&buffer_lock);
+	delete[] input_buffer;
+	input_buffer = NULL;
+	CloseHandle(input_event);
+	input_event = NULL;
+}  // end Win32InputHandler::Close()
+
+unsigned int __stdcall Win32InputHandler::DoThreadOpen(void* param)
+{
+	Win32InputHandler* inputHandler = reinterpret_cast<Win32InputHandler*>(param);
+	unsigned int exitStatus = inputHandler->Run();
+	_endthreadex(exitStatus);
+	return exitStatus;
+}  // end  Win32InputHandler::DoThreadOpen()
+
+unsigned int Win32InputHandler::Run()
+{
+	// This loop is for the child thread to read input one byte at a time
+	// (Since consoles/anonymous pipes don't support async i/o)
+	// The input bytes are copied to a circularly-managed "input_buffer"
+	// that can be accessed via the "ReadData()" method by the parent
+	while (is_running)
+	{
+		// Read input one byte at a time and buffer
+		DWORD dwRead = 0;
+		BOOL result = ReadFile(input_handle, input_buffer+input_index, 1, &dwRead, NULL);
+		EnterCriticalSection(&buffer_lock);
+		if (result)
+		{
+			if (0 != dwRead)
+			{
+				input_length++;
+				input_index++;
+				if (input_index >= input_buflen)
+					input_index = 0;
+			}
+			else
+			{
+				LeaveCriticalSection(&buffer_lock);
+				continue;
+			}
+		}
+		else
+		{
+			// What do we do on error?!
+			input_error = true;
+			is_running = false;
+		}
+		// Signals the parent of data ready (or error)
+		if (!input_ready)
+		{
+			SetEvent(input_event);
+			input_ready = true;
+		}
+		// If the buffer has been filled, this blocks this child
+		// loop until buffer space is available or it is stopped.
+		while ((input_index == input_outdex) && (0 != input_length) && is_running)
+			SleepConditionVariableCS(&buffer_vacancy, &buffer_lock, INFINITE);
+		LeaveCriticalSection(&buffer_lock);
+	}  // end while(is_running)
+	return (input_error ? 1 : 0);
+}  // end Win32InputHandler::Run()
+
+// Called by parent to gobble up data (non-blocking read)
+// Returns number of bytes read or -1 on error (e.g., STDIN pipe closed)
+int Win32InputHandler::ReadData(char* buffer, int numBytes)
+{
+	EnterCriticalSection(&buffer_lock); 
+	if ((0 != input_length) && (0 != numBytes))
+	{
+		if (numBytes > input_length)
+			numBytes = input_length;
+		input_length -= numBytes;
+		if (0 == input_length)
+		{
+			// On error condition, the event is left triggered
+			// after the input_buffer is emptied so parent gets
+			// all data buffered before the error (e.g., eof) 
+			// occurred.
+			if (input_ready && !input_error)
+			{
+				input_ready = false;
+				ResetEvent(input_event);
+			}
+		}
+		// Copy "numBytes" of available data to caller's "buffer"
+		// (two-step process when input_buffer wraps)
+		unsigned int offset = input_outdex;
+		input_outdex += numBytes;
+		if (input_outdex >= input_buflen)
+		{
+			input_outdex -= input_buflen;
+			unsigned int count = numBytes - input_outdex;
+			memcpy(buffer, input_buffer+offset, count);
+			if (0 != input_outdex)
+				memcpy(buffer+count, input_buffer, input_outdex);
+		}
+		else
+		{
+			memcpy(buffer, input_buffer + offset, numBytes);
+		}
+	}
+	else
+	{
+		// Doing the error check
+		if (input_error && (0 != numBytes))
+			numBytes = -1;  // to indicate an error has occurred and child is dead
+		else
+			numBytes = 0;
+	}
+	// This wakes up the child if it is sleeping because the buffer was full
+	if (numBytes > 0)
+		WakeConditionVariable(&buffer_vacancy);
+	LeaveCriticalSection(&buffer_lock);
+	return numBytes;
+}  // end Win32InputHandler::ReadData()

examples/wintest.cpp

@@ -0,0 +1,57 @@
+ 
+
+#include "win32InputHandler.cpp"  // for class Win32InputHandler
+
+#include <windows.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define BUFSIZE 4096
+
+int main(int argc, char** argv)
+{
+	HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
+
+	if (argc > 1)
+	{
+		// Generate random output
+		const char* text = "This is some text to output\n";
+		int textlen = strlen(text);
+
+		while (true)
+		{
+			int count = rand() % 100;
+		    while (count > 0)
+			{
+				printf(text);
+				DWORD dwWritten;
+				BOOL fSuccess = WriteFile(hStdout, text, textlen, &dwWritten, NULL);
+				count -= textlen;
+			}
+
+			int delay = (rand() % 1000);
+			Sleep(1000);
+
+		}
+
+		return 0;
+	}
+
+	Win32InputHandler inputHandler;
+
+	inputHandler.Open();
+
+	while (true)
+	{
+		char buffer[BUFSIZE];
+		WaitForSingleObject(inputHandler.GetEventHandle(), INFINITE);
+		int numBytes = inputHandler.ReadData(buffer, 1024);
+		if (numBytes < 0) break;
+
+		DWORD dwWritten;
+		BOOL fSuccess = WriteFile(hStdout, buffer, numBytes, &dwWritten, NULL);
+	}
+	inputHandler.Close();
+
+    return 0;
+}  //  end main()

include/normApi.h

@@ -33,7 +33,11 @@ typedef int32_t INT32;
 typedef uint8_t UINT8;
 typedef uint16_t UINT16;
 typedef uint32_t UINT32;
+#if (defined __GNUC__ && __GNUC__ >= 4)
+#define NORM_API_LINKAGE __attribute__ ((visibility ("default")))
+#else
 #define NORM_API_LINKAGE
+#endif
 #endif // if/else WIN32/UNIX
 
 ////////////////////////////////////////////////////////////
@@ -48,10 +52,10 @@ typedef uint32_t UINT32;
 //  is removed, the API shouldn't be considered final.
 
 #ifndef __cplusplus
-#  include <stdbool.h>
-#  define DEFAULT(arg)
+#include <stdbool.h>
+#define DEFAULT(arg)
 #else
-#  define DEFAULT(arg) = arg
+#define DEFAULT(arg) = arg
 #endif
 
 #ifdef __cplusplus
@@ -59,8 +63,8 @@ extern "C" {
 #endif /* __cplusplus */
 
 #define NORM_VERSION_MAJOR 1
-#define NORM_VERSION_MINOR 0
-#define NORM_VERSION_PATCH 0
+#define NORM_VERSION_MINOR 5
+#define NORM_VERSION_PATCH 10
 
 /** NORM API Types */
 typedef const void* NormInstanceHandle;
@@ -92,74 +96,65 @@ typedef __int64 NormSize;
 typedef off_t NormSize;
 #endif // WIN32
 
-NORM_API_LINKAGE
 typedef enum NormObjectType
 {
     NORM_OBJECT_NONE,
     NORM_OBJECT_DATA,
     NORM_OBJECT_FILE,
     NORM_OBJECT_STREAM
-} NormObjectType;
+} NORM_API_LINKAGE NormObjectType;
 
-NORM_API_LINKAGE
 typedef enum NormFlushMode
 {
     NORM_FLUSH_NONE,
     NORM_FLUSH_PASSIVE,
     NORM_FLUSH_ACTIVE
-} NormFlushMode;
+} NORM_API_LINKAGE NormFlushMode;
 
-NORM_API_LINKAGE
 typedef enum NormNackingMode
 {
     NORM_NACK_NONE,
     NORM_NACK_INFO_ONLY,
     NORM_NACK_NORMAL
-} NormNackingMode;
+} NORM_API_LINKAGE NormNackingMode;
 
-NORM_API_LINKAGE
 typedef enum NormAckingStatus
 {
     NORM_ACK_INVALID,
     NORM_ACK_FAILURE,
     NORM_ACK_PENDING,
     NORM_ACK_SUCCESS
-} NormAckingStatus;
+} NORM_API_LINKAGE NormAckingStatus;
 
-NORM_API_LINKAGE
 typedef enum NormTrackingStatus
 {
     NORM_TRACK_NONE,
     NORM_TRACK_RECEIVERS,
     NORM_TRACK_SENDERS,
     NORM_TRACK_ALL
-} NormTrackingStatus;
+} NORM_API_LINKAGE NormTrackingStatus;
 
 
-NORM_API_LINKAGE
 typedef enum NormProbingMode
 {
     NORM_PROBE_NONE,
     NORM_PROBE_PASSIVE,
     NORM_PROBE_ACTIVE
-} NormProbingMode;
+} NORM_API_LINKAGE NormProbingMode;
 
-NORM_API_LINKAGE
 typedef enum NormSyncPolicy
 {
     NORM_SYNC_CURRENT,  // attempt to receiver current/new objects only, join mid-stream
     NORM_SYNC_STREAM,   // sync to current stream, but to beginning of stream
     NORM_SYNC_ALL       // attempt to receive old and new objects
-} NormSyncPolicy;
+} NORM_API_LINKAGE NormSyncPolicy;
 
-NORM_API_LINKAGE
 typedef enum NormRepairBoundary
 {
     NORM_BOUNDARY_BLOCK,
     NORM_BOUNDARY_OBJECT
-} NormRepairBoundary;
+} NORM_API_LINKAGE NormRepairBoundary;
 
-NORM_API_LINKAGE
 typedef enum NormEventType
 {
     NORM_EVENT_INVALID = 0,
@@ -184,13 +179,14 @@ typedef enum NormEventType
     NORM_RX_OBJECT_UPDATED,
     NORM_RX_OBJECT_COMPLETED,
     NORM_RX_OBJECT_ABORTED,
+    NORM_RX_ACK_REQUEST,         // upon receipt of app-extended watermark ack request
     NORM_GRTT_UPDATED,
     NORM_CC_ACTIVE,
     NORM_CC_INACTIVE,
-    NORM_ACKING_NODE_NEW,        // whe NormSetAutoAcking xxx
+    NORM_ACKING_NODE_NEW,        // whe NormSetAutoAcking
     NORM_SEND_ERROR,             // ICMP error (e.g. destination unreachable)
     NORM_USER_TIMEOUT            // issues when timeout set by NormSetUserTimer() expires
-} NormEventType;
+} NORM_API_LINKAGE NormEventType;
 
 typedef struct
 {
@@ -201,6 +197,10 @@ typedef struct
 } NormEvent;
 
 
+// For setting custom NORM_OBJECT_DATA alloc/free functions
+typedef char* (*NormAllocFunctionHandle)(size_t);
+typedef void (*NormFreeFunctionHandle)(char*);
+
 /** NORM API General Initialization and Operation Functions */
 
 NORM_API_LINKAGE
@@ -233,8 +233,8 @@ NORM_API_LINKAGE
 bool NormSetCacheDirectory(NormInstanceHandle instanceHandle,
                            const char*        cachePath);
 
-// This call blocks until the next NormEvent is ready unless asynchronous
-// notification is used (see below)
+// This call blocks until the next NormEvent is ready unless 
+// "waitForEvent" is set to "false"
 NORM_API_LINKAGE
 bool NormGetNextEvent(NormInstanceHandle instanceHandle, NormEvent* theEvent, bool waitForEvent DEFAULT(true));
 
@@ -256,7 +256,12 @@ const NormDescriptor NORM_DESCRIPTOR_INVALID;
 NORM_API_LINKAGE
 NormDescriptor NormGetDescriptor(NormInstanceHandle instanceHandle);
 
-/** NORM Session Creation and Control Functions */
+NORM_API_LINKAGE
+void NormSetAllocationFunctions(NormInstanceHandle      instance,
+                                NormAllocFunctionHandle allocFunc,
+                                NormFreeFunctionHandle  freeFunc);
+
+// NORM Session Creation and Control Functions
 
 NORM_API_LINKAGE
 NormSessionHandle NormCreateSession(NormInstanceHandle instanceHandle,
@@ -288,6 +293,11 @@ void NormCancelUserTimer(NormSessionHandle sessionHandle);
 NORM_API_LINKAGE
 NormNodeId NormGetLocalNodeId(NormSessionHandle sessionHandle);
 
+NORM_API_LINKAGE
+bool NormGetAddress(NormSessionHandle   sessionHandle,
+                    char*               addrBuffer,
+                    unsigned int*       bufferLen,
+                    UINT16*             port DEFAULT((UINT16*)0));
 
 NORM_API_LINKAGE
 UINT16 NormGetRxPort(NormSessionHandle sessionHandle);
@@ -306,6 +316,18 @@ void NormSetTxOnly(NormSessionHandle sessionHandle,
                    bool              txOnly,
                    bool              connectToSessionAddress DEFAULT(false));
 
+NORM_API_LINKAGE
+void NormLimitObjectInfo(NormSessionHandle sessionHandle, bool state);  // if true, FEC OTI in NORM_INFO only
+
+NORM_API_LINKAGE
+bool NormPresetObjectInfo(NormSessionHandle  sessionHandle,             // FEC OTI is preset and not sent
+                          unsigned long      objectSize,                // (most useful for NORM_OBJECT_STREAM)
+                          UINT16             segmentSize,
+                          UINT16             numData,
+                          UINT16             numParity);
+
+NORM_API_LINKAGE
+void NormSetId(NormSessionHandle sessionHandle, NormNodeId normId);
 
 // This does not affect the rx_socket binding if already bound (sender or receiver already started)
 // (i.e., just affects where NORM packets are sent)
@@ -315,6 +337,13 @@ bool NormChangeDestination(NormSessionHandle sessionHandle,
                            UINT16            sessionPort,
                            bool              connectToSessionAddress DEFAULT(false));
 
+NORM_API_LINKAGE
+void NormSetServerListener(NormSessionHandle sessionHandle, bool state);
+
+NORM_API_LINKAGE
+bool NormTransferSender(NormSessionHandle sessionHandle, NormNodeHandle sender);
+
+
 NORM_API_LINKAGE
 void NormSetRxPortReuse(NormSessionHandle sessionHandle,
                         bool              enableReuse,
@@ -325,6 +354,9 @@ void NormSetRxPortReuse(NormSessionHandle sessionHandle,
 NORM_API_LINKAGE
 UINT16 NormGetRxPort(NormSessionHandle sessionHandle);
 
+NORM_API_LINKAGE
+bool NormGetRxBindAddress(NormSessionHandle sessionHandle, char* addr, unsigned int& addrLen, UINT16& port);
+
 // TBD - We should probably have a "NormSetCCMode(NormCCMode ccMode)" function for users
 NORM_API_LINKAGE
 void NormSetEcnSupport(NormSessionHandle  sessionHandle, 
@@ -410,7 +442,8 @@ bool NormStartSender(NormSessionHandle  sessionHandle,
                      UINT32             bufferSpace,
                      UINT16             segmentSize,
                      UINT16             numData,
-                     UINT16             numParity);
+                     UINT16             numParity,
+                     UINT8              fecId DEFAULT(0));
 
 NORM_API_LINKAGE
 void NormStopSender(NormSessionHandle sessionHandle);
@@ -469,6 +502,10 @@ void NormSetGrttProbingInterval(NormSessionHandle sessionHandle,
                                 double            intervalMin,
                                 double            intervalMax);
 
+NORM_API_LINKAGE
+void NormSetGrttProbingTOS(NormSessionHandle sessionHandle,
+                           UINT8              probeTOS);
+
 NORM_API_LINKAGE
 void NormSetBackoffFactor(NormSessionHandle sessionHandle,
                           double            backoffFactor);
@@ -537,6 +574,9 @@ void NormStreamSetPushEnable(NormObjectHandle streamHandle,
 NORM_API_LINKAGE
 bool NormStreamHasVacancy(NormObjectHandle streamHandle);
 
+NORM_API_LINKAGE
+unsigned int NormStreamGetVacancy(NormObjectHandle streamHandle, unsigned int bytesWanted = 0);
+
 NORM_API_LINKAGE
 void NormStreamMarkEom(NormObjectHandle streamHandle);
 
@@ -544,6 +584,15 @@ NORM_API_LINKAGE
 bool NormSetWatermark(NormSessionHandle  sessionHandle,
                       NormObjectHandle   objectHandle,
                       bool               overrideFlush DEFAULT(false));
+
+NORM_API_LINKAGE
+bool NormSetWatermarkEx(NormSessionHandle  sessionHandle,
+                        NormObjectHandle   objectHandle,
+                        const char*        buffer,
+                        unsigned int       numBytes,
+                        bool               overrideFlush DEFAULT(false));
+
+
 NORM_API_LINKAGE
 bool NormResetWatermark(NormSessionHandle sessionHandle);
 
@@ -575,6 +624,12 @@ bool NormGetNextAckingNode(NormSessionHandle    sessionHandle,
                            NormNodeId*          nodeId,   
                            NormAckingStatus*    ackingStatus DEFAULT(0));
 
+NORM_API_LINKAGE
+bool NormGetAckEx(NormSessionHandle sessionHandle,
+                  NormNodeId        nodeId,   
+                  char*             buffer,
+                  unsigned int*     buflen);
+
 NORM_API_LINKAGE
 bool NormSendCommand(NormSessionHandle  sessionHandle,
                      const char*        cmdBuffer, 
@@ -584,6 +639,9 @@ bool NormSendCommand(NormSessionHandle  sessionHandle,
 NORM_API_LINKAGE
 void NormCancelCommand(NormSessionHandle  sessionHandle);
 
+NORM_API_LINKAGE
+void NormSetSynStatus(NormSessionHandle sessionHandle, bool state);
+        
 /* NORM Receiver Functions */
 
 NORM_API_LINKAGE
@@ -646,6 +704,14 @@ NORM_API_LINKAGE
 void NormNodeSetRxRobustFactor(NormNodeHandle   remoteSender,
                                int              robustFactor);
 
+NORM_API_LINKAGE
+bool NormPreallocateRemoteSender(NormSessionHandle  sessionHandle,
+                                 unsigned long      bufferSize,
+                                 UINT16             segmentSize, 
+                                 UINT16             numData, 
+                                 UINT16             numParity,
+                                 unsigned int       streamBufferSize DEFAULT(0));
+
 NORM_API_LINKAGE
 bool NormStreamRead(NormObjectHandle   streamHandle,
                     char*              buffer,
@@ -706,6 +772,12 @@ const char* NormDataAccessData(NormObjectHandle objectHandle);
 NORM_API_LINKAGE
 char* NormDataDetachData(NormObjectHandle objectHandle);
 
+NORM_API_LINKAGE
+char* NormAlloc(size_t numBytes);
+
+NORM_API_LINKAGE
+void NormFree(char* dataPtr);
+
 NORM_API_LINKAGE
 NormNodeHandle NormObjectGetSender(NormObjectHandle objectHandle);
 
@@ -735,6 +807,16 @@ bool NormNodeGetCommand(NormNodeHandle remoteSender,
                         char*          buffer,
                         unsigned int*  buflen);
 
+NORM_API_LINKAGE
+bool NormNodeSendAckEx(NormNodeHandle remoteSender,
+                       const char*    buffer,
+                       unsigned int   numBytes);
+
+NORM_API_LINKAGE
+bool NormNodeGetWatermarkEx(NormNodeHandle remoteSender,
+                            char*          buffer,
+                            unsigned int*  buflen);
+
 NORM_API_LINKAGE
 void NormNodeFreeBuffers(NormNodeHandle remoteSender);
 

include/normMessage.h

@@ -69,12 +69,12 @@ inline UINT16 NormQuantizeLoss(double lossFraction)
     lossFraction = MIN(lossFraction, 65535.0);
     return (UINT16)lossFraction;
 }  // end NormQuantizeLossFraction()
+
 inline double NormUnquantizeLoss(UINT16 lossQuantized)
 {
     return (((double)lossQuantized) / 65535.0);
 }  // end NormUnquantizeLossFraction()
 
-
 // Extended precision Norm loss quantize/unquantize with
 // 32-bit precision (needed for low BER, high bandwidth*delay) 
 inline UINT32 NormQuantizeLoss32(double lossFraction)
@@ -134,6 +134,8 @@ class NormObjectSize
         // Operators
         bool operator==(const NormObjectSize& b) const
             {return (b.size == size);}
+        bool operator!=(const NormObjectSize& b) const
+            {return (b.size != size);}
         NormObjectSize operator+(const NormObjectSize& b) const
         {
             NormObjectSize result(size);
@@ -181,7 +183,7 @@ const NormNodeId NORM_NODE_ANY  = 0xffffffff;
 class NormObjectId
 {
     public:
-        NormObjectId() {};
+        NormObjectId() : value(0) {};
         NormObjectId(UINT16 id) {value = id;}
         NormObjectId(const NormObjectId& id) {value = id.value;}
         operator UINT16() const {return value;}
@@ -193,17 +195,15 @@ class NormObjectId
             UINT16 diff = value - id.value;
             return ((diff > 0x8000) || ((0x8000 == diff) && (value > id.value)));
         }
-        
         bool operator>(const NormObjectId& id) const
         {
             UINT16 diff = id.value - value;
             return ((diff > 0x8000) || ((0x8000 == diff) && (id.value > value)));
         }
-        
         bool operator<=(const NormObjectId& id) const
-            {return ((value == id.value) || (*this<id));}
+            {return ((value == id.value) || (*this < id));}
         bool operator>=(const NormObjectId& id) const
-            {return ((value == id.value) || (*this>id));}
+            {return ((value == id.value) || (*this > id));}
         bool operator==(const NormObjectId& id) const
             {return (value == id.value);}
         bool operator!=(const NormObjectId& id) const
@@ -212,6 +212,8 @@ class NormObjectId
         void operator-=(UINT16 delta)
             {value -= delta;}
         
+        const char* GetValuePtr() const
+            {return (const char*)(&value);}
         
         NormObjectId& operator++(int) {value++; return *this;}
         NormObjectId& operator--(int) {value--; return *this;}
@@ -224,33 +226,93 @@ class NormBlockId
 {
     public:
         NormBlockId() {};
-        NormBlockId(UINT32 id) {value = id;}
-        NormBlockId(const NormObjectId& id) {value = (UINT32)id;}
-        operator UINT32() const {return value;}
+        NormBlockId(UINT32 id) : value(id) {}
+        
+        UINT32 GetValue() const 
+            {return value;}
+        
+        const char* GetValuePtr() const
+            {return ((const char*)&value);}
+        
         bool operator==(const NormBlockId& id) const
-            {return (value == (UINT32)id);}
+            {return (value == id.value);}
         bool operator!=(const NormBlockId& id) const
-            {return (value != (UINT32)id);}
+            {return (value != id.value);}
         
+        // These static helper methods provide a "mask" parameter to allow
+        // for different bit-length NormBlockId values (depends upon FEC encoding scheme).
+        // The "mask" is in the eye-of-the-beholder, i.e., the given NormObject
+        // or NormBlockBuffer that is manipulating block id values for protocol purposes
+        // provide the FEC block "mask" to these methods.
+        // We may reconsider adding the "mask" as a NormBlockId member variable, but since
+        // NormBlockId usage is so ubiquitous, it could be added overhead consider within
+        // the context of sender, object, etc a common fec_block_mask is used.
         
-        //INT32 operator-(const NormBlockId& id) const
-        //    {return ((INT32)value - id.value);}
-        
-        bool operator<(const NormBlockId& id) const
+        // Compute difference of (a - b). If a non-zero bit "mask" is given, the difference
+        // is for the masked word size (e.g., mask = 0x00ffffff is a 24-bit integer).
+        static INT32 Difference(const NormBlockId& a, const NormBlockId& b, UINT32 mask)
         {
-            UINT32 diff = value - id.value;
-            return ((diff > 0x80000000) || ((0x80000000 == diff) && (value > id.value)));
+            if (mask)
+            {
+                UINT32 sign = (mask ^ (mask >> 1));
+                UINT32 result = a.value - b.value;
+                if (0 == (result & sign))
+                    return (INT32)(result & mask);
+                else if ((result != sign) || (a.value < b.value))
+                    return (INT32)(result | ~mask);
+                else
+                    return (INT32)(result & mask);
+            }
+            else
+            {
+                return ((INT32)(a.value - b.value));
+            }
         }
         
-        bool operator>(const NormBlockId& id) const
+        // Compare two block ids.  If a non-zero bit "mask" is given, the comparison
+        // is a "sliding window" (signed) over the bit space.  Otherwise, it is 
+        // simply an unsigned value comparison.
+        // Returns -1, 0, +1 for (a < b), (a == b), and (a > b), respectively
+        static int Compare(const NormBlockId& a, const NormBlockId& b, UINT32 mask)
         {
-            UINT32 diff = id.value - value;
-            return ((diff > 0x80000000) || ((0x80000000 == diff) && (id.value > value)));
+            if (mask)
+            {
+                // "Sliding window" comparison
+                INT32 delta = Difference(a, b, mask);
+                if (delta < 0)
+                    return -1;
+                else if (0 == delta)
+                    return 0;
+                else  // if delta > 0
+                    return 1;
+            }
+            else if (a.value < b.value)
+            {
+                return  -1;
+            }
+            else if (a.value == b.value)
+            {
+                return 0;
+            }
+            else // if (a > b)
+            {
+                return 1;
+            }
         }
         
+        void Increment(UINT32 i, UINT32 mask)
+        {
+            value = value + i;
+            if (mask) value &= mask;
+        }
         
-        NormBlockId& operator++(int ) {value++; return *this;}
-        
+        void Decrement(UINT32 i, UINT32 mask)
+        {
+            if (mask && (value < i))
+                value = (mask - (i - value) + 1);
+            else
+                value -= i;
+        }
     private:
         UINT32  value;  
 };  // end class NormBlockId
@@ -267,14 +329,16 @@ class NormHeaderExtension
             INVALID     =   0,
             FTI         =  64,  // FEC Object Transmission Information (FTI) extension
             CC_FEEDBACK =   3,  // NORM-CC Feedback extension
-            CC_RATE     = 128   // NORM-CC Rate extension
+            CC_RATE     = 128,  // NORM-CC Rate extension
+            APP_ACK     =  65   // app-defined ACK extension (see NormSetWatermarkEx())
         }; 
             
         NormHeaderExtension();
         virtual ~NormHeaderExtension() {}
-        virtual void Init(UINT32* theBuffer) 
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes) 
         {
-            buffer = theBuffer;
+            // TBD - should we confirm that 'numBytes' is sufficient
+            AttachBuffer(theBuffer, numBytes);
             SetType(INVALID);
             SetWords(0);
         }
@@ -283,13 +347,17 @@ class NormHeaderExtension
         void SetWords(UINT8 words) 
             {((UINT8*)buffer)[LENGTH_OFFSET] = words;}
         
-        void AttachBuffer(const UINT32* theBuffer) {buffer = (UINT32*)theBuffer;}
-        const UINT32* GetBuffer() {return buffer;}
+        void AttachBuffer(const UINT32* theBuffer, UINT16 bufferLength) 
+        {
+                buffer = (UINT32*)theBuffer;
+                buffer_length = bufferLength;
+        }
+        const UINT32* GetBuffer() 
+            {return buffer;}
          
         Type GetType() const
-        {
-            return buffer ? (Type)(((UINT8*)buffer)[TYPE_OFFSET]) : INVALID;  
-        }
+            {return buffer ? (Type)(((UINT8*)buffer)[TYPE_OFFSET]) : INVALID;}
+        
         UINT16 GetLength() const
         {
             return (buffer ? 
@@ -297,18 +365,31 @@ class NormHeaderExtension
                             ((((UINT8*)buffer)[LENGTH_OFFSET]) << 2) : 4) : 
                         0);  
         }
+        // These currently only used for APP_ACK extension
+        const char* GetContent() 
+            {return (((const char*)buffer) + CONTENT_OFFSET);}
+        
+        UINT16 GetContentLength() const
+        {
+            UINT16 totalLen = GetLength();
+            return ((totalLen > CONTENT_OFFSET) ? (totalLen - CONTENT_OFFSET) : 0);
+        }
+        static UINT16 GetContentOffset() 
+            {return (UINT16)CONTENT_OFFSET;}
                   
     protected:   
         enum
         {
-            TYPE_OFFSET     = 0,               // UINT8 offset
-            LENGTH_OFFSET   = TYPE_OFFSET + 1  // UINT8 offset
+            TYPE_OFFSET     = 0,                // UINT8 offset
+            LENGTH_OFFSET   = TYPE_OFFSET + 1,  // UINT8 offset
+            CONTENT_OFFSET  = LENGTH_OFFSET + 1 // UINT8 offset
         };
         UINT32*   buffer;
+        UINT16    buffer_length;
 };  // end class NormHeaderExtension
 
     
-// This class is some we use to set/get
+// This class is what we use to set/get
 // FEC Payload Id content.  The FEC Payload
 // Id format is dependent upon the "fec_id" (FEC Type)
 // and, in some cases, its field size ("m") parameter
@@ -523,13 +604,22 @@ class NormMsg
         {
             buffer[SOURCE_ID_OFFSET] = htonl(sourceId); 
         } 
+        // For messages to be sent, "addr" is destination
         void SetDestination(const ProtoAddress& dst) {addr = dst;}
+        // For message received, "addr" is source
+        void SetSource(const ProtoAddress& src) {addr = src;}
         
         void AttachExtension(NormHeaderExtension& extension)
         {
-            extension.Init(buffer+(header_length/4));
+            extension.Init(buffer+(header_length/4), MAX_SIZE - header_length);
             ExtendHeaderLength(extension.GetLength());
         }
+        // Only use this for extensions that have content appended after attachment
+        // (Fixed-length extensions should set their length upon Init())
+        void PackExtension(NormHeaderExtension& extension)
+        {
+            ExtendHeaderLength(2 + extension.GetContentLength());
+        }
         
         // Message processing routines
         bool InitFromBuffer(UINT16 msgLength);
@@ -543,8 +633,10 @@ class NormMsg
             {return (((UINT8*)buffer)[VERSION_OFFSET] >> 4);}
         NormMsg::Type GetType() const 
             {return (Type)(((UINT8*)buffer)[TYPE_OFFSET] & 0x0f);}
-        UINT16 GetHeaderLength() 
+        UINT16 GetHeaderLength() const
             {return ((UINT8*)buffer)[HDR_LEN_OFFSET] << 2;}
+        UINT16 GetBaseHeaderLength()
+            {return header_length_base;}
         UINT16 GetSequence() const
         {
             return (ntohs((((UINT16*)buffer)[SEQUENCE_OFFSET])));   
@@ -555,19 +647,31 @@ class NormMsg
         }
         const ProtoAddress& GetDestination() const {return addr;}
         const ProtoAddress& GetSource() const {return addr;}
-        const char* GetBuffer() {return ((char*)buffer);}
+        const char* GetBuffer() const {return ((char*)buffer);}
         UINT16 GetLength() const {return length;}     
         
+        void Display() const; // hex output to log
+        
         // To retrieve any attached header extensions
         bool HasExtensions() const {return (header_length > header_length_base);}
+        bool HasExtension(NormHeaderExtension::Type extType);
         bool GetNextExtension(NormHeaderExtension& ext) const
         {
-            const UINT32* currentBuffer =  ext.GetBuffer();
+            const UINT32* currentBuffer = ext.GetBuffer();
+            // 'nextOffset' here is a UINT32 offset
             UINT16 nextOffset = 
                 (UINT16)(currentBuffer ? (currentBuffer - buffer + (ext.GetLength()/4)) : 
-			                    (header_length_base/4));
-            bool result = (nextOffset < (header_length/4));
-            ext.AttachBuffer(result ? (buffer+nextOffset) : (UINT32*)NULL);
+			                             (header_length_base/4));
+            bool result = HasExtensions() ? (nextOffset < (header_length/4)) : false;
+            if (result)
+            {
+                UINT16 nextLength = ((UINT8*)(buffer + nextOffset))[1] << 2;
+                ext.AttachBuffer(buffer+nextOffset, nextLength);
+            }
+            else
+            {
+                ext.AttachBuffer((UINT32*)NULL, 0);
+            }
             return result;
         }
         
@@ -627,7 +731,8 @@ class NormObjectMsg : public NormMsg
             FLAG_INFO       = 0x04,
             FLAG_UNRELIABLE = 0x08,
             FLAG_FILE       = 0x10,
-            FLAG_STREAM     = 0x20
+            FLAG_STREAM     = 0x20,
+            FLAG_SYN        = 0x40
             //FLAG_MSG_START  = 0x40 deprecated
         }; 
         UINT16 GetInstanceId() const
@@ -681,9 +786,9 @@ class NormFtiExtension2 : public NormHeaderExtension
 {
     public:
         // To build the FTI Header Extension
-        virtual void Init(UINT32* theBuffer)
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes)
         {
-            AttachBuffer(theBuffer);
+            AttachBuffer(theBuffer, numBytes);
             SetType(FTI);  // HET = 64
             SetWords(4);
         }    
@@ -733,6 +838,60 @@ class NormFtiExtension2 : public NormHeaderExtension
         };  
 };  // end class NormFtiExtension2
 
+// Helper class for containing key FTI params
+class NormFtiData
+{
+    public:
+        NormFtiData()
+          : object_size(NormObjectSize(0)), segment_size(0),
+            num_data(0), num_parity(0), fec_m(0), instance_id(0) {}
+        ~NormFtiData() {}
+        
+        bool IsValid() const
+            {return (0 != segment_size);}
+        
+        void Invalidate()
+        {
+            object_size = 0;
+            segment_size = num_data = num_parity = instance_id = 0;
+            fec_m = 0;
+        }       
+        
+        void SetObjectSize(const NormObjectSize& objectSize)
+            {object_size = objectSize;}
+        void SetSegmentSize(UINT16 segmentSize)
+            {segment_size = segmentSize;}
+        void SetFecMaxBlockLen(UINT16 numData)
+            {num_data = numData;}
+        void SetFecNumParity(UINT16 numParity)
+            {num_parity = numParity;}
+        void SetFecFieldSize(UINT8 fecM)
+            {fec_m = fecM;}
+        void SetFecInstanceId(UINT16 instanceId)
+            {instance_id = instanceId;}
+        
+        const NormObjectSize& GetObjectSize() const 
+            {return object_size;}
+        UINT16 GetSegmentSize() const
+            {return segment_size;}
+        UINT16 GetFecMaxBlockLen() const
+            {return num_data;}
+        UINT16 GetFecNumParity() const
+            {return num_parity;}
+        UINT8 GetFecFieldSize() const
+            {return fec_m;}
+        UINT16 GetFecInstanceId() const
+            {return instance_id;}
+        
+    private:
+        NormObjectSize  object_size;
+        UINT16          segment_size;
+        UINT16          num_data;
+        UINT16          num_parity;
+        UINT8           fec_m;
+        UINT16          instance_id;
+        
+};  // end class NormFtiData
 
 // This FEC Object Transmission Information assumes "fec_id" == 5 (RFC 5510)
 // (this is the fully-defined 8-bit Reed-Solomon codec)
@@ -740,9 +899,9 @@ class NormFtiExtension5 : public NormHeaderExtension
 {
     public:
         // To build the fec_id=5 FTI Header Extension
-        virtual void Init(UINT32* theBuffer)
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes)
         {
-            AttachBuffer(theBuffer);
+            AttachBuffer(theBuffer, numBytes);
             SetType(FTI);  // HET = 64
             SetWords(3);
         }    
@@ -774,14 +933,45 @@ class NormFtiExtension5 : public NormHeaderExtension
     private:
         enum
         {
-            OBJ_SIZE_MSB_OFFSET = (LENGTH_OFFSET + 1)/2,
-            OBJ_SIZE_LSB_OFFSET = ((OBJ_SIZE_MSB_OFFSET*2)+2)/4,
-            SEG_SIZE_OFFSET = ((OBJ_SIZE_LSB_OFFSET*4)+4)/2,
-            FEC_NDATA_OFFSET    = ((SEG_SIZE_OFFSET+1)*2),
-            FEC_NPARITY_OFFSET  = (FEC_NDATA_OFFSET+1)
+            OBJ_SIZE_MSB_OFFSET = (CONTENT_OFFSET)/2,           // UINT16 offset
+            OBJ_SIZE_LSB_OFFSET = ((OBJ_SIZE_MSB_OFFSET*2)+2)/4,// UINT32 offset
+            SEG_SIZE_OFFSET = ((OBJ_SIZE_LSB_OFFSET*4)+4)/2,    // UINT16 offset
+            FEC_NDATA_OFFSET    = ((SEG_SIZE_OFFSET+1)*2),      // UINT8 offset
+            FEC_NPARITY_OFFSET  = (FEC_NDATA_OFFSET+1)          // UINT8 offset
         };  
 };  // end class NormFtiExtension5
 
+class NormAppAckExtension : public NormHeaderExtension
+{
+    public:
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes)
+        {
+            AttachBuffer(theBuffer, numBytes);
+            SetType(APP_ACK);
+            SetWords(0);
+        }  
+        bool SetContent(const char* data, UINT16 dataLen)
+        {
+            if (dataLen > (buffer_length - CONTENT_OFFSET)) return false;
+            memcpy(((char*)buffer) + CONTENT_OFFSET, data, dataLen);
+            if (dataLen > 2)
+            {
+                // Pad out to get 32-bit alignment of extension
+                dataLen += CONTENT_OFFSET;
+                UINT16 padLen = dataLen % 4;
+                if (padLen)
+                    padLen = 4 - padLen;
+                dataLen += padLen;
+                SetWords(dataLen/4);
+            }    
+            else
+            {
+                SetWords(1);
+            }       
+            return true;       
+        }
+};  // end class NormAppAckExtension
+
 
 // This FEC Object Transmission Information assumes "fec_id" == 129
 class NormFtiExtension129 : public NormHeaderExtension
@@ -789,9 +979,9 @@ class NormFtiExtension129 : public NormHeaderExtension
     public:
         // To build the FTI Header Extension
         // (TBD) allow for different "fec_id" types in the future
-        virtual void Init(UINT32* theBuffer)
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes)
         {
-            AttachBuffer(theBuffer);
+            AttachBuffer(theBuffer, numBytes);
             SetType(FTI);
             SetWords(4);
         }        
@@ -1087,7 +1277,7 @@ class NormCmdFlushMsg : public NormCmdMsg
             {length = header_length;}
         bool AppendAckingNode(NormNodeId nodeId, UINT16 segmentSize)
         {
-            if ((length-header_length+ 4) > segmentSize) return false;
+            if ((length-header_length + 4) > segmentSize) return false;
             buffer[length/4] = htonl((UINT32)nodeId);
             length += 4;
             return true;
@@ -1282,6 +1472,13 @@ class NormCmdCCMsg : public NormCmdMsg
         } 
         bool GetCCNode(NormNodeId nodeId, UINT8& flags, UINT8& rtt, UINT16& rate) const;
         
+        // This function uses the "reserved" field of the NORM_CMD(CC) message
+        // and is not strictly compliant with RFC 5740 when invoked.
+        enum {FLAG_SYN = 0x01};
+        bool SynIsSet() const
+                {return (0 != (FLAG_SYN & ((UINT8*)buffer)[RESERVED_OFFSET]));} 
+        void SetSyn()
+            {((UINT8*)buffer)[RESERVED_OFFSET] = FLAG_SYN;}
         
         class Iterator;
         friend class Iterator;
@@ -1324,9 +1521,9 @@ class NormCCRateExtension : public NormHeaderExtension
 {
     public:
             
-        virtual void Init(UINT32* theBuffer)
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes)
         {
-            AttachBuffer(theBuffer);
+            AttachBuffer(theBuffer, numBytes);
             SetType(CC_RATE);
             ((UINT8*)buffer)[RESERVED_OFFSET] = 0;
         }
@@ -1390,6 +1587,8 @@ class NormRepairRequest
             {flags |= theFlag;} 
         void ClearFlag(NormRepairRequest::Flag theFlag) 
             {flags &= ~theFlag;}    
+        void SetFlags(int theFlags)
+            {flags = theFlags;}   
         
         // Returns length (each repair item requires 8 bytes of space)
         bool AppendRepairItem(UINT8               fecId,
@@ -1425,7 +1624,13 @@ class NormRepairRequest
             {return form;}
         bool FlagIsSet(NormRepairRequest::Flag theFlag) const
             {return (0 != (theFlag & flags));}
-        UINT16 GetLength() const {return length;}
+        int GetFlags() const
+            {return flags;}
+        UINT16 GetLength() const 
+            {return (ITEM_LIST_OFFSET + length);}
+        
+        UINT32* GetBuffer() const
+            {return buffer;}
         
         // Outputs textual representation of RepairRequest content
         void Log(UINT8 fecId, UINT8 fecM) const;
@@ -1436,10 +1641,10 @@ class NormRepairRequest
                  // Checks for matching fecId and assumes constant 'm' ?!?!
                 Iterator(const NormRepairRequest& theRequest, UINT8 fecId, UINT8 fecM);  
                 void Reset() {offset = 0;}
-                bool NextRepairItem(NormObjectId* objectId,
-                                    NormBlockId*  blockId,
-                                    UINT16*       blockLen,
-                                    UINT16*       symbolId);
+                UINT16 NextRepairItem(NormObjectId* objectId,
+                                      NormBlockId*  blockId,
+                                      UINT16*       blockLen,
+                                      UINT16*       symbolId);
             private:
                 const NormRepairRequest& request;
                 UINT8                    fec_id;
@@ -1536,9 +1741,9 @@ class NormCmdRepairAdvMsg : public NormCmdMsg
 class NormCCFeedbackExtension : public NormHeaderExtension
 {
     public:
-        virtual void Init(UINT32* theBuffer)
+        virtual void Init(UINT32* theBuffer, UINT16 numBytes)
         {
-            AttachBuffer(theBuffer);
+            AttachBuffer(theBuffer, numBytes);
             SetType(CC_FEEDBACK);
             SetWords(3);
             ((UINT8*)buffer)[CC_FLAGS_OFFSET] = 0;
@@ -1596,7 +1801,7 @@ class NormCCFeedbackExtension : public NormHeaderExtension
             CC_LOSS_OFFSET      = (CC_RTT_OFFSET + 1)/2,
             CC_RATE_OFFSET      = ((CC_LOSS_OFFSET*2)+2)/2,
             //CC_RESERVED_OFFSET  = ((CC_RATE_OFFSET*2)+2)/2
-            CC_LOSS_EX_OFFSET  = ((CC_RATE_OFFSET*2)+2)/2  // extended precision loss estimate (non-RFC5940 compliant, but compatible)
+            CC_LOSS_EX_OFFSET  = ((CC_RATE_OFFSET*2)+2)/2  // extended precision loss estimate (non-RFC5740 compliant, but compatible)
         };
                             
 };  // end class NormCCFeedbackExtension
@@ -1732,6 +1937,22 @@ class NormNackMsg : public NormMsg
             return requestLength;
         }
         
+        // TBD - add some safety checks to these methods 
+        void InitFrom(NormNackMsg nack)
+        {
+            // Copy header from "nack"
+            memcpy(buffer, nack.buffer, nack.GetHeaderLength());
+            header_length_base = nack.header_length_base;
+            length = header_length = nack.GetHeaderLength();
+        }
+        void AppendRepairRequest(const NormRepairRequest request)
+        {
+            memcpy(buffer+length/4, request.GetBuffer(), request.GetLength());
+            length += request.GetLength();
+        }
+        void ResetPayload()
+            {length = GetHeaderLength();}
+                        
         // Message processing 
         NormNodeId GetSenderId() const
             {return (ntohl(buffer[SENDER_ID_OFFSET]));}
@@ -1767,7 +1988,7 @@ class NormNackMsg : public NormMsg
         };    
 };  // end class NormNackMsg
 
-class NormAckMsg : public NormMsg
+class NormAckMsg : public NormAck, public NormMsg
 {
     public:
         

include/normNode.h

@@ -134,13 +134,17 @@ class NormLossEstimator2
             history[1] = (unsigned int)((1.0 / lossFraction) + 0.5);
         }
         unsigned long CurrentLossInterval() {return history[0];}
-        unsigned int LastLossInterval() {return history[1];}
+        unsigned long LastLossInterval() {return history[1];}
         
         void SetIgnoreLoss(bool state) 
             {ignore_loss = state;}
         void SetTolerateLoss(bool state)
             {tolerate_loss = state;}
         
+        unsigned short GetLagIndex() const
+            {return lag_index;}
+        
+
     private:
         enum {DEPTH = 8};
     
@@ -174,7 +178,6 @@ class NormLossEstimator2
             {init = true; Sync(theSequence);}
         void Sync(unsigned short theSequence)
             {lag_index = theSequence;}
-        
         void ChangeLagDepth(unsigned int theDepth)
         {
             theDepth = (theDepth > 20) ? 20 : theDepth;
@@ -185,6 +188,28 @@ class NormLossEstimator2
     
 };  // end class NormLossEstimator2
 
+class NormCongestionDetector
+{
+    // This class implements a delay-based congestion indicator
+    // based on feedback from NORM receivers.  Similar to loss-based
+    // congestion detection, this detector monitors the history of
+    // RTT measurement
+    
+    // NOTES:
+    // 1) One challenge to consider is reverse path contribution
+    //    to RTT may create "false positives" on RTT change detection.
+    //    A technique to mitigate is to use reference timing that the
+    //    receiver can establish on expected sender packet arrivals
+    //    given rate information to helps separate sender->receiver
+    //    contribution to RTT versus the reverse path receiver->sender 
+    //    aspect.  E.g., the receiver could compare the stats of arriving
+    //    sender packets in reference to advertised rate to have an indication
+    //    of increasing (or decreasing) sender->receiver latency.
+    //
+    // 2)
+    //
+};  // end class NormCongestionDetector
+
 class NormAckingNode : public NormNode
 {
     public:
@@ -204,9 +229,20 @@ class NormAckingNode : public NormNode
         bool AckReceived() const {return ack_received;}
         void MarkAckReceived() {ack_received = true;}
         
+        bool SetAckEx(const char* buffer, UINT16 numBytes);
+        bool GetAckEx(char* buffer, unsigned int* buflen);
+        
+        /*
+        const char* GetAppAckContent() const
+            {return (const char*)ack_ex_buffer;}
+        unsigned int GetAppAckLength() const
+            {return ack_ex_length;} */
+                
     private:
         bool            ack_received; // was ack received?
         unsigned int    req_count;    // remaining request attempts
+        char*           ack_ex_buffer;
+        unsigned int    ack_ex_length;
         
 };  // end NormAckingNode
 
@@ -270,14 +306,13 @@ class NormCCNode : public NormNode
         UINT16          cc_sequence;
 };  // end class NormCCNode
 
-class NormSenderNode : public NormNode
+class NormSenderNode : public NormNode, public ProtoTree::Item
 {
     public:
         enum ObjectStatus {OBJ_INVALID, OBJ_NEW, OBJ_PENDING, OBJ_COMPLETE};
     
         enum RepairBoundary {BLOCK_BOUNDARY, OBJECT_BOUNDARY};
         
-        
         enum SyncPolicy
         {
             SYNC_CURRENT,  // sync to detect transmit point, iff NORM_DATA from first FEC block unless stream
@@ -291,6 +326,12 @@ class NormSenderNode : public NormNode
         void SetInstanceId(UINT16 instanceId)
             {instance_id = instanceId;}
         
+        bool PreallocateRxStream(unsigned int bufferSize,
+                                 UINT16       segmentSize  ,     
+                                 UINT16       numData,           
+                                 UINT16       numParity);
+        bool GetFtiData(const NormObjectMsg& msg, NormFtiData& ftiData);
+        
         // Parameters
         NormObject::NackingMode GetDefaultNackingMode() const 
             {return default_nacking_mode;}
@@ -317,11 +358,15 @@ class NormSenderNode : public NormNode
         
         void UpdateGrttEstimate(UINT8 grttQuantized);
         double GetGrttEstimate() const {return grtt_estimate;}
+        void ResetGrttNotification() {notify_on_grtt_update = true;}
         
         bool UpdateLossEstimate(const struct timeval&   currentTime,
                                 unsigned short          theSequence, 
                                 bool                    ecnStatus = false);
         double LossEstimate() {return loss_estimator.LossFraction();}
+        unsigned short GetCurrentSequence() const
+            {return loss_estimator.GetLagIndex();}
+        double GetRttEstimate() const {return rtt_estimate;}
         
         void CheckCCFeedback();
         
@@ -340,8 +385,14 @@ class NormSenderNode : public NormNode
         UINT16 GetInstanceId() {return instance_id;}
         bool IsOpen() const {return is_open;} 
         void Close();
-        bool AllocateBuffers(UINT8 fecId, UINT16 fecInstanceId, UINT8 fecM, UINT16 segmentSize, UINT16 numData, UINT16 numParity);
-        bool BuffersAllocated() {return (0 != segment_size);}
+        bool AllocateBuffers(unsigned int   bufferSpace,
+                             UINT8          fecId, 
+                             UINT16         fecInstanceId, 
+                             UINT8          fecM, 
+                             UINT16         segmentSize,
+                             UINT16         numData, 
+                             UINT16         numParity);
+        bool BuffersAllocated() {return (NULL != retrieval_pool);}
         void FreeBuffers();
         void Activate(bool isObjectMsg);
         
@@ -352,7 +403,7 @@ class NormSenderNode : public NormNode
         
         
         UINT8 GetFecFieldSize() const 
-            {return fec_m;}
+            {return fti_data.GetFecFieldSize();}
         
         bool GetFirstPending(NormObjectId& objectId) const
         {
@@ -381,9 +432,18 @@ class NormSenderNode : public NormNode
         
         void DeleteObject(NormObject* obj);
         
-        UINT16 SegmentSize() {return segment_size;}
-        UINT16 BlockSize() {return ndata;}
-        UINT16 NumParity() {return nparity;}
+        NormObject* GetNextPendingObject()
+        {
+            NormObjectId objid;
+            if (GetNextPending(objid))
+                return rx_table.Find(objid);
+            else
+                return NULL;
+        }        
+        
+        UINT16 SegmentSize() const {return fti_data.GetSegmentSize();}
+        UINT16 BlockSize() const {return fti_data.GetFecMaxBlockLen();}
+        UINT16 NumParity() const {return fti_data.GetFecNumParity();}
         
         NormBlock* GetFreeBlock(NormObjectId objectId, NormBlockId blockId);
         void PutFreeBlock(NormBlock* block)
@@ -398,7 +458,7 @@ class NormSenderNode : public NormNode
         
         void SetErasureLoc(UINT16 index, UINT16 value)
         {
-            ASSERT(index < nparity);
+            ASSERT(index < NumParity());
             erasure_loc[index] = value;
         }
         UINT16 GetErasureLoc(UINT16 index) 
@@ -407,7 +467,7 @@ class NormSenderNode : public NormNode
         }
         void SetRetrievalLoc(UINT16 index, UINT16 value)
         {
-            ASSERT(index < ndata);
+            ASSERT(index < BlockSize());
             retrieval_loc[index] = value;
         }
         UINT16 GetRetrievalLoc(UINT16 index) 
@@ -417,7 +477,7 @@ class NormSenderNode : public NormNode
         char* GetRetrievalSegment()
         {
             char* s = retrieval_pool[retrieval_index++];
-            retrieval_index = (retrieval_index >= ndata) ? 0 : retrieval_index;
+            retrieval_index = (retrieval_index >= BlockSize()) ? 0 : retrieval_index;
             return s;   
         }
         
@@ -431,15 +491,15 @@ class NormSenderNode : public NormNode
         
         // Statistics kept on sender
         unsigned long CurrentBufferUsage() const
-            {return (segment_size * segment_pool.CurrentUsage());}
+            {return (SegmentSize() * segment_pool.CurrentUsage());}
         unsigned long PeakBufferUsage() const
-            {return (segment_size * segment_pool.PeakUsage());}
+            {return (SegmentSize() * segment_pool.PeakUsage());}
         unsigned long BufferOverunCount() const
             {return segment_pool.OverunCount() + block_pool.OverrunCount();}
         
-        unsigned long CurrentStreamBufferUsage() const;
-        unsigned long PeakStreamBufferUsage() const;
-        unsigned long StreamBufferOverunCount() const;
+        unsigned long CurrentStreamBufferUsage();
+        unsigned long PeakStreamBufferUsage();
+        unsigned long StreamBufferOverunCount();
         
         
         //unsigned long RecvTotal() const {return recv_total;}
@@ -461,6 +521,7 @@ class NormSenderNode : public NormNode
             recv_goodput.Reset();
         }
         void IncrementResyncCount() {resync_count++;}
+        void DecrementResyncCount() {resync_count--;}
         unsigned long ResyncCount() const {return resync_count;}
         unsigned long NackCount() const {return nack_count;}
         unsigned long SuppressCount() const {return suppress_count;}
@@ -502,9 +563,37 @@ class NormSenderNode : public NormNode
         
         bool ReadNextCmd(char* buffer, unsigned int* buflen);
         
+        bool SendAckEx(const char* data, unsigned int numBytes);   
+        bool GetWatermarkEx(char* buffer, unsigned int* buflen);
+        
+        void SetAddress(const ProtoAddress& address)
+        {
+            unsigned int len = address.GetLength();
+            memcpy(key_buffer, address.GetRawHostAddress(), len);
+            UINT16 port = htons(address.GetPort());
+            memcpy(key_buffer+len, &port, 2);
+            key_size = (len+2) << 3;
+            NormNode::SetAddress(address);
+        }
+        
+        UINT8 GetGrttQuantized() const
+            {return grtt_quantized;}
+        double GetBackoffFactor() const
+            {return backoff_factor;}
+        UINT8 GetGroupSizeQuantized() const
+            {return gsize_quantized;}
+        UINT16 GetCCSequence() const
+            {return cc_sequence;}
+        double GetSendRate() const
+            {return send_rate;}
         
         
     private:
+        const char* GetKey() const
+            {return key_buffer;}    
+        unsigned int GetKeysize() const
+            {return key_size;}
+            
         static const double DEFAULT_NOMINAL_INTERVAL;
         static const double ACTIVITY_INTERVAL_MIN;
         
@@ -523,6 +612,9 @@ class NormSenderNode : public NormNode
         
         void AttachCCFeedback(NormAckMsg& ack);
         void HandleRepairContent(const UINT32* buffer, UINT16 bufferLen);
+        void FragmentNack(NormNackMsg& superNack);
+        
+        
          
         UINT16                  instance_id;
         int                     robust_factor;
@@ -535,11 +627,16 @@ class NormSenderNode : public NormNode
         UINT16                  max_pending_range;  // max range of pending objs allowed
         
         bool                    is_open;
-        UINT16                  segment_size;
+        // TBD - embed the FTI parameters into a NormFtiData object
         UINT8                   fec_id;
-        UINT8                   fec_m;
-        unsigned int            ndata;
-        unsigned int            nparity;
+        NormFtiData             fti_data;
+        bool                    preset_fti;
+        //UINT16                  segment_size;
+        //UINT8                   fec_id;
+        //UINT8                   fec_m;
+        //unsigned int            ndata;
+        //unsigned int            nparity;
+        NormStreamObject*       preset_stream;
         
         NormObjectTable         rx_table;
         ProtoSlidingMask        rx_pending_mask;
@@ -561,10 +658,13 @@ class NormSenderNode : public NormNode
         
         // Watermark acknowledgement
         ProtoTimer              ack_timer;
-	bool			ack_pending;
+	    bool			        ack_pending;
         NormObjectId            watermark_object_id;
         NormBlockId             watermark_block_id;
         NormSegmentId           watermark_segment_id;
+        bool                    ack_ex_pending;
+        char*                   ack_ex_buffer;
+        unsigned int            ack_ex_length;
         
         // Remote sender grtt measurement state       
         double                  grtt_estimate;
@@ -574,6 +674,7 @@ class NormSenderNode : public NormNode
         double                  gsize_estimate;
         UINT8                   gsize_quantized;
         double                  backoff_factor;
+        bool                    notify_on_grtt_update;  // for API
         
         // Remote sender congestion control state
         NormLossEstimator2      loss_estimator;
@@ -600,6 +701,10 @@ class NormSenderNode : public NormNode
         CmdBuffer*              cmd_buffer_tail;  // newly-received commands appended here
         CmdBuffer*              cmd_buffer_pool;  // we "pool" allocated buffers for possible reuse here
         
+        // Used for NormSocket API extension to index by addr/port
+        char                    key_buffer[16+2];  // big enough for IPv6 plus port
+        unsigned int            key_size;
+        
         // For statistics tracking
         Accumulator             recv_total;        // total recvd accumulator
         Accumulator             recv_goodput;      // goodput recvd accumulator
@@ -612,7 +717,8 @@ class NormSenderNode : public NormNode
 };  // end class NormSenderNode
     
     
-    // Used for binary trees of NormNodes
+// Used for binary trees of NormNodes sorted by NormNodeId
+// (TBD - update to use ProtoTree instead?
 class NormNodeTree
 {    
     friend class NormNodeTreeIterator;
@@ -639,7 +745,6 @@ class NormNodeTreeIterator
         void Reset(NormNode* prevNode = NULL);
         NormNode* GetNextNode();  
 
-
     private:
         const NormNodeTree& tree;
         NormNode*           next;
@@ -689,4 +794,30 @@ class NormNodeListIterator
         const NormNodeList& list;
         NormNode*           next;
 };  // end class NormNodeListIterator
+
+// Used to track remote client sender nodes for server/listener sessions
+class NormClientTree : public ProtoTreeTemplate<NormSenderNode>
+{
+    public:
+        void InsertNode(NormSenderNode& sender)
+        {
+            sender.Retain();
+            Insert(sender);
+        }      
+        void RemoveNode(NormSenderNode& sender)
+        {
+            Remove(sender);
+            sender.Release();
+        }   
+        NormSenderNode* FindNodeByAddress(const ProtoAddress& addr)
+        {
+            char key[16+2];
+            unsigned int len = addr.GetLength();
+            memcpy(key, addr.GetRawHostAddress(), len);
+            UINT16 port = htons(addr.GetPort());
+            memcpy(key+len, &port, 2);
+            return Find(key, (len+2) << 3);
+        }
+};  // end class NormClientTree
+
 #endif // NORM_NODE

include/normObject.h

@@ -7,7 +7,15 @@
 
 #include <stdio.h>
 
+#define USE_PROTO_TREE 1  // for more better performing NormObjectTable?
+
+#ifdef USE_PROTO_TREE
+#include "protoTree.h"
+
+class NormObject : public ProtoSortedTree::Item
+#else
 class NormObject
+#endif // if/else USE_PROTO_TREE
 {
     friend class NormObjectTable;
     
@@ -22,6 +30,7 @@ class NormObject
             
         enum CheckLevel
         {
+            BLIND_CHECK,
             TO_OBJECT,
             THRU_INFO,
             TO_BLOCK,
@@ -44,16 +53,50 @@ class NormObject
         
         // This must be reset after each update
         void SetNotifyOnUpdate(bool state)
-        {
-            notify_on_update = state;
-        }
+            {notify_on_update = state;}
         
         // Object information
         NormObject::Type GetType() const {return type;}
         const NormObjectId& GetId() const {return transport_id;}  
         const NormObjectSize& GetSize() const {return object_size;}
+        
+        void SetId(const NormObjectId transportId)
+            {transport_id = transportId;}
+        
+        
+        UINT16 GetSegmentSize() const {return segment_size;}
+        UINT8 GetFecId() const {return fec_id;}
+        UINT16 GetFecMaxBlockLen() const {return ndata;}
+        UINT16 GetFecNumParity() const {return nparity;}
+        UINT8 GetFecFieldSize() const {return fec_m;}
+        
+        // returns difference (a - b)
+        INT32 Difference(NormBlockId a, NormBlockId b) const
+            {return NormBlockId::Difference(a, b, fec_block_mask);}
+        // returns -1, 0, or 1 for (a < b), (a == b), or (a > b), respectively
+        int Compare(NormBlockId a, NormBlockId b) const
+            {return NormBlockId::Compare(a, b, fec_block_mask);}
+        void Increment(NormBlockId& b, UINT32 i = 1) const
+            {b.Increment(i, fec_block_mask);}
+        void Decrement(NormBlockId& b, UINT32 i = 1) const
+            {b.Decrement(i, fec_block_mask);}
+        
         bool HaveInfo() const {return (info_len > 0);}
         bool HasInfo() const {return (NULL != info_ptr);}
+        void ClearInfo()
+        {
+            if (NULL != info_ptr) delete[] info_ptr;
+            info_ptr = NULL;
+            info_len = 0;
+            pending_info = false;
+        }
+        // This is used to bootstrap reception sender
+        // is using FTI_INFO FtiMode
+        void SetPendingInfo(bool state, UINT8 fecId)
+        {
+            pending_info = true;
+            fec_id = fecId;
+        }
 
         const char* GetInfo() const {return info_ptr;}
         UINT16 GetInfoLength() const {return info_len;}
@@ -94,8 +137,6 @@ class NormObject
         }
         void Close();
         
-        
-        
         virtual bool WriteSegment(NormBlockId   blockId, 
                                   NormSegmentId segmentId, 
                                   const char*   buffer) = 0;
@@ -119,17 +160,19 @@ class NormObject
         NormBlockId GetFinalBlockId() const {return final_block_id;}
         UINT32 GetBlockSize(NormBlockId blockId) const
         {
-            return (((UINT32)blockId < large_block_count) ? large_block_size : 
-                                                            small_block_size);
+            return ((blockId.GetValue() < large_block_count) ? large_block_size : 
+                                                               small_block_size);
         }
         
         NormObjectSize GetBytesPending() const;
         
         bool IsPending(bool flush = true) const;
-        bool IsRepairPending() const;
+        bool IsRepairPending();
         bool IsPendingInfo() {return pending_info;}
         bool PendingMaskIsSet() const
             {return pending_mask.IsSet();}
+        unsigned int GetPendingMaskSize() const
+            {return pending_mask.GetSize();}
         bool GetFirstPending(NormBlockId& blockId) const 
         {
             UINT32 index = 0;
@@ -139,7 +182,7 @@ class NormObject
         }
         bool GetNextPending(NormBlockId& blockId) const
         {
-            UINT32 index = (UINT32)blockId;
+            UINT32 index = blockId.GetValue();
             bool result = pending_mask.GetNextSet(index);
             blockId = NormBlockId(index);
             return result;
@@ -161,7 +204,7 @@ class NormObject
         
         bool GetNextRepair(NormBlockId& blockId) const
         {
-            UINT32 index = (UINT32)blockId;
+            UINT32 index = blockId.GetValue();
             bool result = repair_mask.GetNextSet(index);
             blockId = NormBlockId(index);
             return result;
@@ -174,7 +217,7 @@ class NormObject
             return result;
         }
         
-        bool FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId) const;
+        bool FindRepairIndex(NormBlockId& blockId, NormSegmentId& segmentId);
                     
         // Methods available to sender for transmission
         bool NextSenderMsg(NormObjectMsg* msg);
@@ -185,26 +228,17 @@ class NormObject
         void ClearFirstPass() {first_pass = false};*/
         
         bool TxReset(NormBlockId firstBlock = NormBlockId(0), bool requeue = false);
-        bool TxResetBlocks(NormBlockId nextId, NormBlockId lastId);
+        bool TxResetBlocks(const NormBlockId& nextId, const NormBlockId& lastId);
         bool TxUpdateBlock(NormBlock*       theBlock, 
                            NormSegmentId    firstSegmentId, 
                            NormSegmentId    lastSegmentId,
-                           UINT16           numErasures)
-        {
-            NormBlockId blockId = theBlock->GetId();
-            bool result = theBlock->TxUpdate(firstSegmentId, lastSegmentId, 
-                                             GetBlockSize(blockId), nparity, 
-                                             numErasures);
-            ASSERT(result ? pending_mask.Set(blockId) : true);
-            result = result ? pending_mask.Set(blockId) : false;
-            return result; 
-        }  
+                           UINT16           numErasures);
         bool HandleInfoRequest(bool holdoff);
-        bool HandleBlockRequest(NormBlockId nextId, NormBlockId lastId);
+        bool HandleBlockRequest(const NormBlockId& nextId, const NormBlockId& lastId);
         NormBlock* FindBlock(NormBlockId blockId) {return block_buffer.Find(blockId);}
         bool ActivateRepairs();
-        bool IsRepairSet(NormBlockId blockId) {return repair_mask.Test(blockId);}
-        bool IsPendingSet(NormBlockId blockId) {return pending_mask.Test(blockId);}
+        bool IsRepairSet(NormBlockId blockId) {return repair_mask.Test(blockId.GetValue());}
+        bool IsPendingSet(NormBlockId blockId) {return pending_mask.Test(blockId.GetValue());}
         bool AppendRepairAdv(NormCmdRepairAdvMsg& cmd);
         
         NormBlockId GetMaxPendingBlockId() const {return max_pending_block;}
@@ -236,12 +270,12 @@ class NormObject
         bool ReceiverRewindCheck(NormBlockId    blockId,
                                  NormSegmentId  segmentId);
         bool IsRepairPending(bool flush);
-        bool AppendRepairRequest(NormNackMsg& nack, bool flush);
+        bool AppendRepairRequest(NormNackMsg& nack, bool flush, UINT16 payloadMax);
         void SetRepairInfo() {repair_info = true;}
         bool SetRepairs(NormBlockId first, NormBlockId last)
         {
-            return (first == last) ? repair_mask.Set(first) :
-                                     repair_mask.SetBits(first, repair_mask.Delta(last,first)+1); 
+            return ((first == last) ? repair_mask.Set(first.GetValue()) :
+                                      repair_mask.SetBits(first.GetValue(), repair_mask.Difference(last.GetValue(),first.GetValue())+1)); 
         }
         void SetLastNackTime(const ProtoTime& theTime)
             {last_nack_time = theTime;}
@@ -259,6 +293,16 @@ class NormObject
     
         void Accept() {accepted = true;}
 
+#ifdef USE_PROTO_TREE    
+        // Proto::Tree item required overrides
+        const char* GetKey() const
+            {return transport_id.GetValuePtr();}
+        unsigned int GetKeysize() const
+            {return (8*sizeof(UINT16));}
+        ProtoTree::Endian GetEndian() const
+            {return ProtoTree::GetNativeEndian();}
+#endif // USE PROTO_TREE
+                
         NormObject::Type      type;
         class NormSession&    session;
         class NormSenderNode* sender;  // NULL value indicates local (tx) object
@@ -269,6 +313,7 @@ class NormObject
         UINT16                segment_size;
         UINT8                 fec_id;
         UINT8                 fec_m;
+        UINT32                fec_block_mask;
         UINT16                ndata;
         UINT16                nparity;
         NormBlockBuffer       block_buffer;
@@ -298,8 +343,9 @@ class NormObject
         bool                  notify_on_update;
         
         const void*           user_data;  // for NORM API usage only
-        
+#ifndef USE_PROTO_TREE       
         NormObject*           next;
+#endif  
 };  // end class NormObject
 
 
@@ -315,6 +361,7 @@ class NormFileObject : public NormObject
                   const char* infoPtr = NULL,
                   UINT16      infoLen = 0);
         bool Accept(const char* thePath);
+        void CloseFile();
         void Close();
         
         const char* GetPath() {return path;}
@@ -325,7 +372,7 @@ class NormFileObject : public NormObject
             return result;
         }
         bool PadToSize()
-            {return file.Pad(NormObject::GetSize().GetOffset());}
+            {return file.IsOpen() ? file.Pad(NormObject::GetSize().GetOffset()) : false;}
         
         virtual bool WriteSegment(NormBlockId   blockId, 
                                   NormSegmentId segmentId, 
@@ -349,9 +396,12 @@ class NormDataObject : public NormObject
 {
     // (TBD) allow support of greater than 4GB size data objects
     public:
+        typedef void (*DataFreeFunctionHandle)(char*);
+    
         NormDataObject(class NormSession&       theSession,
                        class NormSenderNode*    theSender,
-                       const NormObjectId&      objectId);
+                       const NormObjectId&      objectId,
+                       DataFreeFunctionHandle   dataFreeFunc);
         ~NormDataObject();
         
         bool Open(char*       dataPtr,
@@ -381,12 +431,16 @@ class NormDataObject : public NormObject
         virtual char* RetrieveSegment(NormBlockId   blockId,
                                       NormSegmentId segmentId);
         
+        
+            
     private:
-        NormObjectSize  large_block_length;
-        NormObjectSize  small_block_length;
-        char*           data_ptr;
-        UINT32          data_max;
-        bool            data_released;   // when true, data_ptr is deleted 
+        NormObjectSize          large_block_length;
+        NormObjectSize          small_block_length;
+        char*                   data_ptr;
+        UINT32                  data_max;
+        bool                    data_released;   // when true, data_ptr is deleted 
+        DataFreeFunctionHandle  data_free_func;
+        
                                          // on NormDataObject destruction
 };  // end class NormDataObject
 
@@ -426,12 +480,16 @@ class NormStreamObject : public NormObject
         bool GetPushMode() const {return push_mode;}
         
         bool IsOldBlock(NormBlockId blockId) const
-            {return (!stream_buffer.IsEmpty() && (blockId < stream_buffer.RangeLo()));}
+            {return (!stream_buffer.IsEmpty() && (Compare(blockId, stream_buffer.RangeLo()) < 0));}
 
         bool IsClosing() {return stream_closing;}
         bool HasVacancy() 
             {return (stream_closing ? false : write_vacancy);}
         
+        // Returns how many bytes can be written to stream without blocking
+        // (up to 'wanted' for non-zero 'wanted', otherwise max vacancy available)
+        unsigned int GetVacancy(unsigned int wanted = 0);
+        
         NormBlock* StreamBlockLo()
             {return stream_buffer.Find(stream_buffer.RangeLo());}
         void SetLastNackTime(NormBlockId blockId, const ProtoTime& theTime)
@@ -459,6 +517,11 @@ class NormStreamObject : public NormObject
         }
         bool StreamAdvance();
         
+        NormBlockId GetSyncId() const
+            {return stream_sync_id;}
+        NormBlockId GetNextId() const
+            {return stream_next_id;}
+        
         virtual bool WriteSegment(NormBlockId   blockId, 
                                   NormSegmentId segmentId, 
                                   const char*   buffer);
@@ -479,15 +542,15 @@ class NormStreamObject : public NormObject
         
         bool LockSegments(NormBlockId blockId, NormSegmentId firstId, NormSegmentId lastId);
         NormBlockId StreamBufferLo() const {return stream_buffer.RangeLo();} 
+        NormBlockId RepairWindowLo() const;
         void Prune(NormBlockId blockId, bool updateStatus);
         
         bool IsFlushPending() {return flush_pending;}
-        NormBlockId FlushBlockId()
-            {return (write_index.segment ? write_index.block : 
-                     (NormBlockId((UINT32)write_index.block-1)));}
-        NormSegmentId FlushSegmentId()
-            {return (write_index.segment ? (write_index.segment-1) : 
-                                           (ndata-1));}
+        
+        NormBlockId FlushBlockId() const;
+        
+        NormSegmentId FlushSegmentId() const
+            {return (write_index.segment ? (write_index.segment-1) : (ndata-1));}
         
         NormBlockId GetNextBlockId() const
             {return (sender ? read_index.block : write_index.block);}
@@ -529,6 +592,7 @@ class NormStreamObject : public NormObject
             public:
                 NormBlockId     block;
                 NormSegmentId   segment; 
+                UINT16          offset;
         };
         // Extra state for STREAM objects
         bool                        stream_sync;
@@ -551,9 +615,10 @@ class NormStreamObject : public NormObject
         bool                        msg_start;
         FlushMode                   flush_mode;
         bool                        push_mode;
-        
+        bool                        stream_broken;
         bool                        stream_closing;
         
+        
         // For threaded API purposes
         UINT32                      block_pool_threshold;
 };  // end class NormStreamObject
@@ -588,6 +653,10 @@ class NormSimObject : public NormObject
 };  // end class NormSimObject
 #endif // SIMULATE
 
+#ifdef USE_PROTO_TREE
+class NormObjectTree : public ProtoSortedTreeTemplate<NormObject> {};
+#endif // USE_PROTO_TREE
+
 class NormObjectTable
 {
     public:
@@ -600,7 +669,6 @@ class NormObjectTable
         void SetRangeMax(UINT16 rangeMax);
         void Destroy();
         
-        bool IsInited() const {return (NULL != table);}
         UINT16 GetRangeMax() const {return range_max;}
         bool CanInsert(NormObjectId objectId) const;
         bool Insert(NormObject* theObject);
@@ -613,6 +681,22 @@ class NormObjectTable
         UINT32 GetCount() const {return count;}
         const NormObjectSize& GetSize() const {return size;}
 
+#ifdef USE_PROTO_TREE
+        class Iterator
+        {
+            public:
+                Iterator(NormObjectTable& objectTable);
+                NormObject* GetNextObject();
+                NormObject* GetPrevObject();
+                void Reset();
+                
+            private:
+                const NormObjectTable&      table;
+                NormObjectTree::Iterator    iterator;
+                NormObject*                 next_object;
+        }; 
+        
+#else        
         class Iterator
         {
             public:
@@ -626,12 +710,19 @@ class NormObjectTable
                 bool                    reset;
                 NormObjectId            index;
         }; 
+#endif // if/else USE_PROTO_TREE
             
     private:
+#ifndef USE_PROTO_TREE
         NormObject* Next(NormObject* o) const {return o->next;}    
+#endif
     
+#ifdef USE_PROTO_TREE
+        NormObjectTree  tree;
+#else        
         NormObject**    table;
         UINT16          hash_mask;       
+#endif // if/else USE_PROTO_TREE
         UINT16          range_max;  // max range of objects that can be kept
         UINT16          range;      // zero if "object table" is empty
         NormObjectId    range_lo;

include/normSegment.h

@@ -4,6 +4,9 @@
 #include "normMessage.h"
 #include "protoBitmask.h"
 
+#define USE_PROTO_TREE 1  // for more better performing NormBlockBuffer?
+
+
 // Norm uses preallocated (or dynamically allocated) pools of 
 // segments (vectors) for different buffering purposes
 
@@ -43,7 +46,11 @@ class NormSegmentPool
         bool            overrun_flag;
 };  // end class NormSegmentPool
 
+#ifdef USE_PROTO_TREE
+class NormBlock : public ProtoSortedTree::Item
+#else
 class NormBlock
+#endif // if/else USE_PROTO_TREE
 {
     friend class NormBlockPool;
     friend class NormBlockBuffer;
@@ -144,7 +151,7 @@ class NormBlock
                              UINT8                fecId,
                              UINT8                fecM,
                              UINT16               numData,
-                             UINT16               segmentSize);
+                             UINT16               payloadMax);
         
         // Receiver routines
         void RxInit(NormBlockId& blockId, UINT16 ndata, UINT16 nparity)
@@ -238,7 +245,7 @@ class NormBlock
                                  UINT16         numParity,
                                  NormObjectId   objectId,
                                  bool           pendingInfo,
-                                 UINT16         segmentSize);
+                                 UINT16         payloadMax);
         
         
         void SetLastNackTime(const ProtoTime& theTime)
@@ -254,6 +261,15 @@ class NormBlock
         void EmptyToPool(NormSegmentPool& segmentPool);
             
     private:
+#ifdef USE_PROTO_TREE
+        const char* GetKey() const
+            {return blk_id.GetValuePtr();}
+        unsigned int GetKeysize() const
+            {return (8*sizeof(UINT32));} 
+        ProtoTree::Endian GetEndian() const
+            {return ProtoTree::GetNativeEndian();}    
+#endif  // USE_PROTO_TREE
+            
         NormBlockId  blk_id;
         UINT16       size;
         char**       segment_table;
@@ -267,8 +283,7 @@ class NormBlock
         ProtoBitmask pending_mask;
         ProtoBitmask repair_mask;
         ProtoTime    last_nack_time;  // for stream flow control
-        
-        NormBlock*   next;
+        NormBlock*   next;            // used for NormBlockPool
 };  // end class NormBlock
 
 class NormBlockPool
@@ -314,6 +329,10 @@ class NormBlockPool
         bool            overrun_flag;
 };  // end class NormBlockPool
 
+#ifdef USE_PROTO_TREE
+class NormBlockTree : public ProtoSortedTreeTemplate<NormBlock> {};
+#endif // USE_PROTO_TREE
+
 class NormBlockBuffer
 {
     public:
@@ -322,18 +341,34 @@ class NormBlockBuffer
             
         NormBlockBuffer();
         ~NormBlockBuffer();
-        bool Init(unsigned long rangeMax, unsigned long tableSize = 256);
+        bool Init(unsigned long rangeMax, unsigned long tableSize, UINT32 fecBlockMask);
         void Destroy();
         
         bool Insert(NormBlock* theBlock);
-        bool Remove(const NormBlock* theBlock);
+        bool Remove(NormBlock* theBlock);
         NormBlock* Find(const NormBlockId& blockId) const;
         
         NormBlockId RangeLo() const {return range_lo;}
         NormBlockId RangeHi() const {return range_hi;}
+        NormBlockId RangeMin() const;
         bool IsEmpty() const {return (0 == range);}
         bool CanInsert(NormBlockId blockId) const;
 
+#ifdef USE_PROTO_TREE
+        class Iterator
+        {
+            public:
+                Iterator(NormBlockBuffer& blockBuffer);
+                NormBlock* GetNextBlock();
+                void Reset();
+                
+            private:
+                NormBlockBuffer&           buffer;
+                NormBlockTree::Iterator    iterator;
+                NormBlock*                 next_block;
+        }; 
+        
+#else        
         class Iterator
         {
             public:
@@ -346,18 +381,29 @@ class NormBlockBuffer
                 bool                    reset;
                 NormBlockId             index;
         }; 
-            
+#endif // if/else USE_PROTO_TREE            
     private:
-        static NormBlock* Next(NormBlock* b) {return b->next;}    
+        int Compare(NormBlockId a, NormBlockId b) const
+            {return NormBlockId::Compare(a, b, fec_block_mask);}
+        INT32 Difference(NormBlockId a, NormBlockId b) const
+            {return NormBlockId::Difference(a, b, fec_block_mask);}
+        void Increment(NormBlockId& b, UINT32 i = 1) const
+            {b.Increment(i, fec_block_mask);}
+        void Decrement(NormBlockId& b, UINT32 i = 1) const
+            {b.Decrement(i, fec_block_mask);}
 
+#ifdef USE_PROTO_TREE
+        NormBlockTree   tree;
+#else    
+        static NormBlock* Next(NormBlock* b) {return b->next;}    
         NormBlock**     table;
         unsigned long   hash_mask; 
+#endif // if/else USE_PROTO_TREE      
         unsigned long   range_max;  // max range of blocks that can be buffered
         unsigned long   range;      // zero if "block buffer" is empty
+        UINT32          fec_block_mask;
         NormBlockId     range_lo;
         NormBlockId     range_hi;
 };  // end class NormBlockBuffer
 
-
-
 #endif // _NORM_SEGMENT

include/normSession.h

@@ -51,12 +51,15 @@ class NormController
             RX_OBJECT_UPDATED,
             RX_OBJECT_COMPLETED,
             RX_OBJECT_ABORTED,
+            RX_ACK_REQUEST,         // upon receipt of app-extended watermark ack request
             GRTT_UPDATED,
             CC_ACTIVE,              // posted when cc feedback is detected
             CC_INACTIVE,            // posted when no cc feedback and min rate reached 
             ACKING_NODE_NEW,
             SEND_ERROR,
-            USER_TIMEOUT
+            USER_TIMEOUT,
+            // The ones below here are not exposed via the NORM API
+            SEND_OK
         };
                   
         virtual void Notify(NormController::Event event,
@@ -103,11 +106,18 @@ class NormSessionMgr
     
         NormController* GetController() const {return controller;}
         
+        
+        void SetDataFreeFunction(NormDataObject::DataFreeFunctionHandle freeFunc)
+            {data_free_func = freeFunc;}
+        NormDataObject::DataFreeFunctionHandle GetDataFreeFunction() const
+            {return data_free_func;}
+        
     private:   
-        ProtoTimerMgr&           timer_mgr;  
-        ProtoSocket::Notifier&   socket_notifier;  
-        ProtoChannel::Notifier*  channel_notifier;
-        NormController*          controller;
+        ProtoTimerMgr&                          timer_mgr;      
+        ProtoSocket::Notifier&                  socket_notifier; 
+        ProtoChannel::Notifier*                 channel_notifier; 
+        NormController*                         controller;     
+        NormDataObject::DataFreeFunctionHandle  data_free_func;
         
         class NormSession*       top_session;  // top of NormSession list
               
@@ -160,7 +170,17 @@ class NormSession
             TRACK_ALL       = 0x03
         };
        
+        // Object FEC Transport Information (FTI) mode
+        enum FtiMode
+        {
+            FTI_PRESET  = 0,  // Receivers have preset FTI, don't send
+            FTI_INFO    = 1,  // Send FTI in NORM_INFO messages only
+            FTI_ALWAYS  = 2   // Send FTI in NORM_DATA and NORM_INFO messages
+        };
+               
         // General methods
+        void SetNodeId(NormNodeId nodeId)
+            {local_node_id = nodeId;}
         const NormNodeId& LocalNodeId() const {return local_node_id;}
         bool Open();
         void Close();
@@ -210,12 +230,15 @@ class NormSession
         UINT16 GetTxPort() const;
             
         bool SetRxPortReuse(bool enableReuse, 
-                            const char*       rxAddress,                      // bind() to <rxAddress>/<sessionPort>
+                            const char*       rxAddress = NULL,               // bind() to <rxAddress>/<sessionPort>
                             const char*       senderAddress = (const char*)0, // connect() to <senderAddress>/<senderPort>
                             UINT16            senderPort = 0);
         
         UINT16 GetRxPort() const;
         
+        const ProtoAddress& GetRxBindAddr() const
+            {return rx_bind_addr;}
+        
         // "SetEcnSupport(true)" sets up raw packet capture (pcap) so that incoming packet
         // ECN status may be checked
         // NOTE: only effective _before_ sndr/rcvr startup!
@@ -249,6 +272,9 @@ class NormSession
         }
         void SetTxRateBounds(double rateMin, double rateMax);
         
+        void ClearSendError()
+            {posted_send_error = false;}
+        
         double BackoffFactor() 
             {return backoff_factor;}
         void SetBackoffFactor(double value) 
@@ -263,6 +289,12 @@ class NormSession
             if (state) probe_proactive = true;
         }
         
+        // This MUST be called before
+        void SetProbeTOS(UINT8 probeTOS)
+            {probe_tos = probeTOS;}
+        UINT8 GetProbeTOS() const
+            {return probe_tos;}
+        
         // This method enables/disables flow control operation.
         void SetFlowControl(double flowControlFactor)  
             {flow_control_factor = flowControlFactor;}
@@ -289,6 +321,7 @@ class NormSession
         // flow control is imposed
         double GetFlowControlDelay() const
         {
+            if (0.0 == flow_control_factor) return 0.0;
             double fdelay =  (flow_control_factor * (SenderGrtt() * (backoff_factor + 1)));
             return ((fdelay > 0.020) ? fdelay : 0.020);  // minimum 20 msec flow control
         }
@@ -301,6 +334,13 @@ class NormSession
         bool SetTxCacheBounds(NormObjectSize sizeMax,
                               unsigned long  countMin,
                               unsigned long  countMax);
+        
+        // For NormSocket API extension support only
+        void SetServerListener(bool state)
+            {is_server_listener = state;}
+        bool IsServerListener() const
+            {return is_server_listener;}
+        
         void Notify(NormController::Event event,
                     class NormNode*       node,
                     class NormObject*     object)
@@ -313,7 +353,14 @@ class NormSession
         NormMsg* GetMessageFromPool() {return message_pool.RemoveHead();}
         void ReturnMessageToPool(NormMsg* msg) {message_pool.Append(msg);}
         void QueueMessage(NormMsg* msg);
-        bool SendMessage(NormMsg& msg);
+        enum MessageStatus
+        {
+            MSG_SEND_FAILED,
+            MSG_SEND_BLOCKED,
+            MSG_SEND_OK
+                    
+        };
+        MessageStatus SendMessage(NormMsg& msg);
         void ActivateTimer(ProtoTimer& timer) {session_mgr.ActivateTimer(timer);}
         
         void SetUserData(const void* userData) 
@@ -334,7 +381,8 @@ class NormSession
                          UINT32         bufferSpace,
                          UINT16         segmentSize,
                          UINT16         numData,
-                         UINT16         numParity);
+                         UINT16         numParity,
+                         UINT8          fecId = 0);
         void StopSender();
         void SetTxOnly(bool txOnly, bool connectToSessionAddress = false);
         bool GetTxOnly() const
@@ -359,11 +407,14 @@ class NormSession
         NormObject* SenderFindTxObject(NormObjectId objectId)
             {return tx_table.Find(objectId);}
         
-        // postive ack mgmnt
-        void SenderSetWatermark(NormObjectId  objectId,
+        // postive ack mgmnt (can only fail when 'appAckReq' is set)
+        bool SenderSetWatermark(NormObjectId  objectId,
                                 NormBlockId   blockId,
                                 NormSegmentId segmentId,
-                                bool          overrideFlush = false);
+                                bool          overrideFlush = false,
+                                const char*   appAckReq = NULL,
+                                unsigned int  appAckReqLen = 0);
+        
         void SenderResetWatermark();
         void SenderCancelWatermark();
         
@@ -375,6 +426,7 @@ class NormSession
         AckingStatus SenderGetAckingStatus(NormNodeId nodeId);
         // Set "prevNodeId = NORM_NODE_NONE" to init this iteration (returns "false" when done)
         bool SenderGetNextAckingNode(NormNodeId& prevNodeId, AckingStatus* ackingStatus = NULL);
+        bool SenderGetAckEx(NormNodeId nodeId, char* buffer, unsigned int* buflen);
         
         NormAckingNode* SenderFindAckingNode(NormNodeId nodeId) const
         {
@@ -385,6 +437,12 @@ class NormSession
         bool SenderSendCmd(const char* cmdBuffer, unsigned int cmdLength, bool robust);
         void SenderCancelCmd();
         
+        // The following method is currently only used for NormSocket purposes
+        bool SenderSendAppCmd(const char* buffer, unsigned int length, const ProtoAddress& dst);
+        
+        void SenderSetSynStatus(bool state)
+            {syn_status = state;}
+        
         // robust factor
         void SetTxRobustFactor(int value)
             {tx_robust_factor = value;}
@@ -407,6 +465,13 @@ class NormSession
         void SenderSetExtraParity(UINT16 extraParity)
             {extra_parity = extraParity;}
         
+        INT32 Difference(NormBlockId a, NormBlockId b) const
+            {return NormBlockId::Difference(a, b, fec_block_mask);}
+        int Compare(NormBlockId a, NormBlockId b) const
+            {return NormBlockId::Compare(a, b, fec_block_mask);}
+        void Increment(NormBlockId& b, UINT32 i = 1) const
+            {b.Increment(i, fec_block_mask);}
+        
         
         // EMCON Sender (useful when there are silent receivers)
         // (NORM_INFO is redundantly sent)
@@ -431,6 +496,8 @@ class NormSession
         }
         
         double SenderGrtt() const {return grtt_advertised;}
+        void ResetGrttNotification() 
+            {notify_on_grtt_update = true;}
         void SenderSetGrtt(double grttValue)
         {
             if (IsSender())
@@ -449,6 +516,11 @@ class NormSession
             gsize_advertised = NormUnquantizeGroupSize(gsize_quantized);
         }
         
+        FtiMode SenderFtiMode() const
+            {return fti_mode;}
+        void SenderSetFtiMode(FtiMode ftiMode)
+            {fti_mode = ftiMode;}
+        
         void SenderEncode(unsigned int segmentId, const char* segment, char** parityVectorList)
             {encoder->Encode(segmentId, segment, parityVectorList);}
         
@@ -482,13 +554,36 @@ class NormSession
         unsigned long RemoteSenderBufferSize() const
             {return remote_sender_buffer_size;}
         
+        bool InsertRemoteSender(NormSenderNode& sender);
+        
         void DeleteRemoteSender(NormSenderNode& senderNode);
         
         // Call this to do remote sender memory allocations ahead of time
-        bool PreallocateRemoteSender(UINT16 segmentSize, UINT16 numData, UINT16 numParity);
+        bool PreallocateRemoteSender(unsigned int   bufferSize,
+                                     UINT16         segmentSize, 
+                                     UINT16         numData, 
+                                     UINT16         numParity, 
+                                     unsigned int   streamBufferSize = 0);
         
-        void ReceiverSetUnicastNacks(bool state) {unicast_nacks = state;}
-        bool ReceiverGetUnicastNacks() const {return unicast_nacks;}
+        bool SetPresetFtiData(unsigned int objectSize,
+                              UINT16       segmentSize,   
+                              UINT16       numData,       
+                              UINT16       numParity);   
+        
+        bool GetPresetFtiData(NormFtiData& ftiData)
+        {
+            if (preset_fti.IsValid())
+            {
+                ftiData = preset_fti;
+                return true;
+            }
+            return false;   
+        }
+        
+        void ReceiverSetUnicastNacks(bool state) 
+            {unicast_nacks = state;}
+        bool ReceiverGetUnicastNacks() const 
+            {return unicast_nacks;}
         
         void ReceiverSetSilent(bool state) 
             {receiver_silent = state;}
@@ -500,7 +595,7 @@ class NormSession
             {return rcvr_ignore_info;}        
         
         // The default "rcvr_max_delay = -1" corresponds to typical 
-        // operation where source data forpartially received FEC blocks 
+        // operation where source data for partially received FEC blocks 
         // are only provided to the app when buffer constraints require it.
         // Otherwise, the "maxDelay" corresponds to the max number
         // of FEC blocks the receiver waits before passing partially
@@ -541,7 +636,7 @@ class NormSession
         
         // Set default "max_pending_range" of NormObjects for reception
         void SetRxCacheMax(UINT16 maxCount)
-            {rx_cache_count_max = maxCount;}
+            {rx_cache_count_max = (maxCount > 0x7fff) ? 0x7fff : maxCount;}
         UINT16 GetRxCacheMax() const
             {return rx_cache_count_max;}
         
@@ -560,6 +655,7 @@ class NormSession
 #endif // SIMULATE
         
         void SetProbeCount(unsigned probeCount) {probe_count = probeCount;}
+        bool SenderQueueSquelch(NormObjectId objectId);
                    
     private:
         // Only NormSessionMgr can create/delete sessions
@@ -585,9 +681,14 @@ class NormSession
         void RxSocketRecvHandler(ProtoSocket& theSocket, ProtoSocket::Event theEvent);        
         void HandleReceiveMessage(NormMsg& msg, bool wasUnicast, bool ecn = false);
 
+#ifdef ECN_SUPPORT        
         // This is used when raw packet capture is enabled
+        bool OpenProtoCap();
+        void CloseProtoCap();
+        bool RawSendTo(const char* buffer, unsigned int& numBytes, const ProtoAddress& dstAddr, UINT8 trafficClass);
         void OnPktCapture(ProtoChannel&              theChannel,
 	                      ProtoChannel::Notification notifyType);
+#endif // ECN_SUPPORT
         
         // Sender message handling routines
         void SenderHandleNackMessage(const struct timeval& currentTime, 
@@ -607,7 +708,7 @@ class NormSession
                                     UINT16         ccSequence);         
         void AdjustRate(bool onResponse);
         void SetTxRateInternal(double txRate);  // here, txRate is bytes/sec
-        bool SenderQueueSquelch(NormObjectId objectId);
+        //bool SenderQueueSquelch(NormObjectId objectId);
         void SenderQueueFlush();
         bool SenderQueueWatermarkFlush();
         bool SenderBuildRepairAdv(NormCmdRepairAdvMsg& cmd);
@@ -633,7 +734,10 @@ class NormSession
         ProtoSocket                     tx_socket_actual;
         ProtoSocket*                    tx_socket;
         ProtoSocket                     rx_socket;
-        ProtoCap*                       rx_cap;        // raw packet capture alternative to "rx_socket"
+#ifdef ECN_SUPPORT
+        ProtoCap*                       proto_cap;        // raw packet capture alternative to "rx_socket"
+        ProtoAddress                    src_addr;         // used for raw packet sendto()
+#endif // ECN_SUPPORT
         bool                            rx_port_reuse; // enable rx_socket port (sessionPort) reuse when true
         ProtoAddress                    rx_bind_addr;
         ProtoAddress                    rx_connect_addr;
@@ -660,6 +764,7 @@ class NormSession
         double                          tx_rate;  // bytes per second
         double                          tx_rate_min;
         double                          tx_rate_max;
+        unsigned int                    tx_residual;    // for NORM_CMD(CC)/NORM_DATA "packet pairing"
         
         
         // Sender parameters and state
@@ -675,6 +780,7 @@ class NormSession
         bool                            sndr_emcon;
         bool                            tx_only;
         bool                            tx_connect;
+        FtiMode                         fti_mode;  
         
         NormObjectTable                 tx_table;
         ProtoSlidingMask                tx_pending_mask;
@@ -685,6 +791,7 @@ class NormSession
         NormEncoder*                    encoder;
         UINT8                           fec_id;
         UINT8                           fec_m;
+        INT32                           fec_block_mask;
         
         NormObjectId                    next_tx_object_id;
         unsigned int                    tx_cache_count_min;
@@ -694,6 +801,7 @@ class NormSession
         int                             flush_count;
         bool                            posted_tx_queue_empty;
         bool                            posted_tx_rate_changed;
+        bool                            posted_send_error;
         
         // For postive acknowledgement collection
         NormNodeTree                    acking_node_tree;
@@ -723,6 +831,7 @@ class NormSession
         bool                            probe_reset;   
         bool                            probe_data_check;  // refrain cc probe until data is send
         struct timeval                  probe_time_last;
+        UINT8                           probe_tos;         // optionally use different IP TOS for GRTT probe/response
         
         double                          grtt_interval;     // current GRTT update interval
         double                          grtt_interval_min; // minimum GRTT update interval
@@ -761,6 +870,11 @@ class NormSession
         char*                           cmd_buffer;
         unsigned int                    cmd_length;
         ProtoTimer                      cmd_timer;
+        bool                            syn_status;
+        
+        // Sender "app-defined" ACK_REQUEST state (for NormSetWatermarkEx())
+        char*                           ack_ex_buffer;
+        unsigned int                    ack_ex_length;
         
         // Receiver parameters
         bool                            is_receiver;
@@ -777,6 +891,14 @@ class NormSession
         NormObject::NackingMode         default_nacking_mode;
         NormSenderNode::SyncPolicy      default_sync_policy;
         UINT16                          rx_cache_count_max;
+        NormFtiData                     preset_fti;
+        
+        // For NormSocket server-listener support
+        bool                            is_server_listener;
+        NormClientTree                  client_tree;
+        
+        // API-specific state variables
+        bool                            notify_on_grtt_update;
         
         // State for some experimental congestion control
         bool                            ecn_ignore_loss;  

include/normVersion.h

@@ -36,6 +36,6 @@
  
 #ifndef _NORM_VERSION
 #define _NORM_VERSION
-#define VERSION "1.5r6"
+#define VERSION "1.5.10"
 #endif // _NORM_VERSION
         

makefiles/Makefile.common

@@ -14,13 +14,13 @@ NS = ../src/sim/ns
 
 INCLUDES = $(SYSTEM_INCLUDES) -I$(UNIX) -I../include -I$(PROTOLIB)/include
 
-CFLAGS = -g -DPROTO_DEBUG -DUNIX -D_FILE_OFFSET_BITS=64 -O $(SYSTEM_CFLAGS) $(SYSTEM_HAVES) $(INCLUDES)
+CFLAGS = -g -DPROTO_DEBUG -DUNIX -D_FILE_OFFSET_BITS=64 -O $(SYSTEM_CFLAGS) $(SYSTEM_HAVES) $(INCLUDES) -Wno-attributes
 #CFLAGS = -g -DPROTO_DEBUG -DUNIX -D_FILE_OFFSET_BITS=64 $(SYSTEM_CFLAGS) $(SYSTEM_HAVES) $(INCLUDES)
 
 LDFLAGS = $(SYSTEM_LDFLAGS)
 
 # Note: Even command line app needs X11 for Netscape post-processing
-LIBS = $(SYSTEM_LIBS) -lm -lpthread
+LIBS = $(SYSTEM_LIBS) -lm
 XLIBS = -lXmu -lXt -lX11
 
 
@@ -178,12 +178,72 @@ normMsgr:    $(MSGR_OBJ) libnorm.a $(LIBPROTO)
 	mkdir -p ../bin
 	cp $@ ../bin/$@
 
+# (normStreamer) stream sender/receiver
+STREAMER_SRC = $(EXAMPLE)/normStreamer.cpp
+STREAMER_OBJ = $(STREAMER_SRC:.cpp=.o)
+
+normStreamer:    $(STREAMER_OBJ) libnorm.a $(LIBPROTO) 
+	$(CC) $(CFLAGS) -o $@ $(STREAMER_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS)
+	mkdir -p ../bin
+	cp $@ ../bin/$@
+    
+# (normCast) file sender/receiver
+CAST_SRC = $(EXAMPLE)/normCast.cpp $(COMMON)/normPostProcess.cpp \
+           $(UNIX)/unixPostProcess.cpp
+CAST_OBJ = $(CAST_SRC:.cpp=.o)
+
+normCast:    $(CAST_OBJ) libnorm.a $(LIBPROTO) 
+	$(CC) $(CFLAGS) -o $@ $(CAST_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS)
+	mkdir -p ../bin
+	cp $@ ../bin/$@
+        
+# (normCastApp) file sender/receiver
+CASTAPP_SRC = $(EXAMPLE)/normCastApp.cpp $(COMMON)/normPostProcess.cpp \
+              $(UNIX)/unixPostProcess.cpp
+CASTAPP_OBJ = $(CASTAPP_SRC:.cpp=.o)
+
+normCastApp:    $(CASTAPP_OBJ) libnorm.a $(LIBPROTO) 
+	$(CC) $(CFLAGS) -o $@ $(CASTAPP_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS)
+	mkdir -p ../bin
+	cp $@ ../bin/$@
+
+# (chant) NORM command-line chat sender/receiver
+CHANT_SRC = $(EXAMPLE)/chant.cpp
+CHANT_OBJ = $(CHANT_SRC:.cpp=.o)
+
+chant:    $(CHANT_OBJ) libnorm.a $(LIBPROTO) 
+	$(CC) $(CFLAGS) -o $@ $(CHANT_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS)
+	mkdir -p ../bin
+	cp $@ ../bin/$@
+
+
+# These are the new "NormSocket" API extension examples
+SERVER_SRC = $(EXAMPLE)/normServer.cpp  $(EXAMPLE)/normSocket.cpp
+SERVER_OBJ = $(SERVER_SRC:.cpp=.o)
+
+normServer:    $(SERVER_OBJ) libnorm.a $(LIBPROTO) 
+	$(CC) $(CFLAGS) -o $@ $(SERVER_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS)
+	mkdir -p ../bin
+	cp $@ ../bin/$@
+    
+    
+CLIENT_SRC = $(EXAMPLE)/normClient.cpp  $(EXAMPLE)/normSocket.cpp
+CLIENT_OBJ = $(CLIENT_SRC:.cpp=.o)
+
+normClient:    $(CLIENT_OBJ) libnorm.a $(LIBPROTO) 
+	$(CC) $(CFLAGS) -o $@ $(CLIENT_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO) $(LIBS)
+	mkdir -p ../bin
+	cp $@ ../bin/$@
+    
+           
+    
+
 # (pcap2norm) - parses pcap (e.g. tcpdump) file and prints NORM trace
 PCAP_SRC = $(COMMON)/pcap2norm.cpp
 PCAP_OBJ = $(PCAP_SRC:.cpp=.o)
 
 pcap2norm:    $(PCAP_OBJ) libnorm.a $(LIBPROTO) 
-	$(CC) $(CFLAGS) -o $@ $(PCAP_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO)  $(LIBS)
+	$(CC) $(CFLAGS) -o $@ $(PCAP_OBJ) $(LDFLAGS) libnorm.a $(LIBPROTO)  $(LIBS) -lpcap
 	mkdir -p ../bin
 	cp $@ ../bin/$@
 

makefiles/Makefile.hpux

@@ -8,7 +8,7 @@
 #
 SYSTEM_INCLUDES = 
 SYSTEM_LDFLAGS = 
-SYTSTEM_LIBS = 
+SYTSTEM_LIBS = -lpthread
 
 # 6) System specific capabilities
 # Must choose appropriate for the following:

makefiles/Makefile.iphone

@@ -12,7 +12,7 @@ SYSROOT = $(SDK)/SDKs/iPhoneOS2.2.1.sdk
 
 SYSTEM_INCLUDES = 
 SYSTEM_LDFLAGS = -L$(SYSROOT)/usr/lib
-SYSTEM_LIBS = -lresolv
+SYSTEM_LIBS = -lresolv -lpthread
 
 # 2) System specific capabilities
 # Must choose appropriate for the following:

makefiles/Makefile.linux

@@ -7,7 +7,7 @@
 #
 SYSTEM_INCLUDES = -Wall -I/usr/X11R6/include 
 SYSTEM_LDFLAGS = -L/usr/X11R6/lib 
-SYSTEM_LIBS = -ldl 
+SYSTEM_LIBS = -ldl -lpthread
 
 # 6) System specific capabilities
 # Must choose appropriate for the following:
@@ -35,15 +35,18 @@ SYSTEM_LIBS = -ldl
 # (We export these for other Makefiles as needed)
 #
 
-SYSTEM_HAVES = -DLINUX -DECN_SUPPORT -DHAVE_IPV6 -DHAVE_GETLOGIN -D_FILE_OFFSET_BITS=64 -DHAVE_LOCKF \
--DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT
+SYSTEM_HAVES = -DLINUX -DECN_SUPPORT  -DHAVE_IPV6 -DHAVE_GETLOGIN -D_FILE_OFFSET_BITS=64 -DHAVE_LOCKF \
+-DHAVE_OLD_SIGNALHANDLER -DHAVE_DIRFD -DHAVE_ASSERT -DNO_SCM_RIGHTS -DHAVE_SCHED -DUNIX \
+-DUSE_SELECT -DUSE_TIMERFD -DUSE_EVENTFD -DHAVE_PSELECT
+
+
 
 SYSTEM_SRC = ../protolib/src/linux/linuxCap.cpp
 
 SYSTEM = linux
 
 export CC = g++
-export SYSTEM_CFLAGS = -Wall -Wcast-align -pedantic -fPIC
+export SYSTEM_CFLAGS = -Wall -Wcast-align -pedantic -fPIC -Wno-variadic-macros
 export SYSTEM_SOFLAGS = -shared -Wl,-soname,libnorm.so.1
 SYSTEM_SOEXT = so
 export RANLIB = ranlib

makefiles/Makefile.macosx

@@ -8,7 +8,7 @@
 
 SYSTEM_INCLUDES = 
 SYSTEM_LDFLAGS = 
-SYSTEM_LIBS = -lresolv -lpcap
+SYSTEM_LIBS = -lresolv -lpcap -lpthread
 
 # 2) System specific capabilities
 # Must choose appropriate for the following:
@@ -44,8 +44,10 @@ SYSTEM_SRC = ../protolib/src/unix/bpfCap.cpp
 # The "SYSTEM" keyword can be used for dependent makes
 SYSTEM = macosx
 
+# TBD - provide means to build universal Mac OSX library (x86/arm64)
+
 CC = g++
-SYSTEM_CFLAGS = -Wall -Wcast-align -fPIC -arch x86_64 -arch i386
+SYSTEM_CFLAGS = -Wall -Wcast-align -fPIC
 SYSTEM_SOFLAGS = -dynamiclib 
 SYSTEM_SOEXT = dylib
 RANLIB = ranlib

makefiles/Makefile.mklinux

@@ -9,7 +9,7 @@
 SYSTEM_INCLUDES = -I/usr/X11R6/include
 
 SYSTEM_LDFLAGS = -L/usr/X11R6/lib 
-SYSTEM_LIBS = -ldl 
+SYSTEM_LIBS = -ldl -lpthread
 
 # 6) System specific capabilities
 # Must choose appropriate for the following:

makefiles/Makefile.netbsd

@@ -7,7 +7,7 @@
 #
 SYSTEM_INCLUDES = -I/usr/X11R6/include 
 SYSTEM_LDFLAGS = -L/usr/X11R6/lib 
-SYSTEM_LIBS =  
+SYSTEM_LIBS = -lpthread
 
 # 6) System specific capabilities
 # Must choose appropriate for the following:

makefiles/Makefile.sgi

@@ -8,7 +8,7 @@
 #
 SYSTEM_INCLUDES = -DIRIX -n32 -woff 3333
 SYSTEM_LDFLAGS = 
-SYSTEM_LIBS = 
+SYSTEM_LIBS = -lpthread
 
 # 6) System specific capabilities
 # Must choose appropriate for the following:

makefiles/Makefile.solaris

@@ -7,7 +7,7 @@
 #
 SYSTEM_INCLUDES = 
 SYSTEM_LDFLAGS = 
-SYSTEM_LIBS = -ldl -lnsl -lsocket -lresolv
+SYSTEM_LIBS = -ldl -lnsl -lsocket -lresolv -lpthread
 
 # 2) System specific capabilities
 # Must choose appropriate for the following: