WIP - NNFDM implementation in AXL

CMakeLists.txt

@@ -39,6 +39,9 @@ MESSAGE(STATUS "ENABLE_IBM_BBAPI: ${ENABLE_IBM_BBAPI}")
 OPTION(ENABLE_CRAY_DW "Whether to enable Cray Datawarp support" OFF)
 MESSAGE(STATUS "ENABLE_CRAY_DW: ${ENABLE_CRAY_DW}")
 
+OPTION(ENABLE_HPE_NNFDM "Whether to enable HPE Data Mover support" ON)
+MESSAGE(STATUS "ENABLE_HPE_NNFDM: ${ENABLE_HPE_NNFDM}")
+
 OPTION(ENABLE_TESTS "Whether to build tests" ON)
 MESSAGE(STATUS "ENABLE_TESTS: ${ENABLE_TESTS}")
 
@@ -80,6 +83,16 @@ IF(ENABLE_CRAY_DW)
     # LIST(APPEND AXL_LINK_LINE " -L${WITH_DATAWARP_PREFIX}/lib64 -ldatawarp")
 ENDIF(ENABLE_CRAY_DW)
 
+## DataMover
+IF(ENABLE_HPE_NNFDM)
+  FIND_PACKAGE(NNFDM)
+  IF(NNFDM_FOUND)
+    SET(HAVE_NNFDM TRUE)
+    INCLUDE_DIRECTORIES(${NNFDM_INCLUDE_DIRS})
+    LIST(APPEND AXL_EXTERNAL_LIBS ${NNFDM_LIBRARIES})
+  ENDIF(NNFDM_FOUND)
+ENDIF(ENABLE_HPE_NNFDM)
+
 ## IBM Burst Buffer API
 IF(ENABLE_IBM_BBAPI)
     FIND_PACKAGE(BBAPI)

cmake/FindNNFDM.cmake

@@ -0,0 +1,41 @@
+# - Try to find libnnfdm
+#
+# Once done this will define
+#  NNFDM_FOUND - System has libdatawarp
+#  NNFDM_INCLUDE_DIRS - The libdatawarp include directories
+#  NNFDM_LIBRARIES - The libraries needed to use libdatawarp
+#
+# This is early days for this library.  For now, we assume that there is
+# a the following directory and contents pointed to by WITH_NNFDM_PREFIX:
+#
+# path/nnfdm/
+#   lib64/
+#    libnnfdm.a
+#   include/
+#    datamovement.pb-c.h
+#    nnfdm.h
+#
+# So, the following cmake line will cause this to be found assuming the prefix exists:
+# cmake -DWITH_NNFDM_PREFIX="/usr/WS2/martymcf/scr/dm/nnfdm" -DMPI=ON ..
+
+FIND_LIBRARY(NNFDM_LIBRARIES
+    NAMES nnfdm
+    HINTS ${WITH_NNFDM_PREFIX}/lib64
+)
+
+FIND_PATH(NNFDM_INCLUDE_DIRS
+    NAMES nnfdm.h
+    HINTS ${WITH_NNFDM_PREFIX}/include
+)
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(NNFDM DEFAULT_MSG
+  NNFDM_LIBRARIES
+  NNFDM_INCLUDE_DIRS
+)
+
+# Hide these vars from ccmake GUI
+MARK_AS_ADVANCED(
+  NNFDM_LIBRARIES
+  NNFDM_INCLUDE_DIRS
+)

cmake/config.h.in

@@ -1,5 +1,6 @@
 // Machine Specific Libs
 #cmakedefine HAVE_PTHREADS
 #cmakedefine HAVE_DATAWARP
+#cmakedefine HAVE_NNFDM
 #cmakedefine HAVE_BBAPI
 #cmakedefine HAVE_BBAPI_FALLBACK

src/CMakeLists.txt

@@ -34,6 +34,10 @@ IF(HAVE_DATAWARP)
     LIST(APPEND libaxl_srcs axl_async_datawarp.c)
 ENDIF(HAVE_DATAWARP)
 
+IF(HAVE_NNFDM)
+    LIST(APPEND libaxl_srcs axl_async_nnfdm.c)
+ENDIF(HAVE_NNFDM)
+
 # Default AXL library is withOUT MPI
 ADD_LIBRARY(axl_o OBJECT ${libaxl_srcs})
 

src/axl.c

@@ -38,6 +38,10 @@
 #include "axl_async_bbapi.h"
 #endif /* HAVE_BBAPI */
 
+#ifdef HAVE_NNFDM
+#include "axl_async_nnfdm.h"
+#endif /* HAVE_NNFDM */
+
 #ifdef HAVE_DATAWARP
 #include "axl_async_datawarp.h"
 #endif /* HAVE_DATAWARP */
@@ -91,6 +95,7 @@ static unsigned int axl_kvtrees_count = 0;
 static int bbapi_is_loaded = 0;
 #endif
 
+
 /* Allocate a new kvtree and return the AXL ID for it.  If state_file is
  * specified, then populate the kvtree with it's data. */
 static int axl_alloc_id(const char* state_file)
@@ -205,10 +210,12 @@ static axl_xfer_t axl_detect_native_xfer(void)
      * DataWarp libraries.  In the real world, our supercomputer is only going
      * to have one of those libraries, so just use whatever we find at
      * build time. */
-#ifdef HAVE_BBAPI
+#if defined(HAVE_BBAPI)
     xtype = AXL_XFER_ASYNC_BBAPI;
-#elif HAVE_DATAWARP
+#elif defined(HAVE_DATAWARP)
     xtype = AXL_XFER_ASYNC_DW;
+#elif defined(HAVE_NNFDM)
+    xtype = AXL_XFER_ASYNC_NNFDM;
 #else
     xtype = AXL_XFER_SYNC;
 #endif
@@ -293,6 +300,11 @@ int AXL_Finalize (void)
     }
 #endif
 
+#ifdef HAVE_NNFDM
+    axl_async_finalize_nnfdm();
+#endif
+
+
     /* decrement reference count and free data structures on last call */
     axl_init_count--;
     if (axl_init_count == 0) {
@@ -688,6 +700,15 @@ int AXL_Create(axl_xfer_t xtype, const char* name, const char* state_file)
 #endif /* HAVE_DATAWARP */
         break;
 
+    case AXL_XFER_ASYNC_NNFDM:
+#ifndef HAVE_NNFDM
+        AXL_ERR("NNFDM requested but not enabled during build");
+        rc = AXL_FAILURE;
+#else
+        axl_async_init_nnfdm();
+#endif /* HAVE_NNFDM */
+        break;
+
     default:
         AXL_ERR("Unknown transfer type (%d)", (int) xtype);
         rc = AXL_FAILURE;
@@ -826,6 +847,11 @@ static int __AXL_Add (int id, const char* src, const char* dest)
         break;
 #endif /* HAVE_DATAWARP */
 
+#ifdef HAVE_NNFDM
+    case AXL_XFER_ASYNC_NNFDM:
+        break;
+#endif /* HAVE_NNFDM */
+
     default:
         AXL_ERR("Unknown transfer type (%d)", (int) xtype);
         rc = AXL_FAILURE;
@@ -1146,6 +1172,17 @@ int __AXL_Dispatch (int id, int resume)
         break;
 #endif /* HAVE_DATAWARP */
 
+#ifdef HAVE_NNFDM
+    case AXL_XFER_ASYNC_NNFDM:
+        if (resume) {
+            AXL_ERR("AXL_Resume() isn't supported yet for NNFDM");
+            rc = AXL_FAILURE;
+            break;
+        }
+        rc = axl_async_start_nnfdm(id);
+        break;
+#endif /* HAVE_NNFDM */
+
     default:
         AXL_ERR("Unknown transfer type (%d)", (int) xtype);
         rc = AXL_FAILURE;
@@ -1297,6 +1334,12 @@ int AXL_Test (int id)
         break;
 #endif /* HAVE_DATAWARP */
 
+#ifdef HAVE_NNFDM
+    case AXL_XFER_ASYNC_NNFDM:
+        rc = axl_async_test_nnfdm(id);
+        break;
+#endif /* HAVE_NNFDM */
+
     default:
         AXL_ERR("Unknown transfer type (%d)", (int) xtype);
         rc = AXL_FAILURE;
@@ -1367,6 +1410,12 @@ int AXL_Wait (int id)
         break;
 #endif /* HAVE_DATAWARP */
 
+#ifdef HAVE_NNFDM
+    case AXL_XFER_ASYNC_NNFDM:
+        rc = axl_async_wait_nnfdm(id);
+        break;
+#endif /* HAVE_NNDFM */
+
     default:
         AXL_ERR("Unknown transfer type (%d)", (int) xtype);
         rc = AXL_FAILURE;
@@ -1465,6 +1514,12 @@ int AXL_Cancel (int id)
         break;
 #endif
 
+#if 0
+    case AXL_XFER_ASYNC_NNFDM:
+        rc = axl_async_cancel_nnfdm(id);
+        break;
+#endif
+
     default:
         AXL_ERR("Unknown transfer type (%d)", (int) xtype);
         rc = AXL_FAILURE;

src/axl.h

@@ -42,6 +42,7 @@ typedef enum {
     AXL_XFER_SYNC,          /* synchronous copy */
     AXL_XFER_ASYNC_DAEMON,  /* async daemon process (not used, but kept to maintain enum values) */
     AXL_XFER_ASYNC_DW,      /* Cray Datawarp */
+    AXL_XFER_ASYNC_NNFDM,   /* HPE DataMover */
     AXL_XFER_ASYNC_BBAPI,   /* IBM Burst Buffer API */
     AXL_XFER_NATIVE,        /* Autodetect and use the native API (BBAPI, DW,
                              * etc) for this node type.  It may or may not