Add ENABLE_BBAPI_FALLBACK, check FS type on BBAPI transfers

Previously if you attempted to copy a file using BBAPI, and either
the source and destination didn't support file extents, then it
would automatically fallback to using pthreads.  This change removes
the fallback by default, and returns an error instead.  The fallback
can be re-enabled by passing -DENABLE_BBAPI_FALLBACK to cmake.

This patch also checks the source and destination filesystem type
against a whitelist instead of checking if they support extents.
This was more effective, as there were some filesystems that BBAPI
could not copy to/from even though they supported extents.

CMakeLists.txt

@@ -68,6 +68,12 @@ ENDIF(${AXL_ASYNC_API} STREQUAL "CRAY_DW")
 FIND_PACKAGE(BBAPI)
 IF(BBAPI_FOUND)
     SET(HAVE_BBAPI TRUE)
+
+    SET(ENABLE_BBAPI_FALLBACK OFF CACHE BOOL "Fallback to a different transfer type if BBAPI not supported")
+    IF(${ENABLE_BBAPI_FALLBACK})
+        SET(HAVE_BBAPI_FALLBACK TRUE)
+    ENDIF(${ENABLE_BBAPI_FALLBACK})
+
     INCLUDE_DIRECTORIES(${BBAPI_INCLUDE_DIRS})
     LIST(APPEND AXL_EXTERNAL_LIBS ${BBAPI_LIBRARIES})
     LIST(APPEND AXL_LINK_LINE " -L${WITH_BBAPI_PREFIX}/lib -lbbAPI")

cmake/config.h.in

@@ -3,6 +3,7 @@
 #cmakedefine HAVE_LIBCPPR
 #cmakedefine HAVE_DATAWARP
 #cmakedefine HAVE_BBAPI
+#cmakedefine HAVE_BBAPI_FALLBACK
 
 // Flush ASYNC API
 #define AXL_FLUSH_ASYNC_@AXL_ASYNC_API@

src/axl_async_bbapi.c

@@ -19,6 +19,90 @@
  */
 #ifdef HAVE_BBAPI
 #include <bbapi.h>
+#include <sys/statfs.h>
+#include <linux/magic.h>
+#include <libgen.h>
+
+/* These aren't always defined in linux/magic.h */
+#ifndef GPFS_SUPER_MAGIC
+#define GPFS_SUPER_MAGIC 0x47504653     /* "GPFS" in ASCII */
+#endif
+#ifndef XFS_SUPER_MAGIC
+#define XFS_SUPER_MAGIC 0x58465342      /* "XFSB" in ASCII */
+#endif
+
+/* Return the filesystem magic value for a given file */
+static __fsword_t axl_get_fs_magic(char *path)
+{
+    struct statfs st;
+    int rc;
+    char *dir = NULL;
+
+    /* Stat the path itself */
+    rc = statfs(path, &st);
+    if (rc != 0) {
+        /*
+         * Couldn't statfs the path, which could be normal if the path is the
+         * destination path.  Next, try stating the underlying path's directory
+         * (which should exist for both the source and destination), for the
+         * FS type.
+         */
+        dir = strdup(path);
+        if (!dir)
+            return 0;
+        path = dirname(dir);
+        rc = statfs(path, &st);
+        if (rc != 0) {
+            free(dir);
+            return 0;
+        }
+    }
+
+    free(dir);
+    return st.f_type;
+}
+
+/*
+ * Returns 1 if its possible to transfer a file from the src to dst using the
+ * BBAPI. In general (but not all cases) BBAPI can transfer between filesystems
+ * if they both support extents.  EXT4 <-> gpfs is one exception.
+ *
+ * Returns 0 if BBAPI can not transfer from src to dst.
+ */
+int bbapi_copy_is_compatible(char *src, char *dst)
+{
+    /* List all filesystem types that are (somewhat) compatible with BBAPI */
+    const __fsword_t whitelist[] = {
+        XFS_SUPER_MAGIC,
+        GPFS_SUPER_MAGIC,
+        EXT4_SUPER_MAGIC,
+    };
+    __fsword_t source, dest;
+    int found_source = 0, found_dest = 0;
+    int i;
+
+    source = axl_get_fs_magic(src);
+    dest = axl_get_fs_magic(dst);
+
+    /* Exception: EXT4 <-> GPFS transfers are not supported by BBAPI */
+    if ((source == EXT4_SUPER_MAGIC && dest == GPFS_SUPER_MAGIC) ||
+        (source == GPFS_SUPER_MAGIC && dest == EXT4_SUPER_MAGIC)) {
+            return 0;
+    }
+
+    for (i = 0; i < sizeof(whitelist) / sizeof(whitelist[0]); i++) {
+        if (source == whitelist[i])
+            found_source = 1;
+
+        if (dest == whitelist[i])
+            found_dest = 1;
+    }
+
+    if (found_source && found_dest)
+        return 1;
+
+    return 0;
+}
 
 static void getLastErrorDetails(BBERRORFORMAT pFormat, char** pBuffer)
 {
@@ -356,9 +440,18 @@ int axl_async_wait_bbapi (int id) {
             usleep(100 * 1000);   /* 100ms */
         }
     }
-
     /* we're done now, either with error or success */
     if (status == AXL_STATUS_DEST) {
+
+        char *src;
+        char *dst;
+        kvtree_elem *elem = NULL;
+
+        /* Look though all our list of files */
+        while ((elem = axl_get_next_path(id, elem, &src, &dst))) {
+            AXL_DBG(2, "Read and copied %s to %s sucessfully", src, dst);
+        }
+
         return AXL_SUCCESS;
     } else {
         return AXL_FAILURE;
@@ -405,15 +498,19 @@ axl_all_paths_are_bbapi_compatible(int id)
     char *dst;
     kvtree_elem *elem = NULL;
 
+    /* Look though all our list of files */
     while ((elem = axl_get_next_path(id, elem, &src, &dst))) {
-        if (axl_file_supports_fiemap(src) == 0 ||
-            axl_file_supports_fiemap(dst) == 0) {
-                /* One of our paths doesn't support fiemap */
+        if (!bbapi_copy_is_compatible(src, dst)) {
+            /* This file copy isn't compatible with BBAPI */
             return (0);
         }
     }
-#endif
+
+    /* All files copies are BBAPI compatible */
     return (1);
+
+#endif
+    return (0);
 }
 
 /*
@@ -422,17 +519,22 @@ axl_all_paths_are_bbapi_compatible(int id)
  * Fallback mode happens when we can't transfer the files using the BBAPI due
  * to the source or destination nor supporting extents (which BBAPI requires).
  * If we're in fallback mode, we use a more compatible transfer method.
+ *
+ * Fallback mode is DISABLED by default.  You need to pass
+ * -DENABLE_BBAPI_FALLBACK to cmake to enable it.
  */
 int
 axl_bbapi_in_fallback(int id)
 {
-    kvtree* file_list = kvtree_get_kv_int(axl_file_lists, AXL_KEY_HANDLE_UID, id);
     int bbapi_fallback = 0;
+#ifdef HAVE_BBAPI_FALLBACK
+    kvtree* file_list = kvtree_get_kv_int(axl_file_lists, AXL_KEY_HANDLE_UID, id);
 
     if (kvtree_util_get_int(file_list, AXL_BBAPI_KEY_FALLBACK, &bbapi_fallback) !=
         KVTREE_SUCCESS) {
         /* Value isn't set, so we're not in fallback mode */
         return 0;
     }
+#endif
     return (bbapi_fallback);
 }

src/axl_io.c

@@ -19,15 +19,6 @@
 
 #include "axl_internal.h"
 
-/* fiemap() - currently only used by BBAPI. */
-#ifdef HAVE_BBAPI
-#include <linux/fiemap.h>
-#include <sys/ioctl.h>
-#include <linux/fs.h>
-#include <libgen.h>
-#endif
-
-
 /* Configurations */
 #ifndef AXL_OPEN_TRIES
 #define AXL_OPEN_TRIES (5)
@@ -37,63 +28,6 @@
 #define AXL_OPEN_USLEEP (100)
 #endif
 
-#ifdef HAVE_BBAPI
-/*
- * Returns 1 if the filesystem for a particular path supports the fiemap ioctl
- * (the filesystem for the file is able to report extents).  If the path does
- * not exist, attempt to do the ioctl on basename(path).  This allows us to
- * easily check a destination path to where a file will go.
- *
- * Returns 0 if extents are not supported on the path.
- */
-int axl_file_supports_fiemap(char *path)
-{
-    int fd;
-    struct fiemap fiemap;
-    int rc;
-    char *dir = NULL;
-
-    fd = open(path, O_RDONLY);
-    if (fd < 0) {
-        if (errno == ENOENT) {
-            /* The path to the file/dir doesn't exist.  Try one level up */
-            dir = strdup(path);
-            path = dirname(dir);
-            fd = open(path, O_RDONLY);
-            if (fd < 0) {
-                free(dir);
-                return (0);
-            }
-        } else {
-            return (0);
-        }
-    }
-
-    memset(&fiemap, 0, sizeof(fiemap));
-    fiemap.fm_length = FIEMAP_MAX_OFFSET;
-
-    if (ioctl(fd, FS_IOC_FIEMAP, &fiemap) == 0) {
-        /* Successful ioctl */
-        rc = 1;
-    } else {
-        /*
-         * Some kind of error, most likely error 95 (Operation not supported)
-         * if fiemap isn't supported by the underlying file system.
-         */
-        rc = 0;
-    }
-
-    /* Close file descriptors */
-    close(fd);
-
-    if (dir) {
-        free(dir);
-    }
-
-    return rc;
-}
-#endif
-
 /* returns user's current mode as determine by their umask */
 mode_t axl_getmode(int read, int write, int execute) {
     /* lookup current mask and set it back */

test/axl_cp.c

@@ -108,7 +108,7 @@ main(int argc, char **argv) {
     sigaction(SIGTERM, &action, NULL);
     char *state_file = NULL;
 
-    while ((opt = getopt(argc, argv, "rRX:")) != -1) {
+    while ((opt = getopt(argc, argv, "rRSX:")) != -1) {
         switch (opt) {
             case 'X':
                 xfer_str = optarg;