Merge branch 'develop'

.github/workflows/github_actions.yml

@@ -50,9 +50,10 @@ jobs:
             experimental: true
             mpi_impl: intel
             armci_network: sockets
-            config_opts: --enable-i4 --with-blas=-lmkl_rt --with-scalapack=-lmkl_scalapack_lp64
+            config_opts: --enable-i4
             f77: ifort
-            cc: gcc
+            cc: icc
+            oneapi: /opt/intel/oneapi
           - os: ubuntu-latest
             experimental: true
             mpi_impl: openmpi
@@ -66,6 +67,8 @@ jobs:
             armci_network: mpi-pr
             f77: ifort
             cc: icc
+            config_opts: LIBS=-lifcore
+            oneapi: /Users/runner/apps/oneapi
           - os: ubuntu-20.04
             experimental: true 
             mpi_impl: mpich
@@ -114,6 +117,7 @@ jobs:
             armci_network: mpi-ts
             f77: ifort
             cc: gcc
+            oneapi: /opt/intel/oneapi
         exclude:
           - armci_network: mpi-pr
             mpi_impl: openmpi
@@ -134,12 +138,12 @@ jobs:
     continue-on-error: ${{ matrix.experimental }}
     steps: 
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 40
       - name: Cache install steps
         id: ga-cache-install
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         env:
           cache-name: cache-install-steps
         with:
@@ -153,41 +157,51 @@ jobs:
           case "${{ matrix.os }}" in
           ubuntu*|jessie|stretch|buster)
           sudo apt-get update -q -y
-          if [[ "$FC" == "gfortran-11" ]] || [[ "$CC" == "gcc-11" ]]; then sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get -y install gcc-11 gfortran-11 g++-11; fi
+          if [[ "$F77" == "gfortran-11" ]] || [[ "$CC" == "gcc-11" ]]; then sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get -y install gcc-11 gfortran-11 g++-11; fi
           sudo apt-get install -q -y gfortran
           ;;
           macos*)
           brew update
           brew install gcc coreutils automake || true
           ;;
           esac
-          if [[ "$FC" == "ifort" ]] || [[ "$CC" == "icc" ]] ; then ./travis/install-intel.sh; else true;  fi
-          echo FC is `which "$FC"`
-          echo FC compiler version `"$FC" -v`
+          if [[ "$F77" == "ifort" ]] || [[ "$CC" == "icc" ]] ; then ./travis/install-intel.sh; source ${{ matrix.oneapi }}/setvars.sh --force;  fi
+          echo F77 is `which "$F77"`
+          echo F77 compiler version `"$F77" -v`
       - name: before_install
         if: steps.ga-cache-install.outputs.cache-hit != 'true'
         run: |
           pwd
           ls -lart
           mkdir -p $GITHUB_WORKSPACE/apps
+          if [[ "$F77" == "ifort" ]] || [[ "$CC" == "icc" ]] ; then source ${{ matrix.oneapi }}/setvars.sh --force;  fi
           sh ./travis/install-autotools.sh $GITHUB_WORKSPACE/apps
       - name: install
         run: |
           export CI_ROOT=$GITHUB_WORKSPACE/apps
+          if [[ "$F77" == "ifort" ]] || [[ "$CC" == "icc" ]] ; then source ${{ matrix.oneapi }}/setvars.sh --force;  fi
           ./travis/install-mpi.sh $CI_ROOT $MPI_IMPL
           if [[ "$PORT" == "ofi" ]]; then ./travis/install-libfabric.sh $CI_ROOT; else true;  fi
           if [[ "$PORT" == "armci" ]]; then ./travis/install-armci-mpi.sh $CI_ROOT; else true; fi
           if [[ "$USE_SICM" == "Y" ]]; then ./travis/install-sicm.sh $HOME/no_cache; else true; fi
       - name: compile and test global arrays
         if: ${{ success() }}
         run: |
+          if [[ "$F77" == "ifort" ]] || [[ "$CC" == "icc" ]] ; then  source ${{ matrix.oneapi }}/setvars.sh --force;  fi
           ./travis/build-run.sh $GITHUB_WORKSPACE/apps $PORT $MPI_IMPL $USE_CMAKE $F77
       - name: after_failure
         if: ${{ failure() }}
         run: |
           pwd
           ls
+          echo '***** cat config.log ****'
           find . -name config.log -exec cat {} ";" || true
+          echo '***** cat global.fh *****'
+          find . -name "global.fh"  -exec cat {} ";" || true
+          echo '***** global.fh* location *****'
+          find ../.. -name "global.fh*" || true
+          echo '***** cat global.fh.in *****'
+          find ../.. -name "global.fh.in" -exec cat {} ";" || true
           cat ./test-suite.log || true
           echo '***** CMakeOutput.log *****'
           find . -name CMakeOutput.log -exec cat {} ";" || true

CHANGELOG.md

@@ -8,6 +8,17 @@ This project follows the [Gitflow Workflow model](https://www.atlassian.com/git/
 
 ## [Unreleased]
 The Unreleased section will be empty for tagged releases. Unreleased functionality appears in the develop branch.
+- Known Bugs
+- Added
+  - Setting ARMCI_VERBOSE=1 at runtime will also dump configuration details for
+    ComEx runtime
+- Changed
+  - Updated compiler settings in CMake build if Fujitsu compilers are detected
+- Fixed
+  - Fixed gcc toolchain checks in CMake for clang build
+  - Fixed tiled arrays so that they work with restricted arrays and fixed some
+    additional bugs in block cyclic distributions
+  - Removed several memory leaks
 
 ## [5.8.2]
 - Known Bugs

CMakeLists.txt

@@ -60,6 +60,7 @@ ga_option(CMAKE_BUILD_TYPE Release)
 ga_option(LINALG_VENDOR BLIS)
 ga_option(ENABLE_TESTS ON)
 ga_option(ENABLE_PROFILING OFF)
+ga_option(ENABLE_SYSV OFF)
 #Options for user provided LinAlg libraries
 ga_option(ENABLE_BLAS OFF)
 ga_option(ENABLE_SCALAPACK OFF)
@@ -230,7 +231,11 @@ if (ENABLE_FORTRAN)
      message(STATUS "Fujitsu Fortran compiler settings")
      set (FORTRAN_I8_FLAG "-CcdII8 -CcdLL8")
      set (FORTRAN_I4_FLAG "")
-  endif()
+  elseif (CMAKE_Fortran_COMPILER_ID MATCHES "Cray")
+     message(STATUS "Cray Fortran compiler settings")
+     set (FORTRAN_I8_FLAG "-default64")
+     set (FORTRAN_I4_FLAG "")     
+  endif() 
 else()
 # need to set these variable even if only compiling C/C++
    set (F77_GETARG GETARG)
@@ -287,9 +292,9 @@ if (CMAKE_SIZEOF_VOID_P EQUAL 8)
   if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
     set(NOUSE_MMAP 1)
   endif()
-  set(GA_FORTRAN_ACCESS_INDEX integer*8)
+  set(GA_ACCESS_INDEX_TYPE integer*8)
 else()
-  set(GA_FORTRAN_ACCESS_INDEX integer*4)
+  set(GA_ACCESS_INDEX_TYPE integer*4)
 endif()
 # set (CM_BLAS_SIZE ${CM_SIZEOF_F77_INTEGER})
 
@@ -306,7 +311,7 @@ CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/matypes.h.in
                 ${CMAKE_CURRENT_BINARY_DIR}/ma/matypes.h )
 CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/mafdecls.fh.in
                 ${CMAKE_CURRENT_BINARY_DIR}/ma/mafdecls.fh )
-CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/global.fh.in
+CONFIGURE_FILE( ${CMAKE_CURRENT_SOURCE_DIR}/global/src/global.fh.in
                 ${CMAKE_CURRENT_BINARY_DIR}/global/src/global.fh )
 
 if (ENABLE_FORTRAN)
@@ -400,13 +405,18 @@ if(DEFINED GA_GCC_TOOLCHAIN_FLAG)
   target_compile_options(ga PUBLIC $<$<COMPILE_LANGUAGE:C,CXX>:${GA_GCC_TOOLCHAIN_FLAG}>)
 endif()
 
+if(ENABLE_TESTS)
+  # turn testing on
+  enable_testing()
+endif()
+
 # -------------------------------------------------------------
 # Subdirectories
 # -------------------------------------------------------------
 add_subdirectory(global/src)
 add_subdirectory(comex)
 if(ENABLE_CXX)
-add_subdirectory(ga++/src)
+add_subdirectory(ga++)
 endif()
 add_subdirectory(ma)
 add_subdirectory(pario)
@@ -420,12 +430,8 @@ if(ENABLE_PROFILING)
 endif()
 
 if(ENABLE_TESTS)
-  if (NOT GA_TEST_NPROCS)
-    set(GA_TEST_NPROCS ${MPIEXEC_MAX_NUMPROCS})
-  endif()
-  # turn testing on
-  enable_testing()
   add_subdirectory(global/testing)
+  add_subdirectory(global/examples)
 endif()
 
 
@@ -488,12 +494,6 @@ if(ENABLE_BLAS)
     ${CMAKE_CURRENT_LIST_DIR}/cmake/linalg-modules/FindStandardFortran.cmake
     DESTINATION ${INSTALL_CONFIGDIR}/linalg-cmake-modules
   )
-  if(ENABLE_DPCPP)
-    install(FILES
-      ${CMAKE_CURRENT_LIST_DIR}/cmake/FindIntelSYCL.cmake
-      DESTINATION ${INSTALL_CONFIGDIR}
-    )
-  endif()
 endif()
 
 list(REMOVE_AT CMAKE_MODULE_PATH 0)

Makefile.am

@@ -446,7 +446,9 @@ AM_CPPFLAGS += -I$(top_srcdir)/global/testing
 
 check_PROGRAMS += global/testing/big
 check_PROGRAMS += global/testing/elempatch
+check_PROGRAMS += global/testing/tiled_irreg_test
 check_PROGRAMS += global/testing/gatscat
+check_PROGRAMS += global/testing/comm_init
 check_PROGRAMS += global/testing/getmem
 check_PROGRAMS += global/testing/mtest
 check_PROGRAMS += global/testing/mulmatpatchc
@@ -637,7 +639,9 @@ endif
 
 #GLOBAL_PARALLEL_TESTS += global/testing/big$(EXEEXT) # needs lots of memory
 GLOBAL_PARALLEL_TESTS += global/testing/elempatch$(EXEEXT)
+GLOBAL_PARALLEL_TESTS += global/testing/tiled_irreg_test$(EXEEXT)
 #GLOBAL_PARALLEL_TESTS += global/testing/gatscat$(EXEEXT) # broken
+GLOBAL_PARALLEL_TESTS += global/testing/comm_init$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/getmem$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/mtest$(EXEEXT)
 GLOBAL_PARALLEL_TESTS += global/testing/mulmatpatchc$(EXEEXT)
@@ -883,13 +887,15 @@ global_testing_bin_SOURCES                 = global/testing/bin.F $(gtsrcf)
 global_testing_blktest_SOURCES             = global/testing/blktest.F $(gtsrcf)
 global_testing_d2test_SOURCES              = global/testing/d2test.F $(gtsrcf)
 global_testing_elempatch_SOURCES           = global/testing/elempatch.c
+global_testing_tiled_irreg_test_SOURCES    = global/testing/tiled_irreg_test.c
 global_testing_field_test_SOURCES          = global/testing/field-test.F $(gtsrcf)
 global_testing_g2test_SOURCES              = global/testing/g2test.F $(gtsrcf)
 global_testing_g3test_SOURCES              = global/testing/g3test.F $(gtsrcf)
 global_testing_ga_lu_SOURCES               = global/testing/ga_lu.c
 global_testing_ga_mpi_SOURCES              = global/testing/ga-mpi.c
 global_testing_ga_shift_SOURCES            = global/testing/ga_shift.F $(gtsrcf)
 global_testing_gatscat_SOURCES             = global/testing/gatscat.c
+global_testing_comm_init_SOURCES           = global/testing/comm_init.c
 global_testing_getmem_SOURCES              = global/testing/getmem.c
 global_testing_ghosts_SOURCES              = global/testing/ghosts.F $(gtsrcf)
 global_testing_ipc_clean_SOURCES           = global/testing/ipc.clean.c
@@ -1360,6 +1366,7 @@ check_PROGRAMS += ga++/testing/mtest
 check_PROGRAMS += ga++/testing/ntestc
 check_PROGRAMS += ga++/testing/testc
 check_PROGRAMS += ga++/testing/testmult
+check_PROGRAMS += ga++/testing/create_irreg_test
 check_PROGRAMS += ga++/testing/threadsafecpp
 
 CXX_SERIAL_TESTS =
@@ -1374,13 +1381,15 @@ CXX_PARALLEL_TESTS += ga++/testing/mtest$(EXEEXT)
 CXX_PARALLEL_TESTS += ga++/testing/ntestc$(EXEEXT)
 CXX_PARALLEL_TESTS += ga++/testing/testc$(EXEEXT)
 CXX_PARALLEL_TESTS += ga++/testing/testmult$(EXEEXT)
+CXX_PARALLEL_TESTS += ga++/testing/create_irreg_test$(EXEEXT)
 CXX_PARALLEL_TESTS += ga++/testing/threadsafecpp$(EXEEXT)
 
 ga___testing_elempatch_SOURCES      = ga++/testing/elempatch.cc
 ga___testing_mtest_SOURCES          = ga++/testing/mtest.cc
 ga___testing_ntestc_SOURCES         = ga++/testing/ntestc.cc
 ga___testing_testc_SOURCES          = ga++/testing/testc.cc
 ga___testing_testmult_SOURCES       = ga++/testing/testmult.cc
+ga___testing_create_irreg_test_SOURCES       = ga++/testing/create_irreg_test.cc
 ga___testing_threadsafecpp_SOURCES  = ga++/testing/thread-safe.cc
 
 ga___testing_threadsafecpp_CXXFLAGS = $(AM_CXXFLAGS) $(OPENMP_CXXFLAGS)
@@ -1394,6 +1403,7 @@ ga___testing_mtest_LDADD            = libga++.la
 ga___testing_ntestc_LDADD           = libga++.la
 ga___testing_testc_LDADD            = libga++.la
 ga___testing_testmult_LDADD         = libga++.la
+ga___testing_create_irreg_test_LDADD         = libga++.la
 
 endif # CXX_BINDINGS
 

README.md

@@ -63,7 +63,7 @@ Please refer to the INSTALL file for generic build instructions.  That is a good
 
 [back to top]
 
-Please submit issues to our [GitHub issue tracker](https://github.com/GlobalArrays/ga/issues).  We use Google Groups to host [our discussion forum](https://groups.google.com/forum/#!forum/hpctools), or send an email to [email protected] as an alias for that group.
+Please submit issues to our [GitHub issue tracker](https://github.com/GlobalArrays/ga/issues).  We use Google Groups to host [our discussion forum](https://groups.google.com/forum/#!forum/hpctools).
 
 ## WHERE IS THE DOCUMENTATION?
 
@@ -233,9 +233,10 @@ Any application code which only makes GA function calls can remain unchanged.
                         the software. If you experience mysterious faults,
                         consider rebuilding without optimization by using this
                         option.
+--enable-sysv           Enable System V Shared Memory.
 --enable-peigs          Enable Parallel Eigensystem Solver interface. This
                         will build the stubs required to call into the peigs
-                        library (external). 
+                        library (external).
 --enable-checkpoint     Enable checkpointing.  Untested.  For use with old
                         X-based visualization tool.
 --enable-profile        Enable profiling. Not sure what this does, sorry.
@@ -423,6 +424,7 @@ The CMake build only supports the MPI-based runtimes so GA can only be built usi
   * MPI_MULTITHREADED Use thread multiple runtime
   * MPI_PROGRESS_THREAD Use progress thread runtime
   * MPI_RMA Use MPI RMA based runtime.
+* `ENABLE_SYSV` Enable System V Shared Memory
 * `ENABLE_PROFILING` Build GA operation profiler. Does not work when using Clang compilers. [Default:OFF]
 * `GA_EXTRA_LIBS` Specify additional libraries or linker options when building GA.
 * `GCCROOT` Specify root of GCC installation. Only required when building with Clang compilers.
@@ -437,6 +439,9 @@ The CMake build only supports the MPI-based runtimes so GA can only be built usi
     * `LINALG_REQUIRED_COMPONENTS`: Options are `lp64` or `ilp64`. [Default:lp64]
     * `LINALG_OPTIONAL_COMPONENTS`: `sycl` [Default:none]
     * `ENABLE_SCALAPACK`: To enable ScaLAPACK discovery.
+* `[OPTIONAL]` CTEST options for handling different types of job launchers and their parameters.
+   * `GA_JOB_LAUNCH_CMD`: `mpirun`
+   * `GA_JOB_LAUNCH_ARGS`: `"-n 5"`
 #### The following options are standard CMake parameters. More information about them can be found in the CMake documentation.
 
 * `CMAKE_INSTALL_PREFIX` Specify the install location for GA.

armci/README

@@ -42,9 +42,9 @@ types of builds.  Detailed instructions are covered later in this file.
 QUESTIONS/HELP/SUPPORT/BUG-REPORT
 =================================
 
-email: [email protected]
+bug reports: https://github.com/GlobalArrays/ga/issues
 
-website: http://www.emsl.pnl.gov/docs/parsoft/armci
+website: https://hpc.pnl.gov/armci/
 
 ABOUT THIS SOFTWARE
 ===================

armci/configure.ac

@@ -5,8 +5,8 @@
 ###############################################################################
 AC_PREREQ([2.67])
 AC_INIT([Aggregate Remote Memory Copy Interface (ARMCI)], [1.5],
-        [[email protected]], [armci],
-        [http://www.emsl.pnl.gov/docs/parsoft/armci/])
+        [https://github.com/GlobalArrays/ga/issues], [armci],
+        [https://hpc.pnl.gov/armci/])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_HEADER([config.h])
 AC_CONFIG_MACRO_DIR([m4])

armci/src-gemini/armci-onesided.c

@@ -575,7 +575,7 @@ armci_onesided_remove_from_remote_mdh_list(void *tgt_ptr)
         long total_bytes;
         remote_mdh_node_t *rm_ll, *ll = remote_mdh_base_node;
 
-        NTK_MPI_GetComm(MPI_COMM_WORLD, &info);
+        NTK_MPI_GetComm(ARMCI_COMM_WORLD, &info);
 
      // get the onesided v2.0 api handle for the compute process
         cpGetOnesidedHandle(&cp_hnd);

armci/src-gemini/armci.c

@@ -358,7 +358,7 @@ int PARMCI_Init_args(int *argc, char ***argv)
 extern void *sbrk(intptr_t);
 extern void code_summary();
 
-int PARMCI_Init()
+int _armci_init(MPI_Comm comm)
 {
     caddr_t atbeginbrval = (caddr_t)sbrk(0);
     if(_armci_initialized>0) return 0;
@@ -405,6 +405,18 @@ int PARMCI_Init()
 }
 
 
+int PARMCI_Init()
+{
+  return _armci_init(MPI_COMM_WORLD);
+}
+
+
+int PARMCI_Init_mpi_comm(MPI_Comm comm)
+{
+  return _armci_init(comm);
+}
+
+
 void PARMCI_Finalize()
 {
     if(!_armci_initialized)return;

armci/src-gemini/armci.h

@@ -25,6 +25,7 @@ typedef long armci_size_t;
 extern int armci_notify(int proc);
 extern int armci_notify_wait(int proc,int *pval);
 extern int ARMCI_Init(void);    /* initialize ARMCI */
+extern int ARMCI_Init_mpi_comm(MPI_Comm comm);    /* initialize ARMCI */
 extern int ARMCI_Init_args(int *argc, char ***argv);
 extern void ARMCI_Barrier(void);    /* ARMCI Barrier*/
 

armci/src-gemini/memory.c

@@ -678,7 +678,7 @@ int PARMCI_Malloc(void *ptr_arr[], armci_size_t bytes)
       remote_mdh_node_t *ll;
       int node_master = armci_me;
       long total_bytes, lbytes = (long) bytes;
-      NTK_MPI_GetComm(MPI_COMM_WORLD, &info);
+      NTK_MPI_GetComm(ARMCI_COMM_WORLD, &info);
       long *bytes_per_rank = (long *) malloc(info.numa_np*sizeof(long));
 
    // not a wonderfully scalable solution