diff --git a/armci/src-gemini/armci.c b/armci/src-gemini/armci.c index d7e760199..8aa4213a7 100644 --- a/armci/src-gemini/armci.c +++ b/armci/src-gemini/armci.c @@ -238,6 +238,12 @@ void ARMCI_Error(char *msg, int code) armci_die(msg,code); } +/* Define this function so that all codes compile */ +int ARMCI_Active_rank() +{ + return 1; +} + void armci_allocate_locks() { diff --git a/armci/src-gemini/armci.h b/armci/src-gemini/armci.h index 643240cb2..4a5e4245f 100644 --- a/armci/src-gemini/armci.h +++ b/armci/src-gemini/armci.h @@ -26,6 +26,7 @@ extern int armci_notify(int proc); extern int armci_notify_wait(int proc,int *pval); extern int ARMCI_Init(void); /* initialize ARMCI */ extern int ARMCI_Init_mpi_comm(MPI_Comm comm); /* initialize ARMCI */ +extern int ARMCI_Active_rank(); extern int ARMCI_Init_args(int *argc, char ***argv); extern void ARMCI_Barrier(void); /* ARMCI Barrier*/ diff --git a/armci/src-portals/armci.c b/armci/src-portals/armci.c index 99d55a72e..3fcffedad 100644 --- a/armci/src-portals/armci.c +++ b/armci/src-portals/armci.c @@ -234,6 +234,12 @@ void ARMCI_Error(char *msg, int code) armci_die(msg,code); } +/* Define this function so that all codes compile */ +int ARMCI_Active_rank() +{ + return 1; +} + void armci_allocate_locks() { diff --git a/armci/src-portals/armci.h b/armci/src-portals/armci.h index fe5fef018..88ad805f6 100644 --- a/armci/src-portals/armci.h +++ b/armci/src-portals/armci.h @@ -26,6 +26,7 @@ extern int armci_notify(int proc); extern int armci_notify_wait(int proc,int *pval); extern int ARMCI_Init(void); /* initialize ARMCI */ extern int ARMCI_Init_mpi_comm(MPI_Comm comm); /* initialize ARMCI */ +extern int ARMCI_Active_rank(); extern int ARMCI_Init_args(int *argc, char ***argv); extern void ARMCI_Barrier(void); /* ARMCI Barrier*/ diff --git a/armci/src/common/armci.c b/armci/src/common/armci.c index 5d4876dd4..c91126886 100644 --- a/armci/src/common/armci.c +++ b/armci/src/common/armci.c @@ -208,6 +208,14 @@ void ARMCI_Error(char *msg, int code) armci_die(msg,code); } +/* This is used in some of the MPI-based runtimes to support initialization + * using a communicator. Include it here so all codes compile. + */ +int ARMCI_Active_rank() +{ + return 1; +} + void armci_allocate_locks() diff --git a/armci/src/include/armci.h b/armci/src/include/armci.h index fddfefe56..369eff989 100644 --- a/armci/src/include/armci.h +++ b/armci/src/include/armci.h @@ -30,6 +30,7 @@ extern int ARMCI_Init(void); /* initialize ARMCI */ #ifdef MSG_COMMS_MPI extern int ARMCI_Init_mpi_comm(MPI_Comm comm); /* initialize ARMCI */ #endif +extern int ARMCI_Active_rank(); extern int ARMCI_Init_args(int *argc, char ***argv); /* initialize ARMCI */ extern int ARMCI_Initialized(); diff --git a/comex/src-armci/armci.c b/comex/src-armci/armci.c index 47649f353..ee4bba4b3 100644 --- a/comex/src-armci/armci.c +++ b/comex/src-armci/armci.c @@ -445,6 +445,10 @@ int PARMCI_Initialized() return comex_initialized(); } +int ARMCI_Active_rank() +{ + return comex_active_rank(); +} void PARMCI_Lock(int mutex, int proc) { diff --git a/comex/src-armci/armci.h b/comex/src-armci/armci.h index da5ed1300..234baed4a 100644 --- a/comex/src-armci/armci.h +++ b/comex/src-armci/armci.h @@ -26,6 +26,7 @@ extern int armci_notify_wait(int proc,int *pval); extern int ARMCI_Init(); /* initialize ARMCI */ extern int ARMCI_Init_args(int *argc, char ***argv); /* initialize ARMCI */ extern int ARMCI_Init_mpi_comm(MPI_Comm comm); /* initialize ARMCI with external communicator */ +extern int ARMCI_Active_rank(); extern int ARMCI_Initialized(); extern void ARMCI_Barrier(); /* ARMCI Barrier*/ diff --git a/comex/src-common/comex.h b/comex/src-common/comex.h index 726b62b8f..c985688c2 100644 --- a/comex/src-common/comex.h +++ b/comex/src-common/comex.h @@ -64,6 +64,17 @@ extern int comex_init_args(int *argc, char ***argv); */ extern int comex_init_comm(MPI_Comm); +/** + * Returns status of rank with respect to user visibility. + * This function is true if this rank is visible to the + * application, false otherwise. Used in conjunction with + * comex_init_comm. + * + * @return 1 if rank is visible to application + * 0 otherwise + */ +extern int comex_active_rank(); + /** * Test whether comex has been initialized. * diff --git a/comex/src-dmapp/comex.c b/comex/src-dmapp/comex.c index 63dff0a12..6357c8330 100644 --- a/comex/src-dmapp/comex.c +++ b/comex/src-dmapp/comex.c @@ -1075,6 +1075,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return 1; +} + int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-mpi-mt/comex.c b/comex/src-mpi-mt/comex.c index b58da872d..e3ba76f6e 100644 --- a/comex/src-mpi-mt/comex.c +++ b/comex/src-mpi-mt/comex.c @@ -363,6 +363,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return 1; +} + int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-mpi-pr/comex.c b/comex/src-mpi-pr/comex.c index 633eef364..709d0ec69 100644 --- a/comex/src-mpi-pr/comex.c +++ b/comex/src-mpi-pr/comex.c @@ -722,6 +722,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return g_state.master[g_state.rank] != g_state.rank ? 1 : 0; +} + int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-mpi-pt/comex.c b/comex/src-mpi-pt/comex.c index 8143b30cf..14ba946e1 100644 --- a/comex/src-mpi-pt/comex.c +++ b/comex/src-mpi-pt/comex.c @@ -422,6 +422,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return g_state.master[g_state.rank] != g_state.rank ? 1 : 0; +} + int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-mpi/comex.c b/comex/src-mpi/comex.c index edcc98110..adbff98ac 100644 --- a/comex/src-mpi/comex.c +++ b/comex/src-mpi/comex.c @@ -1427,6 +1427,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_init_comm() +{ + return 1; +} + int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-mpi3/comex.c b/comex/src-mpi3/comex.c index 787bcc326..14e04d5dd 100644 --- a/comex/src-mpi3/comex.c +++ b/comex/src-mpi3/comex.c @@ -319,6 +319,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return 1; +} + int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-ofa/comex.c b/comex/src-ofa/comex.c index 53855e000..88044b42c 100644 --- a/comex/src-ofa/comex.c +++ b/comex/src-ofa/comex.c @@ -289,6 +289,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return 1; +} + int comex_init_args(int *argc, char ***argv) { int rc; diff --git a/comex/src-ofi/comex.c b/comex/src-ofi/comex.c index 3dc5a4197..dd4cbc7c8 100644 --- a/comex/src-ofi/comex.c +++ b/comex/src-ofi/comex.c @@ -1250,6 +1250,11 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int comex_active_rank() +{ + return 1; +} + int comex_init_args(int* argc, char*** argv) { int init_flag; diff --git a/comex/src-portals4/comex.c b/comex/src-portals4/comex.c index 4f7ba5de8..00149291a 100644 --- a/comex/src-portals4/comex.c +++ b/comex/src-portals4/comex.c @@ -346,6 +346,10 @@ int comex_init_comm(MPI_Comm comm) return _comex_init(comm); } +int return comex_active_rank() +{ + return 1; +} int comex_init_args(int *argc, char ***argv) { diff --git a/comex/src-template/comex.c b/comex/src-template/comex.c index 48a06fcf2..525d3ee9c 100644 --- a/comex/src-template/comex.c +++ b/comex/src-template/comex.c @@ -98,6 +98,48 @@ int comex_init_args(int *argc, char ***argv) return comex_init(); } +int comex_init_comm(MPI_Comm comm) +{ + int status; + + if (initialized) { + return 0; + } + initialized = 1; + + /* Assert MPI has been initialized */ + int init_flag; + status = MPI_Initialized(&init_flag); + assert(MPI_SUCCESS == status); + assert(init_flag); + + /* Duplicate the World Communicator */ + status = MPI_Comm_dup(comm, &(l_state.world_comm)); + assert(MPI_SUCCESS == status); + assert(l_state.world_comm); + + /* My Rank */ + status = MPI_Comm_rank(l_state.world_comm, &(l_state.rank)); + assert(MPI_SUCCESS == status); + + /* World Size */ + status = MPI_Comm_size(l_state.world_comm, &(l_state.size)); + assert(MPI_SUCCESS == status); + + /* groups */ + comex_group_init(); + + /* Synch - Sanity Check */ + MPI_Barrier(l_state.world_comm); + + return COMEX_SUCCESS; +} + +int comex_active_rank() +{ + return 1; +} + int comex_initialized() { diff --git a/global/src/base.c b/global/src/base.c index fe0e93cc8..db190c2d8 100644 --- a/global/src/base.c +++ b/global/src/base.c @@ -525,6 +525,18 @@ int pnga_initialize_comm(MPI_Comm comm) } #endif +/** + * Return status of processor with respect to visibility to user application. + * This function returns 1 if application can see this processor, 0 otherwise. + */ +#if HAVE_SYS_WEAK_ALIAS_PRAGMA +# pragma weak wnga_active_rank = pnga_active_rank +#endif +logical pnga_active_rank() +{ + return ARMCI_Active_rank(); +} + #if HAVE_SYS_WEAK_ALIAS_PRAGMA # pragma weak wnga_initialized = pnga_initialized #endif diff --git a/global/src/capi.c b/global/src/capi.c index a80787f87..f71dc391a 100644 --- a/global/src/capi.c +++ b/global/src/capi.c @@ -131,6 +131,16 @@ int NGA_Initialize_comm(MPI_Comm comm) return wnga_initialize_comm(comm); } +int GA_Active_rank() +{ + return wnga_active_rank(); +} + +int NGA_Active_rank() +{ + return wnga_active_rank(); +} + int GA_Initialized() { return wnga_initialized(); diff --git a/global/src/cnames.h b/global/src/cnames.h index 0752d9c1d..a708070a9 100644 --- a/global/src/cnames.h +++ b/global/src/cnames.h @@ -1,3 +1,15 @@ +#define ga_active_rank_ F77_FUNC_(ga_active_rank, GA_ACTIVE_RANK) +#define ga_cactive_rank_ F77_FUNC_(ga_cactive_rank, GA_CACTIVE_RANK) +#define ga_dactive_rank_ F77_FUNC_(ga_dactive_rank, GA_DACTIVE_RANK) +#define ga_iactive_rank_ F77_FUNC_(ga_iactive_rank, GA_IACTIVE_RANK) +#define ga_sactive_rank_ F77_FUNC_(ga_sactive_rank, GA_SACTIVE_RANK) +#define ga_zactive_rank_ F77_FUNC_(ga_zactive_rank, GA_ZACTIVE_RANK) +#define nga_active_rank_ F77_FUNC_(nga_active_rank, NGA_ACTIVE_RANK) +#define nga_cactive_rank_ F77_FUNC_(nga_cactive_rank, NGA_CACTIVE_RANK) +#define nga_dactive_rank_ F77_FUNC_(nga_dactive_rank, NGA_DACTIVE_RANK) +#define nga_iactive_rank_ F77_FUNC_(nga_iactive_rank, NGA_IACTIVE_RANK) +#define nga_sactive_rank_ F77_FUNC_(nga_sactive_rank, NGA_SACTIVE_RANK) +#define nga_zactive_rank_ F77_FUNC_(nga_zactive_rank, NGA_ZACTIVE_RANK) #define ga_allocate_ F77_FUNC_(ga_allocate, GA_ALLOCATE) #define ga_deallocate_ F77_FUNC_(ga_deallocate, GA_DEALLOCATE) #define ga_callocate_ F77_FUNC_(ga_callocate,GA_CALLOCATE) diff --git a/global/src/fapi.c b/global/src/fapi.c index 66dd84a48..cf4123d43 100644 --- a/global/src/fapi.c +++ b/global/src/fapi.c @@ -480,6 +480,16 @@ logical FATR nga_initialize_comm_(MPI_Comm comm) return wnga_initialize_comm(comm); } +logical FATR ga_active_rank() +{ + return wnga_active_rank(); +} + +logical FATR nga_active_rank() +{ + return wnga_active_rank(); +} + void FATR ga_initialize_ltd_(Integer *limit) { _ga_initialize_f=1; diff --git a/global/src/ga-papi.h b/global/src/ga-papi.h index 6baedbe78..52e9b89d5 100644 --- a/global/src/ga-papi.h +++ b/global/src/ga-papi.h @@ -13,6 +13,7 @@ typedef intp AccessIndex; /* Routines from base.c */ extern void pnga_version(Integer *major, Integer *minor, Integer *patch); extern logical pnga_allocate(Integer g_a); +extern logical pnga_active_rank(); extern logical pnga_deallocate(Integer g_a); extern logical pnga_compare_distr(Integer g_a, Integer g_b); extern logical pnga_create(Integer type, Integer ndim, diff --git a/global/src/ga.h b/global/src/ga.h index e19eb1a17..aa9e03b36 100644 --- a/global/src/ga.h +++ b/global/src/ga.h @@ -19,6 +19,7 @@ typedef Integer ga_nbhdl_t; extern void GA_Abs_value(int g_a); extern void GA_Abs_value_patch(int g_a, int *lo, int *hi); +extern int GA_Active_rank(); extern void GA_Add_constant(int g_a, void* alpha); extern void GA_Add_constant_patch(int g,int *lo,int *hi,void *alpha); extern void GA_Add_diagonal(int g_a, int g_v); @@ -199,6 +200,7 @@ extern void NGA_Access_ghost_element(int g_a, void *ptr, int subscript extern void NGA_Access_ghosts(int g_a, int dims[], void *ptr, int ld[]); extern void NGA_Access(int g_a, int lo[], int hi[], void *ptr, int ld[]); extern void NGA_Acc(int g_a, int lo[], int hi[],void* buf,int ld[],void* alpha); +extern int NGA_Active_rank(); extern void NGA_Add_patch(void * alpha, int g_a, int alo[], int ahi[], void * beta, int g_b, int blo[], int bhi[], int g_c, int clo[], int chi[]); extern int NGA_Allocate(int g_a); extern void NGA_Alloc_gatscat_buf(int nelems); diff --git a/global/src/global.fh.in b/global/src/global.fh.in index a316022fb..26c900ca4 100644 --- a/global/src/global.fh.in +++ b/global/src/global.fh.in @@ -4,6 +4,7 @@ ! logical ga_allocate logical ga_deallocate + logical ga_active_rank complex ga_cdot complex ga_cdot_patch integer ga_cluster_nnodes @@ -76,6 +77,7 @@ double complex ga_zdot_patch logical nga_allocate logical nga_deallocate + logical nga_active_rank complex nga_cdot complex nga_cdot_patch integer nga_cluster_nnodes