-
Notifications
You must be signed in to change notification settings - Fork 47
/
dbcsr_mpiwrap.F
5568 lines (4656 loc) · 200 KB
/
dbcsr_mpiwrap.F
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
!--------------------------------------------------------------------------------------------------!
! Copyright (C) by the DBCSR developers group - All rights reserved !
! This file is part of the DBCSR library. !
! !
! For information on the license, see the LICENSE file. !
! For further information please visit https://dbcsr.cp2k.org !
! SPDX-License-Identifier: GPL-2.0+ !
!--------------------------------------------------------------------------------------------------!
MODULE dbcsr_mpiwrap
!! Interface to the message passing library MPI
USE ISO_C_BINDING, ONLY: C_F_POINTER, &
C_PTR
USE dbcsr_kinds, ONLY: &
dp, int_4, int_4_size, int_8, int_8_size, real_4, real_4_size, real_8, &
real_8_size
USE dbcsr_machine, ONLY: m_abort, m_hostnm
#include "base/dbcsr_base_uses.f90"
#:include 'dbcsr_mpiwrap.fypp'
#if defined(__parallel) && defined(__USE_MPI_F08)
USE mpi_f08, ONLY: mpi_datatype, mpi_comm, mpi_request, mpi_win, mpi_file, mpi_info, mpi_status, mpi_group, MPI_ANY_TAG, &
MPI_ANY_SOURCE, MPI_COMM_NULL, MPI_COMM_SELF, MPI_COMM_WORLD, MPI_REQUEST_NULL, MPI_WIN_NULL, &
MPI_FILE_NULL, MPI_INFO_NULL, MPI_DATATYPE_NULL, MPI_STATUS_SIZE, MPI_PROC_NULL, &
MPI_MAX_LIBRARY_VERSION_STRING, MPI_OFFSET_KIND, MPI_ADDRESS_KIND, MPI_MODE_CREATE, &
MPI_MODE_RDONLY, MPI_MODE_WRONLY, MPI_MODE_RDWR, MPI_MODE_EXCL, MPI_MODE_APPEND, &
MPI_MAX_ERROR_STRING, MPI_IDENT, MPI_CONGRUENT, MPI_SIMILAR, MPI_UNEQUAL, MPI_COMPLEX, MPI_DOUBLE_COMPLEX, &
MPI_INTEGER, MPI_LOGICAL, MPI_DOUBLE_PRECISION, MPI_STATUS_IGNORE, MPI_TYPE_SIZE, MPI_FILE_READ_AT_ALL, &
MPI_FILE_READ_AT, mpi_type_indexed, mpi_irecv, mpi_recv, mpi_isend, mpi_send, mpi_sendrecv, mpi_allreduce, &
mpi_reduce, mpi_barrier, mpi_ibarrier, mpi_iallreduce, mpi_test, mpi_probe, mpi_wait, mpi_iprobe, &
mpi_testany, mpi_testall, mpi_waitany, mpi_waitall, mpi_allgather, mpi_allgatherv, mpi_iallgather, &
mpi_iallgatherv, mpi_gather, mpi_gatherv, mpi_scatter, mpi_scatterv, mpi_iscatterv, mpi_iscatter, &
mpi_scan, mpi_alltoall, mpi_alltoallv, mpi_type_indexed, mpi_bcast, mpi_ibcast, mpi_group_free, &
mpi_comm_free, mpi_comm_create, mpi_win_create, mpi_rget, mpi_free_mem, mpi_get_address, &
MPI_FILE_WRITE_AT, MPI_FILE_WRITE_AT_ALL, mpi_comm_group, mpi_init, mpi_init_thread, mpi_bottom, &
MPI_IN_PLACE, MPI_MIN, MPI_MAX, MPI_SUM, MPI_PROD, MPI_SOURCE, MPI_TAG, MPI_REAL, MPI_INTEGER8, &
MPI_MODE_NOCHECK, MPI_CHARACTER, MPI_ERRORS_RETURN, MPI_2DOUBLE_PRECISION, MPI_MAXLOC, MPI_LOR, &
MPI_MINLOC, MPI_SUCCESS, MPI_THREAD_FUNNELED
#endif
#if defined(__parallel) && ! defined(__USE_MPI_F08)
USE mpi
#endif
! subroutines: unfortunately, mpi implementations do not provide interfaces for all subroutines (problems with types and ranks explosion),
! we do not quite know what is in the module, so we can not include any....
! to nevertheless get checking for what is included, we use the mpi module without use clause, getting all there is
! USE mpi, ONLY: mpi_allgather, mpi_allgatherv, mpi_alloc_mem, mpi_allreduce, mpi_alltoall, mpi_alltoallv, mpi_bcast,&
! mpi_cart_coords, mpi_cart_create, mpi_cart_get, mpi_cart_rank, mpi_cart_sub, mpi_dims_create, mpi_file_close,&
! mpi_file_get_size, mpi_file_open, mpi_file_read_at_all, mpi_file_read_at, mpi_file_write_at_all,&
! mpi_file_write_at, mpi_free_mem, mpi_gather, mpi_gatherv, mpi_get_address, mpi_group_translate_ranks, mpi_irecv,&
! mpi_isend, mpi_recv, mpi_reduce, mpi_reduce_scatter, mpi_rget, mpi_scatter, mpi_send,&
! mpi_sendrecv, mpi_sendrecv_replace, mpi_testany, mpi_waitall, mpi_waitany, mpi_win_create
! functions
! USE mpi, ONLY: mpi_wtime
! constants
! USE mpi, ONLY: MPI_DOUBLE_PRECISION, MPI_DOUBLE_COMPLEX, MPI_REAL, MPI_COMPLEX, MPI_ANY_TAG,&
! MPI_ANY_SOURCE, MPI_COMM_NULL, MPI_REQUEST_NULL, MPI_WIN_NULL, MPI_STATUS_SIZE, MPI_STATUS_IGNORE, MPI_STATUSES_IGNORE, &
! MPI_ADDRESS_KIND, MPI_OFFSET_KIND, MPI_MODE_CREATE, MPI_MODE_RDONLY, MPI_MODE_WRONLY,&
! MPI_MODE_RDWR, MPI_MODE_EXCL, MPI_COMM_SELF, MPI_COMM_WORLD, MPI_THREAD_FUNNELED,&
! MPI_ERRORS_RETURN, MPI_SUCCESS, MPI_MAX_PROCESSOR_NAME, MPI_MAX_ERROR_STRING, MPI_IDENT,&
! MPI_UNEQUAL, MPI_MAX, MPI_SUM, MPI_INFO_NULL, MPI_IN_PLACE, MPI_CONGRUENT, MPI_SIMILAR, MPI_MIN, MPI_SOURCE,&
! MPI_TAG, MPI_INTEGER8, MPI_INTEGER, MPI_MAXLOC, MPI_2INTEGER, MPI_MINLOC, MPI_LOGICAL, MPI_2DOUBLE_PRECISION,&
! MPI_LOR, MPI_CHARACTER, MPI_BOTTOM, MPI_MODE_NOCHECK, MPI_2REAL
! To simplify the transition between the old MPI module and the F08-style module, we introduce these macros to switch between the required handle types
! Unfortunately, Fortran does not offer something like typedef in C/C++
!
! MPI_STATUS_ARRAY is a macro to provide the appropriate type of arrays of status variables because with mpi.
!
! MPI_STATUS_EXTRACT is a macro to provide an extraction method from the respective MPI_Status objects/ status arrays depending on the MPI library in use.
! Use it as "<name of status variable> MPI_STATUS_EXTRACT(<name of component of interest>)".
! The space before MPI_STATUS_EXTRACT is compulsory to allow the C-preprocessor to identify the macro.
! In Fortran, this space is ignored according to the standards.
#if defined(__parallel) && defined(__USE_MPI_F08)
#define MPI_DATA_TYPE TYPE(MPI_Datatype)
#define MPI_COMM_TYPE TYPE(MPI_Comm)
#define MPI_REQUEST_TYPE TYPE(MPI_Request)
#define MPI_WIN_TYPE TYPE(MPI_Win)
#define MPI_FILE_TYPE TYPE(MPI_File)
#define MPI_INFO_TYPE TYPE(MPI_Info)
#define MPI_STATUS_TYPE TYPE(MPI_Status)
#define MPI_STATUS_TYPE_ARRAY(X) TYPE(MPI_Status),DIMENSION(X)
#define MPI_GROUP_TYPE TYPE(MPI_Group)
#define MPI_STATUS_EXTRACT(X) %X
#else
#define MPI_DATA_TYPE INTEGER
#define MPI_COMM_TYPE INTEGER
#define MPI_REQUEST_TYPE INTEGER
#define MPI_WIN_TYPE INTEGER
#define MPI_FILE_TYPE INTEGER
#define MPI_INFO_TYPE INTEGER
#define MPI_STATUS_TYPE INTEGER,DIMENSION(MPI_STATUS_SIZE)
#define MPI_STATUS_TYPE_ARRAY(X) INTEGER,DIMENSION(MPI_STATUS_SIZE,X)
#define MPI_GROUP_TYPE INTEGER
#define MPI_STATUS_EXTRACT(X) (X)
#endif
IMPLICIT NONE
PRIVATE
! parameters that might be needed
#if defined(__parallel)
LOGICAL, PARAMETER :: dbcsr_is_parallel = .TRUE.
INTEGER, PARAMETER, PUBLIC :: mp_any_tag = MPI_ANY_TAG
INTEGER, PARAMETER, PUBLIC :: mp_any_source = MPI_ANY_SOURCE
MPI_COMM_TYPE, PARAMETER :: mp_comm_null_handle = MPI_COMM_NULL
MPI_COMM_TYPE, PARAMETER :: mp_comm_self_handle = MPI_COMM_SELF
MPI_COMM_TYPE, PARAMETER :: mp_comm_world_handle = MPI_COMM_WORLD
MPI_REQUEST_TYPE, PARAMETER :: mp_request_null_handle = MPI_REQUEST_NULL
MPI_WIN_TYPE, PARAMETER :: mp_win_null_handle = MPI_WIN_NULL
MPI_FILE_TYPE, PARAMETER :: mp_file_null_handle = MPI_FILE_NULL
MPI_INFO_TYPE, PARAMETER :: mp_info_null_handle = MPI_INFO_NULL
MPI_DATA_TYPE, PARAMETER :: mp_datatype_null_handle = MPI_DATATYPE_NULL
INTEGER, PARAMETER, PUBLIC :: mp_status_size = MPI_STATUS_SIZE
INTEGER, PARAMETER, PUBLIC :: mp_proc_null = MPI_PROC_NULL
! Set max allocatable memory by MPI to 2 GiByte
INTEGER(KIND=MPI_ADDRESS_KIND), PARAMETER, PRIVATE :: mp_max_memory_size = HUGE(INT(1, KIND=int_4))
INTEGER, PARAMETER, PUBLIC :: mp_max_library_version_string = MPI_MAX_LIBRARY_VERSION_STRING
INTEGER, PARAMETER, PUBLIC :: file_offset = MPI_OFFSET_KIND
INTEGER, PARAMETER, PUBLIC :: address_kind = MPI_ADDRESS_KIND
INTEGER, PARAMETER, PUBLIC :: file_amode_create = MPI_MODE_CREATE
INTEGER, PARAMETER, PUBLIC :: file_amode_rdonly = MPI_MODE_RDONLY
INTEGER, PARAMETER, PUBLIC :: file_amode_wronly = MPI_MODE_WRONLY
INTEGER, PARAMETER, PUBLIC :: file_amode_rdwr = MPI_MODE_RDWR
INTEGER, PARAMETER, PUBLIC :: file_amode_excl = MPI_MODE_EXCL
INTEGER, PARAMETER, PUBLIC :: file_amode_append = MPI_MODE_APPEND
#else
LOGICAL, PARAMETER :: dbcsr_is_parallel = .FALSE.
INTEGER, PARAMETER, PUBLIC :: mp_any_tag = -1
INTEGER, PARAMETER, PUBLIC :: mp_any_source = -2
MPI_COMM_TYPE, PARAMETER :: mp_comm_null_handle = -3
MPI_COMM_TYPE, PARAMETER :: mp_comm_self_handle = -11
MPI_COMM_TYPE, PARAMETER :: mp_comm_world_handle = -12
MPI_REQUEST_TYPE, PARAMETER :: mp_request_null_handle = -4
MPI_WIN_TYPE, PARAMETER :: mp_win_null_handle = -5
MPI_FILE_TYPE, PARAMETER :: mp_file_null_handle = -6
MPI_INFO_TYPE, PARAMETER :: mp_info_null_handle = -7
MPI_DATA_TYPE, PARAMETER :: mp_datatype_null_handle = -13
INTEGER, PARAMETER, PUBLIC :: mp_status_size = -7
INTEGER, PARAMETER, PUBLIC :: mp_proc_null = -8
INTEGER, PARAMETER, PUBLIC :: mp_max_library_version_string = 1
INTEGER, PARAMETER, PUBLIC :: file_offset = int_8
INTEGER, PARAMETER, PUBLIC :: address_kind = int_8
INTEGER, PARAMETER, PUBLIC :: file_amode_create = 1
INTEGER, PARAMETER, PUBLIC :: file_amode_rdonly = 2
INTEGER, PARAMETER, PUBLIC :: file_amode_wronly = 4
INTEGER, PARAMETER, PUBLIC :: file_amode_rdwr = 8
INTEGER, PARAMETER, PUBLIC :: file_amode_excl = 64
INTEGER, PARAMETER, PUBLIC :: file_amode_append = 128
#endif
! MPI wrapper types (keep the handles private for to switch between serial mode/old mpi module and mpi_f08!)
TYPE mp_comm_type
PRIVATE
MPI_COMM_TYPE :: handle = mp_comm_null_handle
CONTAINS
PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: get_handle => mp_get_comm_handle
PROCEDURE, PUBLIC, PASS(comm), NON_OVERRIDABLE :: set_handle => mp_set_comm_handle
PROCEDURE, PRIVATE, PASS(comm1), NON_OVERRIDABLE :: mp_comm_op_eq
GENERIC, PUBLIC :: OPERATOR(.EQ.) => mp_comm_op_eq
PROCEDURE, PRIVATE, PASS(comm1), NON_OVERRIDABLE :: mp_comm_op_ne
GENERIC, PUBLIC :: OPERATOR(.NE.) => mp_comm_op_ne
END TYPE mp_comm_type
TYPE mp_request_type
PRIVATE
MPI_REQUEST_TYPE :: handle = mp_request_null_handle
CONTAINS
PROCEDURE, PUBLIC, PASS(request), NON_OVERRIDABLE :: get_handle => mp_get_request_handle
PROCEDURE, PUBLIC, PASS(request), NON_OVERRIDABLE :: set_handle => mp_set_request_handle
PROCEDURE, PRIVATE, PASS(request1), NON_OVERRIDABLE :: mp_request_op_eq
GENERIC, PUBLIC :: OPERATOR(.EQ.) => mp_request_op_eq
PROCEDURE, PRIVATE, PASS(request1), NON_OVERRIDABLE :: mp_request_op_ne
GENERIC, PUBLIC :: OPERATOR(.NE.) => mp_request_op_ne
END TYPE mp_request_type
TYPE mp_win_type
PRIVATE
MPI_WIN_TYPE :: handle = mp_win_null_handle
CONTAINS
PROCEDURE, PUBLIC, PASS(win), NON_OVERRIDABLE :: get_handle => mp_get_win_handle
PROCEDURE, PUBLIC, PASS(win), NON_OVERRIDABLE :: set_handle => mp_set_win_handle
PROCEDURE, PRIVATE, PASS(win1), NON_OVERRIDABLE :: mp_win_op_eq
GENERIC, PUBLIC :: OPERATOR(.EQ.) => mp_win_op_eq
PROCEDURE, PRIVATE, PASS(win1), NON_OVERRIDABLE :: mp_win_op_ne
GENERIC, PUBLIC :: OPERATOR(.NE.) => mp_win_op_ne
END TYPE mp_win_type
TYPE mp_file_type
PRIVATE
MPI_FILE_TYPE :: handle = mp_file_null_handle
CONTAINS
PROCEDURE, PUBLIC, PASS(file), NON_OVERRIDABLE :: get_handle => mp_get_file_handle
PROCEDURE, PUBLIC, PASS(file), NON_OVERRIDABLE :: set_handle => mp_set_file_handle
PROCEDURE, PRIVATE, PASS(file1), NON_OVERRIDABLE :: mp_file_op_eq
GENERIC, PUBLIC :: OPERATOR(.EQ.) => mp_file_op_eq
PROCEDURE, PRIVATE, PASS(file1), NON_OVERRIDABLE :: mp_file_op_ne
GENERIC, PUBLIC :: OPERATOR(.NE.) => mp_file_op_ne
END TYPE mp_file_type
TYPE mp_info_type
PRIVATE
MPI_INFO_TYPE :: handle = mp_info_null_handle
CONTAINS
PROCEDURE, PUBLIC, PASS(info), NON_OVERRIDABLE :: get_handle => mp_get_info_handle
PROCEDURE, PUBLIC, PASS(info), NON_OVERRIDABLE :: set_handle => mp_set_info_handle
PROCEDURE, PRIVATE, PASS(info1), NON_OVERRIDABLE :: mp_info_op_eq
GENERIC, PUBLIC :: OPERATOR(.EQ.) => mp_info_op_eq
PROCEDURE, PRIVATE, PASS(info1), NON_OVERRIDABLE :: mp_info_op_ne
GENERIC, PUBLIC :: OPERATOR(.NE.) => mp_info_op_ne
END TYPE mp_info_type
! The actual MPI wrapper constants
TYPE(mp_comm_type), PARAMETER, PUBLIC :: mp_comm_null = mp_comm_type(mp_comm_null_handle)
TYPE(mp_comm_type), PARAMETER, PUBLIC :: mp_comm_self = mp_comm_type(mp_comm_self_handle)
TYPE(mp_comm_type), PARAMETER, PUBLIC :: mp_comm_world = mp_comm_type(mp_comm_world_handle)
TYPE(mp_request_type), PARAMETER, PUBLIC :: mp_request_null = mp_request_type(mp_request_null_handle)
TYPE(mp_win_type), PARAMETER, PUBLIC :: mp_win_null = mp_win_type(mp_win_null_handle)
TYPE(mp_file_type), PARAMETER, PUBLIC :: mp_file_null = mp_file_type(mp_file_null_handle)
TYPE(mp_info_type), PARAMETER, PUBLIC :: mp_info_null = mp_info_type(mp_info_null_handle)
! we need to fix this to a given number (crossing fingers)
! so that the serial code using Fortran stream IO and the MPI have the same sizes.
INTEGER, PARAMETER, PUBLIC :: mpi_character_size = 1
INTEGER, PARAMETER, PUBLIC :: mpi_integer_size = 4
CHARACTER(LEN=*), PARAMETER, PRIVATE :: moduleN = 'dbcsr_mpiwrap'
#if defined(__parallel)
! internal reference counter used to debug communicator leaks
INTEGER, PRIVATE, SAVE :: debug_comm_count = 0
#endif
! types
PUBLIC :: mp_comm_type
PUBLIC :: mp_request_type
PUBLIC :: mp_win_type
PUBLIC :: mp_file_type
PUBLIC :: mp_info_type
! init and error
PUBLIC :: mp_world_init, mp_world_finalize
PUBLIC :: mp_get_comm_count
PUBLIC :: mp_abort
! performance gathering
PUBLIC :: mp_perf_env_type
PUBLIC :: mp_perf_env_retain, mp_perf_env_release
PUBLIC :: add_mp_perf_env, rm_mp_perf_env, get_mp_perf_env, describe_mp_perf_env
PUBLIC :: has_mp_perf_env
! informational / generation of sub comms
PUBLIC :: mp_environ, mp_comm_compare, mp_cart_coords, mp_rank_compare
PUBLIC :: mp_cart_create, mp_dims_create, mp_cart_rank, mp_cart_sub, mp_comm_free
PUBLIC :: mp_comm_dup, mp_comm_split, mp_comm_split_direct
PUBLIC :: dbcsr_is_parallel
PUBLIC :: mp_probe
! message passing
PUBLIC :: mp_bcast, mp_sum, mp_sum_partial, mp_max, mp_maxloc, mp_minloc, mp_min, mp_prod, mp_sync
PUBLIC :: mp_isync, mp_isum
PUBLIC :: mp_gather, mp_alltoall, mp_sendrecv, mp_allgather, mp_iallgather
PUBLIC :: mp_isend, mp_irecv, mp_ibcast
PUBLIC :: mp_isendrecv, mp_wait, mp_waitall, mp_waitany, mp_testany
PUBLIC :: mp_testall, mp_iscatter, mp_test
PUBLIC :: mp_gatherv
PUBLIC :: mp_send, mp_recv
! Memory management
PUBLIC :: mp_allocate, mp_deallocate
! MPI re-ordering
PUBLIC :: mp_reordering
! I/O
PUBLIC :: mp_file_open, mp_file_close
PUBLIC :: mp_file_delete
PUBLIC :: mp_file_write_at
PUBLIC :: mp_file_write_at_all, mp_file_read_at_all
PUBLIC :: mp_file_get_size
PUBLIC :: mp_file_get_position
! some 'advanced types' currently only used for dbcsr
PUBLIC :: mp_type_descriptor_type
PUBLIC :: mp_type_make
PUBLIC :: mp_type_size
! one-sided communication
PUBLIC :: mp_win_create, mp_win_free, mp_win_lock_all, &
mp_win_unlock_all, mp_rget, mp_win_flush_all
! vector types
PUBLIC :: mp_type_indexed_make_r, mp_type_indexed_make_d, &
mp_type_indexed_make_c, mp_type_indexed_make_z, &
mp_type_indexed_make_i, mp_type_indexed_make_l
! misc
PUBLIC :: mp_get_library_version, mp_get_processor_name
! assumed to be private
! Interface declarations for non-data-oriented subroutines.
INTERFACE mp_environ
MODULE PROCEDURE mp_environ_l, mp_environ_c, mp_environ_c2
END INTERFACE
INTERFACE mp_waitall
MODULE PROCEDURE mp_waitall_1, mp_waitall_2
END INTERFACE
INTERFACE mp_testall
MODULE PROCEDURE mp_testall_tv
END INTERFACE
INTERFACE mp_test
MODULE PROCEDURE mp_test_1
END INTERFACE
INTERFACE mp_testany
MODULE PROCEDURE mp_testany_1, mp_testany_2
END INTERFACE
!
! interfaces to deal easily with scalars / vectors / matrices / ...
! of the different types (integers, doubles, logicals, characters)
!
INTERFACE mp_minloc
MODULE PROCEDURE mp_minloc_dv
END INTERFACE
INTERFACE mp_maxloc
MODULE PROCEDURE mp_maxloc_dv
END INTERFACE
$:gen_mp_iface('bcast', suffixes=['', 'v', 'm', '3'], extra_suffixes=['b', 'bv', 'av', 'am'])
$:gen_mp_iface('ibcast', suffixes=['', 'v'])
INTERFACE mp_sum
MODULE PROCEDURE mp_sum_i, mp_sum_iv, mp_sum_im, mp_sum_im3, mp_sum_im4, &
mp_sum_l, mp_sum_lv, mp_sum_lm, mp_sum_lm3, mp_sum_lm4, &
mp_sum_r, mp_sum_rv, mp_sum_rm, mp_sum_rm3, mp_sum_rm4, &
mp_sum_d, mp_sum_dv, mp_sum_dm, mp_sum_dm3, mp_sum_dm4, &
mp_sum_c, mp_sum_cv, mp_sum_cm, mp_sum_cm3, mp_sum_cm4, &
mp_sum_z, mp_sum_zv, mp_sum_zm, mp_sum_zm3, mp_sum_zm4, &
mp_sum_root_iv, mp_sum_root_im, &
mp_sum_root_lv, mp_sum_root_lm, &
mp_sum_root_rv, mp_sum_root_rm, &
mp_sum_root_dv, mp_sum_root_dm, &
mp_sum_root_cv, mp_sum_root_cm, &
mp_sum_root_zv, mp_sum_root_zm
MODULE PROCEDURE mp_sum_b, mp_sum_bv
END INTERFACE
$:gen_mp_iface('isum', suffixes=['v'], extra_suffixes=['bv'])
$:gen_mp_iface('sum_partial', suffixes=['m'])
$:gen_mp_iface('max', suffixes=['', 'v'])
$:gen_mp_iface('min', suffixes=['', 'v'])
$:gen_mp_iface('prod')
$:gen_mp_iface('gather', suffixes=['', 'm', 'v'])
$:gen_mp_iface('gatherv', suffixes=['v'])
INTERFACE mp_allgather
!! @todo move allgatherv to a separate declaration
MODULE PROCEDURE &
mp_allgather_i, mp_allgather_i2, &
mp_allgather_i12, mp_allgather_i23, mp_allgather_i34, &
mp_allgather_i22, &
mp_allgather_l, mp_allgather_l2, &
mp_allgather_l12, mp_allgather_l23, mp_allgather_l34, &
mp_allgather_l22, &
mp_allgather_r, mp_allgather_r2, &
mp_allgather_r12, mp_allgather_r23, mp_allgather_r34, &
mp_allgather_r22, &
mp_allgather_d, mp_allgather_d2, &
mp_allgather_d12, mp_allgather_d23, mp_allgather_d34, &
mp_allgather_d22, &
mp_allgather_c, mp_allgather_c2, &
mp_allgather_c12, mp_allgather_c23, mp_allgather_c34, &
mp_allgather_c22, &
mp_allgather_z, mp_allgather_z2, &
mp_allgather_z12, mp_allgather_z23, mp_allgather_z34, &
mp_allgather_z22, &
mp_allgatherv_iv, &
mp_allgatherv_lv, &
mp_allgatherv_rv, &
mp_allgatherv_dv, &
mp_allgatherv_cv, &
mp_allgatherv_zv
END INTERFACE
INTERFACE mp_iallgather
MODULE PROCEDURE &
mp_iallgather_i, mp_iallgather_l, &
mp_iallgather_r, mp_iallgather_d, &
mp_iallgather_c, mp_iallgather_z, &
mp_iallgather_i11, mp_iallgather_l11, &
mp_iallgather_r11, mp_iallgather_d11, &
mp_iallgather_c11, mp_iallgather_z11, &
mp_iallgather_i13, mp_iallgather_l13, &
mp_iallgather_r13, mp_iallgather_d13, &
mp_iallgather_c13, mp_iallgather_z13, &
mp_iallgather_i22, mp_iallgather_l22, &
mp_iallgather_r22, mp_iallgather_d22, &
mp_iallgather_c22, mp_iallgather_z22, &
mp_iallgather_i24, mp_iallgather_l24, &
mp_iallgather_r24, mp_iallgather_d24, &
mp_iallgather_c24, mp_iallgather_z24, &
mp_iallgather_i33, mp_iallgather_l33, &
mp_iallgather_r33, mp_iallgather_d33, &
mp_iallgather_c33, mp_iallgather_z33, &
mp_iallgatherv_iv, mp_iallgatherv_iv2, &
mp_iallgatherv_lv, mp_iallgatherv_lv2, &
mp_iallgatherv_rv, mp_iallgatherv_rv2, &
mp_iallgatherv_dv, mp_iallgatherv_dv2, &
mp_iallgatherv_cv, mp_iallgatherv_cv2, &
mp_iallgatherv_zv, mp_iallgatherv_zv2
END INTERFACE
INTERFACE mp_iscatter
MODULE PROCEDURE mp_iscatter_i, &
mp_iscatter_l, &
mp_iscatter_r, &
mp_iscatter_d, &
mp_iscatter_c, &
mp_iscatter_z, &
mp_iscatter_iv2, &
mp_iscatter_lv2, &
mp_iscatter_rv2, &
mp_iscatter_dv2, &
mp_iscatter_cv2, &
mp_iscatter_zv2, &
mp_iscatterv_iv, &
mp_iscatterv_lv, &
mp_iscatterv_rv, &
mp_iscatterv_dv, &
mp_iscatterv_cv, &
mp_iscatterv_zv
END INTERFACE
$:gen_mp_iface('alltoall', suffixes=['', '22', '44', '11v'])
$:gen_mp_iface('send', suffixes=['', 'v'])
$:gen_mp_iface('recv', suffixes=['', 'v'])
$:gen_mp_iface('sendrecv', suffixes=['v', ])
$:gen_mp_iface('isendrecv', suffixes=['', 'v'])
$:gen_mp_iface('isend', suffixes=['v', 'm2'], extra_suffixes=['bv', 'custom'])
$:gen_mp_iface('irecv', suffixes=['v', 'm2'], extra_suffixes=['bv', 'custom'])
$:gen_mp_iface('win_create', suffixes=['v'])
$:gen_mp_iface('rget', suffixes=['v'])
$:gen_mp_iface('allocate')
$:gen_mp_iface('deallocate')
$:gen_mp_iface('type_make', extra_suffixes=['struct'])
$:gen_mp_iface('file_write_at', suffixes=['', 'v'], extra_suffixes=['ch'])
$:gen_mp_iface('file_write_at_all', suffixes=['', 'v'], extra_suffixes=['ch'])
$:gen_mp_iface('file_read_at_all', suffixes=['', 'v'], extra_suffixes=['ch'])
#if defined(__parallel)
$:gen_mp_iface('alloc_mem')
$:gen_mp_iface('free_mem')
#endif
! Type declarations
TYPE mp_indexing_meta_type
INTEGER, DIMENSION(:), POINTER :: index => NULL(), chunks => NULL()
END TYPE mp_indexing_meta_type
TYPE mp_type_descriptor_type
MPI_DATA_TYPE :: type_handle = mp_datatype_null_handle
INTEGER :: length = -1
#if defined(__parallel)
INTEGER(kind=mpi_address_kind) :: base = -1_mpi_address_kind
#endif
INTEGER(kind=int_4), DIMENSION(:), POINTER :: data_i => NULL()
INTEGER(kind=int_8), DIMENSION(:), POINTER :: data_l => NULL()
REAL(kind=real_4), DIMENSION(:), POINTER :: data_r => NULL()
REAL(kind=real_8), DIMENSION(:), POINTER :: data_d => NULL()
COMPLEX(kind=real_4), DIMENSION(:), POINTER :: data_c => NULL()
COMPLEX(kind=real_8), DIMENSION(:), POINTER :: data_z => NULL()
TYPE(mp_type_descriptor_type), DIMENSION(:), POINTER :: subtype => NULL()
INTEGER :: vector_descriptor(2) = -1
LOGICAL :: has_indexing = .FALSE.
TYPE(mp_indexing_meta_type) :: index_descriptor = mp_indexing_meta_type()
END TYPE mp_type_descriptor_type
TYPE mp_file_indexing_meta_type
INTEGER, DIMENSION(:), POINTER :: index => NULL()
INTEGER(kind=address_kind), &
DIMENSION(:), POINTER :: chunks => NULL()
END TYPE mp_file_indexing_meta_type
! type internally used to store message passing performance indicators
! **************************************************************************************************
TYPE mp_perf_type
CHARACTER(LEN=20) :: name = ""
INTEGER :: count = -1
REAL(KIND=dp) :: msg_size = -1.0_dp
END TYPE mp_perf_type
INTEGER, PARAMETER :: MAX_PERF = 28
! **************************************************************************************************
TYPE mp_perf_env_type
!private
INTEGER :: ref_count = -1, id_nr = -1
TYPE(mp_perf_type), DIMENSION(MAX_PERF) :: mp_perfs = mp_perf_type()
END TYPE mp_perf_env_type
! **************************************************************************************************
TYPE mp_perf_env_p_type
TYPE(mp_perf_env_type), POINTER :: mp_perf_env => Null()
END TYPE mp_perf_env_p_type
! introduce a stack of mp_perfs, first index is the stack pointer, for convenience is replacing
INTEGER, PARAMETER :: max_stack_size = 10
INTEGER :: stack_pointer = 0
! target attribute needed as a hack around ifc 7.1 bug
TYPE(mp_perf_env_p_type), DIMENSION(max_stack_size), TARGET, SAVE :: mp_perf_stack
CHARACTER(LEN=20), PARAMETER :: sname(MAX_PERF) = &
(/"MP_Group ", "MP_Bcast ", "MP_Allreduce ", &
"MP_Gather ", "MP_Sync ", "MP_Alltoall ", &
"MP_SendRecv ", "MP_ISendRecv ", "MP_Wait ", &
"MP_comm_split ", "MP_ISend ", "MP_IRecv ", &
"MP_Send ", "MP_Recv ", "MP_Memory ", &
"MP_Put ", "MP_Get ", "MP_Fence ", &
"MP_Win_Lock ", "MP_Win_Create ", "MP_Win_Free ", &
"MP_IBcast ", "MP_IAllreduce ", "MP_IScatter ", &
"MP_RGet ", "MP_Isync ", "MP_Read_All ", &
"MP_Write_All "/)
! we make some assumptions on the length of INTEGERS, REALS and LOGICALS
INTEGER, PARAMETER :: intlen = BIT_SIZE(0)/8
INTEGER, PARAMETER :: reallen = 8
INTEGER, PARAMETER :: loglen = BIT_SIZE(0)/8
INTEGER, PARAMETER :: charlen = 1
INTEGER, SAVE, PRIVATE :: last_mp_perf_env_id = 0
CONTAINS
#:mute
#:set types = ["comm", "request", "win", "file", "info"]
#:endmute
#:for type in types
ELEMENTAL INTEGER FUNCTION mp_get_${type}$_handle(${type}$)
CLASS(mp_${type}$_type), INTENT(IN) :: ${type}$
#if defined(__parallel) && defined(__USE_MPI_F08)
mp_get_${type}$_handle = ${type}$%handle%mpi_val
#else
mp_get_${type}$_handle = ${type}$%handle
#endif
END FUNCTION mp_get_${type}$_handle
ELEMENTAL SUBROUTINE mp_set_${type}$_handle(${type}$, handle)
CLASS(mp_${type}$_type), INTENT(INOUT) :: ${type}$
INTEGER, INTENT(IN) :: handle
#if defined(__parallel) && defined(__USE_MPI_F08)
${type}$%handle%mpi_val = handle
#else
${type}$%handle = handle
#endif
END SUBROUTINE mp_set_${type}$_handle
ELEMENTAL IMPURE LOGICAL FUNCTION mp_${type}$_op_eq(${type}$1, ${type}$2)
CLASS(mp_${type}$_type), INTENT(IN) :: ${type}$1, ${type}$2
#if defined(__parallel) && defined(__USE_MPI_F08)
mp_${type}$_op_eq = (${type}$1%handle%mpi_val .EQ. ${type}$2%handle%mpi_val)
#else
mp_${type}$_op_eq = (${type}$1%handle .EQ. ${type}$2%handle)
#endif
END FUNCTION mp_${type}$_op_eq
ELEMENTAL IMPURE LOGICAL FUNCTION mp_${type}$_op_ne(${type}$1, ${type}$2)
CLASS(mp_${type}$_type), INTENT(IN) :: ${type}$1, ${type}$2
#if defined(__parallel) && defined(__USE_MPI_F08)
mp_${type}$_op_ne = (${type}$1%handle%mpi_val .NE. ${type}$2%handle%mpi_val)
#else
mp_${type}$_op_ne = (${type}$1%handle .NE. ${type}$2%handle)
#endif
END FUNCTION mp_${type}$_op_ne
#:endfor
SUBROUTINE mp_world_init(mp_comm)
!! initializes the system default communicator
!! @note
!! should only be called once
TYPE(mp_comm_type), INTENT(OUT) :: mp_comm
!! [output] : handle of the default communicator
#if defined(__parallel)
INTEGER :: ierr
!$ INTEGER :: provided_tsl
!$ LOGICAL :: no_threading_support
#if defined(__NO_MPI_THREAD_SUPPORT_CHECK)
! Hack that does not request or check MPI thread support level.
! User asserts that the MPI library will work correctly with
! threads.
!
!$ no_threading_support = .TRUE.
#else
! Does the right thing when using OpenMP: requests that the MPI
! library supports funneled mode and verifies that the MPI library
! provides that support.
!
! Developers: Only the master thread will ever make calls to the
! MPI library.
!
!$ no_threading_support = .FALSE.
#endif
!$ IF (no_threading_support) THEN
CALL mpi_init(ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_init @ mp_world_init")
!$ ELSE
!$OMP MASTER
!$ CALL mpi_init_thread(MPI_THREAD_FUNNELED, provided_tsl, ierr)
!$ IF (ierr /= 0) CALL mp_stop(ierr, "mpi_init_thread @ mp_world_init")
!$ IF (provided_tsl .LT. MPI_THREAD_FUNNELED) THEN
!$ CALL mp_stop(0, "MPI library does not support the requested level of threading (MPI_THREAD_FUNNELED).")
!$ END IF
!$OMP END MASTER
!$ END IF
CALL mpi_comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_set_errhandler @ mp_world_init")
debug_comm_count = 1
#endif
mp_comm = mp_comm_world
CALL add_mp_perf_env()
END SUBROUTINE mp_world_init
FUNCTION mp_get_comm_count()
!! Return the current number of communicators
INTEGER :: mp_get_comm_count
mp_get_comm_count = 0
#if defined(__parallel)
mp_get_comm_count = debug_comm_count
#endif
END FUNCTION mp_get_comm_count
SUBROUTINE mp_reordering(mp_comm, mp_new_comm, ranks_order)
!! re-create the system default communicator with a different MPI
!! rank order
!! @note
!! should only be called once, at very beginning of CP2K run
TYPE(mp_comm_type), INTENT(IN) :: mp_comm
!! [output] : handle of the default communicator
TYPE(mp_comm_type), INTENT(out) :: mp_new_comm
INTEGER, DIMENSION(:), CONTIGUOUS :: ranks_order
CHARACTER(LEN=*), PARAMETER :: routineN = 'mp_reordering'
INTEGER :: handle, ierr
#if defined(__parallel)
MPI_GROUP_TYPE :: newgroup, oldgroup
TYPE(mp_comm_type) :: newcomm
#endif
CALL timeset(routineN, handle)
ierr = 0
#if defined(__parallel)
CALL mpi_comm_group(mp_comm%handle, oldgroup, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_group @ "//routineN)
CALL mpi_group_incl(oldgroup, SIZE(ranks_order), ranks_order, newgroup, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_group_incl @ "//routineN)
CALL mpi_comm_create(mp_comm%handle, newgroup, newcomm%handle, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_comm_create @ "//routineN)
CALL mpi_group_free(oldgroup, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_group_free @ "//routineN)
CALL mpi_group_free(newgroup, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_group_free @ "//routineN)
! update the system default communicator
mp_new_comm = newcomm
debug_comm_count = debug_comm_count + 1
#else
MARK_USED(ranks_order)
mp_new_comm = mp_comm
#endif
CALL timestop(handle)
END SUBROUTINE mp_reordering
SUBROUTINE mp_world_finalize()
!! finalizes the system default communicator
#if defined(__parallel)
INTEGER :: ierr
CALL mpi_barrier(MPI_COMM_WORLD, ierr) ! call mpi directly to avoid 0 stack pointer
CALL rm_mp_perf_env()
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_barrier @ mp_world_finalize")
debug_comm_count = debug_comm_count - 1
IF (debug_comm_count .NE. 0) THEN
! A bug, we're leaking or double-freeing communicators. Needs to be fixed where the leak happens.
! Memory leak checking might be helpful to locate the culprit
DBCSR_ABORT("mp_world_finalize: assert failed: leaking communicators")
END IF
CALL mpi_finalize(ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_finalize @ mp_world_finalize")
#else
CALL rm_mp_perf_env()
#endif
END SUBROUTINE mp_world_finalize
! all the following routines should work for a given communicator, not MPI_WORLD
SUBROUTINE add_mp_perf_env(perf_env)
!! start and stop the performance indicators
!! for every call to start there has to be (exactly) one call to stop
!! @note
!! can be used to measure performance of a sub-part of a program.
!! timings measured here will not show up in the outer start/stops
!! Doesn't need a fresh communicator
TYPE(mp_perf_env_type), OPTIONAL, POINTER :: perf_env
stack_pointer = stack_pointer + 1
IF (stack_pointer > max_stack_size) THEN
DBCSR_ABORT("stack_pointer too large : mpiwrap @ add_mp_perf_env")
END IF
NULLIFY (mp_perf_stack(stack_pointer)%mp_perf_env)
IF (PRESENT(perf_env)) THEN
mp_perf_stack(stack_pointer)%mp_perf_env => perf_env
IF (ASSOCIATED(perf_env)) CALL mp_perf_env_retain(perf_env)
END IF
IF (.NOT. ASSOCIATED(mp_perf_stack(stack_pointer)%mp_perf_env)) THEN
CALL mp_perf_env_create(mp_perf_stack(stack_pointer)%mp_perf_env)
END IF
END SUBROUTINE add_mp_perf_env
SUBROUTINE mp_perf_env_create(perf_env)
TYPE(mp_perf_env_type), OPTIONAL, POINTER :: perf_env
INTEGER :: i, stat
NULLIFY (perf_env)
ALLOCATE (perf_env, stat=stat)
IF (stat /= 0) THEN
DBCSR_ABORT("allocation failed in mp_perf_env_create")
END IF
last_mp_perf_env_id = last_mp_perf_env_id + 1
perf_env%id_nr = last_mp_perf_env_id
perf_env%ref_count = 1
DO i = 1, MAX_PERF
perf_env%mp_perfs(i)%name = sname(i)
perf_env%mp_perfs(i)%count = 0
perf_env%mp_perfs(i)%msg_size = 0.0_dp
END DO
END SUBROUTINE mp_perf_env_create
SUBROUTINE mp_perf_env_release(perf_env)
TYPE(mp_perf_env_type), POINTER :: perf_env
IF (ASSOCIATED(perf_env)) THEN
IF (perf_env%ref_count < 1) THEN
DBCSR_ABORT("invalid ref_count: mpiwrap @ mp_perf_env_release")
END IF
perf_env%ref_count = perf_env%ref_count - 1
IF (perf_env%ref_count == 0) THEN
DEALLOCATE (perf_env)
END IF
END IF
NULLIFY (perf_env)
END SUBROUTINE mp_perf_env_release
SUBROUTINE mp_perf_env_retain(perf_env)
TYPE(mp_perf_env_type), POINTER :: perf_env
IF (.NOT. ASSOCIATED(perf_env)) THEN
DBCSR_ABORT("unassociated perf_env: mpiwrap @ mp_perf_env_retain")
END IF
IF (perf_env%ref_count < 1) THEN
DBCSR_ABORT("invalid ref_count: mpiwrap @ mp_perf_env_retain")
END IF
perf_env%ref_count = perf_env%ref_count + 1
END SUBROUTINE mp_perf_env_retain
!.. reports the performance counters for the MPI run
SUBROUTINE mp_perf_env_describe(perf_env, iw)
TYPE(mp_perf_env_type), POINTER :: perf_env
INTEGER, INTENT(IN) :: iw
#if defined(__parallel)
INTEGER :: i
REAL(KIND=dp) :: vol
#endif
IF (.NOT. ASSOCIATED(perf_env)) THEN
DBCSR_ABORT("unassociated perf_env : mpiwrap @ mp_perf_env_describe")
END IF
IF (perf_env%ref_count < 1) THEN
DBCSR_ABORT("invalid perf_env%ref_count : mpiwrap @ mp_perf_env_describe")
END IF
#if defined(__parallel)
IF (iw > 0) THEN
WRITE (iw, '( " -", 77X, "-" )')
WRITE (iw, '( " -", 21X, A, 21X, "-" )') ' DBCSR MESSAGE PASSING PERFORMANCE '
WRITE (iw, '( " -", 77X, "-" )')
WRITE (iw, '( 1X, 79("-"))')
WRITE (iw, '( A, A, A )') ' ROUTINE', ' CALLS ', &
' AVE VOLUME [Bytes]'
DO i = 1, MAX_PERF
IF (perf_env%mp_perfs(i)%count > 0) THEN
vol = perf_env%mp_perfs(i)%msg_size/REAL(perf_env%mp_perfs(i)%count, KIND=dp)
IF (vol < 1.0_dp) THEN
WRITE (iw, '(1X,A15,T17,I10)') &
ADJUSTL(perf_env%mp_perfs(i)%name), perf_env%mp_perfs(i)%count
ELSE
WRITE (iw, '(1X,A15,T17,I10,T40,F11.0)') &
ADJUSTL(perf_env%mp_perfs(i)%name), perf_env%mp_perfs(i)%count, &
vol
END IF
END IF
END DO
WRITE (iw, '( 1X, 79("-"))')
END IF
#else
MARK_USED(iw)
#endif
END SUBROUTINE mp_perf_env_describe
SUBROUTINE rm_mp_perf_env()
IF (stack_pointer < 1) THEN
DBCSR_ABORT("no perf_env in the stack : mpiwrap @ rm_mp_perf_env")
END IF
CALL mp_perf_env_release(mp_perf_stack(stack_pointer)%mp_perf_env)
stack_pointer = stack_pointer - 1
END SUBROUTINE rm_mp_perf_env
PURE FUNCTION has_mp_perf_env() RESULT(res)
LOGICAL :: res
res = .FALSE.
IF (stack_pointer < 1) RETURN
IF (.NOT. ASSOCIATED(mp_perf_stack(stack_pointer)%mp_perf_env)) RETURN
res = .TRUE.
END FUNCTION has_mp_perf_env
FUNCTION get_mp_perf_env() RESULT(res)
TYPE(mp_perf_env_type), POINTER :: res
IF (stack_pointer < 1) THEN
DBCSR_ABORT("no perf_env in the stack : mpiwrap @ get_mp_perf_env")
END IF
res => mp_perf_stack(stack_pointer)%mp_perf_env
END FUNCTION get_mp_perf_env
SUBROUTINE describe_mp_perf_env(scr)
INTEGER, INTENT(in) :: scr
TYPE(mp_perf_env_type), POINTER :: perf_env
perf_env => get_mp_perf_env()
CALL mp_perf_env_describe(perf_env, scr)
END SUBROUTINE describe_mp_perf_env
#if defined(__parallel)
SUBROUTINE add_perf(perf_id, msg_size)
!! adds the performance informations of one call
INTEGER, INTENT(in) :: perf_id
INTEGER, INTENT(in) :: msg_size
TYPE(mp_perf_type), POINTER :: mp_perf
IF (stack_pointer < 1) return
IF (.NOT. ASSOCIATED(mp_perf_stack(stack_pointer)%mp_perf_env)) return
mp_perf => mp_perf_stack(stack_pointer)%mp_perf_env%mp_perfs(perf_id)
mp_perf%count = mp_perf%count + 1
mp_perf%msg_size = mp_perf%msg_size + REAL(msg_size, dp)
END SUBROUTINE add_perf
#endif
SUBROUTINE mp_abort()
!! globally stops all tasks
!! this is intended to be low level, most of CP2K should call dbcsr_abort()
INTEGER :: ierr
ierr = 0
#if !defined(__NO_ABORT)
#if defined(__parallel)
CALL mpi_abort(MPI_COMM_WORLD, 1, ierr)
#else
CALL m_abort()
#endif
#endif
! this routine never returns and levels with non-zero exit code
STOP 1
END SUBROUTINE mp_abort
SUBROUTINE mp_stop(ierr, prg_code)
!! stops *after an mpi error* translating the error code
!! @note
!! this function is private to mpiwrap.F
INTEGER, INTENT(IN) :: ierr
!! an error code * returned by an mpi call *
CHARACTER(LEN=*) :: prg_code
#if defined(__parallel)
INTEGER :: istat, len
CHARACTER(LEN=MPI_MAX_ERROR_STRING) :: error_string
CHARACTER(LEN=MPI_MAX_ERROR_STRING + 512) :: full_error
#else
CHARACTER(LEN=512) :: full_error
#endif
#if defined(__parallel)
CALL mpi_error_string(ierr, error_string, len, istat)
WRITE (full_error, '(A,I0,A)') ' MPI error ', ierr, ' in '//TRIM(prg_code)//' : '//error_string(1:len)
#else
WRITE (full_error, '(A,I0,A)') ' MPI error (!?) ', ierr, ' in '//TRIM(prg_code)
#endif
DBCSR_ABORT(full_error)
END SUBROUTINE mp_stop
SUBROUTINE mp_sync(group)
!! synchronizes with a barrier a given group of mpi tasks
TYPE(mp_comm_type), INTENT(IN) :: group
!! mpi communicator
CHARACTER(LEN=*), PARAMETER :: routineN = 'mp_sync'
INTEGER :: handle, ierr
ierr = 0
CALL timeset(routineN, handle)
#if defined(__parallel)
CALL mpi_barrier(group%handle, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_barrier @ "//routineN)
#else
MARK_USED(group)
#endif
CALL timestop(handle)
END SUBROUTINE mp_sync
SUBROUTINE mp_isync(group, request)
!! synchronizes with a barrier a given group of mpi tasks
TYPE(mp_comm_type), INTENT(IN) :: group
!! mpi communicator
TYPE(mp_request_type), INTENT(OUT) :: request
CHARACTER(LEN=*), PARAMETER :: routineN = 'mp_isync'
INTEGER :: handle, ierr
ierr = 0
CALL timeset(routineN, handle)
#if defined(__parallel)
CALL mpi_ibarrier(group%handle, request%handle, ierr)
IF (ierr /= 0) CALL mp_stop(ierr, "mpi_ibarrier @ "//routineN)
#else
MARK_USED(group)
request = mp_request_null
#endif
CALL timestop(handle)
END SUBROUTINE mp_isync
RECURSIVE SUBROUTINE mp_environ_l(numtask, taskid, groupid)
!! returns number of tasks and task id for a given mpi group
!! simple and cartesian version.. recursive needed in case of failing mpi_comm_rank.
!! @note
!! ..mp_world_setup is gone, use mp_environ instead (i.e. give a groupid explicitly)
INTEGER, OPTIONAL, INTENT(OUT) :: numtask, taskid
TYPE(mp_comm_type), INTENT(IN) :: groupid
!! mpi communicator