Neko 1.99.1
A portable framework for high-order spectral element flow simulations
Loading...
Searching...
No Matches
comm.F90
Go to the documentation of this file.
1module comm
2 use mpi_f08, only : mpi_comm, mpi_datatype, mpi_initialized, mpi_init_thread, &
3 mpi_init, mpi_thread_multiple, mpi_thread_serialized, mpi_comm_rank, &
4 mpi_comm_split, mpi_comm_dup, mpi_barrier, mpi_comm_free, mpi_finalize, &
5 mpi_comm_world, mpi_double_precision, mpi_real
6 use utils, only : neko_error
8 !$ use omp_lib
9 implicit none
10 private
11
12 interface
13 subroutine neko_comm_wrapper_init(fcomm) &
14 bind(c, name='neko_comm_wrapper_init')
15 integer, value :: fcomm
16 end subroutine neko_comm_wrapper_init
17
18#ifdef HAVE_NVSHMEM
19 subroutine neko_comm_nvshmem_init() &
20 bind(c, name='neko_comm_nvshmem_init')
21 end subroutine neko_comm_nvshmem_init
22
23 subroutine neko_comm_nvshmem_finalize() &
24 bind(c, name='neko_comm_nvshmem_finalize')
25 end subroutine neko_comm_nvshmem_finalize
26#endif
27
28#if defined(HAVE_NCCL) || defined(HAVE_RCCL)
29 subroutine neko_comm_nccl_init() &
30 bind(c, name='neko_comm_nccl_init')
31 end subroutine neko_comm_nccl_init
32
33 subroutine neko_comm_nccl_finalize() &
34 bind(c, name='neko_comm_nccl_finalize')
35 end subroutine neko_comm_nccl_finalize
36#endif
37
38 end interface
39
40
42 type(mpi_comm), public :: neko_comm
43 type(mpi_comm), public :: neko_global_comm
44
46#ifdef HAVE_MPI_PARAM_DTYPE
47 type(mpi_datatype), public, parameter :: mpi_real_precision = mpi_double_precision
48 type(mpi_datatype), public, parameter :: mpi_extra_precision = mpi_double_precision
49#else
50 type(mpi_datatype), public :: mpi_real_precision
51 type(mpi_datatype), public :: mpi_extra_precision
52#endif
53
55 integer, public :: pe_rank
56
58 integer, public :: pe_size
59
61 logical, public :: nio
62
64 integer, public :: global_pe_rank
65
67 integer, public :: global_pe_size
68
69 public :: comm_init, comm_free
70
71contains
72 subroutine comm_init
73 integer :: ierr
74 logical :: initialized
75 integer :: provided, nthrds
76 integer :: color = 0
77 integer :: envvar_len
78 character(len=255) :: color_str
79
80 pe_rank = -1
81 pe_size = 0
82 nio = .false.
83
84 call mpi_initialized(initialized, ierr)
85
86 call get_environment_variable("NEKO_COMM_ID", color_str, envvar_len)
87 if (envvar_len .gt. 0) then
88 read(color_str(1:envvar_len), *) color
89 else
90 color = 0
91 end if
92
93 nthrds = 1
94 !$omp parallel
95 !$omp master
96 !$ nthrds = omp_get_num_threads()
97 !$omp end master
98 !$omp end parallel
99
100 if (.not.initialized) then
101 if (nthrds .gt. 1) then
102 call mpi_init_thread(mpi_thread_multiple, provided, ierr)
103 if (provided .ne. mpi_thread_multiple) then
104 ! MPI_THREAD_MULTIPLE is required for mt. device backends
105 if (neko_bcknd_device .eq. 1) then
106 call neko_error('Invalid thread support provided by MPI')
107 else
108 call mpi_init_thread(mpi_thread_serialized, provided, ierr)
109 if (provided .ne. mpi_thread_serialized) then
110 call neko_error('Invalid thread support provided by MPI')
111 end if
112 end if
113 end if
114 else
115 call mpi_init(ierr)
116 end if
117 end if
118
119#ifndef HAVE_MPI_PARAM_DTYPE
120 mpi_real_precision = mpi_double_precision
121 mpi_extra_precision = mpi_double_precision
122#endif
123
124
125#ifdef HAVE_ADIOS2
126 ! We split the communicator it to work asynchronously (MPMD)
127 call mpi_comm_rank(mpi_comm_world, pe_rank, ierr)
128 call mpi_comm_split(mpi_comm_world, 0, pe_rank, neko_global_comm, ierr)
129#else
130 ! Original version duplicates the communicator:
131 call mpi_comm_dup(mpi_comm_world, neko_global_comm, ierr)
132#endif
133
134 call mpi_comm_rank(neko_global_comm, global_pe_rank, ierr)
135 call mpi_comm_size(neko_global_comm, global_pe_size, ierr)
136 if (envvar_len .gt. 0) then
137 call mpi_comm_split(neko_global_comm, color, global_pe_rank, &
138 neko_comm, ierr)
139 else
140 call mpi_comm_dup(neko_global_comm, neko_comm, ierr)
141 end if
142 call mpi_comm_rank(neko_comm, pe_rank, ierr)
143 call mpi_comm_size(neko_comm, pe_size, ierr)
144
145 ! Setup C/C++ wrapper
147
148
149#ifdef HAVE_NVSHMEM
150 ! Setup NVSHMEM (if requested)
151 call neko_comm_nvshmem_init()
152#endif
153
154#if defined(HAVE_NCCL) | defined(HAVE_RCCL)
155 ! Setup NCCL (if requested)
156 call neko_comm_nccl_init()
157#endif
158
159
160 end subroutine comm_init
161
162 subroutine comm_free
163 integer :: ierr
164
165 call mpi_barrier(neko_comm, ierr)
166 call mpi_comm_free(neko_comm, ierr)
167
168#ifdef HAVE_NCCL
169 call neko_comm_nccl_finalize()
170#endif
171
172#ifdef HAVE_NVSHMEM
173 call neko_comm_nvshmem_finalize()
174#endif
175
176 call mpi_finalize(ierr)
177
178 end subroutine comm_free
179
180end module comm
Definition comm.F90:1
subroutine, public comm_free
Definition comm.F90:163
subroutine, public comm_init
Definition comm.F90:73
logical, public nio
I/O node.
Definition comm.F90:61
type(mpi_comm), public neko_global_comm
Definition comm.F90:43
type(mpi_datatype), public mpi_real_precision
MPI type for working precision of REAL types.
Definition comm.F90:50
integer, public global_pe_rank
Global MPI rank.
Definition comm.F90:64
integer, public pe_size
MPI size of communicator.
Definition comm.F90:58
integer, public pe_rank
MPI rank.
Definition comm.F90:55
integer, public global_pe_size
Global MPI size of communicator.
Definition comm.F90:67
type(mpi_comm), public neko_comm
MPI communicator.
Definition comm.F90:42
type(mpi_datatype), public mpi_extra_precision
Definition comm.F90:51
Build configurations.
integer, parameter neko_bcknd_device
Utilities.
Definition utils.f90:35