Neko 1.99.6
A portable framework for high-order spectral element flow simulations
Loading...
Searching...
No Matches
lpt_migration.f90
Go to the documentation of this file.
1! Copyright (c) 2026, The Neko Authors
2! All rights reserved.
3!
4! Redistribution and use in source and binary forms, with or without
5! modification, are permitted provided that the following conditions
6! are met:
7!
8! * Redistributions of source code must retain the above copyright
9! notice, this list of conditions and the following disclaimer.
10!
11! * Redistributions in binary form must reproduce the above
12! copyright notice, this list of conditions and the following
13! disclaimer in the documentation and/or other materials provided
14! with the distribution.
15!
16! * Neither the name of the authors nor the names of its
17! contributors may be used to endorse or promote products derived
18! from this software without specific prior written permission.
19!
20! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21! "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22! LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23! FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24! COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25! INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26! BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27! LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28! CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29! LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30! ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31! POSSIBILITY OF SUCH DAMAGE.
32!
34module lpt_migrate
35 use num_types, only : rp
36 use global_interpolation, only : global_interpolation_t
37 use glb_intrp_comm, only : glb_intrp_comm_t
38 use lpt_periodic_bc, only : lpt_periodic_bc_t
39 use comm, only : neko_comm, mpi_real_precision, pe_rank, pe_size
40 use mpi_f08, only : mpi_allreduce, mpi_bcast, mpi_integer, mpi_scatterv, &
41 mpi_sum
42 use vector, only : vector_t
43 use device, only : host_to_device, device_to_host
44 use scratch_registry, only : neko_scratch_registry
45 use particles, only : particles_t
46 implicit none
47 private
48
49 integer, public, parameter :: lpt_migrate_to_owner = 1
50 integer, public, parameter :: lpt_migrate_none = 2
51
52 type, public :: lpt_migrate_t
53 integer :: lag_len = 0
54 integer :: strategy = lpt_migrate_to_owner
55 contains
56 procedure, pass(this) :: init => lpt_migrate_init
57 procedure, pass(this) :: free => lpt_migrate_free
58 procedure, pass(this) :: initialize_particle_distribution
59 procedure, pass(this) :: migrate_particles
60 procedure, private, pass(this) :: distribute_particles_evenly
61 procedure, private, pass(this) :: distribute_particle_ids
62 procedure, private, pass(this) :: distribute_particle_scalar
63 procedure, private, pass(this) :: localize_global_interpolation
64 procedure, private, pass(this) :: migrate_particle_ids
65 procedure, private, pass(this) :: migrate_particle_scalar
66 end type lpt_migrate_t
67
68contains
69
73 subroutine lpt_migrate_init(this, lag_len, strategy)
74 class(lpt_migrate_t), intent(inout) :: this
75 integer, intent(in) :: lag_len
76 integer, intent(in), optional :: strategy
77
78 this%lag_len = lag_len
79 this%strategy = lpt_migrate_to_owner
80 if (present(strategy)) this%strategy = strategy
81 end subroutine lpt_migrate_init
82
84 subroutine lpt_migrate_free(this)
85 class(lpt_migrate_t), intent(inout) :: this
86
87 this%lag_len = 0
88 this%strategy = lpt_migrate_to_owner
89 end subroutine lpt_migrate_free
90
96 subroutine build_even_particle_distribution(n_root, counts, offsets, n_local)
97 integer, intent(in) :: n_root
98 integer, allocatable, intent(out) :: counts(:)
99 integer, allocatable, intent(out) :: offsets(:)
100 integer, intent(out) :: n_local
101 integer :: n_total
102 integer :: rank
103 integer :: ierr
104
105 n_total = n_root
106 call mpi_bcast(n_total, 1, mpi_integer, 0, neko_comm, ierr)
107
108 allocate(counts(0:pe_size - 1))
109 allocate(offsets(0:pe_size - 1))
110 do rank = 0, pe_size - 1
111 counts(rank) = n_total / pe_size
112 if (rank .lt. mod(n_total, pe_size)) counts(rank) = counts(rank) + 1
113 end do
114
115 offsets(0) = 0
116 do rank = 1, pe_size - 1
117 offsets(rank) = offsets(rank - 1) + counts(rank - 1)
118 end do
119 n_local = counts(pe_rank)
120 end subroutine build_even_particle_distribution
121
125 subroutine initialize_particle_distribution(this, inertia, particles)
126 class(lpt_migrate_t), intent(inout) :: this
127 logical, intent(in) :: inertia
128 type(particles_t), intent(inout) :: particles
129
130 if (this%strategy .eq. lpt_migrate_none) then
131 call this%distribute_particles_evenly(inertia, particles)
132 end if
133 end subroutine initialize_particle_distribution
134
140 subroutine migrate_particles(this, global_interp, periodic_bc, inertia, &
141 particles)
142 class(lpt_migrate_t), intent(inout) :: this
143 type(global_interpolation_t), intent(inout) :: global_interp
144 type(lpt_periodic_bc_t), intent(inout) :: periodic_bc
145 logical, intent(in) :: inertia
146 type(particles_t), intent(inout) :: particles
147
148 type(glb_intrp_comm_t) :: migrate_comm
149 integer :: n_particles_old
150 integer :: n_particles_local
151 integer, allocatable :: particle_ids_local(:)
152 type(vector_t), pointer :: x_local, y_local, z_local
153 type(vector_t), pointer :: u_local, v_local, w_local
154 type(vector_t), pointer :: acc_xlocal, acc_ylocal, acc_zlocal
155 type(vector_t), pointer :: d_local, rho_local
156 type(vector_t), pointer :: u_lag_local, v_lag_local, w_lag_local
157 type(vector_t), pointer :: u_laglag_local, v_laglag_local, w_laglag_local
158 type(vector_t), pointer :: acc_xlag_local, acc_ylag_local, acc_zlag_local
159 type(vector_t), pointer :: acc_xlaglag_local, &
160 acc_ylaglag_local, acc_zlaglag_local
161 integer :: ierr
162 integer :: rank
163 logical :: migration_needed
164 integer :: ind_basic(9)
165 integer :: ind_inertia(14)
166
167 associate( &
168 x => particles%x, y => particles%y, z => particles%z, &
169 u => particles%u, v => particles%v, w => particles%w, &
170 acc_x => particles%acc_x, acc_y => particles%acc_y, &
171 acc_z => particles%acc_z, d => particles%d, rho => particles%rho, &
172 u_lag => particles%u_lag, v_lag => particles%v_lag, &
173 w_lag => particles%w_lag, u_laglag => particles%u_laglag, &
174 v_laglag => particles%v_laglag, w_laglag => particles%w_laglag, &
175 acc_xlag => particles%acc_xlag, acc_ylag => particles%acc_ylag, &
176 acc_zlag => particles%acc_zlag, acc_xlaglag => particles%acc_xlaglag, &
177 acc_ylaglag => particles%acc_ylaglag, &
178 acc_zlaglag => particles%acc_zlaglag, n => particles%n, &
179 n_global => particles%n_global)
180
181 if (inertia) then
182 call periodic_bc%wrap(x, y, z, n, u, v, w, u_lag, &
183 v_lag, w_lag, u_laglag, v_laglag, w_laglag, &
184 acc_xlag, acc_ylag, acc_zlag, acc_xlaglag, &
185 acc_ylaglag, acc_zlaglag)
186 else
187 call periodic_bc%wrap(x, y, z, n, u, v, w, u_lag, &
188 v_lag, w_lag, u_laglag, v_laglag, w_laglag)
189 end if
190 if (this%strategy .eq. lpt_migrate_none) then
191 call global_interp%find_points(x%x, y%x, z%x, n)
192 call mpi_allreduce(n, n_global, 1, mpi_integer, mpi_sum, neko_comm, &
193 ierr)
194 return
195 end if
196
197 n_particles_old = n
198 call global_interp%find_points(x%x, y%x, z%x, n)
199 n_particles_local = global_interp%n_points_local
200
201 migration_needed = .false.
202 do rank = 0, global_interp%pe_size - 1
203 if (rank .ne. pe_rank) then
204 migration_needed = migration_needed .or. &
205 global_interp%n_points_pe(rank) .gt. 0
206 migration_needed = migration_needed .or. &
207 global_interp%n_points_pe_local(rank) .gt. 0
208 end if
209 end do
210 if (.not. migration_needed .and. n_particles_local .eq. n) then
211 call this%localize_global_interpolation(global_interp, n)
212 call mpi_allreduce(n, n_global, 1, mpi_integer, mpi_sum, neko_comm, &
213 ierr)
214 return
215 end if
216
217 allocate(particle_ids_local(n_particles_local))
218 particle_ids_local = 0
219 call neko_scratch_registry%request_vector(x_local, ind_basic(1), &
220 n_particles_local, .false.)
221 call neko_scratch_registry%request_vector(y_local, ind_basic(2), &
222 n_particles_local, .false.)
223 call neko_scratch_registry%request_vector(z_local, ind_basic(3), &
224 n_particles_local, .false.)
225 call neko_scratch_registry%request_vector(u_lag_local, ind_basic(4), &
226 n_particles_local, .false.)
227 call neko_scratch_registry%request_vector(v_lag_local, ind_basic(5), &
228 n_particles_local, .false.)
229 call neko_scratch_registry%request_vector(w_lag_local, ind_basic(6), &
230 n_particles_local, .false.)
231 call neko_scratch_registry%request_vector(u_laglag_local, ind_basic(7), &
232 n_particles_local, .false.)
233 call neko_scratch_registry%request_vector(v_laglag_local, ind_basic(8), &
234 n_particles_local, .false.)
235 call neko_scratch_registry%request_vector(w_laglag_local, ind_basic(9), &
236 n_particles_local, .false.)
237 if (inertia) then
238 call neko_scratch_registry%request_vector(u_local, ind_inertia(1), &
239 n_particles_local, .false.)
240 call neko_scratch_registry%request_vector(v_local, ind_inertia(2), &
241 n_particles_local, .false.)
242 call neko_scratch_registry%request_vector(w_local, ind_inertia(3), &
243 n_particles_local, .false.)
244 call neko_scratch_registry%request_vector(acc_xlocal, ind_inertia(4), &
245 n_particles_local, .false.)
246 call neko_scratch_registry%request_vector(acc_ylocal, ind_inertia(5), &
247 n_particles_local, .false.)
248 call neko_scratch_registry%request_vector(acc_zlocal, ind_inertia(6), &
249 n_particles_local, .false.)
250 call neko_scratch_registry%request_vector(d_local, ind_inertia(7), &
251 n_particles_local, .false.)
252 call neko_scratch_registry%request_vector(rho_local, ind_inertia(8), &
253 n_particles_local, .false.)
254 call neko_scratch_registry%request_vector(acc_xlag_local, &
255 ind_inertia(9), n_particles_local, .false.)
256 call neko_scratch_registry%request_vector(acc_ylag_local, &
257 ind_inertia(10), n_particles_local, .false.)
258 call neko_scratch_registry%request_vector(acc_zlag_local, &
259 ind_inertia(11), n_particles_local, .false.)
260 call neko_scratch_registry%request_vector(acc_xlaglag_local, &
261 ind_inertia(12), n_particles_local, .false.)
262 call neko_scratch_registry%request_vector(acc_ylaglag_local, &
263 ind_inertia(13), n_particles_local, .false.)
264 call neko_scratch_registry%request_vector(acc_zlaglag_local, &
265 ind_inertia(14), n_particles_local, .false.)
266 end if
267
268 call global_interp%init_redist_comm(migrate_comm)
269 call this%migrate_particle_ids(migrate_comm, particles%ids, &
270 n_particles_old, n_particles_local, particle_ids_local)
271 call this%migrate_particle_scalar(migrate_comm, x, n_particles_old, &
272 n_particles_local, x_local)
273 call this%migrate_particle_scalar(migrate_comm, y, n_particles_old, &
274 n_particles_local, y_local)
275 call this%migrate_particle_scalar(migrate_comm, z, n_particles_old, &
276 n_particles_local, z_local)
277 call this%migrate_particle_scalar(migrate_comm, u_lag, n_particles_old, &
278 n_particles_local, u_lag_local)
279 call this%migrate_particle_scalar(migrate_comm, v_lag, n_particles_old, &
280 n_particles_local, v_lag_local)
281 call this%migrate_particle_scalar(migrate_comm, w_lag, n_particles_old, &
282 n_particles_local, w_lag_local)
283 call this%migrate_particle_scalar(migrate_comm, u_laglag, n_particles_old, &
284 n_particles_local, u_laglag_local)
285 call this%migrate_particle_scalar(migrate_comm, v_laglag, n_particles_old, &
286 n_particles_local, v_laglag_local)
287 call this%migrate_particle_scalar(migrate_comm, w_laglag, n_particles_old, &
288 n_particles_local, w_laglag_local)
289 if (inertia) then
290 call this%migrate_particle_scalar(migrate_comm, u, n_particles_old, &
291 n_particles_local, u_local)
292 call this%migrate_particle_scalar(migrate_comm, v, n_particles_old, &
293 n_particles_local, v_local)
294 call this%migrate_particle_scalar(migrate_comm, w, n_particles_old, &
295 n_particles_local, w_local)
296 call this%migrate_particle_scalar(migrate_comm, acc_x, n_particles_old, &
297 n_particles_local, acc_xlocal)
298 call this%migrate_particle_scalar(migrate_comm, acc_y, n_particles_old, &
299 n_particles_local, acc_ylocal)
300 call this%migrate_particle_scalar(migrate_comm, acc_z, n_particles_old, &
301 n_particles_local, acc_zlocal)
302 call this%migrate_particle_scalar(migrate_comm, d, &
303 n_particles_old, n_particles_local, d_local)
304 call this%migrate_particle_scalar(migrate_comm, rho, &
305 n_particles_old, n_particles_local, rho_local)
306 call this%migrate_particle_scalar(migrate_comm, acc_xlag, &
307 n_particles_old, n_particles_local, acc_xlag_local)
308 call this%migrate_particle_scalar(migrate_comm, acc_ylag, &
309 n_particles_old, n_particles_local, acc_ylag_local)
310 call this%migrate_particle_scalar(migrate_comm, acc_zlag, &
311 n_particles_old, n_particles_local, acc_zlag_local)
312 call this%migrate_particle_scalar(migrate_comm, acc_xlaglag, &
313 n_particles_old, n_particles_local, acc_xlaglag_local)
314 call this%migrate_particle_scalar(migrate_comm, acc_ylaglag, &
315 n_particles_old, n_particles_local, acc_ylaglag_local)
316 call this%migrate_particle_scalar(migrate_comm, acc_zlaglag, &
317 n_particles_old, n_particles_local, acc_zlaglag_local)
318 end if
319 call migrate_comm%free()
320
321 n = n_particles_local
322 x = x_local
323 y = y_local
324 z = z_local
325 call move_alloc(particle_ids_local, particles%ids)
326 u_lag = u_lag_local
327 v_lag = v_lag_local
328 w_lag = w_lag_local
329 u_laglag = u_laglag_local
330 v_laglag = v_laglag_local
331 w_laglag = w_laglag_local
332 if (inertia) then
333 u = u_local
334 v = v_local
335 w = w_local
336 acc_x = acc_xlocal
337 acc_y = acc_ylocal
338 acc_z = acc_zlocal
339 d = d_local
340 rho = rho_local
341 acc_xlag = acc_xlag_local
342 acc_ylag = acc_ylag_local
343 acc_zlag = acc_zlag_local
344 acc_xlaglag = acc_xlaglag_local
345 acc_ylaglag = acc_ylaglag_local
346 acc_zlaglag = acc_zlaglag_local
347 end if
348
349 if (allocated(particle_ids_local)) deallocate(particle_ids_local)
350 call neko_scratch_registry%relinquish(ind_basic)
351 if (inertia) call neko_scratch_registry%relinquish(ind_inertia)
352
353 call this%localize_global_interpolation(global_interp, n)
354 call mpi_allreduce(n, n_global, 1, mpi_integer, mpi_sum, neko_comm, ierr)
355
356 end associate
357
358 end subroutine migrate_particles
359
363 subroutine distribute_particles_evenly(this, inertia, particles)
364 class(lpt_migrate_t), intent(inout) :: this
365 logical, intent(in) :: inertia
366 type(particles_t), intent(inout) :: particles
367
368 integer, allocatable :: ids_local(:)
369 type(vector_t), pointer :: x_local, y_local, z_local
370 type(vector_t), pointer :: u_lag_local, v_lag_local, w_lag_local
371 type(vector_t), pointer :: u_laglag_local, v_laglag_local, w_laglag_local
372 type(vector_t), pointer :: u_local, v_local, w_local
373 type(vector_t), pointer :: acc_xlocal, acc_ylocal, acc_zlocal
374 type(vector_t), pointer :: d_local, rho_local
375 type(vector_t), pointer :: acc_xlag_local, acc_ylag_local, acc_zlag_local
376 type(vector_t), pointer :: acc_xlaglag_local, &
377 acc_ylaglag_local, acc_zlaglag_local
378 integer, allocatable :: counts(:)
379 integer, allocatable :: offsets(:)
380 integer :: n_local
381 integer :: ind_basic(9), ind_inertia(14)
382
383 associate( &
384 x => particles%x, y => particles%y, z => particles%z, &
385 u => particles%u, v => particles%v, w => particles%w, &
386 acc_x => particles%acc_x, acc_y => particles%acc_y, &
387 acc_z => particles%acc_z, d => particles%d, rho => particles%rho, &
388 u_lag => particles%u_lag, v_lag => particles%v_lag, &
389 w_lag => particles%w_lag, u_laglag => particles%u_laglag, &
390 v_laglag => particles%v_laglag, w_laglag => particles%w_laglag, &
391 acc_xlag => particles%acc_xlag, acc_ylag => particles%acc_ylag, &
392 acc_zlag => particles%acc_zlag, acc_xlaglag => particles%acc_xlaglag, &
393 acc_ylaglag => particles%acc_ylaglag, &
394 acc_zlaglag => particles%acc_zlaglag, n => particles%n)
395
396 call build_even_particle_distribution(n, counts, offsets, n_local)
397
398 allocate(ids_local(n_local))
399 call neko_scratch_registry%request_vector(x_local, ind_basic(1), &
400 n_local, .false.)
401 call neko_scratch_registry%request_vector(y_local, ind_basic(2), &
402 n_local, .false.)
403 call neko_scratch_registry%request_vector(z_local, ind_basic(3), &
404 n_local, .false.)
405 call neko_scratch_registry%request_vector(u_lag_local, ind_basic(4), &
406 n_local, .false.)
407 call neko_scratch_registry%request_vector(v_lag_local, ind_basic(5), &
408 n_local, .false.)
409 call neko_scratch_registry%request_vector(w_lag_local, ind_basic(6), &
410 n_local, .false.)
411 call neko_scratch_registry%request_vector(u_laglag_local, ind_basic(7), &
412 n_local, .false.)
413 call neko_scratch_registry%request_vector(v_laglag_local, ind_basic(8), &
414 n_local, .false.)
415 call neko_scratch_registry%request_vector(w_laglag_local, ind_basic(9), &
416 n_local, .false.)
417 if (inertia) then
418 call neko_scratch_registry%request_vector(u_local, ind_inertia(1), &
419 n_local, .false.)
420 call neko_scratch_registry%request_vector(v_local, ind_inertia(2), &
421 n_local, .false.)
422 call neko_scratch_registry%request_vector(w_local, ind_inertia(3), &
423 n_local, .false.)
424 call neko_scratch_registry%request_vector(acc_xlocal, ind_inertia(4), &
425 n_local, .false.)
426 call neko_scratch_registry%request_vector(acc_ylocal, ind_inertia(5), &
427 n_local, .false.)
428 call neko_scratch_registry%request_vector(acc_zlocal, ind_inertia(6), &
429 n_local, .false.)
430 call neko_scratch_registry%request_vector(d_local, ind_inertia(7), &
431 n_local, .false.)
432 call neko_scratch_registry%request_vector(rho_local, ind_inertia(8), &
433 n_local, .false.)
434 call neko_scratch_registry%request_vector(acc_xlag_local, &
435 ind_inertia(9), n_local, .false.)
436 call neko_scratch_registry%request_vector(acc_ylag_local, &
437 ind_inertia(10), n_local, .false.)
438 call neko_scratch_registry%request_vector(acc_zlag_local, &
439 ind_inertia(11), n_local, .false.)
440 call neko_scratch_registry%request_vector(acc_xlaglag_local, &
441 ind_inertia(12), n_local, .false.)
442 call neko_scratch_registry%request_vector(acc_ylaglag_local, &
443 ind_inertia(13), n_local, .false.)
444 call neko_scratch_registry%request_vector(acc_zlaglag_local, &
445 ind_inertia(14), n_local, .false.)
446 end if
447
448 call this%distribute_particle_ids(particles%ids, counts, offsets, n_local, &
449 ids_local)
450 call this%distribute_particle_scalar(x, counts, offsets, n_local, x_local)
451 call this%distribute_particle_scalar(y, counts, offsets, n_local, y_local)
452 call this%distribute_particle_scalar(z, counts, offsets, n_local, z_local)
453 call this%distribute_particle_scalar(u_lag, counts, offsets, n_local, &
454 u_lag_local)
455 call this%distribute_particle_scalar(v_lag, counts, offsets, n_local, &
456 v_lag_local)
457 call this%distribute_particle_scalar(w_lag, counts, offsets, n_local, &
458 w_lag_local)
459 call this%distribute_particle_scalar(u_laglag, counts, offsets, n_local, &
460 u_laglag_local)
461 call this%distribute_particle_scalar(v_laglag, counts, offsets, n_local, &
462 v_laglag_local)
463 call this%distribute_particle_scalar(w_laglag, counts, offsets, n_local, &
464 w_laglag_local)
465 if (inertia) then
466 call this%distribute_particle_scalar(u, counts, offsets, n_local, &
467 u_local)
468 call this%distribute_particle_scalar(v, counts, offsets, n_local, &
469 v_local)
470 call this%distribute_particle_scalar(w, counts, offsets, n_local, &
471 w_local)
472 call this%distribute_particle_scalar(acc_x, counts, offsets, n_local, &
473 acc_xlocal)
474 call this%distribute_particle_scalar(acc_y, counts, offsets, n_local, &
475 acc_ylocal)
476 call this%distribute_particle_scalar(acc_z, counts, offsets, n_local, &
477 acc_zlocal)
478 call this%distribute_particle_scalar(d, counts, offsets, n_local, &
479 d_local)
480 call this%distribute_particle_scalar(rho, counts, offsets, n_local, &
481 rho_local)
482 call this%distribute_particle_scalar(acc_xlag, counts, offsets, &
483 n_local, acc_xlag_local)
484 call this%distribute_particle_scalar(acc_ylag, counts, offsets, &
485 n_local, acc_ylag_local)
486 call this%distribute_particle_scalar(acc_zlag, counts, offsets, &
487 n_local, acc_zlag_local)
488 call this%distribute_particle_scalar(acc_xlaglag, counts, offsets, &
489 n_local, acc_xlaglag_local)
490 call this%distribute_particle_scalar(acc_ylaglag, counts, offsets, &
491 n_local, acc_ylaglag_local)
492 call this%distribute_particle_scalar(acc_zlaglag, counts, offsets, &
493 n_local, acc_zlaglag_local)
494 end if
495
496 n = n_local
497 x = x_local
498 y = y_local
499 z = z_local
500 call move_alloc(ids_local, particles%ids)
501 u_lag = u_lag_local
502 v_lag = v_lag_local
503 w_lag = w_lag_local
504 u_laglag = u_laglag_local
505 v_laglag = v_laglag_local
506 w_laglag = w_laglag_local
507 if (inertia) then
508 u = u_local
509 v = v_local
510 w = w_local
511 acc_x = acc_xlocal
512 acc_y = acc_ylocal
513 acc_z = acc_zlocal
514 d = d_local
515 rho = rho_local
516 acc_xlag = acc_xlag_local
517 acc_ylag = acc_ylag_local
518 acc_zlag = acc_zlag_local
519 acc_xlaglag = acc_xlaglag_local
520 acc_ylaglag = acc_ylaglag_local
521 acc_zlaglag = acc_zlaglag_local
522 end if
523
524 if (allocated(ids_local)) deallocate(ids_local)
525 call neko_scratch_registry%relinquish(ind_basic)
526 if (inertia) call neko_scratch_registry%relinquish(ind_inertia)
527
528 deallocate(counts)
529 deallocate(offsets)
530
531 end associate
532
533 end subroutine distribute_particles_evenly
534
541 subroutine distribute_particle_ids(this, ids_old, counts, offsets, n_local, &
542 ids_local)
543 class(lpt_migrate_t), intent(inout) :: this
544 integer, allocatable, intent(in) :: ids_old(:)
545 integer, intent(in) :: counts(0:)
546 integer, intent(in) :: offsets(0:)
547 integer, intent(in) :: n_local
548 integer, allocatable, intent(inout) :: ids_local(:)
549 integer, allocatable :: counts_i(:)
550 integer, allocatable :: offsets_i(:)
551 integer :: ierr
552
553 allocate(counts_i(0:pe_size - 1))
554 allocate(offsets_i(0:pe_size - 1))
555 counts_i = counts
556 offsets_i = offsets
557 call mpi_scatterv(ids_old, counts_i, offsets_i, mpi_integer, ids_local, &
558 n_local, mpi_integer, 0, neko_comm, ierr)
559 deallocate(counts_i)
560 deallocate(offsets_i)
561 end subroutine distribute_particle_ids
562
569 subroutine distribute_particle_scalar(this, scalar_old, counts, offsets, &
570 n_local, scalar_local)
571 class(lpt_migrate_t), intent(inout) :: this
572 type(vector_t), intent(inout) :: scalar_old
573 integer, intent(in) :: counts(0:)
574 integer, intent(in) :: offsets(0:)
575 integer, intent(in) :: n_local
576 type(vector_t), intent(inout) :: scalar_local
577 integer, allocatable :: counts_r(:)
578 integer, allocatable :: offsets_r(:)
579 integer :: ierr
580
581 allocate(counts_r(0:pe_size - 1))
582 allocate(offsets_r(0:pe_size - 1))
583 counts_r = counts
584 offsets_r = offsets
585 call scalar_old%copy_from(device_to_host, .true.)
586 call mpi_scatterv(scalar_old%x, counts_r, offsets_r, mpi_real_precision, &
587 scalar_local%x, n_local, mpi_real_precision, 0, neko_comm, ierr)
588 call scalar_local%copy_from(host_to_device, .true.)
589 deallocate(counts_r)
590 deallocate(offsets_r)
591 end subroutine distribute_particle_scalar
592
596 subroutine localize_global_interpolation(this, global_interp, n_local)
597 class(lpt_migrate_t), intent(inout) :: this
598 type(global_interpolation_t), intent(inout) :: global_interp
599 integer, intent(in) :: n_local
600
601 global_interp%n_points = n_local
602 global_interp%n_points_local = n_local
603 call global_interp%temp_local%init(n_local)
604 call global_interp%local_interp%init(global_interp%Xh, &
605 global_interp%rst_local, n_local)
606 global_interp%all_points_local = .true.
607 end subroutine localize_global_interpolation
608
615 subroutine migrate_particle_ids(this, migrate_comm, ids_old, &
616 n_particles_old, n_local, particle_ids_local)
617 class(lpt_migrate_t), intent(inout) :: this
618 type(glb_intrp_comm_t), intent(inout) :: migrate_comm
619 integer, allocatable, intent(in) :: ids_old(:)
620 integer, intent(in) :: n_particles_old
621 integer, intent(in) :: n_local
622 integer, allocatable, intent(inout) :: particle_ids_local(:)
623 real(kind=rp), allocatable :: sendbuf(:)
624 real(kind=rp), allocatable :: recvbuf(:)
625 integer :: n_sendbuf
626 integer :: n_recvbuf
627
628 if (n_particles_old .eq. 0 .and. n_local .eq. 0) return
629
630 n_sendbuf = max(1, n_particles_old)
631 n_recvbuf = max(1, n_local)
632 allocate(sendbuf(n_sendbuf))
633 allocate(recvbuf(n_recvbuf))
634 sendbuf = 0.0_rp
635 recvbuf = 0.0_rp
636 if (n_particles_old .gt. 0) sendbuf = real(ids_old, rp)
637 call migrate_comm%sendrecv(sendbuf, recvbuf, n_sendbuf, n_recvbuf)
638 if (n_local .gt. 0) particle_ids_local = nint(recvbuf(1:n_local))
639 if (allocated(sendbuf)) deallocate(sendbuf)
640 if (allocated(recvbuf)) deallocate(recvbuf)
641 end subroutine migrate_particle_ids
642
649 subroutine migrate_particle_scalar(this, migrate_comm, scalar_old, &
650 n_particles_old, n_local, scalar_local)
651 class(lpt_migrate_t), intent(inout) :: this
652 type(glb_intrp_comm_t), intent(inout) :: migrate_comm
653 type(vector_t), intent(inout) :: scalar_old
654 integer, intent(in) :: n_particles_old
655 integer, intent(in) :: n_local
656 type(vector_t), intent(inout) :: scalar_local
657 real(kind=rp), allocatable :: sendbuf(:)
658 real(kind=rp), allocatable :: recvbuf(:)
659 integer :: n_sendbuf
660 integer :: n_recvbuf
661
662 if (n_particles_old .eq. 0 .and. n_local .eq. 0) return
663
664 n_sendbuf = max(1, n_particles_old)
665 n_recvbuf = max(1, n_local)
666 allocate(sendbuf(n_sendbuf))
667 allocate(recvbuf(n_recvbuf))
668 sendbuf = 0.0_rp
669 recvbuf = 0.0_rp
670 call scalar_old%copy_from(device_to_host, .true.)
671 if (n_particles_old .gt. 0) sendbuf = scalar_old%x
672 call migrate_comm%sendrecv(sendbuf, recvbuf, n_sendbuf, n_recvbuf)
673 call scalar_local%copy_from(host_to_device, .true.)
674 if (n_local .gt. 0) scalar_local = recvbuf(1:n_local)
675 if (allocated(sendbuf)) deallocate(sendbuf)
676 if (allocated(recvbuf)) deallocate(recvbuf)
677 end subroutine migrate_particle_scalar
678
679end module lpt_migrate
real
double real
Definition device_config.h:12
comm
Definition comm.F90:1
comm::mpi_real_precision
type(mpi_datatype), public mpi_real_precision
MPI type for working precision of REAL types.
Definition comm.F90:54
comm::pe_size
integer, public pe_size
MPI size of communicator.
Definition comm.F90:62
comm::pe_rank
integer, public pe_rank
MPI rank.
Definition comm.F90:59
comm::neko_comm
type(mpi_comm), public neko_comm
MPI communicator.
Definition comm.F90:46
device
Device abstraction, common interface for various accelerators.
Definition device.F90:34
device::host_to_device
integer, parameter, public host_to_device
Definition device.F90:48
device::device_to_host
integer, parameter, public device_to_host
Definition device.F90:48
glb_intrp_comm
Defines global interpolation communication Based on the MPI based gather-scatter kernel.
Definition global_interpolation_comm.f90:35
global_interpolation
Implements global_interpolation given a dofmap.
Definition global_interpolation.f90:35
lpt_migrate
Particle redistribution support for LPT.
Definition lpt_migration.f90:34
lpt_migrate::lpt_migrate_to_owner
integer, parameter, public lpt_migrate_to_owner
Definition lpt_migration.f90:49
lpt_migrate::lpt_migrate_init
subroutine lpt_migrate_init(this, lag_len, strategy)
Initialise particle migration settings.
Definition lpt_migration.f90:74
lpt_migrate::initialize_particle_distribution
subroutine initialize_particle_distribution(this, inertia, particles)
Apply the initial particle distribution required by the strategy.
Definition lpt_migration.f90:126
lpt_migrate::lpt_migrate_free
subroutine lpt_migrate_free(this)
Reset migration settings to defaults.
Definition lpt_migration.f90:85
lpt_migrate::localize_global_interpolation
subroutine localize_global_interpolation(this, global_interp, n_local)
Mark interpolation data as local after migration has completed.
Definition lpt_migration.f90:597
lpt_migrate::distribute_particle_ids
subroutine distribute_particle_ids(this, ids_old, counts, offsets, n_local, ids_local)
Scatter particle ids from rank 0 to the current rank.
Definition lpt_migration.f90:543
lpt_migrate::migrate_particle_ids
subroutine migrate_particle_ids(this, migrate_comm, ids_old, n_particles_old, n_local, particle_ids_local)
Exchange particle ids according to global-interpolation ownership data.
Definition lpt_migration.f90:617
lpt_migrate::distribute_particle_scalar
subroutine distribute_particle_scalar(this, scalar_old, counts, offsets, n_local, scalar_local)
Scatter one particle scalar field from rank 0 to the current rank.
Definition lpt_migration.f90:571
lpt_migrate::migrate_particle_scalar
subroutine migrate_particle_scalar(this, migrate_comm, scalar_old, n_particles_old, n_local, scalar_local)
Exchange one particle scalar according to interpolation ownership data.
Definition lpt_migration.f90:651
lpt_migrate::lpt_migrate_none
integer, parameter, public lpt_migrate_none
Definition lpt_migration.f90:50
lpt_migrate::build_even_particle_distribution
subroutine build_even_particle_distribution(n_root, counts, offsets, n_local)
Build an even rank distribution from the root-rank particle count.
Definition lpt_migration.f90:97
lpt_migrate::migrate_particles
subroutine migrate_particles(this, global_interp, periodic_bc, inertia, particles)
Update particle ownership according to the selected migration strategy.
Definition lpt_migration.f90:142
lpt_migrate::distribute_particles_evenly
subroutine distribute_particles_evenly(this, inertia, particles)
Distribute particles from rank 0 evenly across all ranks.
Definition lpt_migration.f90:364
lpt_periodic_bc
Periodic and cyclic boundary-condition support for LPT.
Definition lpt_periodic_bc.f90:34
num_types
Definition num_types.f90:1
num_types::rp
integer, parameter, public rp
Global precision used in computations.
Definition num_types.f90:12
particles
Defines a collection of Lagrangian particles.
Definition particles.f90:34
scratch_registry
Defines a registry for storing and requesting temporary objects This can be used when you have a func...
Definition scratch_registry.f90:37
scratch_registry::neko_scratch_registry
type(scratch_registry_t), target, public neko_scratch_registry
Global scratch registry.
Definition scratch_registry.f90:138
vector
Defines a vector.
Definition vector.f90:34
glb_intrp_comm::glb_intrp_comm_t
Global interpolation communication method.
Definition global_interpolation_comm.f90:61
global_interpolation::global_interpolation_t
Implements global interpolation for arbitrary points in the domain.
Definition global_interpolation.f90:87
lpt_migrate::lpt_migrate_t
Definition lpt_migration.f90:52
lpt_periodic_bc::lpt_periodic_bc_t
Definition lpt_periodic_bc.f90:54
particles::particles_t
Particle positions, velocities, properties, and time-history data.
Definition particles.f90:42
vector::vector_t
Definition vector.f90:48
max
#define max(a, b)
Definition tensor.cu:40