Neko 1.99.3
A portable framework for high-order spectral element flow simulations
Loading...
Searching...
No Matches
checkpoint.f90
Go to the documentation of this file.
1! Copyright (c) 2021, The Neko Authors
2! All rights reserved.
3!
4! Redistribution and use in source and binary forms, with or without
5! modification, are permitted provided that the following conditions
6! are met:
7!
8! * Redistributions of source code must retain the above copyright
9! notice, this list of conditions and the following disclaimer.
10!
11! * Redistributions in binary form must reproduce the above
12! copyright notice, this list of conditions and the following
13! disclaimer in the documentation and/or other materials provided
14! with the distribution.
15!
16! * Neither the name of the authors nor the names of its
17! contributors may be used to endorse or promote products derived
18! from this software without specific prior written permission.
19!
20! THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21! "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22! LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23! FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24! COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25! INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26! BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
27! LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
28! CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29! LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
30! ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31! POSSIBILITY OF SUCH DAMAGE.
32!
34module checkpoint
35 use neko_config, only : neko_bcknd_device
36 use num_types, only : rp, dp
37 use field_series, only : field_series_t
38 use field_series_list, only : field_series_list_t
39 use space, only : space_t, operator(.ne.)
40 use device, only : device_memcpy, device_to_host, host_to_device, &
41 device_sync, glb_cmd_queue
42 use field, only : field_t, field_ptr_t
43 use utils, only : neko_error
44 use mesh, only : mesh_t
45 use math, only : neko_eps
46 use global_interpolation, only : glob_interp_tol
47 implicit none
48 private
49
50 type, public :: chkp_t
51 type(field_t), pointer :: u => null()
52 type(field_t), pointer :: v => null()
53 type(field_t), pointer :: w => null()
54 type(field_t), pointer :: p => null()
55
56
57 !
58 ! Optional payload
59 !
60 type(field_series_t), pointer :: ulag => null()
61 type(field_series_t), pointer :: vlag => null()
62 type(field_series_t), pointer :: wlag => null()
63
64 real(kind=rp), pointer :: tlag(:) => null()
65 real(kind=rp), pointer :: dtlag(:) => null()
66
68 type(field_t), pointer :: abx1 => null()
69 type(field_t), pointer :: abx2 => null()
70 type(field_t), pointer :: aby1 => null()
71 type(field_t), pointer :: aby2 => null()
72 type(field_t), pointer :: abz1 => null()
73 type(field_t), pointer :: abz2 => null()
74
75 type(field_t), pointer :: s => null()
76 type(field_series_t), pointer :: slag => null()
77 type(field_t), pointer :: abs1 => null()
78 type(field_t), pointer :: abs2 => null()
79
80 type(field_series_list_t) :: scalar_lags
81
83 type(field_ptr_t), allocatable :: scalar_abx1(:)
84 type(field_ptr_t), allocatable :: scalar_abx2(:)
85
86 real(kind=dp) :: t = 0d0
87 type(mesh_t) :: previous_mesh
88 type(space_t) :: previous_xh
89 real(kind=dp) :: mesh2mesh_tol = glob_interp_tol
90
92 type(field_t), pointer :: wm_x => null()
93 type(field_t), pointer :: wm_y => null()
94 type(field_t), pointer :: wm_z => null()
95 type(field_series_t), pointer :: wm_x_lag => null()
96 type(field_series_t), pointer :: wm_y_lag => null()
97 type(field_series_t), pointer :: wm_z_lag => null()
98 real(kind=rp), pointer :: msh_x(:,:,:,:) => null()
99 real(kind=rp), pointer :: msh_y(:,:,:,:) => null()
100 real(kind=rp), pointer :: msh_z(:,:,:,:) => null()
101 real(kind=rp), pointer :: pivot_pos(:) => null()
102 real(kind=rp), pointer :: pivot_vel_lag(:,:) => null()
103 real(kind=rp), pointer :: blag(:,:,:,:) => null()
104 real(kind=rp), pointer :: blaglag(:,:,:,:) => null()
105 real(kind=rp), pointer :: basis_pos(:) => null()
106 real(kind=rp), pointer :: basis_vel_lag(:,:) => null()
107 contains
108 procedure, pass(this) :: init => chkp_init
109 procedure, pass(this) :: sync_host => chkp_sync_host
110 procedure, pass(this) :: sync_device => chkp_sync_device
111 procedure, pass(this) :: add_fluid => chkp_add_fluid
112 procedure, pass(this) :: add_lag => chkp_add_lag
113 procedure, pass(this) :: add_scalar => chkp_add_scalar
114 procedure, pass(this) :: add_ale => chkp_add_ale
115 procedure, pass(this) :: restart_time => chkp_restart_time
116 procedure, pass(this) :: free => chkp_free
117 end type chkp_t
118
119contains
120
122 subroutine chkp_init(this)
123 class(chkp_t), intent(inout) :: this
124
125 ! Make sure the object is clean
126 call this%free()
127
128 end subroutine chkp_init
129
131 subroutine chkp_free(this)
132 class(chkp_t), intent(inout) :: this
133
134 this%t = 0d0
135
136 if (associated(this%u)) nullify(this%u)
137 if (associated(this%v)) nullify(this%v)
138 if (associated(this%w)) nullify(this%w)
139 if (associated(this%p)) nullify(this%p)
140
141 if (associated(this%ulag)) nullify(this%ulag)
142 if (associated(this%vlag)) nullify(this%vlag)
143 if (associated(this%wlag)) nullify(this%wlag)
144
145 if (associated(this%tlag)) nullify(this%tlag)
146 if (associated(this%dtlag)) nullify(this%dtlag)
147
148 ! ALE cleanup
149 if (associated(this%wm_x)) nullify(this%wm_x)
150 if (associated(this%wm_y)) nullify(this%wm_y)
151 if (associated(this%wm_z)) nullify(this%wm_z)
152 if (associated(this%wm_x_lag)) nullify(this%wm_x_lag)
153 if (associated(this%wm_y_lag)) nullify(this%wm_y_lag)
154 if (associated(this%wm_z_lag)) nullify(this%wm_z_lag)
155 if (associated(this%basis_vel_lag)) nullify(this%basis_vel_lag)
156 if (associated(this%msh_x)) nullify(this%msh_x)
157 if (associated(this%msh_y)) nullify(this%msh_y)
158 if (associated(this%msh_z)) nullify(this%msh_z)
159 if (associated(this%pivot_pos)) nullify(this%pivot_pos)
160 if (associated(this%pivot_vel_lag)) nullify(this%pivot_vel_lag)
161 if (associated(this%Blag)) nullify(this%Blag)
162 if (associated(this%Blaglag)) nullify(this%Blaglag)
163 if (associated(this%basis_pos)) nullify(this%basis_pos)
164
165 if (associated(this%abx1)) nullify(this%abx1)
166 if (associated(this%abx2)) nullify(this%abx2)
167 if (associated(this%aby1)) nullify(this%aby1)
168 if (associated(this%aby2)) nullify(this%aby2)
169 if (associated(this%abz1)) nullify(this%abz1)
170 if (associated(this%abz2)) nullify(this%abz2)
171
172 if (associated(this%s)) nullify(this%s)
173 if (associated(this%slag)) nullify(this%slag)
174 if (associated(this%abs1)) nullify(this%abs1)
175 if (associated(this%abs2)) nullify(this%abs2)
176
177 call this%scalar_lags%free()
178
179 if (allocated(this%scalar_abx1)) then
180 deallocate(this%scalar_abx1)
181 end if
182
183 if (allocated(this%scalar_abx2)) then
184 deallocate(this%scalar_abx2)
185 end if
186
187 call this%previous_mesh%free()
188 call this%previous_Xh%free()
189
190 end subroutine chkp_free
191
193 subroutine chkp_sync_host(this)
194 class(chkp_t), intent(inout) :: this
195 integer :: i, j
196
197 if (neko_bcknd_device .eq. 1) then
198 associate(u => this%u, v => this%v, w => this%w, &
199 ulag => this%ulag, vlag => this%vlag, wlag => this%wlag, &
200 p => this%p)
201
202 if (associated(this%u) .and. associated(this%v) .and. &
203 associated(this%w) .and. associated(this%p)) then
204 call u%copy_from(device_to_host, sync = .false.)
205 call v%copy_from(device_to_host, sync = .false.)
206 call w%copy_from(device_to_host, sync = .false.)
207 call p%copy_from(device_to_host, sync = .false.)
208 end if
209
210 if (associated(this%ulag) .and. associated(this%vlag) .and. &
211 associated(this%wlag)) then
212 call ulag%lf(1)%copy_from(device_to_host, sync = .false.)
213 call ulag%lf(2)%copy_from(device_to_host, sync = .false.)
214
215 call vlag%lf(1)%copy_from(device_to_host, sync = .false.)
216 call vlag%lf(2)%copy_from(device_to_host, sync = .false.)
217
218 call wlag%lf(1)%copy_from(device_to_host, sync = .false.)
219 call wlag%lf(2)%copy_from(device_to_host, sync = .false.)
220 call this%abx1%copy_from(device_to_host, sync = .false.)
221 call this%abx2%copy_from(device_to_host, sync = .false.)
222 call this%aby1%copy_from(device_to_host, sync = .false.)
223 call this%aby2%copy_from(device_to_host, sync = .false.)
224 call this%abz1%copy_from(device_to_host, sync = .false.)
225 call this%abz2%copy_from(device_to_host, sync = .false.)
226 end if
227
228 if (associated(this%s)) then
229 call this%s%copy_from(device_to_host, sync = .false.)
230 call this%slag%lf(1)%copy_from(device_to_host, sync = .false.)
231 call this%slag%lf(2)%copy_from(device_to_host, sync = .false.)
232 call this%abs1%copy_from(device_to_host, sync = .false.)
233 call this%abs2%copy_from(device_to_host, sync = .false.)
234 end if
235
236 ! ALE field synchronization
237 if (associated(this%wm_x) .and. associated(this%wm_y) .and. &
238 associated(this%wm_z)) then
239 call this%wm_x%copy_from(device_to_host, sync = .false.)
240 call this%wm_y%copy_from(device_to_host, sync = .false.)
241 call this%wm_z%copy_from(device_to_host, sync = .false.)
242
243 if (associated(this%wm_x_lag) .and. associated(this%wm_y_lag) &
244 .and. associated(this%wm_z_lag)) then
245
246 call this%wm_x_lag%lf(1)%copy_from(device_to_host, &
247 sync = .false.)
248 call this%wm_x_lag%lf(2)%copy_from(device_to_host, &
249 sync = .false.)
250
251 call this%wm_y_lag%lf(1)%copy_from(device_to_host, &
252 sync = .false.)
253 call this%wm_y_lag%lf(2)%copy_from(device_to_host, &
254 sync = .false.)
255
256 call this%wm_z_lag%lf(1)%copy_from(device_to_host, &
257 sync = .false.)
258 call this%wm_z_lag%lf(2)%copy_from(device_to_host, &
259 sync = .false.)
260 end if
261 end if
262
263 ! Multi-scalar lag field synchronization
264 do i = 1, this%scalar_lags%size()
265 block
266 type(field_series_t), pointer :: slag
267 integer :: slag_size
268 slag => this%scalar_lags%get(i)
269 slag_size = slag%size()
270 do j = 1, slag_size
271 call slag%lf(j)%copy_from(device_to_host, sync = .false.)
272 end do
273 end block
274 end do
275
276 ! Multi-scalar ABX field synchronization
277 if (allocated(this%scalar_abx1) .and. allocated(this%scalar_abx2)) then
278 do i = 1, size(this%scalar_abx1)
279 call this%scalar_abx1(i)%ptr%copy_from(device_to_host, &
280 sync = .false.)
281 call this%scalar_abx2(i)%ptr%copy_from(device_to_host, &
282 sync = .false.)
283 end do
284 end if
285 end associate
286 call device_sync(glb_cmd_queue)
287 end if
288
289 end subroutine chkp_sync_host
290
292 subroutine chkp_sync_device(this)
293 class(chkp_t), intent(inout) :: this
294 integer :: i, j
295
296 if (neko_bcknd_device .eq. 1) then
297 associate(u => this%u, v => this%v, w => this%w, &
298 ulag => this%ulag, vlag => this%vlag, wlag => this%wlag, &
299 p => this%p)
300
301 if (associated(this%u) .and. associated(this%v) .and. &
302 associated(this%w)) then
303 call u%copy_from(host_to_device, sync = .false.)
304 call v%copy_from(host_to_device, sync = .false.)
305 call w%copy_from(host_to_device, sync = .false.)
306 call p%copy_from(host_to_device, sync = .false.)
307 end if
308
309 if (associated(this%ulag) .and. associated(this%vlag) .and. &
310 associated(this%wlag)) then
311 call ulag%lf(1)%copy_from(host_to_device, sync = .false.)
312 call ulag%lf(2)%copy_from(host_to_device, sync = .false.)
313
314 call vlag%lf(1)%copy_from(host_to_device, sync = .false.)
315 call vlag%lf(2)%copy_from(host_to_device, sync = .false.)
316
317 call wlag%lf(1)%copy_from(host_to_device, sync = .false.)
318 call wlag%lf(2)%copy_from(host_to_device, sync = .false.)
319 end if
320
321 if (associated(this%s)) then
322 call this%s%copy_from(host_to_device, sync = .false.)
323
324 call this%slag%lf(1)%copy_from(host_to_device, sync = .false.)
325 call this%slag%lf(2)%copy_from(host_to_device, sync = .false.)
326 call this%abs1%copy_from(host_to_device, sync = .false.)
327 call this%abs2%copy_from(host_to_device, sync = .false.)
328 end if
329
330 ! ALE field synchronization
331 if (associated(this%wm_x) .and. associated(this%wm_y) .and. &
332 associated(this%wm_z)) then
333 call this%wm_x%copy_from(host_to_device, sync = .false.)
334 call this%wm_y%copy_from(host_to_device, sync = .false.)
335 call this%wm_z%copy_from(host_to_device, sync = .false.)
336 if (associated(this%wm_x_lag) .and. associated(this%wm_y_lag) .and. &
337 associated(this%wm_z_lag)) then
338 call this%wm_x_lag%lf(1)%copy_from(host_to_device, &
339 sync = .false.)
340 call this%wm_x_lag%lf(2)%copy_from(host_to_device, &
341 sync = .false.)
342
343 call this%wm_y_lag%lf(1)%copy_from(host_to_device, &
344 sync = .false.)
345 call this%wm_y_lag%lf(2)%copy_from(host_to_device, &
346 sync = .false.)
347
348 call this%wm_z_lag%lf(1)%copy_from(host_to_device, &
349 sync = .false.)
350 call this%wm_z_lag%lf(2)%copy_from(host_to_device, &
351 sync = .false.)
352 end if
353 end if
354
355 ! Multi-scalar lag field synchronization
356 if (allocated(this%scalar_lags%items) .and. &
357 this%scalar_lags%size() > 0) then
358 do i = 1, this%scalar_lags%size()
359 block
360 type(field_series_t), pointer :: slag
361 integer :: slag_size, dof_size
362 slag => this%scalar_lags%get(i)
363 slag_size = slag%size()
364 dof_size = slag%f%dof%size()
365 do j = 1, slag_size
366 call slag%lf(j)%copy_from(host_to_device, sync = .false.)
367 end do
368 end block
369 end do
370 end if
371
372 ! Multi-scalar ABX field synchronization
373 if (allocated(this%scalar_abx1) .and. allocated(this%scalar_abx2)) then
374 do i = 1, size(this%scalar_abx1)
375 call this%scalar_abx1(i)%ptr%copy_from(host_to_device, &
376 sync = .false.)
377 call this%scalar_abx2(i)%ptr%copy_from(host_to_device, &
378 sync = .false.)
379 end do
380 end if
381 end associate
382 end if
383
384 end subroutine chkp_sync_device
385
387 subroutine chkp_add_fluid(this, u, v, w, p)
388 class(chkp_t), intent(inout) :: this
389 type(field_t), target :: u
390 type(field_t), target :: v
391 type(field_t), target :: w
392 type(field_t), target :: p
393
394 ! Check that all velocity components are defined on the same
395 ! function space
396 if ( u%Xh .ne. v%Xh .or. &
397 u%Xh .ne. w%Xh ) then
398 call neko_error('Different function spaces for each velocity component')
399 end if
400
401 ! Check that both velocity and pressure is defined on the same mesh
402 if ( u%msh%nelv .ne. p%msh%nelv ) then
403 call neko_error('Velocity and pressure defined on different meshes')
404 end if
405
406 this%u => u
407 this%v => v
408 this%w => w
409 this%p => p
410
411 end subroutine chkp_add_fluid
412
414 subroutine chkp_add_lag(this, ulag, vlag, wlag)
415 class(chkp_t), intent(inout) :: this
416 type(field_series_t), target :: ulag
417 type(field_series_t), target :: vlag
418 type(field_series_t), target :: wlag
419
420 this%ulag => ulag
421 this%vlag => vlag
422 this%wlag => wlag
423
424 end subroutine chkp_add_lag
425
426
427
429 subroutine chkp_add_scalar(this, s, slag, abs1, abs2)
430 class(chkp_t), intent(inout) :: this
431 type(field_t), target, intent(in) :: s
432 type(field_series_t), target, intent(in) :: slag
433 type(field_t), target, intent(in), optional :: abs1, abs2
434
435 this%s => s
436 this%slag => slag
437
438 if (present(abs1)) this%abs1 => abs1
439 if (present(abs2)) this%abs2 => abs2
440
441 end subroutine chkp_add_scalar
442
444 subroutine chkp_add_ale(this, x, y, z, Blag, Blaglag, wm_x, wm_y, wm_z, &
445 wm_x_lag, wm_y_lag, wm_z_lag, pivot_pos, pivot_vel_lag, basis_pos, &
446 basis_vel_lag)
447 class(chkp_t), intent(inout) :: this
448 type(field_t), target, intent(in) :: wm_x, wm_y, wm_z
449 real(kind=rp), intent(in), pointer :: pivot_pos(:), pivot_vel_lag(:,:)
450 type(field_series_t), target, intent(in) :: wm_x_lag, wm_y_lag, wm_z_lag
451 real(kind=rp), intent(in), pointer :: x(:,:,:,:), y(:,:,:,:), z(:,:,:,:)
452 real(kind=rp), pointer, intent(in) :: blag(:,:,:,:), blaglag(:,:,:,:)
453 real(kind=rp), intent(in), pointer :: basis_pos(:)
454 real(kind=rp), intent(in), pointer :: basis_vel_lag(:,:)
455
456 this%msh_x => x
457 this%msh_y => y
458 this%msh_z => z
459 this%wm_x => wm_x
460 this%wm_y => wm_y
461 this%wm_z => wm_z
462 this%wm_x_lag => wm_x_lag
463 this%wm_y_lag => wm_y_lag
464 this%wm_z_lag => wm_z_lag
465 this%Blag => blag
466 this%Blaglag => blaglag
467 this%pivot_pos => pivot_pos
468 this%pivot_vel_lag => pivot_vel_lag
469 this%basis_pos => basis_pos
470 this%basis_vel_lag => basis_vel_lag
471 end subroutine chkp_add_ale
472
474 pure function chkp_restart_time(this) result(rtime)
475 class(chkp_t), intent(in) :: this
476 real(kind=dp) :: rtime
477
478 rtime = this%t
479 end function chkp_restart_time
480
481end module checkpoint
device::device_memcpy
Copy data between host and device (or device and device)
Definition device.F90:71
device::device_sync
Synchronize a device or stream.
Definition device.F90:113
utils::neko_error
Definition utils.f90:42
checkpoint
Defines a checkpoint.
Definition checkpoint.f90:34
checkpoint::chkp_sync_host
subroutine chkp_sync_host(this)
Synchronize checkpoint with device.
Definition checkpoint.f90:194
checkpoint::chkp_sync_device
subroutine chkp_sync_device(this)
Synchronize device with checkpoint.
Definition checkpoint.f90:293
checkpoint::chkp_add_ale
subroutine chkp_add_ale(this, x, y, z, blag, blaglag, wm_x, wm_y, wm_z, wm_x_lag, wm_y_lag, wm_z_lag, pivot_pos, pivot_vel_lag, basis_pos, basis_vel_lag)
Add mesh velocity and other required variables to checkpointing for ALE.
Definition checkpoint.f90:447
checkpoint::chkp_restart_time
pure real(kind=dp) function chkp_restart_time(this)
Return restart time from a loaded checkpoint.
Definition checkpoint.f90:475
checkpoint::chkp_free
subroutine chkp_free(this)
Reset checkpoint.
Definition checkpoint.f90:132
checkpoint::chkp_add_lag
subroutine chkp_add_lag(this, ulag, vlag, wlag)
Add lagged velocity terms.
Definition checkpoint.f90:415
checkpoint::chkp_add_fluid
subroutine chkp_add_fluid(this, u, v, w, p)
Add a fluid to the checkpoint.
Definition checkpoint.f90:388
checkpoint::chkp_init
subroutine chkp_init(this)
Initialize checkpoint structure with mandatory data.
Definition checkpoint.f90:123
checkpoint::chkp_add_scalar
subroutine chkp_add_scalar(this, s, slag, abs1, abs2)
Add a scalar to checkpointing.
Definition checkpoint.f90:430
device
Device abstraction, common interface for various accelerators.
Definition device.F90:34
device::host_to_device
integer, parameter, public host_to_device
Definition device.F90:47
device::device_to_host
integer, parameter, public device_to_host
Definition device.F90:47
device::glb_cmd_queue
type(c_ptr), bind(C), public glb_cmd_queue
Global command queue.
Definition device.F90:51
field_series_list
Contains the field_series_list_t type for managing multiple field series.
Definition field_series_list.f90:34
field_series
Contains the field_serties_t type.
Definition field_series.f90:34
field
Defines a field.
Definition field.f90:34
global_interpolation
Implements global_interpolation given a dofmap.
Definition global_interpolation.f90:35
global_interpolation::glob_interp_tol
real(kind=dp), parameter, public glob_interp_tol
Definition global_interpolation.f90:73
math
Definition math.f90:60
math::neko_eps
real(kind=rp), parameter, public neko_eps
Machine epsilon .
Definition math.f90:69
mesh
Defines a mesh.
Definition mesh.f90:34
neko_config
Build configurations.
Definition neko_config.f90:34
neko_config::neko_bcknd_device
integer, parameter neko_bcknd_device
Definition neko_config.f90:44
num_types
Definition num_types.f90:1
num_types::dp
integer, parameter, public dp
Definition num_types.f90:9
num_types::rp
integer, parameter, public rp
Global precision used in computations.
Definition num_types.f90:12
space
Defines a function space.
Definition space.f90:34
utils
Utilities.
Definition utils.f90:35
checkpoint::chkp_t
Definition checkpoint.f90:50
field::field_ptr_t
field_ptr_t, To easily obtain a pointer to a field
Definition field.f90:82
field::field_t
Definition field.f90:47
field_series::field_series_t
Stores a series (sequence) of fields, logically connected to a base field, and arranged according to ...
Definition field_series.f90:42
field_series_list::field_series_list_t
A list of field series pointers, used for managing multiple scalar lag fields.
Definition field_series_list.f90:41
mesh::mesh_t
Definition mesh.f90:72
space::space_t
The function space for the SEM solution fields.
Definition space.f90:63