da/da7/cuda_2opgrad__kernel_8h_source.html

#ifndef __MATH_OPGRAD_KERNEL_H__

#define __MATH_OPGRAD_KERNEL_H__


#include "elem_block.h"

/*

 Copyright (c) 2021-2023, The Neko Authors

 All rights reserved.


 Redistribution and use in source and binary forms, with or without

 modification, are permitted provided that the following conditions

 are met:


   * Redistributions of source code must retain the above copyright

     notice, this list of conditions and the following disclaimer.


   * Redistributions in binary form must reproduce the above

     copyright notice, this list of conditions and the following

     disclaimer in the documentation and/or other materials provided

     with the distribution.


   * Neither the name of the authors nor the names of its

     contributors may be used to endorse or promote products derived

     from this software without specific prior written permission.


 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS

 FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE

 COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,

 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,

 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

 ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

 POSSIBILITY OF SUCH DAMAGE.

*/


template< typename T, const int LX, const int CHUNKS >


__global__ void opgrad_kernel_1d(T * __restrict__ ux,

                                 T * __restrict__ uy,

                                 T * __restrict__ uz,

                                 const T * __restrict__ u,

                                 const T * __restrict__ dx,

                                 const T * __restrict__ dy,

                                 const T * __restrict__ dz,

                                 const T * __restrict__ drdx,

                                 const T * __restrict__ dsdx,

                                 const T * __restrict__ dtdx,

                                 const T * __restrict__ drdy,

                                 const T * __restrict__ dsdy,

                                 const T * __restrict__ dtdy,

                                 const T * __restrict__ drdz,

                                 const T * __restrict__ dsdz,

                                 const T * __restrict__ dtdz,

                                 const T * __restrict__ w3) {


  __shared__ T shu[LX * LX * LX];


  __shared__ T shdx[LX * LX];

  __shared__ T shdy[LX * LX];

  __shared__ T shdz[LX * LX];


  int i,j,k;


  const int e = blockIdx.x;

  const int iii = threadIdx.x;

  const int nchunks = (LX * LX * LX - 1) / CHUNKS + 1;


  if (iii < (LX * LX)) {

    shdx[iii] = dx[iii];

    shdy[iii] = dy[iii];

    shdz[iii] = dz[iii];

  }


  j = iii;

  while(j < (LX * LX * LX)) {

    shu[j] = u[j + e * LX * LX * LX];

    j = j + CHUNKS;

  }


  __syncthreads();


  for (int n = 0; n < nchunks; n++) {

    const int ijk = iii + n * CHUNKS;

    const int jk = ijk / LX;

    i = ijk - jk * LX;

    k = jk / LX;

    j = jk - k * LX;

    if ( i < LX && j < LX && k < LX ) {

      T rtmp = 0.0;

      T stmp = 0.0;

      T ttmp = 0.0;

      for (int l = 0; l < LX; l++) {

    rtmp += shdx[i + l * LX] * shu[l + j * LX + k * LX * LX];

    stmp += shdy[j + l * LX] * shu[i + l * LX + k * LX * LX];

    ttmp += shdz[k + l * LX] * shu[i + j * LX + l * LX * LX];

      }


      ux[ijk + e * LX * LX * LX] = w3[ijk]

    * (drdx[ijk + e * LX * LX * LX] * rtmp

       + dsdx[ijk + e * LX * LX * LX] * stmp

       + dtdx[ijk + e * LX * LX * LX] * ttmp);


      uy[ijk + e * LX * LX * LX] = w3[ijk]

    * (drdy[ijk + e * LX * LX * LX] * rtmp

       + dsdy[ijk + e * LX * LX * LX] * stmp

       + dtdy[ijk + e * LX * LX * LX] * ttmp);


      uz[ijk + e * LX * LX * LX] = w3[ijk]

    * (drdz[ijk + e * LX * LX * LX] * rtmp

       + dsdz[ijk + e * LX * LX * LX] * stmp

       + dtdz[ijk + e * LX * LX * LX] * ttmp);


    }

  }


}


template< typename T, const int LX, const int EB >

__global__ void NEKO_EB_BOUNDS(LX*LX*EB)

opgrad_kernel_kstep(T * __restrict__ ux,

                    T * __restrict__ uy,

                    T * __restrict__ uz,

                    const T * __restrict__ u,

                    const T * __restrict__ dx,

                    const T * __restrict__ dy,

                    const T * __restrict__ dz,

                    const T * __restrict__ drdx,

                    const T * __restrict__ dsdx,

                    const T * __restrict__ dtdx,

                    const T * __restrict__ drdy,

                    const T * __restrict__ dsdy,

                    const T * __restrict__ dtdy,

                    const T * __restrict__ drdz,

                    const T * __restrict__ dsdz,

                    const T * __restrict__ dtdz,

                    const T * __restrict__ w3,

                    const int nelv) {


  /* One slice per element in the block */

  __shared__ T shu[EB * LX * LX];


  /* Element independent, one copy per block */

  __shared__ T shdx[LX * LX];

  __shared__ T shdy[LX * LX];

  __shared__ T shdz[LX * LX];


  static_assert(sizeof(shu) + sizeof(shdx) + sizeof(shdy) + sizeof(shdz)

                <= NEKO_EB_MAX_SMEM,

                "kstep block exceeds the shared memory budget");


  /* Threads past the last element still have to reach the barriers in the k

     loop, so clamp their reads and drop their stores rather than returning

     early. At EB == 1 all of this is constant folded away */

  const int eb = (EB == 1) ? 0 : threadIdx.z;

  const int e_blk = blockIdx.x * EB + eb;

  const bool active = (EB == 1) ? true : (e_blk < nelv);

  const int e = active ? e_blk : (nelv - 1);

  const int j = threadIdx.y;

  const int i = threadIdx.x;

  const int ij = i + j * LX;

  const int sh = eb * LX * LX;

  const int ele = e*LX*LX*LX;


  if (eb == 0) {

    shdx[ij] = dx[ij];

    shdy[ij] = dy[ij];

    shdz[ij] = dz[ij];

  }


  T ru[LX];


#pragma unroll LX


  for (int k = 0; k < LX; ++k) {

    ru[k] = u[ij + k*LX*LX + ele];

  }


  __syncthreads();


  #pragma unroll

  for (int k = 0; k < LX; ++k) {

    const int ijk = ij + k*LX*LX;

    const T W3 = w3[ijk];

    T ttmp = 0.0;

    shu[sh + ij] = ru[k];

#pragma unroll

    for (int l = 0; l < LX; l++) {

      ttmp += shdz[k+l*LX] * ru[l];

    }

    __syncthreads();


    T rtmp = 0.0;

    T stmp = 0.0;

#pragma unroll

    for (int l = 0; l < LX; l++) {

      rtmp += shdx[i+l*LX] * shu[sh + l+j*LX];

      stmp += shdy[j+l*LX] * shu[sh + i+l*LX];

    }


    if (active) {

      ux[ijk + ele] = W3 * (drdx[ijk + ele] * rtmp

                            + dsdx[ijk + ele] * stmp

                            + dtdx[ijk + ele] * ttmp);


      uy[ijk + ele] = W3 * (drdy[ijk + ele] * rtmp

                            + dsdy[ijk + ele] * stmp

                            + dtdy[ijk + ele] * ttmp);


      uz[ijk + ele] = W3 * (drdz[ijk + ele] * rtmp

                            + dsdz[ijk + ele] * stmp

                            + dtdz[ijk + ele] * ttmp);

    }

    __syncthreads();

  }

}


#endif // __MATH_OPGRAD_KERNEL_H__

ale_add_kinematics_kernel
__global__ void ale_add_kinematics_kernel(const int n, T *__restrict__ wx, T *__restrict__ wy, T *__restrict__ wz, const T *__restrict__ x_ref, const T *__restrict__ y_ref, const T *__restrict__ z_ref, const T *__restrict__ phi, const T *__restrict__ x, const T *__restrict__ y, const T *__restrict__ z, const kinematics_params_t kin_params)
Definition ale_kinematics_kernel.h:52

shu
__shared__ T shu[LX *LX]
Definition ax_helm_full_kernel.h:69

NEKO_EB_MAX_SMEM
#define NEKO_EB_MAX_SMEM
Definition elem_block.h:60

NEKO_EB_BOUNDS
#define NEKO_EB_BOUNDS(NT)
Definition elem_block.h:95

active
const bool active
Definition opgrad_kernel.h:163

dsdx
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dsdx
Definition opgrad_kernel.h:135

dtdz
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dtdz
Definition opgrad_kernel.h:142

uy
__global__ void T *__restrict__ uy
Definition opgrad_kernel.h:128

opgrad_kernel_1d
__global__ void opgrad_kernel_1d(T *__restrict__ ux, T *__restrict__ uy, T *__restrict__ uz, const T *__restrict__ u, const T *__restrict__ dx, const T *__restrict__ dy, const T *__restrict__ dz, const T *__restrict__ drdx, const T *__restrict__ dsdx, const T *__restrict__ dtdx, const T *__restrict__ drdy, const T *__restrict__ dsdy, const T *__restrict__ dtdy, const T *__restrict__ drdz, const T *__restrict__ dsdz, const T *__restrict__ dtdz, const T *__restrict__ w3)
Definition opgrad_kernel.h:44

dx
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ dx
Definition opgrad_kernel.h:131

sh
const int sh
Definition opgrad_kernel.h:168

uz
__global__ void T *__restrict__ T *__restrict__ uz
Definition opgrad_kernel.h:129

dsdz
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dsdz
Definition opgrad_kernel.h:141

nelv
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const int nelv
Definition opgrad_kernel.h:144

dz
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dz
Definition opgrad_kernel.h:133

ru
T ru[LX]
Definition opgrad_kernel.h:177

eb
const int eb
Definition opgrad_kernel.h:161

shdy
__shared__ T shdy[LX *LX]
Definition opgrad_kernel.h:151

drdy
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ drdy
Definition opgrad_kernel.h:137

drdz
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ drdz
Definition opgrad_kernel.h:140

i
const int i
Definition opgrad_kernel.h:166

dtdx
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dtdx
Definition opgrad_kernel.h:136

ij
const int ij
Definition opgrad_kernel.h:167

shdx
__shared__ T shdx[LX *LX]
Definition opgrad_kernel.h:150

e
const int e
Definition opgrad_kernel.h:164

dtdy
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dtdy
Definition opgrad_kernel.h:139

drdx
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ drdx
Definition opgrad_kernel.h:134

shdz
__shared__ T shdz[LX *LX]
Definition opgrad_kernel.h:152

e_blk
const int e_blk
Definition opgrad_kernel.h:162

dsdy
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dsdy
Definition opgrad_kernel.h:138

dy
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dy
Definition opgrad_kernel.h:132

u
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ u
Definition opgrad_kernel.h:130

ele
const int ele
Definition opgrad_kernel.h:169

w3
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ w3
Definition opgrad_kernel.h:143

j
const int j
Definition opgrad_kernel.h:165

__syncthreads
__syncthreads()

elem_block.h