dc/dd2/cuda_2tensor__kernel_8h_source.html

#ifndef __MATH_TENSOR_KERNEL_H__

#define __MATH_TENSOR_KERNEL_H__

/*

 Copyright (c) 2021-2022, The Neko Authors

 All rights reserved.


 Redistribution and use in source and binary forms, with or without

 modification, are permitted provided that the following conditions

 are met:


   * Redistributions of source code must retain the above copyright

     notice, this list of conditions and the following disclaimer.


   * Redistributions in binary form must reproduce the above

     copyright notice, this list of conditions and the following

     disclaimer in the documentation and/or other materials provided

     with the distribution.


   * Neither the name of the authors nor the names of its

     contributors may be used to endorse or promote products derived

     from this software without specific prior written permission.


 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS

 FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE

 COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,

 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,

 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER

 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

 ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

 POSSIBILITY OF SUCH DAMAGE.

*/

template< typename T, const int N >


__global__ void tnsr3d_el_kernel(T  * __restrict__  v,

                              const int nv,

                              const T * __restrict__ u,

                              const int nu,

                              const T * __restrict__ A,

                              const T * __restrict__ Bt,

                              const T * __restrict__ Ct,

                              const int *  elements,

                              const int n_points) {

  __shared__ T shwork[N*N*N];

  __shared__ T shwork2[N*N*N];


  const int idx = threadIdx.x;

  const int str = blockDim.x;

  const int pt = blockIdx.x;

  const int e = elements[pt];


  /* Stage the element into shared memory before the first contraction.

     Read straight from global, phase 1 fetches every value of u nv times --

     once per output row i -- and fetches them badly: with ii decomposing as

     i + j*nv, consecutive threads vary i and share j, so a warp asks for

     values nu apart and pulls a whole sector for each one. Staged, the global

     read is a single contiguous pass and the redundancy is served from shared

     memory. shwork2 is unused until phase 2 writes it, by which point u is

     dead, so this costs no extra shared memory. */

  for (int p = idx; p < nu*nu*nu; p += str)

    shwork2[p] = u[p + e*nu*nu*nu];


  __syncthreads();


  for (int ii = idx; ii< nu*nu*nv; ii += str) {

    T tmp = 0.0;

    int j = ii/nv;

    int i = ii - j*nv;

    for( int l = 0; l < nu; l++){

      tmp += A[i+l*nv+pt*nv*nu]*shwork2[l+nu*j];

    }

    shwork[ii] = tmp;

  }


  __syncthreads();


  for (int ijk = idx; ijk< nu*nv*nv; ijk += str) {

    const int jk = ijk / nv;

    const int i = ijk - jk * nv;

    const int k = jk / nv;

    const int j = jk - k * nv;

    T tmp = 0.0;

    const int ik2 = i + k*nv*nu;

    for( int l = 0; l < nu; l++){

      tmp += Bt[l+j*nu+pt*nv*nu]*shwork[l*nv+ik2];

    }

    shwork2[ijk] = tmp;

  }


  __syncthreads();


  for (int ijk = idx; ijk< nv*nv*nv; ijk += str) {

    const int jk = ijk / nv;

    const int i = ijk - jk * nv;

    const int k = jk / nv;

    const int j = jk - k * nv;

    T tmp = 0.0;

    const int ij2 = i + j*nv;

    for( int l = 0; l < nu; l++){

      tmp += Ct[l+k*nu+pt*nv*nu]*shwork2[ij2 + l*nv*nv];

    }

    v[ijk+pt*nv*nv*nv] = tmp;

  }


}


template< typename T, const int N >


__global__ void tnsr3d_kernel(T  * __restrict__  v,

                              const int nv,

                              const T * __restrict__ u,

                              const int nu,

                              const T * __restrict__ A,

                              const T * __restrict__ Bt,

                              const T * __restrict__ Ct) {

  __shared__ T shwork[N*N*N];

  __shared__ T shwork2[N*N*N];


  const int idx = threadIdx.x;

  const int str = blockDim.x;

  const int e = blockIdx.x;


  /* Stage the element into shared memory before the first contraction.

     Read straight from global, phase 1 fetches every value of u nv times --

     once per output row i -- and fetches them badly: with ii decomposing as

     i + j*nv, consecutive threads vary i and share j, so a warp asks for

     values nu apart and pulls a whole sector for each one. Staged, the global

     read is a single contiguous pass and the redundancy is served from shared

     memory. shwork2 is unused until phase 2 writes it, by which point u is

     dead, so this costs no extra shared memory. */

  for (int p = idx; p < nu*nu*nu; p += str)

    shwork2[p] = u[p + e*nu*nu*nu];


  __syncthreads();


  for (int ii = idx; ii< nu*nu*nv; ii += str) {

    T tmp = 0.0;

    int j = ii/nv;

    int i = ii - j*nv;

    for( int l = 0; l < nu; l++){

      tmp += A[i+l*nv]*shwork2[l+nu*j];

    }

    shwork[ii] = tmp;

  }


  __syncthreads();


  for (int ijk = idx; ijk< nu*nv*nv; ijk += str) {

    const int jk = ijk / nv;

    const int i = ijk - jk * nv;

    const int k = jk / nv;

    const int j = jk - k * nv;

    T tmp = 0.0;

    const int ik2 = i + k*nv*nu;

    for( int l = 0; l < nu; l++){

      tmp += Bt[l+j*nu]*shwork[l*nv+ik2];

    }

    shwork2[ijk] = tmp;

  }


  __syncthreads();


  for (int ijk = idx; ijk< nv*nv*nv; ijk += str) {

    const int jk = ijk / nv;

    const int i = ijk - jk * nv;

    const int k = jk / nv;

    const int j = jk - k * nv;

    T tmp = 0.0;

    const int ij2 = i + j*nv;

    for( int l = 0; l < nu; l++){

      tmp += Ct[l+k*nu]*shwork2[ij2 + l*nv*nv];

    }

    v[ijk+e*nv*nv*nv] = tmp;

  }

}


template< typename T, const int N >


__global__ void tnsr3d_kernel_large(T  * __restrict__  v,

                                    const int nv,

                                    const T * __restrict__ u,

                                    const int nu,

                                    const T * __restrict__ A,

                                    const T * __restrict__ Bt,

                                    const T * __restrict__ Ct) {

  extern __shared__ T shmem[];

  T *shwork = shmem;

  T *shwork2 = shmem + N*N*N;


  const int idx = threadIdx.x;

  const int str = blockDim.x;

  const int e = blockIdx.x;


  /* Stage the element into shared memory before the first contraction.

     Read straight from global, phase 1 fetches every value of u nv times --

     once per output row i -- and fetches them badly: with ii decomposing as

     i + j*nv, consecutive threads vary i and share j, so a warp asks for

     values nu apart and pulls a whole sector for each one. Staged, the global

     read is a single contiguous pass and the redundancy is served from shared

     memory. shwork2 is unused until phase 2 writes it, by which point u is

     dead, so this costs no extra shared memory. */

  for (int p = idx; p < nu*nu*nu; p += str)

    shwork2[p] = u[p + e*nu*nu*nu];


  __syncthreads();


  for (int ii = idx; ii< nu*nu*nv; ii += str) {

    T tmp = 0.0;

    int j = ii/nv;

    int i = ii - j*nv;

    for( int l = 0; l < nu; l++){

      tmp += A[i+l*nv]*shwork2[l+nu*j];

    }

    shwork[ii] = tmp;

  }


  __syncthreads();


  for (int ijk = idx; ijk< nu*nv*nv; ijk += str) {

    const int jk = ijk / nv;

    const int i = ijk - jk * nv;

    const int k = jk / nv;

    const int j = jk - k * nv;

    T tmp = 0.0;

    const int ik2 = i + k*nv*nu;

    for( int l = 0; l < nu; l++){

      tmp += Bt[l+j*nu]*shwork[l*nv+ik2];

    }

    shwork2[ijk] = tmp;

  }


  __syncthreads();


  for (int ijk = idx; ijk< nv*nv*nv; ijk += str) {

    const int jk = ijk / nv;

    const int i = ijk - jk * nv;

    const int k = jk / nv;

    const int j = jk - k * nv;

    T tmp = 0.0;

    const int ij2 = i + j*nv;

    for( int l = 0; l < nu; l++){

      tmp += Ct[l+k*nu]*shwork2[ij2 + l*nv*nv];

    }

    v[ijk+e*nv*nv*nv] = tmp;

  }

}


#endif // __MATH_TENSOR_KERNEL_H__

ale_add_kinematics_kernel
__global__ void ale_add_kinematics_kernel(const int n, T *__restrict__ wx, T *__restrict__ wy, T *__restrict__ wz, const T *__restrict__ x_ref, const T *__restrict__ y_ref, const T *__restrict__ z_ref, const T *__restrict__ phi, const T *__restrict__ x, const T *__restrict__ y, const T *__restrict__ z, const kinematics_params_t kin_params)
Definition ale_kinematics_kernel.h:52

i
const int i
Definition ax_helm_full_kernel.h:95

u
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ u
Definition ax_helm_full_kernel.h:46

e
const int e
Definition ax_helm_full_kernel.h:93

v
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ v
Definition ax_helm_full_kernel.h:47

j
const int j
Definition ax_helm_full_kernel.h:94

__syncthreads
__syncthreads()

tnsr3d_el_kernel
__global__ void tnsr3d_el_kernel(T *__restrict__ v, const int nv, const T *__restrict__ u, const int nu, const T *__restrict__ A, const T *__restrict__ Bt, const T *__restrict__ Ct, const int *elements, const int n_points)
Definition tensor_kernel.h:37

tnsr3d_kernel
__global__ void tnsr3d_kernel(T *__restrict__ v, const int nv, const T *__restrict__ u, const int nu, const T *__restrict__ A, const T *__restrict__ Bt, const T *__restrict__ Ct)
Definition tensor_kernel.h:112

tnsr3d_kernel_large
__global__ void tnsr3d_kernel_large(T *__restrict__ v, const int nv, const T *__restrict__ u, const int nu, const T *__restrict__ A, const T *__restrict__ Bt, const T *__restrict__ Ct)
Definition tensor_kernel.h:181

shmem
Fortran bindings to SHMEM's C API.
Definition shmem.F90:34