Neko 1.99.9
A portable framework for high-order spectral element flow simulations
Loading...
Searching...
No Matches
tensor_kernel.h
Go to the documentation of this file.
1#ifndef __MATH_TENSOR_KERNEL_H__
2#define __MATH_TENSOR_KERNEL_H__
3/*
4 Copyright (c) 2022, The Neko Authors
5 All rights reserved.
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions
9 are met:
10
11 * Redistributions of source code must retain the above copyright
12 notice, this list of conditions and the following disclaimer.
13
14 * Redistributions in binary form must reproduce the above
15 copyright notice, this list of conditions and the following
16 disclaimer in the documentation and/or other materials provided
17 with the distribution.
18
19 * Neither the name of the authors nor the names of its
20 contributors may be used to endorse or promote products derived
21 from this software without specific prior written permission.
22
23 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26 FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27 COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
29 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
31 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
33 ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 POSSIBILITY OF SUCH DAMAGE.
35*/
36template< typename T, const int N >
37__global__ void tnsr3d_el_kernel(T * __restrict__ v,
38 const int nv,
39 const T * __restrict__ u,
40 const int nu,
41 const T * __restrict__ A,
42 const T * __restrict__ Bt,
43 const T * __restrict__ Ct,
44 const int * elements,
45 const int n_points) {
46 __shared__ T shwork[N*N*N];
47 __shared__ T shwork2[N*N*N];
48
49 const int idx = threadIdx.x;
50 const int str = blockDim.x;
51 const int pt = blockIdx.x;
52 const int e = elements[pt];
53
54 /* Stage the element into shared memory before the first contraction.
55 Read straight from global, phase 1 fetches every value of u nv times --
56 once per output row i -- and fetches them badly: with ii decomposing as
57 i + j*nv, consecutive threads vary i and share j, so a warp asks for
58 values nu apart and pulls a whole sector for each one. Staged, the global
59 read is a single contiguous pass and the redundancy is served from shared
60 memory. shwork2 is unused until phase 2 writes it, by which point u is
61 dead, so this costs no extra shared memory. */
62 for (int p = idx; p < nu*nu*nu; p += str)
63 shwork2[p] = u[p + e*nu*nu*nu];
64
65 __syncthreads();
66
67 for (int ii = idx; ii< nu*nu*nv; ii += str) {
68 T tmp = 0.0;
69 int j = ii/nv;
70 int i = ii - j*nv;
71 for( int l = 0; l < nu; l++){
72 tmp += A[i+l*nv+pt*nv*nu]*shwork2[l+nu*j];
73 }
74 shwork[ii] = tmp;
75 }
76
77 __syncthreads();
78
79 for (int ijk = idx; ijk< nu*nv*nv; ijk += str) {
80 const int jk = ijk / nv;
81 const int i = ijk - jk * nv;
82 const int k = jk / nv;
83 const int j = jk - k * nv;
84 T tmp = 0.0;
85 const int ik2 = i + k*nv*nu;
86 for( int l = 0; l < nu; l++){
87 tmp += Bt[l+j*nu+pt*nv*nu]*shwork[l*nv+ik2];
88 }
89 shwork2[ijk] = tmp;
90 }
91
92 __syncthreads();
93
94 for (int ijk = idx; ijk< nv*nv*nv; ijk += str) {
95 const int jk = ijk / nv;
96 const int i = ijk - jk * nv;
97 const int k = jk / nv;
98 const int j = jk - k * nv;
99 T tmp = 0.0;
100 const int ij2 = i + j*nv;
101 for( int l = 0; l < nu; l++){
102 tmp += Ct[l+k*nu+pt*nv*nu]*shwork2[ij2 + l*nv*nv];
103 }
104 v[ijk+pt*nv*nv*nv] = tmp;
105 }
106
107}
108
109
110template< typename T, const int N >
111__global__ void tnsr3d_kernel(T * __restrict__ v,
112 const int nv,
113 const T * __restrict__ u,
114 const int nu,
115 const T * __restrict__ A,
116 const T * __restrict__ Bt,
117 const T * __restrict__ Ct) {
118 __shared__ T shwork[N*N*N];
119 __shared__ T shwork2[N*N*N];
120
121 const int idx = threadIdx.x;
122 const int str = blockDim.x;
123 const int e = blockIdx.x;
124
125 /* Stage the element into shared memory before the first contraction.
126 Read straight from global, phase 1 fetches every value of u nv times --
127 once per output row i -- and fetches them badly: with ii decomposing as
128 i + j*nv, consecutive threads vary i and share j, so a warp asks for
129 values nu apart and pulls a whole sector for each one. Staged, the global
130 read is a single contiguous pass and the redundancy is served from shared
131 memory. shwork2 is unused until phase 2 writes it, by which point u is
132 dead, so this costs no extra shared memory. */
133 for (int p = idx; p < nu*nu*nu; p += str)
134 shwork2[p] = u[p + e*nu*nu*nu];
135
136 __syncthreads();
137
138 for (int ii = idx; ii< nu*nu*nv; ii += str) {
139 T tmp = 0.0;
140 int j = ii/nv;
141 int i = ii - j*nv;
142 for( int l = 0; l < nu; l++){
143 tmp += A[i+l*nv]*shwork2[l+nu*j];
144 }
145 shwork[ii] = tmp;
146 }
147
148 __syncthreads();
149
150 for (int ijk = idx; ijk< nu*nv*nv; ijk += str) {
151 const int jk = ijk / nv;
152 const int i = ijk - jk * nv;
153 const int k = jk / nv;
154 const int j = jk - k * nv;
155 T tmp = 0.0;
156 const int ik2 = i + k*nv*nu;
157 for( int l = 0; l < nu; l++){
158 tmp += Bt[l+j*nu]*shwork[l*nv+ik2];
159 }
160 shwork2[ijk] = tmp;
161 }
162
163 __syncthreads();
164
165 for (int ijk = idx; ijk< nv*nv*nv; ijk += str) {
166 const int jk = ijk / nv;
167 const int i = ijk - jk * nv;
168 const int k = jk / nv;
169 const int j = jk - k * nv;
170 T tmp = 0.0;
171 const int ij2 = i + j*nv;
172 for( int l = 0; l < nu; l++){
173 tmp += Ct[l+k*nu]*shwork2[ij2 + l*nv*nv];
174 }
175 v[ijk+e*nv*nv*nv] = tmp;
176 }
177
178}
179
180
181
182#endif // __MATH_TENSOR_KERNEL_H__
ale_add_kinematics_kernel
__global__ void ale_add_kinematics_kernel(const int n, T *__restrict__ wx, T *__restrict__ wy, T *__restrict__ wz, const T *__restrict__ x_ref, const T *__restrict__ y_ref, const T *__restrict__ z_ref, const T *__restrict__ phi, const T *__restrict__ x, const T *__restrict__ y, const T *__restrict__ z, const kinematics_params_t kin_params)
Definition ale_kinematics_kernel.h:52
i
const int i
Definition ax_helm_full_kernel.h:95
u
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ u
Definition ax_helm_full_kernel.h:46
e
const int e
Definition ax_helm_full_kernel.h:93
v
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ v
Definition ax_helm_full_kernel.h:47
j
const int j
Definition ax_helm_full_kernel.h:94
__syncthreads
__syncthreads()
tnsr3d_el_kernel
__global__ void tnsr3d_el_kernel(T *__restrict__ v, const int nv, const T *__restrict__ u, const int nu, const T *__restrict__ A, const T *__restrict__ Bt, const T *__restrict__ Ct, const int *elements, const int n_points)
Definition tensor_kernel.h:37
tnsr3d_kernel
__global__ void tnsr3d_kernel(T *__restrict__ v, const int nv, const T *__restrict__ u, const int nu, const T *__restrict__ A, const T *__restrict__ Bt, const T *__restrict__ Ct)
Definition tensor_kernel.h:112