1#ifndef __MATH_AX_HELM_COMPR_KERNEL_H__
2#define __MATH_AX_HELM_COMPR_KERNEL_H__
87template<
typename T, const
int LX, const
int CHUNKS >
89 const T * __restrict__
u,
90 const T * __restrict__
dx,
91 const T * __restrict__
dy,
92 const T * __restrict__
dz,
93 const T * __restrict__
dxt,
94 const T * __restrict__
dyt,
95 const T * __restrict__
dzt,
96 const T * __restrict__
h1,
97 const T * __restrict__
g11,
98 const T * __restrict__
g22,
99 const T * __restrict__
g33,
100 const T * __restrict__
g12,
101 const T * __restrict__
g13,
102 const T * __restrict__
g23,
103 const int * __restrict__
ind) {
105 __shared__ T
shdx[LX*LX];
106 __shared__ T
shdy[LX*LX];
107 __shared__ T
shdzt[LX*LX];
109 __shared__ T shdxt[LX*LX];
110 __shared__ T
shdyt[LX*LX];
111 __shared__ T
shdz[LX*LX];
113 __shared__ T
shu[LX*LX*LX];
114 __shared__ T
shur[LX*LX*LX];
115 __shared__ T
shus[LX*LX*LX];
116 __shared__ T shut[LX*LX*LX];
118 const int e = blockIdx.x;
119 const int ec =
ind[
e] - 1;
120 const int iii = threadIdx.x;
121 const int nchunks = (LX * LX * LX - 1)/CHUNKS + 1;
131 while (
i < LX * LX * LX){
140 shdxt[iii] =
dxt[iii];
145 for (
int n=0; n<nchunks; n++){
146 const int ijk = iii+n*CHUNKS;
147 const int jk = ijk/LX;
148 const int i = ijk-jk*LX;
150 const int j = jk-k*LX;
151 if (
i<LX &&
j<LX && k<LX && ijk < LX*LX*LX){
152 const T G00 =
g11[ijk+ec*LX*LX*LX];
153 const T G11 =
g22[ijk+ec*LX*LX*LX];
154 const T G22 =
g33[ijk+ec*LX*LX*LX];
155 const T G01 =
g12[ijk+ec*LX*LX*LX];
156 const T G02 =
g13[ijk+ec*LX*LX*LX];
157 const T G12 =
g23[ijk+ec*LX*LX*LX];
158 const T H1 =
h1[ijk+
e*LX*LX*LX];
163 for (
int l = 0; l<LX; l++){
164 rtmp = rtmp +
shdx[
i+l*LX] *
shu[l+
j*LX+k*LX*LX];
165 stmp = stmp +
shdy[
j+l*LX] *
shu[
i+l*LX+k*LX*LX];
166 ttmp = ttmp +
shdz[k+l*LX] *
shu[
i+
j*LX+l*LX*LX];
168 shur[ijk] = H1 * (G00 * rtmp + G01 * stmp + G02 * ttmp);
169 shus[ijk] = H1 * (G01 * rtmp + G11 * stmp + G12 * ttmp);
170 shut[ijk] = H1 * (G02 * rtmp + G12 * stmp + G22 * ttmp);
176 for (
int n=0; n<nchunks; n++){
177 const int ijk = iii+n*CHUNKS;
178 const int jk = ijk/LX;
180 const int j = jk-k*LX;
181 const int i = ijk-jk*LX;
182 if (
i<LX &&
j<LX && k<LX && ijk <LX*LX*LX){
185 for (
int l = 0; l<LX; l++){
187 + shdxt[
i+l*LX] *
shur[l+
j*LX+k*LX*LX]
189 +
shdzt[k+l*LX] * shut[
i+
j*LX+l*LX*LX];
191 w[ijk+
e*LX*LX*LX] = wijke;
196template<
typename T, const
int LX, const
int EB >
198ax_helm_compr_kernel_kstep(T * __restrict__
w,
199 const T * __restrict__
u,
200 const T * __restrict__
dx,
201 const T * __restrict__
dy,
202 const T * __restrict__
dz,
203 const T * __restrict__
h1,
204 const T * __restrict__
g11,
205 const T * __restrict__
g22,
206 const T * __restrict__
g33,
207 const T * __restrict__
g12,
208 const T * __restrict__
g13,
209 const T * __restrict__
g23,
210 const int * __restrict__
ind,
214 __shared__ T
shdx[LX * LX];
219 __shared__ T
shu[EB * LX * LX];
220 __shared__ T
shur[EB * LX * LX];
221 __shared__ T
shus[EB * LX * LX];
223 static_assert(
sizeof(
shdx) +
230 "kstep block exceeds the shared memory budget");
241 const int eb = (EB == 1) ? 0 : threadIdx.z;
245 const int j = threadIdx.y;
246 const int i = threadIdx.x;
259 for(
int k = 0; k < LX; ++k){
267 for (
int k = 0; k < LX; ++k){
268 const int ijk =
ij + k*LX*LX;
275 const T H1 =
h1[ijk+
ele];
279 for (
int l = 0; l < LX; l++){
280 ttmp +=
shdz[k+l*LX] *
ru[l];
287 for (
int l = 0; l < LX; l++){
308 for (
int l = 0; l < LX; l++){
317 for (
int k = 0; k < LX; ++k){
328template<
typename T, const
int LX, const
int EB >
330ax_helm_compr_kernel_kstep_padded(T * __restrict__
w,
331 const T * __restrict__
u,
332 const T * __restrict__
dx,
333 const T * __restrict__
dy,
334 const T * __restrict__
dz,
335 const T * __restrict__
h1,
336 const T * __restrict__
g11,
337 const T * __restrict__
g22,
338 const T * __restrict__
g33,
339 const T * __restrict__
g12,
340 const T * __restrict__
g13,
341 const T * __restrict__
g23,
342 const int * __restrict__
ind,
346 __shared__ T
shdx[LX * (LX+1)];
347 __shared__ T
shdy[LX * (LX+1)];
348 __shared__ T
shdz[LX * (LX+1)];
351 __shared__ T
shu[EB * LX * (LX+1)];
352 __shared__ T
shur[EB * LX * LX];
353 __shared__ T
shus[EB * LX * (LX+1)];
355 static_assert(
sizeof(
shdx) +
362 "kstep block exceeds the shared memory budget");
370 const int eb = (EB == 1) ? 0 : threadIdx.z;
371 const int e_blk = blockIdx.x * EB +
eb;
374 const int j = threadIdx.y;
375 const int i = threadIdx.x;
376 const int ij =
i +
j*LX;
378 const int sh =
eb*LX*LX;
380 const int ele =
e*LX*LX*LX;
381 const int gele = (
ind[
e] - 1)*LX*LX*LX;
390 for(
int k = 0; k < LX; ++k){
398 for (
int k = 0; k < LX; ++k){
399 const int ijk =
ij + k*LX*LX;
406 const T H1 =
h1[ijk+
ele];
410 for (
int l = 0; l < LX; l++){
411 ttmp +=
shdz[k+l*(LX+1)] *
ru[l];
418 for (
int l = 0; l < LX; l++){
439 for (
int l = 0; l < LX; l++){
448 for (
int k = 0; k < LX; ++k){
467#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ < 1000)
469template< const
int LX, const
int NW >
470__device__ __forceinline__
471void ax_helm_compr_dmma_elem(
double * __restrict__
w,
472 const double * __restrict__
u,
473 const double * __restrict__
dx,
474 const double * __restrict__
dy,
475 const double * __restrict__
dz,
476 const double * __restrict__
h1,
477 const double * __restrict__
g11,
478 const double * __restrict__
g22,
479 const double * __restrict__
g33,
480 const double * __restrict__
g12,
481 const double * __restrict__
g13,
482 const double * __restrict__
g23,
483 const int * __restrict__
ind,
495 __shared__ __align__(16)
double shr[
DMMA_CUBE];
496 __shared__ __align__(16)
double shs[
DMMA_CUBE];
497 __shared__ __align__(16)
double sht[
DMMA_CUBE];
499 static_assert(sizeof(
shdx) +
507 "dmma block exceeds the shared memory budget");
514 PACK = PPA * PPA * PPA,
518 typedef dmma_pack< LX, PPA > pack;
520 const int nthrds = 32 * NW;
521 const int tid = threadIdx.x;
522 const int wf = tid >> 5;
523 const int ebase = pack::ebase();
528 for (
int p = tid; p <
DMMA_CUBE; p += nthrds) {
533 for (
int p = tid; p <
DMMA_MAT; p += nthrds) {
544 for (
int p = tid; p < LX * LX; p += nthrds) {
545 const int i = p % LX;
546 const int l = p / LX;
548 for (
int b = 0; b < PPA; b++) {
549 const int m = (b * LX +
i) +
DMMA_P * (b * LX + l);
556 for (
int p = tid; p < NP; p += nthrds) {
557 const dmma_idx
x = pack::map(p, ebase,
nelv);
564 dmma_contract< 0, false, false, NW >(shr,
shdx,
shu, wf);
565 dmma_contract< 1, false, false, NW >(shs,
shdy,
shu, wf);
566 dmma_contract< 2, false, false, NW >(sht,
shdz,
shu, wf);
570 for (
int p = tid; p < NP; p += nthrds) {
571 const dmma_idx
x = pack::map(p, ebase,
nelv);
574 const int gq =
x.l + (
ind[
x.g / LX3] - 1) * LX3;
576 const double G00 =
g11[gq];
577 const double G11 =
g22[gq];
578 const double G22 =
g33[gq];
579 const double G01 =
g12[gq];
580 const double G02 =
g13[gq];
581 const double G12 =
g23[gq];
582 const double H1 =
h1[gp];
584 const double rtmp = shr[c];
585 const double stmp = shs[c];
586 const double ttmp = sht[c];
608 dmma_contract< 0, true, false, NW >(
shu,
shdx, shr, wf);
610 dmma_contract< 1, true, true, NW >(
shu,
shdy, shs, wf);
612 dmma_contract< 2, true, true, NW >(
shu,
shdz, sht, wf);
615 for (
int p = tid; p < NP; p += nthrds) {
616 const dmma_idx
x = pack::map(p, ebase,
nelv);
635template<
typename T, const
int LX, const
int NW >
637 __device__
static void run(T * __restrict__,
638 const T * __restrict__,
639 const T * __restrict__,
640 const T * __restrict__,
641 const T * __restrict__,
642 const T * __restrict__,
643 const T * __restrict__,
644 const T * __restrict__,
645 const T * __restrict__,
646 const T * __restrict__,
647 const T * __restrict__,
648 const T * __restrict__,
649 const int * __restrict__,
653#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ < 1000)
656#define NEKO_AX_HELM_COMPR_DMMA_DISPATCH(LXV) \
657 template< const int NW > \
658 struct ax_helm_compr_dmma_dispatch< double, LXV, NW > { \
659 __device__ static void run(double * __restrict__ w, \
660 const double * __restrict__ u, \
661 const double * __restrict__ dx, \
662 const double * __restrict__ dy, \
663 const double * __restrict__ dz, \
664 const double * __restrict__ h1, \
665 const double * __restrict__ g11, \
666 const double * __restrict__ g22, \
667 const double * __restrict__ g33, \
668 const double * __restrict__ g12, \
669 const double * __restrict__ g13, \
670 const double * __restrict__ g23, \
671 const int * __restrict__ ind, \
673 ax_helm_compr_dmma_elem< LXV, NW >(w, u, dx, dy, dz, h1, g11, g22, g33, \
674 g12, g13, g23, ind, nelv); \
678NEKO_AX_HELM_COMPR_DMMA_DISPATCH(2);
679NEKO_AX_HELM_COMPR_DMMA_DISPATCH(3);
680NEKO_AX_HELM_COMPR_DMMA_DISPATCH(4);
681NEKO_AX_HELM_COMPR_DMMA_DISPATCH(5);
682NEKO_AX_HELM_COMPR_DMMA_DISPATCH(6);
683NEKO_AX_HELM_COMPR_DMMA_DISPATCH(7);
684NEKO_AX_HELM_COMPR_DMMA_DISPATCH(8);
688template<
typename T, const
int LX, const
int NW >
690ax_helm_compr_kernel_dmma(T * __restrict__
w,
691 const T * __restrict__
u,
692 const T * __restrict__
dx,
693 const T * __restrict__
dy,
694 const T * __restrict__
dz,
695 const T * __restrict__
h1,
696 const T * __restrict__
g11,
697 const T * __restrict__
g22,
698 const T * __restrict__
g33,
699 const T * __restrict__
g12,
700 const T * __restrict__
g13,
701 const T * __restrict__
g23,
702 const int * __restrict__
ind,
705 ax_helm_compr_dmma_dispatch< T, LX, NW >::run(
w,
u,
dx,
dy,
dz,
h1,
g11,
g22,
730#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && \
731 (__CUDA_ARCH__ < 1000) && NEKO_TMA_TOOLKIT
733template< const
int LX, const
int NW >
734__device__ __forceinline__
735void ax_helm_compr_dmma_tma_elem(
double * __restrict__
w,
736 const double * __restrict__
u,
737 const double * __restrict__
dx,
738 const double * __restrict__
dy,
739 const double * __restrict__
dz,
740 const double * __restrict__
h1,
741 const double * __restrict__
g11,
742 const double * __restrict__
g22,
743 const double * __restrict__
g33,
744 const double * __restrict__
g12,
745 const double * __restrict__
g13,
746 const double * __restrict__
g23,
747 const int * __restrict__
ind) {
759 __shared__ __align__(128)
double shr[
DMMA_CUBE];
760 __shared__ __align__(128)
double shs[
DMMA_CUBE];
761 __shared__ __align__(128)
double sht[
DMMA_CUBE];
766 __shared__ __align__(8)
unsigned long long bar_u;
767 __shared__ __align__(8)
unsigned long long bar_g;
769 static_assert(sizeof(
shdx) +
780 "dmma tma block exceeds the shared memory budget");
785 static_assert(LX ==
DMMA_P,
786 "the dmma tma variant stages whole cubes only");
788 enum { CUBE_BYTES =
DMMA_CUBE * (int)
sizeof(double) };
790 const int nthrds = 32 * NW;
791 const int tid = threadIdx.x;
792 const int wf = tid >> 5;
793 const int ebase = blockIdx.x *
DMMA_CUBE;
797 tma_barrier_init(&bar_u, 1);
798 tma_barrier_init(&bar_g, 1);
803 for (
int p = tid; p <
DMMA_MAT; p += nthrds) {
814 tma_expect(&bar_u, CUBE_BYTES);
815 tma_load(
shu,
u + ebase, CUBE_BYTES, &bar_u);
817 tma_expect(&bar_g,
DMMA_NG * CUBE_BYTES);
818 tma_load(shg[0],
h1 + ebase, CUBE_BYTES, &bar_g);
819 tma_load(shg[1],
g11 + gbase, CUBE_BYTES, &bar_g);
820 tma_load(shg[2],
g22 + gbase, CUBE_BYTES, &bar_g);
821 tma_load(shg[3],
g33 + gbase, CUBE_BYTES, &bar_g);
822 tma_load(shg[4],
g12 + gbase, CUBE_BYTES, &bar_g);
823 tma_load(shg[5],
g13 + gbase, CUBE_BYTES, &bar_g);
824 tma_load(shg[6],
g23 + gbase, CUBE_BYTES, &bar_g);
830 dmma_contract< 0, false, false, NW >(shr,
shdx,
shu, wf);
831 dmma_contract< 1, false, false, NW >(shs,
shdy,
shu, wf);
832 dmma_contract< 2, false, false, NW >(sht,
shdz,
shu, wf);
838 for (
int p = tid; p <
DMMA_CUBE; p += nthrds) {
839 const double H1 = shg[0][p];
840 const double G00 = shg[1][p];
841 const double G11 = shg[2][p];
842 const double G22 = shg[3][p];
843 const double G01 = shg[4][p];
844 const double G02 = shg[5][p];
845 const double G12 = shg[6][p];
847 const double rtmp = shr[p];
848 const double stmp = shs[p];
849 const double ttmp = sht[p];
869 dmma_contract< 0, true, false, NW >(
shu,
shdx, shr, wf);
871 dmma_contract< 1, true, true, NW >(
shu,
shdy, shs, wf);
873 dmma_contract< 2, true, true, NW >(
shu,
shdz, sht, wf);
882 tma_store(
w + ebase,
shu, CUBE_BYTES);
899template<
typename T, const
int LX, const
int NW >
901 __device__
static void run(T * __restrict__,
902 const T * __restrict__,
903 const T * __restrict__,
904 const T * __restrict__,
905 const T * __restrict__,
906 const T * __restrict__,
907 const T * __restrict__,
908 const T * __restrict__,
909 const T * __restrict__,
910 const T * __restrict__,
911 const T * __restrict__,
912 const T * __restrict__,
913 const int * __restrict__) { }
916#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && \
917 (__CUDA_ARCH__ < 1000) && NEKO_TMA_TOOLKIT
920#define NEKO_AX_HELM_COMPR_DMMA_TMA_DISPATCH(LXV) \
921 template< const int NW > \
922 struct ax_helm_compr_dmma_tma_dispatch< double, LXV, NW > { \
923 __device__ static void run(double * __restrict__ w, \
924 const double * __restrict__ u, \
925 const double * __restrict__ dx, \
926 const double * __restrict__ dy, \
927 const double * __restrict__ dz, \
928 const double * __restrict__ h1, \
929 const double * __restrict__ g11, \
930 const double * __restrict__ g22, \
931 const double * __restrict__ g33, \
932 const double * __restrict__ g12, \
933 const double * __restrict__ g13, \
934 const double * __restrict__ g23, \
935 const int * __restrict__ ind) { \
936 ax_helm_compr_dmma_tma_elem< LXV, NW >(w, u, dx, dy, dz, h1, g11, g22, \
937 g33, g12, g13, g23, ind); \
941NEKO_AX_HELM_COMPR_DMMA_TMA_DISPATCH(8);
945template<
typename T, const
int LX, const
int NW >
947ax_helm_compr_kernel_dmma_tma(T * __restrict__
w,
948 const T * __restrict__
u,
949 const T * __restrict__
dx,
950 const T * __restrict__
dy,
951 const T * __restrict__
dz,
952 const T * __restrict__
h1,
953 const T * __restrict__
g11,
954 const T * __restrict__
g22,
955 const T * __restrict__
g33,
956 const T * __restrict__
g12,
957 const T * __restrict__
g13,
958 const T * __restrict__
g23,
959 const int * __restrict__
ind) {
971template<
typename T, const
int LX, const
int EB >
973ax_helm_compr_kernel_vector_kstep(T * __restrict__ au,
976 const T * __restrict__
u,
977 const T * __restrict__
v,
978 const T * __restrict__
w,
979 const T * __restrict__
dx,
980 const T * __restrict__
dy,
981 const T * __restrict__
dz,
982 const T * __restrict__
h1,
983 const T * __restrict__
g11,
984 const T * __restrict__
g22,
985 const T * __restrict__
g33,
986 const T * __restrict__
g12,
987 const T * __restrict__
g13,
988 const T * __restrict__
g23,
989 const int * __restrict__
ind,
993 __shared__ T
shdx[LX * LX];
994 __shared__ T
shdy[LX * LX];
995 __shared__ T
shdz[LX * LX];
998 __shared__ T
shu[EB * LX * LX];
999 __shared__ T
shur[EB * LX * LX];
1000 __shared__ T
shus[EB * LX * LX];
1002 __shared__ T
shv[EB * LX * LX];
1006 __shared__ T
shw[EB * LX * LX];
1010 static_assert(
sizeof(
shdx) +
1023 "kstep block exceeds the shared memory budget");
1039 const int eb = (EB == 1) ? 0 : threadIdx.z;
1040 const int e_blk = blockIdx.x * EB +
eb;
1043 const int j = threadIdx.y;
1044 const int i = threadIdx.x;
1045 const int ij =
i +
j*LX;
1046 const int sh =
eb*LX*LX;
1047 const int ele =
e*LX*LX*LX;
1048 const int gele = (
ind[
e] - 1)*LX*LX*LX;
1057 for(
int k = 0; k < LX; ++k){
1071 for (
int k = 0; k < LX; ++k){
1072 const int ijk =
ij + k*LX*LX;
1079 const T H1 =
h1[ijk+
ele];
1087 for (
int l = 0; l < LX; l++){
1088 uttmp +=
shdz[k+l*LX] *
ru[l];
1089 vttmp +=
shdz[k+l*LX] *
rv[l];
1090 wttmp +=
shdz[k+l*LX] *
rw[l];
1103 for (
int l = 0; l < LX; l++){
1159 for (
int l = 0; l < LX; l++){
1178 for (
int k = 0; k < LX; ++k){
1186template<
typename T, const
int LX, const
int EB >
1188ax_helm_compr_kernel_vector_kstep_padded(T * __restrict__ au,
1189 T * __restrict__
av,
1190 T * __restrict__
aw,
1191 const T * __restrict__
u,
1192 const T * __restrict__
v,
1193 const T * __restrict__
w,
1194 const T * __restrict__
dx,
1195 const T * __restrict__
dy,
1196 const T * __restrict__
dz,
1197 const T * __restrict__
h1,
1198 const T * __restrict__
g11,
1199 const T * __restrict__
g22,
1200 const T * __restrict__
g33,
1201 const T * __restrict__
g12,
1202 const T * __restrict__
g13,
1203 const T * __restrict__
g23,
1204 const int * __restrict__
ind,
1208 __shared__ T
shdx[LX * (LX+1)];
1209 __shared__ T
shdy[LX * (LX+1)];
1210 __shared__ T
shdz[LX * (LX+1)];
1213 __shared__ T
shu[EB * LX * (LX+1)];
1214 __shared__ T
shur[EB * LX * LX];
1215 __shared__ T
shus[EB * LX * (LX+1)];
1217 __shared__ T
shv[EB * LX * (LX+1)];
1218 __shared__ T
shvr[EB * LX * LX];
1219 __shared__ T
shvs[EB * LX * (LX+1)];
1221 __shared__ T
shw[EB * LX * (LX+1)];
1222 __shared__ T
shwr[EB * LX * LX];
1223 __shared__ T
shws[EB * LX * (LX+1)];
1225 static_assert(
sizeof(
shdx) +
1238 "kstep block exceeds the shared memory budget");
1254 const int eb = (EB == 1) ? 0 : threadIdx.z;
1255 const int e_blk = blockIdx.x * EB +
eb;
1258 const int j = threadIdx.y;
1259 const int i = threadIdx.x;
1260 const int ij =
i +
j*LX;
1261 const int ij_p =
i +
j*(LX+1);
1262 const int sh =
eb*LX*LX;
1263 const int sh_p =
eb*LX*(LX+1);
1264 const int ele =
e*LX*LX*LX;
1265 const int gele = (
ind[
e] - 1)*LX*LX*LX;
1274 for(
int k = 0; k < LX; ++k){
1288 for (
int k = 0; k < LX; ++k){
1289 const int ijk =
ij + k*LX*LX;
1296 const T H1 =
h1[ijk+
ele];
1304 for (
int l = 0; l < LX; l++){
1305 uttmp +=
shdz[k+l*(LX+1)] *
ru[l];
1306 vttmp +=
shdz[k+l*(LX+1)] *
rv[l];
1307 wttmp +=
shdz[k+l*(LX+1)] *
rw[l];
1320 for (
int l = 0; l < LX; l++){
1376 for (
int l = 0; l < LX; l++){
1395 for (
int k = 0; k < LX; ++k){
1430#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ < 1000)
1432template< const
int LX, const
int NW >
1433__device__ __forceinline__
1434void ax_helm_compr_dmma_vector_elem(
double * __restrict__ au,
1435 double * __restrict__
av,
1436 double * __restrict__
aw,
1437 const double * __restrict__
u,
1438 const double * __restrict__
v,
1439 const double * __restrict__
w,
1440 const double * __restrict__
dx,
1441 const double * __restrict__
dy,
1442 const double * __restrict__
dz,
1443 const double * __restrict__
h1,
1444 const double * __restrict__
g11,
1445 const double * __restrict__
g22,
1446 const double * __restrict__
g33,
1447 const double * __restrict__
g12,
1448 const double * __restrict__
g13,
1449 const double * __restrict__
g23,
1450 const int * __restrict__
ind) {
1459 __shared__ __align__(16)
double shc[
DMMA_CUBE];
1460 __shared__ __align__(16)
double shr[
DMMA_CUBE];
1461 __shared__ __align__(16)
double shs[
DMMA_CUBE];
1462 __shared__ __align__(16)
double sht[
DMMA_CUBE];
1464 static_assert(sizeof(
shdx) +
1472 "dmma vector block exceeds the shared memory budget");
1474 enum { NTHRDS = 32 * NW,
1476 PPT = (LX3 + NTHRDS - 1) / NTHRDS };
1478 const int tid = threadIdx.x;
1479 const int wf = tid >> 5;
1480 const int ele = blockIdx.x * LX3;
1481 const int gele = (
ind[blockIdx.x] - 1) * LX3;
1484 double rG00[PPT], rG11[PPT], rG22[PPT];
1485 double rG01[PPT], rG02[PPT], rG12[PPT];
1492 for (
int p = tid; p <
DMMA_MAT; p += NTHRDS) {
1497 for (
int p = tid; p <
DMMA_CUBE; p += NTHRDS) {
1503 for (
int p = tid; p < LX * LX; p += NTHRDS) {
1504 const int i = p % LX;
1505 const int l = p / LX;
1513 for (
int q = 0; q < PPT; q++) {
1514 const int p = tid + q * NTHRDS;
1517 const int i = p % LX;
1518 const int jk = p / LX;
1519 const int j = jk % LX;
1520 const int k = jk / LX;
1528 rH1[q] =
h1[p +
ele];
1543 const double *
const cin[3] = {
u,
v,
w };
1544 double *
const cout[3] = { au,
av,
aw };
1547 for (
int c = 0; c < 3; c++) {
1549 for (
int p = tid; p < LX3; p += NTHRDS) {
1550 const int i = p % LX;
1551 const int jk = p / LX;
1552 const int j = jk % LX;
1553 const int k = jk / LX;
1559 dmma_contract< 0, false, false, NW >(shr,
shdx, shc, wf);
1560 dmma_contract< 1, false, false, NW >(shs,
shdy, shc, wf);
1561 dmma_contract< 2, false, false, NW >(sht,
shdz, shc, wf);
1566 for (
int q = 0; q < PPT; q++) {
1567 const int p = tid + q * NTHRDS;
1570 const int idx = rc[q];
1571 const double rtmp = shr[idx];
1572 const double stmp = shs[idx];
1573 const double ttmp = sht[idx];
1592 dmma_contract< 0, true, false, NW >(shc,
shdx, shr, wf);
1594 dmma_contract< 1, true, true, NW >(shc,
shdy, shs, wf);
1596 dmma_contract< 2, true, true, NW >(shc,
shdz, sht, wf);
1599 for (
int p = tid; p < LX3; p += NTHRDS) {
1600 const int i = p % LX;
1601 const int jk = p / LX;
1602 const int j = jk % LX;
1603 const int k = jk / LX;
1618template<
typename T, const
int LX, const
int NW >
1620 __device__
static void run(T * __restrict__,
1623 const T * __restrict__,
1624 const T * __restrict__,
1625 const T * __restrict__,
1626 const T * __restrict__,
1627 const T * __restrict__,
1628 const T * __restrict__,
1629 const T * __restrict__,
1630 const T * __restrict__,
1631 const T * __restrict__,
1632 const T * __restrict__,
1633 const T * __restrict__,
1634 const T * __restrict__,
1635 const T * __restrict__,
1636 const int * __restrict__) { }
1639#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 800) && (__CUDA_ARCH__ < 1000)
1643#define NEKO_AX_HELM_COMPR_DMMA_VECTOR_DISPATCH(LXV) \
1644 template< const int NW > \
1645 struct ax_helm_compr_dmma_vector_dispatch< double, LXV, NW > { \
1646 __device__ static void run(double * __restrict__ au, \
1647 double * __restrict__ av, \
1648 double * __restrict__ aw, \
1649 const double * __restrict__ u, \
1650 const double * __restrict__ v, \
1651 const double * __restrict__ w, \
1652 const double * __restrict__ dx, \
1653 const double * __restrict__ dy, \
1654 const double * __restrict__ dz, \
1655 const double * __restrict__ h1, \
1656 const double * __restrict__ g11, \
1657 const double * __restrict__ g22, \
1658 const double * __restrict__ g33, \
1659 const double * __restrict__ g12, \
1660 const double * __restrict__ g13, \
1661 const double * __restrict__ g23, \
1662 const int * __restrict__ ind) { \
1663 ax_helm_compr_dmma_vector_elem< LXV, NW >(au, av, aw, u, v, w, dx, dy, \
1664 dz, h1, g11, g22, g33, g12, \
1669NEKO_AX_HELM_COMPR_DMMA_VECTOR_DISPATCH(4);
1670NEKO_AX_HELM_COMPR_DMMA_VECTOR_DISPATCH(5);
1671NEKO_AX_HELM_COMPR_DMMA_VECTOR_DISPATCH(6);
1672NEKO_AX_HELM_COMPR_DMMA_VECTOR_DISPATCH(7);
1673NEKO_AX_HELM_COMPR_DMMA_VECTOR_DISPATCH(8);
1677template<
typename T, const
int LX, const
int NW >
1679ax_helm_compr_kernel_dmma_vector(T * __restrict__ au,
1680 T * __restrict__
av,
1681 T * __restrict__
aw,
1682 const T * __restrict__
u,
1683 const T * __restrict__
v,
1684 const T * __restrict__
w,
1685 const T * __restrict__
dx,
1686 const T * __restrict__
dy,
1687 const T * __restrict__
dz,
1688 const T * __restrict__
h1,
1689 const T * __restrict__
g11,
1690 const T * __restrict__
g22,
1691 const T * __restrict__
g33,
1692 const T * __restrict__
g12,
1693 const T * __restrict__
g13,
1694 const T * __restrict__
g23,
1695 const int * __restrict__
ind) {
1733#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && \
1734 (__CUDA_ARCH__ < 1000) && NEKO_TMA_TOOLKIT
1736template< const
int LX, const
int NW >
1737__device__ __forceinline__
1738void ax_helm_compr_dmma_tma_vector_elem(
double * __restrict__ au,
1739 double * __restrict__
av,
1740 double * __restrict__
aw,
1741 const double * __restrict__
u,
1742 const double * __restrict__
v,
1743 const double * __restrict__
w,
1744 const double * __restrict__
dx,
1745 const double * __restrict__
dy,
1746 const double * __restrict__
dz,
1747 const double * __restrict__
h1,
1748 const double * __restrict__
g11,
1749 const double * __restrict__
g22,
1750 const double * __restrict__
g33,
1751 const double * __restrict__
g12,
1752 const double * __restrict__
g13,
1753 const double * __restrict__
g23,
1754 const int * __restrict__
ind) {
1763 __shared__ __align__(128)
double shc[
DMMA_CUBE];
1764 __shared__ __align__(128)
double shr[
DMMA_CUBE];
1765 __shared__ __align__(128)
double shs[
DMMA_CUBE];
1766 __shared__ __align__(128)
double sht[
DMMA_CUBE];
1771 __shared__ __align__(8)
unsigned long long bar_c;
1772 __shared__ __align__(8)
unsigned long long bar_g;
1774 static_assert(sizeof(
shdx) +
1785 "dmma tma vector block exceeds the shared memory budget");
1788 static_assert(LX ==
DMMA_P,
1789 "the dmma tma vector variant stages whole cubes only");
1791 enum { CUBE_BYTES =
DMMA_CUBE * (int)
sizeof(double) };
1793 const int nthrds = 32 * NW;
1794 const int tid = threadIdx.x;
1795 const int wf = tid >> 5;
1800 tma_barrier_init(&bar_c, 1);
1801 tma_barrier_init(&bar_g, 1);
1806 for (
int p = tid; p <
DMMA_MAT; p += nthrds) {
1819 tma_expect(&bar_g,
DMMA_NG * CUBE_BYTES);
1820 tma_load(shg[0],
h1 +
ele, CUBE_BYTES, &bar_g);
1821 tma_load(shg[1],
g11 +
gele, CUBE_BYTES, &bar_g);
1822 tma_load(shg[2],
g22 +
gele, CUBE_BYTES, &bar_g);
1823 tma_load(shg[3],
g33 +
gele, CUBE_BYTES, &bar_g);
1824 tma_load(shg[4],
g12 +
gele, CUBE_BYTES, &bar_g);
1825 tma_load(shg[5],
g13 +
gele, CUBE_BYTES, &bar_g);
1826 tma_load(shg[6],
g23 +
gele, CUBE_BYTES, &bar_g);
1829 const double *
const cin[3] = {
u,
v,
w };
1830 double *
const cout[3] = { au,
av,
aw };
1833 for (
int c = 0; c < 3; c++) {
1836 tma_expect(&bar_c, CUBE_BYTES);
1837 tma_load(shc, cin[c] +
ele, CUBE_BYTES, &bar_c);
1844 tma_wait(&bar_c, c & 1);
1846 dmma_contract< 0, false, false, NW >(shr,
shdx, shc, wf);
1847 dmma_contract< 1, false, false, NW >(shs,
shdy, shc, wf);
1848 dmma_contract< 2, false, false, NW >(sht,
shdz, shc, wf);
1854 tma_wait(&bar_g, 0);
1857 for (
int p = tid; p <
DMMA_CUBE; p += nthrds) {
1858 const double H1 = shg[0][p];
1859 const double G00 = shg[1][p];
1860 const double G11 = shg[2][p];
1861 const double G22 = shg[3][p];
1862 const double G01 = shg[4][p];
1863 const double G02 = shg[5][p];
1864 const double G12 = shg[6][p];
1866 const double rtmp = shr[p];
1867 const double stmp = shs[p];
1868 const double ttmp = sht[p];
1886 dmma_contract< 0, true, false, NW >(shc,
shdx, shr, wf);
1888 dmma_contract< 1, true, true, NW >(shc,
shdy, shs, wf);
1890 dmma_contract< 2, true, true, NW >(shc,
shdz, sht, wf);
1898 tma_store(cout[c] +
ele, shc, CUBE_BYTES);
1914template<
typename T, const
int LX, const
int NW >
1916 __device__
static void run(T * __restrict__,
1919 const T * __restrict__,
1920 const T * __restrict__,
1921 const T * __restrict__,
1922 const T * __restrict__,
1923 const T * __restrict__,
1924 const T * __restrict__,
1925 const T * __restrict__,
1926 const T * __restrict__,
1927 const T * __restrict__,
1928 const T * __restrict__,
1929 const T * __restrict__,
1930 const T * __restrict__,
1931 const T * __restrict__,
1932 const int * __restrict__) { }
1935#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && \
1936 (__CUDA_ARCH__ < 1000) && NEKO_TMA_TOOLKIT
1939#define NEKO_AX_HELM_COMPR_DMMA_TMA_VECTOR_DISPATCH(LXV) \
1940 template< const int NW > \
1941 struct ax_helm_compr_dmma_tma_vector_dispatch< double, LXV, NW > { \
1942 __device__ static void run(double * __restrict__ au, \
1943 double * __restrict__ av, \
1944 double * __restrict__ aw, \
1945 const double * __restrict__ u, \
1946 const double * __restrict__ v, \
1947 const double * __restrict__ w, \
1948 const double * __restrict__ dx, \
1949 const double * __restrict__ dy, \
1950 const double * __restrict__ dz, \
1951 const double * __restrict__ h1, \
1952 const double * __restrict__ g11, \
1953 const double * __restrict__ g22, \
1954 const double * __restrict__ g33, \
1955 const double * __restrict__ g12, \
1956 const double * __restrict__ g13, \
1957 const double * __restrict__ g23, \
1958 const int * __restrict__ ind) { \
1959 ax_helm_compr_dmma_tma_vector_elem< LXV, NW >(au, av, aw, u, v, w, \
1962 g12, g13, g23, ind); \
1966NEKO_AX_HELM_COMPR_DMMA_TMA_VECTOR_DISPATCH(8);
1970template<
typename T, const
int LX, const
int NW >
1972ax_helm_compr_kernel_dmma_tma_vector(T * __restrict__ au,
1973 T * __restrict__
av,
1974 T * __restrict__
aw,
1975 const T * __restrict__
u,
1976 const T * __restrict__
v,
1977 const T * __restrict__
w,
1978 const T * __restrict__
dx,
1979 const T * __restrict__
dy,
1980 const T * __restrict__
dz,
1981 const T * __restrict__
h1,
1982 const T * __restrict__
g11,
1983 const T * __restrict__
g22,
1984 const T * __restrict__
g33,
1985 const T * __restrict__
g12,
1986 const T * __restrict__
g13,
1987 const T * __restrict__
g23,
1988 const int * __restrict__
ind) {
2030#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && \
2031 (__CUDA_ARCH__ < 1000) && NEKO_TMA_TOOLKIT
2033template< const
int LX, const
int NW >
2034__device__ __forceinline__
2035void ax_helm_compr_dmma_tma_batch_elem(
double * __restrict__ au,
2036 double * __restrict__
av,
2037 double * __restrict__
aw,
2038 const double * __restrict__
u,
2039 const double * __restrict__
v,
2040 const double * __restrict__
w,
2041 const double * __restrict__
dx,
2042 const double * __restrict__
dy,
2043 const double * __restrict__
dz,
2044 const double * __restrict__
h1,
2045 const double * __restrict__
g11,
2046 const double * __restrict__
g22,
2047 const double * __restrict__
g33,
2048 const double * __restrict__
g12,
2049 const double * __restrict__
g13,
2050 const double * __restrict__
g23,
2051 const int * __restrict__
ind) {
2054 static_assert(LX ==
DMMA_P,
2055 "the dmma tma batch variant stages whole cubes only");
2057 extern __shared__ __align__(128) char neko_ax_compr_dyn_smem[];
2061 enum { CUBE_BYTES =
DMMA_CUBE * (int)
sizeof(double),
2063 REST_BYTES = (
DMMA_NG + 2) * CUBE_BYTES };
2065 const int nthrds = 32 * NW;
2066 const int tid = threadIdx.x;
2067 const int wf = tid >> 5;
2072 tma_barrier_init(&sm.bar_a, 1);
2073 tma_barrier_init(&sm.bar_b, 1);
2078 for (
int p = tid; p <
DMMA_MAT; p += nthrds) {
2091 tma_expect(&sm.bar_a, CUBE_BYTES);
2092 tma_load(sm.c[0],
u +
ele, CUBE_BYTES, &sm.bar_a);
2094 tma_expect(&sm.bar_b, REST_BYTES);
2095 tma_load(sm.c[1],
v +
ele, CUBE_BYTES, &sm.bar_b);
2096 tma_load(sm.c[2],
w +
ele, CUBE_BYTES, &sm.bar_b);
2097 tma_load(sm.g[0],
h1 +
ele, CUBE_BYTES, &sm.bar_b);
2098 tma_load(sm.g[1],
g11 +
gele, CUBE_BYTES, &sm.bar_b);
2099 tma_load(sm.g[2],
g22 +
gele, CUBE_BYTES, &sm.bar_b);
2100 tma_load(sm.g[3],
g33 +
gele, CUBE_BYTES, &sm.bar_b);
2101 tma_load(sm.g[4],
g12 +
gele, CUBE_BYTES, &sm.bar_b);
2102 tma_load(sm.g[5],
g13 +
gele, CUBE_BYTES, &sm.bar_b);
2103 tma_load(sm.g[6],
g23 +
gele, CUBE_BYTES, &sm.bar_b);
2107 tma_wait(&sm.bar_a, 0);
2109 double *
const cout[3] = { au,
av,
aw };
2112 for (
int c = 0; c < 3; c++) {
2114 dmma_contract< 0, false, false, NW >(sm.r, sm.dx, sm.c[c], wf);
2115 dmma_contract< 1, false, false, NW >(sm.s, sm.dy, sm.c[c], wf);
2116 dmma_contract< 2, false, false, NW >(sm.t, sm.dz, sm.c[c], wf);
2122 tma_wait(&sm.bar_b, 0);
2125 for (
int p = tid; p <
DMMA_CUBE; p += nthrds) {
2126 const double H1 = sm.g[0][p];
2127 const double G00 = sm.g[1][p];
2128 const double G11 = sm.g[2][p];
2129 const double G22 = sm.g[3][p];
2130 const double G01 = sm.g[4][p];
2131 const double G02 = sm.g[5][p];
2132 const double G12 = sm.g[6][p];
2134 const double rtmp = sm.r[p];
2135 const double stmp = sm.s[p];
2136 const double ttmp = sm.t[p];
2156 dmma_contract< 0, true, false, NW >(sm.c[c], sm.dx, sm.r, wf);
2158 dmma_contract< 1, true, true, NW >(sm.c[c], sm.dy, sm.s, wf);
2160 dmma_contract< 2, true, true, NW >(sm.c[c], sm.dz, sm.t, wf);
2173 tma_store(cout[c] +
ele, sm.c[c], CUBE_BYTES);
2192template<
typename T, const
int LX, const
int NW >
2194 __device__
static void run(T * __restrict__,
2197 const T * __restrict__,
2198 const T * __restrict__,
2199 const T * __restrict__,
2200 const T * __restrict__,
2201 const T * __restrict__,
2202 const T * __restrict__,
2203 const T * __restrict__,
2204 const T * __restrict__,
2205 const T * __restrict__,
2206 const T * __restrict__,
2207 const T * __restrict__,
2208 const T * __restrict__,
2209 const T * __restrict__,
2210 const int * __restrict__) { }
2213#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900) && \
2214 (__CUDA_ARCH__ < 1000) && NEKO_TMA_TOOLKIT
2217#define NEKO_AX_HELM_COMPR_DMMA_TMA_BATCH_DISPATCH(LXV) \
2218 template< const int NW > \
2219 struct ax_helm_compr_dmma_tma_batch_dispatch< double, LXV, NW > { \
2220 __device__ static void run(double * __restrict__ au, \
2221 double * __restrict__ av, \
2222 double * __restrict__ aw, \
2223 const double * __restrict__ u, \
2224 const double * __restrict__ v, \
2225 const double * __restrict__ w, \
2226 const double * __restrict__ dx, \
2227 const double * __restrict__ dy, \
2228 const double * __restrict__ dz, \
2229 const double * __restrict__ h1, \
2230 const double * __restrict__ g11, \
2231 const double * __restrict__ g22, \
2232 const double * __restrict__ g33, \
2233 const double * __restrict__ g12, \
2234 const double * __restrict__ g13, \
2235 const double * __restrict__ g23, \
2236 const int * __restrict__ ind) { \
2237 ax_helm_compr_dmma_tma_batch_elem< LXV, NW >(au, av, aw, u, v, w, \
2240 g12, g13, g23, ind); \
2244NEKO_AX_HELM_COMPR_DMMA_TMA_BATCH_DISPATCH(8);
2248template<
typename T, const
int LX, const
int NW >
2250ax_helm_compr_kernel_dmma_tma_batch(T * __restrict__ au,
2251 T * __restrict__
av,
2252 T * __restrict__
aw,
2253 const T * __restrict__
u,
2254 const T * __restrict__
v,
2255 const T * __restrict__
w,
2256 const T * __restrict__
dx,
2257 const T * __restrict__
dy,
2258 const T * __restrict__
dz,
2259 const T * __restrict__
h1,
2260 const T * __restrict__
g11,
2261 const T * __restrict__
g22,
2262 const T * __restrict__
g33,
2263 const T * __restrict__
g12,
2264 const T * __restrict__
g13,
2265 const T * __restrict__
g23,
2266 const int * __restrict__
ind) {
2288template<
typename T, const
int LX, const
int NW >
2291 static int state = -1;
2294 const void *
const fn =
2295 (
const void *) ax_helm_compr_kernel_dmma_tma_batch< T, LX, NW >;
2297 cudaFuncSetAttribute(fn, cudaFuncAttributeMaxDynamicSharedMemorySize,
2300 if (err == cudaSuccess) {
2301 err = cudaFuncSetAttribute(fn,
2302 cudaFuncAttributePreferredSharedMemoryCarveout,
2303 cudaSharedmemCarveoutMaxShared);
2305 state = (err == cudaSuccess) ? 1 : 0;
__shared__ T shv[EB *LX *LX]
__shared__ T shdy[LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g23
__global__ void T *__restrict__ T *__restrict__ aw
__shared__ T shu[EB *LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g22
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const int *__restrict__ ind
__shared__ T shdz[LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g13
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ w
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const int *__restrict__ const int nelv
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g12
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dz
__global__ void const T *__restrict__ const T *__restrict__ dx
__global__ void const T *__restrict__ u
__shared__ T shus[EB *LX *LX]
__global__ void T *__restrict__ av
__shared__ T shur[EB *LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ dy
__shared__ T shvs[EB *LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g33
__shared__ T shw[EB *LX *LX]
__shared__ T shwr[EB *LX *LX]
__global__ void ax_helm_compr_kernel_1d(T *__restrict__ w, const T *__restrict__ u, const T *__restrict__ dx, const T *__restrict__ dy, const T *__restrict__ dz, const T *__restrict__ dxt, const T *__restrict__ dyt, const T *__restrict__ dzt, const T *__restrict__ h1, const T *__restrict__ g11, const T *__restrict__ g22, const T *__restrict__ g33, const T *__restrict__ g12, const T *__restrict__ g13, const T *__restrict__ g23, const int *__restrict__ ind)
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ v
__shared__ T shvr[EB *LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g11
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ h1
__shared__ T shws[EB *LX *LX]
static bool ax_helm_compr_dmma_tma_batch_optin()
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dyt
__shared__ T shdzt[LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dzt
__global__ void const T *__restrict__ x
__shared__ T shdyt[LX *LX]
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dxt
#define NEKO_EB_BOUNDS(NT)
#define NEKO_DMMA_TMA_BATCH_SMEM
static __device__ void run(T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const int *__restrict__, const int)
static __device__ void run(T *__restrict__, T *__restrict__, T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const int *__restrict__)
static __device__ void run(T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const int *__restrict__)
static __device__ void run(T *__restrict__, T *__restrict__, T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const int *__restrict__)
static __device__ void run(T *__restrict__, T *__restrict__, T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const T *__restrict__, const int *__restrict__)