Neko 1.99.9
A portable framework for high-order spectral element flow simulations
Loading...
Searching...
No Matches
ax_helm_compr.c
Go to the documentation of this file.
1/*
2 Copyright (c) 2026, The Neko Authors
3 All rights reserved.
4
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions
7 are met:
8
9 * Redistributions of source code must retain the above copyright
10 notice, this list of conditions and the following disclaimer.
11
12 * Redistributions in binary form must reproduce the above
13 copyright notice, this list of conditions and the following
14 disclaimer in the documentation and/or other materials provided
15 with the distribution.
16
17 * Neither the name of the authors nor the names of its
18 contributors may be used to endorse or promote products derived
19 from this software without specific prior written permission.
20
21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
29 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
31 ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 POSSIBILITY OF SUCH DAMAGE.
33*/
34
35#ifdef __APPLE__
36#include <OpenCL/cl.h>
37#else
38#include <CL/cl.h>
39#endif
40
41#include <stdlib.h>
42#include <stdio.h>
43#include <string.h>
44#include <device/device_config.h>
45#include <device/opencl/jit.h>
46#include <device/opencl/prgm_lib.h>
47#include <device/opencl/check.h>
48#include <common/neko_log.h>
49
50#include "ax_helm_compr_kernel.cl.h"
51
52/*
53 * Ax helm on compressed geometric factors, generated from ax_helm.c: the same
54 * strategies and the same tuner, on kernels where element e reads g11..g23
55 * from its copy ind[e] - 1. The tuner state is separate from the standard
56 * one.
57 */
58static int *autotune_ax_helm_compr = NULL;
59
63void opencl_ax_helm_compr(void *w, void *u, void *dx, void *dy, void *dz,
64 void *dxt, void *dyt, void *dzt, void *h1,
65 void *g11, void *g22, void *g33, void *g12,
66 void *g13, void *g23, void *ind,
67 int *nelv, int *lx) {
68
69 cl_int err;
70
71 if (ax_helm_compr_program == NULL)
72 opencl_kernel_jit(ax_helm_compr_kernel,
73 (cl_program *) &ax_helm_compr_program);
74
75 const size_t global_item_size = 256 * (*nelv);
76 const size_t local_item_size = 256;
77
78 size_t global_kstep[2];
79 size_t local_kstep[2];
80 local_kstep[0] = (*lx);
81 local_kstep[1] = (*lx);
82 global_kstep[0] = (*nelv) * (*lx);
83 global_kstep[1] = (*lx);
84
85 if (autotune_ax_helm_compr == NULL) {
86 autotune_ax_helm_compr = malloc(13 * sizeof(int));
87 memset(autotune_ax_helm_compr, 0, 13 * sizeof(int));
88 }
89
90#define STR(X) #X
91#define CASE_1D(LX, QUEUE, EVENT) \
92 { \
93 cl_kernel kernel = clCreateKernel(ax_helm_compr_program, \
94 STR(ax_helm_compr_lx##LX), &err); \
95 CL_CHECK(err); \
96 \
97 CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *) &w)); \
98 CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *) &u)); \
99 CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *) &dx)); \
100 CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), (void *) &dy)); \
101 CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *) &dz)); \
102 CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *) &dxt)); \
103 CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), (void *) &dyt)); \
104 CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), (void *) &dzt)); \
105 CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), (void *) &h1)); \
106 CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), (void *) &g11)); \
107 CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), (void *) &g22)); \
108 CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_mem), (void *) &g33)); \
109 CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), (void *) &g12)); \
110 CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_mem), (void *) &g13)); \
111 CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), (void *) &g23)); \
112 CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_mem), (void *) &ind)); \
113 \
114 CL_CHECK(clEnqueueNDRangeKernel((cl_command_queue) QUEUE, \
115 kernel, 1, NULL, &global_item_size, \
116 &local_item_size, 0, NULL, EVENT)); \
117 CL_CHECK(clReleaseKernel(kernel)); \
118 }
119
120
121#define CASE_KSTEP(LX, QUEUE, EVENT) \
122 { \
123 cl_kernel kernel = clCreateKernel(ax_helm_compr_program, \
124 STR(ax_helm_compr_kstep_lx##LX), &err); \
125 CL_CHECK(err); \
126 \
127 CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *) &w)); \
128 CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *) &u)); \
129 CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *) &dx)); \
130 CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), (void *) &dy)); \
131 CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *) &dz)); \
132 CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *) &h1)); \
133 CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), (void *) &g11)); \
134 CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), (void *) &g22)); \
135 CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), (void *) &g33)); \
136 CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), (void *) &g12)); \
137 CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), (void *) &g13)); \
138 CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_mem), (void *) &g23)); \
139 CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), (void *) &ind)); \
140 \
141 CL_CHECK(clEnqueueNDRangeKernel((cl_command_queue) QUEUE, \
142 kernel, 2, NULL, global_kstep, \
143 local_kstep, 0, NULL, EVENT)); \
144 CL_CHECK(clReleaseKernel(kernel)); \
145 }
146
147
148#define CASE(LX) \
149 case LX: \
150 if(autotune_ax_helm_compr[LX] == 0 ) { \
151 char *env_value = NULL; \
152 char neko_log_buf[80]; \
153 env_value = getenv("NEKO_AUTOTUNE"); \
154 \
155 sprintf(neko_log_buf, "Autotune Ax compr (lx: %d)", *lx); \
156 log_section(neko_log_buf); \
157 if(env_value) { \
158 if( !strcmp(env_value,"1D") ) { \
159 CASE_1D(LX, glb_cmd_queue, NULL); \
160 sprintf(neko_log_buf,"Set by env : 1 (1D)"); \
161 log_message(neko_log_buf); \
162 autotune_ax_helm_compr[LX] = 1; \
163 } else if( !strcmp(env_value,"KSTEP") ) { \
164 CASE_KSTEP(LX, glb_cmd_queue, NULL); \
165 sprintf(neko_log_buf,"Set by env : 2 (KSTEP)"); \
166 log_message(neko_log_buf); \
167 autotune_ax_helm_compr[LX] = 2; \
168 } else { \
169 sprintf(neko_log_buf, "Invalid value set for NEKO_AUTOTUNE"); \
170 log_error(neko_log_buf); \
171 } \
172 } \
173 else { \
174 CL_CHECK(clFinish(glb_cmd_queue)); \
175 cl_event perf_event, sync_event; \
176 cl_ulong start, end; \
177 CL_CHECK(clEnqueueMarker(glb_cmd_queue, &sync_event)); \
178 CL_CHECK(clEnqueueBarrier(prf_cmd_queue)); \
179 CL_CHECK(clEnqueueWaitForEvents(prf_cmd_queue, 1, &sync_event)); \
180 \
181 double elapsed1 = 0.0; \
182 for(int i = 0; i < 100; i++) { \
183 CASE_1D(LX, prf_cmd_queue, &perf_event); \
184 CL_CHECK(clWaitForEvents(1, &perf_event)); \
185 CL_CHECK(clGetEventProfilingInfo(perf_event, \
186 CL_PROFILING_COMMAND_START, \
187 sizeof(cl_ulong), &start, NULL)); \
188 CL_CHECK(clGetEventProfilingInfo(perf_event, \
189 CL_PROFILING_COMMAND_END, \
190 sizeof(cl_ulong), &end, NULL)); \
191 elapsed1 += (end - start)*1.0e-6; \
192 } \
193 \
194 double elapsed2 = 0.0; \
195 for(int i = 0; i < 100; i++) { \
196 CASE_KSTEP(LX, prf_cmd_queue, &perf_event); \
197 CL_CHECK(clWaitForEvents(1, &perf_event)); \
198 CL_CHECK(clGetEventProfilingInfo(perf_event, \
199 CL_PROFILING_COMMAND_START, \
200 sizeof(cl_ulong), &start, NULL)); \
201 CL_CHECK(clGetEventProfilingInfo(perf_event, \
202 CL_PROFILING_COMMAND_END, \
203 sizeof(cl_ulong), &end, NULL)); \
204 elapsed2 += (end - start)*1.0e-6; \
205 } \
206 \
207 CL_CHECK(clFinish(prf_cmd_queue)); \
208 CL_CHECK(clEnqueueMarker(prf_cmd_queue, &sync_event)); \
209 int krnl_strtgy = (elapsed1 < elapsed2 ? 1 : 2); \
210 sprintf(neko_log_buf, "Chose : %d (%s)", krnl_strtgy, \
211 (krnl_strtgy > 1 ? "KSTEP" : "1D")); \
212 autotune_ax_helm_compr[LX] = krnl_strtgy; \
213 log_message(neko_log_buf); \
214 clEnqueueBarrier(glb_cmd_queue); \
215 clEnqueueWaitForEvents(glb_cmd_queue, 1, &sync_event) ; \
216 } \
217 log_end_section(); \
218 } else if (autotune_ax_helm_compr[LX] == 1 ) { \
219 CASE_1D(LX, glb_cmd_queue, NULL); \
220 } else if (autotune_ax_helm_compr[LX] == 2 ) { \
221 CASE_KSTEP(LX, glb_cmd_queue, NULL); \
222 } \
223 break
224
225#define CASE_LARGE(LX) \
226 case LX: \
227 CASE_KSTEP(LX, glb_cmd_queue, NULL); \
228 break
229
230 if((*lx) < 13) {
231 switch(*lx) {
232 CASE(2);
233 CASE(3);
234 CASE(4);
235 CASE(5);
236 CASE(6);
237 CASE(7);
238 CASE(8);
239 CASE(9);
240 CASE(10);
241 CASE(11);
242 CASE(12);
243 default:
244 {
245 fprintf(stderr, __FILE__ ": size not supported: %d\n", *lx);
246 exit(1);
247 }
248 }
249 }
250 else {
251 switch(*lx) {
252 CASE_LARGE(13);
253 CASE_LARGE(14);
254 CASE_LARGE(15);
255 CASE_LARGE(16);
256 default:
257 {
258 fprintf(stderr, __FILE__ ": size not supported: %d\n", *lx);
259 exit(1);
260 }
261 }
262 }
263}
264
269void opencl_ax_helm_compr_vector(void *au, void *av, void *aw,
270 void *u, void *v, void *w,
271 void *dx, void *dy, void *dz,
272 void *dxt, void *dyt, void *dzt, void *h1,
273 void *g11, void *g22, void *g33, void *g12,
274 void *g13, void *g23, void *ind,
275 int *nelv, int *lx) {
276
277 cl_int err;
278
279 if (ax_helm_compr_program == NULL)
280 opencl_kernel_jit(ax_helm_compr_kernel,
281 (cl_program *) &ax_helm_compr_program);
282
283 size_t * global_kstep = (size_t *) malloc(sizeof(size_t) * 2);
284 size_t * local_kstep = (size_t *) malloc(sizeof(size_t) * 2);
285 local_kstep[0] = (*lx);
286 local_kstep[1] = (*lx);
287 global_kstep[0] = (*nelv) * (*lx);
288 global_kstep[1] = (*lx);
289
290
291#define STR(X) #X
292#define CASE_VECTOR(LX) \
293 case LX: \
294 { \
295 cl_kernel kernel = \
296 clCreateKernel(ax_helm_compr_program, \
297 STR(ax_helm_compr_vector_kstep_lx##LX), &err); \
298 CL_CHECK(err); \
299 \
300 CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), (void *) &au)); \
301 CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), (void *) &av)); \
302 CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), (void *) &aw)); \
303 CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), (void *) &u)); \
304 CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), (void *) &v)); \
305 CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), (void *) &w)); \
306 CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_mem), (void *) &dx)); \
307 CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), (void *) &dy)); \
308 CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_mem), (void *) &dz)); \
309 CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_mem), (void *) &h1)); \
310 CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_mem), (void *) &g11)); \
311 CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_mem), (void *) &g22)); \
312 CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_mem), (void *) &g33)); \
313 CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_mem), (void *) &g12)); \
314 CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_mem), (void *) &g13)); \
315 CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_mem), (void *) &g23)); \
316 CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_mem), (void *) &ind)); \
317 \
318 CL_CHECK(clEnqueueNDRangeKernel((cl_command_queue) glb_cmd_queue, \
319 kernel, 2, NULL, global_kstep, \
320 local_kstep, 0, NULL, NULL)); \
321 CL_CHECK(clReleaseKernel(kernel)); \
322 } \
323 break
324
325 switch(*lx) {
326 CASE_VECTOR(2);
327 CASE_VECTOR(3);
328 CASE_VECTOR(4);
329 CASE_VECTOR(5);
330 CASE_VECTOR(6);
331 CASE_VECTOR(7);
332 CASE_VECTOR(8);
333 CASE_VECTOR(9);
334 CASE_VECTOR(10);
335 CASE_VECTOR(11);
336 CASE_VECTOR(12);
337 CASE_VECTOR(13);
338 CASE_VECTOR(14);
339 CASE_VECTOR(15);
340 CASE_VECTOR(16);
341 default:
342 {
343 fprintf(stderr, __FILE__ ": size not supported: %d\n", *lx);
344 exit(1);
345 }
346 }
347
348 free(global_kstep);
349 free(local_kstep);
350}
CASE_VECTOR
#define CASE_VECTOR(LX)
autotune_ax_helm_compr
static int * autotune_ax_helm_compr
Definition ax_helm_compr.c:58
CASE
#define CASE(LX)
opencl_ax_helm_compr
void opencl_ax_helm_compr(void *w, void *u, void *dx, void *dy, void *dz, void *dxt, void *dyt, void *dzt, void *h1, void *g11, void *g22, void *g33, void *g12, void *g13, void *g23, void *ind, int *nelv, int *lx)
Definition ax_helm_compr.c:63
CASE_LARGE
#define CASE_LARGE(LX)
opencl_ax_helm_compr_vector
void opencl_ax_helm_compr_vector(void *au, void *av, void *aw, void *u, void *v, void *w, void *dx, void *dy, void *dz, void *dxt, void *dyt, void *dzt, void *h1, void *g11, void *g22, void *g33, void *g12, void *g13, void *g23, void *ind, int *nelv, int *lx)
Definition ax_helm_compr.c:269
ale_add_kinematics_kernel
__global__ void ale_add_kinematics_kernel(const int n, T *__restrict__ wx, T *__restrict__ wy, T *__restrict__ wz, const T *__restrict__ x_ref, const T *__restrict__ y_ref, const T *__restrict__ z_ref, const T *__restrict__ phi, const T *__restrict__ x, const T *__restrict__ y, const T *__restrict__ z, const kinematics_params_t kin_params)
Definition ale_kinematics_kernel.h:52
g23
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g23
Definition ax_helm_compr_kernel.h:209
aw
__global__ void T *__restrict__ T *__restrict__ aw
Definition ax_helm_compr_kernel.h:975
g22
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g22
Definition ax_helm_compr_kernel.h:205
ind
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const int *__restrict__ ind
Definition ax_helm_compr_kernel.h:210
g13
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g13
Definition ax_helm_compr_kernel.h:208
w
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ w
Definition ax_helm_compr_kernel.h:978
nelv
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const int *__restrict__ const int nelv
Definition ax_helm_compr_kernel.h:211
g12
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g12
Definition ax_helm_compr_kernel.h:207
dz
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dz
Definition ax_helm_compr_kernel.h:202
dx
__global__ void const T *__restrict__ const T *__restrict__ dx
Definition ax_helm_compr_kernel.h:200
u
__global__ void const T *__restrict__ u
Definition ax_helm_compr_kernel.h:199
av
__global__ void T *__restrict__ av
Definition ax_helm_compr_kernel.h:974
dy
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ dy
Definition ax_helm_compr_kernel.h:201
g33
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g33
Definition ax_helm_compr_kernel.h:206
v
__global__ void T *__restrict__ T *__restrict__ const T *__restrict__ const T *__restrict__ v
Definition ax_helm_compr_kernel.h:977
g11
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ g11
Definition ax_helm_compr_kernel.h:204
h1
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ h1
Definition ax_helm_compr_kernel.h:203
dyt
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dyt
Definition cdtp_kernel.h:115
dzt
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dzt
Definition cdtp_kernel.h:116
dxt
__global__ void const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ const T *__restrict__ dxt
Definition cdtp_kernel.h:114
device_config.h
opencl_kernel_jit
void opencl_kernel_jit(const char *kernel, cl_program *program)
Definition jit.c:50
neko_log.h
prgm_lib.h
ax_helm_compr_program
void * ax_helm_compr_program