Neko
1.99.10
A portable framework for high-order spectral element flow simulations
Loading...
Searching...
No Matches
elem_block.h
Go to the documentation of this file.
1
#ifndef __MATH_ELEM_BLOCK_H__
2
#define __MATH_ELEM_BLOCK_H__
3
/*
4
Copyright (c) 2026, The Neko Authors
5
All rights reserved.
6
7
Redistribution and use in source and binary forms, with or without
8
modification, are permitted provided that the following conditions
9
are met:
10
11
* Redistributions of source code must retain the above copyright
12
notice, this list of conditions and the following disclaimer.
13
14
* Redistributions in binary form must reproduce the above
15
copyright notice, this list of conditions and the following
16
disclaimer in the documentation and/or other materials provided
17
with the distribution.
18
19
* Neither the name of the authors nor the names of its
20
contributors may be used to endorse or promote products derived
21
from this software without specific prior written permission.
22
23
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26
FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
29
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
31
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
33
ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34
POSSIBILITY OF SUCH DAMAGE.
35
*/
36
54
#ifndef NEKO_EB_MAX_BLOCK_THREADS
55
#define NEKO_EB_MAX_BLOCK_THREADS 512
56
#endif
57
58
/* Static shared memory limit per block without the opt-in dynamic path */
59
#ifndef NEKO_EB_MAX_SMEM
60
#define NEKO_EB_MAX_SMEM 49152
61
#endif
62
63
#define NEKO_EB_CANDIDATES 3
64
65
template
<
int
LX,
int
C >
66
struct
elem_block
{
67
static
const
int
raw
= 1 << C;
68
static
const
int
value
=
69
(LX * LX *
raw
<=
NEKO_EB_MAX_BLOCK_THREADS
) ?
raw
: 1;
70
};
71
72
/*
73
* Launch bounds.
74
*
75
* Max threads only, deliberately. CUDA's second argument is
76
* minBlocksPerMultiprocessor, and setting it caps registers at
77
* 65536/(minBlocks * LX*LX*EB) -- which under blocking gets tight enough to
78
* force spilling (at lx = 8, EB = 4 with minBlocks = 3 that is 85 registers,
79
* against opgrad's measured 152). The form below can only ever relax the
80
* cap, never tighten it.
81
*
82
* Note the operator kernels historically carried __launch_bounds__(LX*LX, 3),
83
* which is the HIP form where the second argument means minimum waves per
84
* execution unit -- something quite different. On CUDA that squeezed
85
* registers to 80 at lx >= 14 and made cdtp spill 736 bytes per thread.
86
*
87
* Defining NEKO_EB_THREADS_SM pins an occupancy target instead: ptxas is held
88
* to 65536/NEKO_EB_THREADS_SM registers per thread and spills to meet it.
89
*/
90
#ifdef NEKO_EB_THREADS_SM
91
#define NEKO_EB_BOUNDS(NT) \
92
__launch_bounds__((NT), ((NEKO_EB_THREADS_SM)/(NT) > 0 ? \
93
(NEKO_EB_THREADS_SM)/(NT) : 1))
94
#else
95
#define NEKO_EB_BOUNDS(NT) __launch_bounds__((NT))
96
#endif
97
98
/*
99
* Launch geometry helpers. NELV is the element count; the grid is sized so
100
* the tail block is partially filled and clamps, rather than the kernel
101
* returning early -- these kernels have __syncthreads() in the k loop, so an
102
* early return would leave the barrier unmatched.
103
*/
104
#define NEKO_EB(LX, C) (elem_block<LX, C>::value)
105
#define NEKO_EB_NTHRDS(LX, C) dim3((LX), (LX), NEKO_EB(LX, C))
106
#define NEKO_EB_NBLCKS(NELV, LX, C) \
107
dim3(((NELV) + NEKO_EB(LX, C) - 1)/NEKO_EB(LX, C), 1, 1)
108
#define NEKO_EB_SEL(LX, SEL) \
109
((SEL) == 0 ? NEKO_EB(LX, 0) : (SEL) == 1 ? NEKO_EB(LX, 1) : NEKO_EB(LX, 2))
110
125
#define NEKO_CHUNKS_CANDIDATES 4
126
127
/*
128
* Candidate C selects 1024 >> C, i.e. 1024, 512, 256, 128. Candidate 0 is the
129
* historical value, so it is always available as an A/B baseline.
130
*
131
* A heuristic is deliberately avoided here. The best chunk depends on how the
132
* shared memory footprint (which grows as LX^3) caps the resident block count
133
* on the target device, and that differs between architectures -- 164 kB per
134
* SM on A100, 228 kB on H100, 64 kB of LDS per CU on CDNA. Picking by thread
135
* utilisation alone gets it wrong from LX = 6 upward, because once the block
136
* count is capped by shared memory a smaller block simply wastes thread slots.
137
* So all four are instantiated and the tuner measures them.
138
*/
139
template
<
int
LX,
int
C >
140
struct
chunk_block
{
141
static
const
int
raw
= 1024 >> C;
142
static
const
int
value
= (
raw
>= LX * LX) ?
raw
: 1024;
143
};
144
145
#define NEKO_CHUNKS(LX, C) (chunk_block<LX, C>::value)
146
#define NEKO_CHUNKS_NTHRDS(LX, C) dim3(NEKO_CHUNKS(LX, C), 1, 1)
147
#define NEKO_CHUNKS_SEL(LX, SEL) \
148
((SEL) == 0 ? NEKO_CHUNKS(LX, 0) : (SEL) == 1 ? NEKO_CHUNKS(LX, 1) : \
149
(SEL) == 2 ? NEKO_CHUNKS(LX, 2) : NEKO_CHUNKS(LX, 3))
150
151
#endif
// __MATH_ELEM_BLOCK_H__
NEKO_EB_MAX_BLOCK_THREADS
#define NEKO_EB_MAX_BLOCK_THREADS
Definition
elem_block.h:55
chunk_block
Definition
elem_block.h:140
chunk_block::raw
static const int raw
Definition
elem_block.h:141
chunk_block::value
static const int value
Definition
elem_block.h:142
elem_block
Definition
elem_block.h:66
elem_block::raw
static const int raw
Definition
elem_block.h:67
elem_block::value
static const int value
Definition
elem_block.h:68
src
math
bcknd
device
cuda
elem_block.h
Generated on
for Neko by
1.15.0