#include <stdlib.h>
#include <stdint.h>
#include <utofu.h>

Include dependency graph for gs_utofu_aux.c:

Classes
struct	gs_utofu_ctx_t

Macros
#define	GS_UTOFU_PUT_FLAGS_BASE (UTOFU_ONESIDED_FLAG_TCQ_NOTICE \| UTOFU_ONESIDED_FLAG_REMOTE_MRQ_NOTICE)

#define	GS_UTOFU_PEND_STRIDE 32

Functions
static long *	gs_utofu_pend_slot (gs_utofu_ctx_t *ctx, size_t k, int half)

static void	gs_utofu_drain_tcq (size_t k)

static void	gs_utofu_stash_push (gs_utofu_ctx_t *ctx, uint64_t edata)

void	gs_utofu_init (int ntni_req, int nthreads, int ntni_out, int nvcq_out, int nrvcq_out, uint64_t edata_max, int *ierr)

void	gs_utofu_ctx_create (void send_buf, size_t send_bytes, void recv_buf, size_t recv_bytes, void *ctx_out, int nrvcq_out, uint64_t recv_vcq_id, uint64_t recv_stadd, int *ierr)

void	gs_utofu_ctx_free (void vctx, int ierr)

static int	gs_utofu_put_one (gs_utofu_ctx_t ctx, size_t k, int i, const int64_t send_off, const int64_t send_len, const uint64_t rmt_vcq_id, const uint64_t rmt_stadd, const int64_t dst_off0, const int64_t dst_off1, const int dst_tag, int parity, int64_t send_base, int elem_size)

void	gs_utofu_post_puts (void vctx, int tid, int nteam, int nsend, const int64_t send_off, const int64_t send_len, const uint64_t rmt_vcq_id, const uint64_t rmt_stadd, const int64_t dst_off0, const int64_t dst_off1, const int dst_tag, int parity, int64_t send_base, int elem_size, int *ierr)

void	gs_utofu_poll_recv (void vctx, int parity, int cap, int ncompleted, int out_idx, int ierr)

void	gs_utofu_drain_half (void vctx, int half, int tid, int nteam, int ierr)

Variables
static int	gs_utofu_ready = 0

static size_t	gs_utofu_nvcq = 0

static size_t	gs_utofu_ntni = 0

static utofu_vcq_hdl_t *	gs_utofu_vcq = NULL

static utofu_tni_id_t *	gs_utofu_alltni = NULL

static size_t	gs_utofu_nalltni = 0

static int	gs_utofu_recv_rr = 0

static uint64_t	gs_utofu_edata_max = 255

static unsigned long	gs_utofu_put_flags = (UTOFU_ONESIDED_FLAG_TCQ_NOTICE \| UTOFU_ONESIDED_FLAG_REMOTE_MRQ_NOTICE)

static int	gs_utofu_master_inject = 0

static size_t	gs_utofu_nrvcq_def = 1

Macro Definition Documentation

◆ GS_UTOFU_PEND_STRIDE

#define GS_UTOFU_PEND_STRIDE 32

Definition at line 107 of file gs_utofu_aux.c.

◆ GS_UTOFU_PUT_FLAGS_BASE

#define GS_UTOFU_PUT_FLAGS_BASE (UTOFU_ONESIDED_FLAG_TCQ_NOTICE | UTOFU_ONESIDED_FLAG_REMOTE_MRQ_NOTICE)

Native Tofu (uTofu) helper routines for the gs_utofu gather-scatter backend (see gs_utofu.F90).

The transport is a set of one-sided RDMA puts. Each rank registers its receive buffer once and advertises the (vcq_id, stadd) to its neighbours. A send is then a single utofu_put straight into the neighbour's receive buffer, carrying an edata tag in the descriptor; the put's arrival raises a remote MRQ notice carrying that tag, so the signal is fused into the data transfer – no separate atomic/credit message, unlike the coarray backend.

Injection is non-blocking and thread-parallel: one injection VCQ is created per OpenMP thread (dealt round-robin over the TNIs), and each thread fires the puts for its share of the peers on its own VCQ – the uTofu substitute for the per-thread MPI_Isend that Fujitsu MPI's missing MPI_THREAD_MULTIPLE support rules out (cf. the RIKEN-LQCD/jacobi2d reference program). Every VCQ is created with UTOFU_VCQ_FLAG_THREAD_SAFE; without it uTofu takes the unlocked direct-descriptor injection path and any cross-thread touch of the VCQ corrupts its TOQ ("TOQ Direct Descriptor Exception"). With one thread per VCQ the internal lock is uncontended. The number of TNIs to deal the VCQs over is chosen by the caller (NEKO_GS_UTOFU_NTNI, default 1 – multi-TNI is opt-in until validated on a given machine); NEKO_GS_UTOFU_MASTER_INJECT=1 serialises all injection onto thread 0 as an A/B reference.

Build: requires Fujitsu's uTofu headers/runtime (Fugaku). Only compiled when ENABLE_UTOFU. Link with -ltofucom (verify the exact library name in your environment).

Items marked "VERIFY" depend on exact utofu.h symbol/field names that vary slightly between Tofu library versions and should be checked when first building on the target system.

Definition at line 88 of file gs_utofu_aux.c.

Function Documentation

◆ gs_utofu_ctx_create()

void gs_utofu_ctx_create	(	void *	send_buf,
		size_t	send_bytes,
		void *	recv_buf,
		size_t	recv_bytes,
		void **	ctx_out,
		int *	nrvcq_out,
		uint64_t *	recv_vcq_id,
		uint64_t *	recv_stadd,
		int *	ierr
	)

Register an instance's send/recv buffers and allocate its context. send_buf is registered on every injection VCQ (the source STADD must belong to the VCQ that issues the put). The instance gets its OWN set of receive VCQs (private MRQs); recv_buf is registered on each, and the caller binds every receive peer to one of them, advertising that VCQ's id/STADD as the peer's put target – so arrivals land only in this instance's queues, spread over the TNIs.

recv_vcq_id / recv_stadd are arrays of (at least) the nrvcq_out value reported by gs_utofu_init; entries [0, *nrvcq_out) are filled. Fewer than requested may be granted when the VCQ budget runs dry (>= 1, or ierr = 1 – remedy: lower OMP_NUM_THREADS, cap NEKO_GS_UTOFU_NVCQ, or reduce NEKO_GS_UTOFU_NRVCQ).

Definition at line 310 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_ctx_free()

void gs_utofu_ctx_free	(	void *	vctx,
		int *	ierr
	)

Definition at line 382 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_drain_half()

void gs_utofu_drain_half	(	void *	vctx,
		int	half,
		int	tid,
		int	nteam,
		int *	ierr
	)

Block until every put charged to send-buffer half half has completed locally, so that half may be repacked. Called by EVERY thread of the team: the strided map k = tid, tid+nteam, ... covers each injection VCQ exactly once, so each VCQ (and its counter slots) has a single toucher and no atomics are needed. The caller must barrier afterwards – only then is the whole half known to be free. With double buffering the half being reclaimed was posted two rounds ago, so this normally just reaps already-finished completions.

Definition at line 562 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_drain_tcq()

static void gs_utofu_drain_tcq ( size_t k )

static

Definition at line 144 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_init()

void gs_utofu_init	(	int	ntni_req,
		int	nthreads,
		int *	ntni_out,
		int *	nvcq_out,
		int *	nrvcq_out,
		uint64_t *	edata_max,
		int *	ierr
	)

Create the process-global uTofu endpoint (idempotent).

Creates the injection VCQs – one per OpenMP thread, dealt round-robin over the granted TNIs, every one THREAD_SAFE – shared by all instances and driven thread-parallel (thread t owns VCQ t). Receive VCQs are created per gs instance in gs_utofu_ctx_create, so each instance owns a private MRQ.

Parameters

ntni_req	requested number of TNIs (>=1); clamped to availability
nthreads	OpenMP thread count = requested injection VCQs (>=1)
ntni_out	number of TNIs actually in use
nvcq_out	number of injection VCQs actually created
nrvcq_out	receive VCQs requested per instance path (the caller sizes its id/STADD arrays with this; ctx_create may grant fewer)
edata_max	largest usable edata value (for tag-range checking)

Definition at line 180 of file gs_utofu_aux.c.

Here is the call graph for this function:

◆ gs_utofu_pend_slot()

static long * gs_utofu_pend_slot	(	gs_utofu_ctx_t *	ctx,
		size_t	k,
		int	half
	)

static

Definition at line 135 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_poll_recv()

void gs_utofu_poll_recv	(	void *	vctx,
		int	parity,
		int	cap,
		int *	ncompleted,
		int *	out_idx,
		int *	ierr
	)

Collect the receive slabs that have landed for the current parity.

Returns at most cap 1-based receive-peer indices in out_idx. A notice whose parity bit does not match the current round is stashed and replayed on the next call (with the flipped parity), keeping the unpack-as-ready model correct in the face of a neighbour running one round ahead. May return 0 (nothing yet) – the caller polls in a loop.

Definition at line 509 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_post_puts()

void gs_utofu_post_puts	(	void *	vctx,
		int	tid,
		int	nteam,
		int	nsend,
		const int64_t *	send_off,
		const int64_t *	send_len,
		const uint64_t *	rmt_vcq_id,
		const uint64_t *	rmt_stadd,
		const int64_t *	dst_off0,
		const int64_t *	dst_off1,
		const int *	dst_tag,
		int	parity,
		int64_t	send_base,
		int	elem_size,
		int *	ierr
	)

Issue this thread's share of the puts. Called by EVERY thread of the OpenMP team, each passing its own tid: thread tid posts peers i = tid, tid+npost, ... on its own VCQ, where npost = min(nteam, nvcq), so threads beyond the VCQ count post nothing and their peers fall to the others. One thread per VCQ per phase means the completion counters need no atomics; the VCQs are THREAD_SAFE regardless, so a degraded or odd configuration is safe, just slower. Offsets/lengths are in elements; elem_size converts to bytes. The edata tag fuses the slab id (dst_tag) with the parity bit so the receiver can tell this round's notices from a neighbour that has already raced into the next round.

With NEKO_GS_UTOFU_MASTER_INJECT=1, thread 0 instead posts every peer, round-robined over all VCQs (the pre-threading behaviour, kept as an A/B reference).

Definition at line 463 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_put_one()

static int gs_utofu_put_one	(	gs_utofu_ctx_t *	ctx,
		size_t	k,
		int	i,
		const int64_t *	send_off,
		const int64_t *	send_len,
		const uint64_t *	rmt_vcq_id,
		const uint64_t *	rmt_stadd,
		const int64_t *	dst_off0,
		const int64_t *	dst_off1,
		const int *	dst_tag,
		int	parity,
		int64_t	send_base,
		int	elem_size
	)

static

Definition at line 415 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gs_utofu_stash_push()

static void gs_utofu_stash_push	(	gs_utofu_ctx_t *	ctx,
		uint64_t	edata
	)

static

Definition at line 152 of file gs_utofu_aux.c.

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ gs_utofu_alltni

utofu_tni_id_t* gs_utofu_alltni = NULL

static

Definition at line 96 of file gs_utofu_aux.c.

◆ gs_utofu_edata_max

uint64_t gs_utofu_edata_max = 255

static

Definition at line 100 of file gs_utofu_aux.c.

◆ gs_utofu_master_inject

int gs_utofu_master_inject = 0

static

Definition at line 102 of file gs_utofu_aux.c.

◆ gs_utofu_nalltni

size_t gs_utofu_nalltni = 0

static

Definition at line 98 of file gs_utofu_aux.c.

◆ gs_utofu_nrvcq_def

size_t gs_utofu_nrvcq_def = 1

static

Definition at line 103 of file gs_utofu_aux.c.

◆ gs_utofu_ntni

size_t gs_utofu_ntni = 0

static

Definition at line 94 of file gs_utofu_aux.c.

◆ gs_utofu_nvcq

size_t gs_utofu_nvcq = 0

static

Definition at line 93 of file gs_utofu_aux.c.

◆ gs_utofu_put_flags

unsigned long gs_utofu_put_flags = (UTOFU_ONESIDED_FLAG_TCQ_NOTICE | UTOFU_ONESIDED_FLAG_REMOTE_MRQ_NOTICE)

static

Definition at line 101 of file gs_utofu_aux.c.

◆ gs_utofu_ready

int gs_utofu_ready = 0

static

Definition at line 92 of file gs_utofu_aux.c.

◆ gs_utofu_recv_rr

int gs_utofu_recv_rr = 0

static

Definition at line 99 of file gs_utofu_aux.c.

◆ gs_utofu_vcq

utofu_vcq_hdl_t* gs_utofu_vcq = NULL

static

Definition at line 95 of file gs_utofu_aux.c.

Classes

Macros

Functions

Variables

Macro Definition Documentation

◆ GS_UTOFU_PEND_STRIDE

◆ GS_UTOFU_PUT_FLAGS_BASE

Function Documentation

◆ gs_utofu_ctx_create()

◆ gs_utofu_ctx_free()

◆ gs_utofu_drain_half()

◆ gs_utofu_drain_tcq()

◆ gs_utofu_init()

◆ gs_utofu_pend_slot()

◆ gs_utofu_poll_recv()

◆ gs_utofu_post_puts()

◆ gs_utofu_put_one()

◆ gs_utofu_stash_push()

Variable Documentation

◆ gs_utofu_alltni

◆ gs_utofu_edata_max

◆ gs_utofu_master_inject

◆ gs_utofu_nalltni

◆ gs_utofu_nrvcq_def

◆ gs_utofu_ntni

◆ gs_utofu_nvcq

◆ gs_utofu_put_flags

◆ gs_utofu_ready

◆ gs_utofu_recv_rr

◆ gs_utofu_vcq