/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_PMIX_H
#define OPAL_PMIX_H
#include "opal_config.h"
#include "opal/types.h"
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#include "opal/mca/mca.h"
#include "opal/mca/event/event.h"
#include "opal/dss/dss.h"
#include "opal/runtime/opal.h"
#include "opal/dss/dss.h"
#include "opal/util/error.h"
#include "opal/util/proc.h"
#include "opal/hash_string.h"
#include "opal/mca/pmix/pmix_types.h"
#include "opal/mca/pmix/pmix_server.h"
BEGIN_C_DECLS
/* provide access to the framework verbose output without
* exposing the entire base */
extern int opal_pmix_verbose_output;
extern bool opal_pmix_collect_all_data;
extern bool opal_pmix_base_async_modex;
extern int opal_pmix_base_exchange(opal_value_t *info,
opal_pmix_pdata_t *pdat,
int timeout);
/*
* Count the fash for the the external RM
*/
#define OPAL_HASH_JOBID( str, hash ){ \
OPAL_HASH_STR( str, hash ); \
hash &= ~(0x8000); \
}
/**
* Provide a simplified macro for sending data via modex
* to other processes. The macro requires four arguments:
*
* r - the integer return status from the modex op
* sc - the PMIX scope of the data
* s - the key to tag the data being posted
* d - pointer to the data object being posted
* t - the type of the data
*/
#define OPAL_MODEX_SEND_VALUE(r, sc, s, d, t) \
do { \
opal_value_t _kv; \
OBJ_CONSTRUCT(&(_kv), opal_value_t); \
_kv.key = (s); \
if (OPAL_SUCCESS != ((r) = opal_value_load(&(_kv), (d), (t)))) { \
OPAL_ERROR_LOG((r)); \
} else { \
if (OPAL_SUCCESS != ((r) = opal_pmix.put(sc, &(_kv)))) { \
OPAL_ERROR_LOG((r)); \
} \
} \
/* opal_value_load makes a copy of the data, so release it */ \
_kv.key = NULL; \
OBJ_DESTRUCT(&(_kv)); \
} while(0);
/**
* Provide a simplified macro for sending data via modex
* to other processes. The macro requires four arguments:
*
* r - the integer return status from the modex op
* sc - the PMIX scope of the data
* s - the key to tag the data being posted
* d - the data object being posted
* sz - the number of bytes in the data object
*/
#define OPAL_MODEX_SEND_STRING(r, sc, s, d, sz) \
do { \
opal_value_t _kv; \
OBJ_CONSTRUCT(&(_kv), opal_value_t); \
_kv.key = (s); \
_kv.type = OPAL_BYTE_OBJECT; \
_kv.data.bo.bytes = (uint8_t*)(d); \
_kv.data.bo.size = (sz); \
if (OPAL_SUCCESS != ((r) = opal_pmix.put(sc, &(_kv)))) { \
OPAL_ERROR_LOG((r)); \
} \
_kv.data.bo.bytes = NULL; /* protect the data */ \
_kv.key = NULL; /* protect the key */ \
OBJ_DESTRUCT(&(_kv)); \
} while(0);
/**
* Provide a simplified macro for sending data via modex
* to other processes. The macro requires four arguments:
*
* r - the integer return status from the modex op
* sc - the PMIX scope of the data
* s - the MCA component that is posting the data
* d - the data object being posted
* sz - the number of bytes in the data object
*/
#define OPAL_MODEX_SEND(r, sc, s, d, sz) \
do { \
char *_key; \
_key = mca_base_component_to_string((s)); \
OPAL_MODEX_SEND_STRING((r), (sc), _key, (d), (sz)); \
free(_key); \
} while(0);
/**
* Provide a simplified macro for retrieving modex data
* from another process when we don't want the PMIx module
* to request it from the server if not found:
*
* r - the integer return status from the modex op (int)
* s - string key (char*)
* p - pointer to the opal_process_name_t of the proc that posted
* the data (opal_process_name_t*)
* d - pointer to a location wherein the data object
* is to be returned
* t - the expected data type
*/
#define OPAL_MODEX_RECV_VALUE_OPTIONAL(r, s, p, d, t) \
do { \
opal_value_t *_kv, *_info; \
opal_list_t _ilist; \
OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output, \
"%s[%s:%d] MODEX RECV VALUE OPTIONAL FOR PROC %s KEY %s", \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, \
OPAL_NAME_PRINT(*(p)), (s))); \
OBJ_CONSTRUCT(&(_ilist), opal_list_t); \
_info = OBJ_NEW(opal_value_t); \
_info->key = strdup(OPAL_PMIX_OPTIONAL); \
_info->type = OPAL_BOOL; \
_info->data.flag = true; \
opal_list_append(&(_ilist), &(_info)->super); \
if (OPAL_SUCCESS == ((r) = opal_pmix.get((p), (s), &(_ilist), &(_kv)))) { \
if (NULL == _kv) { \
(r) = OPAL_ERR_NOT_FOUND; \
} else { \
(r) = opal_value_unload(_kv, (void**)(d), (t)); \
OBJ_RELEASE(_kv); \
} \
} \
OPAL_LIST_DESTRUCT(&(_ilist)); \
} while(0);
/**
* Provide a simplified macro for retrieving modex data
* from another process when we want the PMIx module
* to request it from the server if not found, but do not
* want the server to go find it if the server doesn't
* already have it:
*
* r - the integer return status from the modex op (int)
* s - string key (char*)
* p - pointer to the opal_process_name_t of the proc that posted
* the data (opal_process_name_t*)
* d - pointer to a location wherein the data object
* is to be returned
* t - the expected data type
*/
#define OPAL_MODEX_RECV_VALUE_IMMEDIATE(r, s, p, d, t) \
do { \
opal_value_t *_kv, *_info; \
opal_list_t _ilist; \
opal_output_verbose(1, opal_pmix_verbose_output, \
"%s[%s:%d] MODEX RECV VALUE IMMEDIATE FOR PROC %s KEY %s", \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, \
OPAL_NAME_PRINT(*(p)), (s)); \
OBJ_CONSTRUCT(&(_ilist), opal_list_t); \
_info = OBJ_NEW(opal_value_t); \
_info->key = strdup(OPAL_PMIX_IMMEDIATE); \
_info->type = OPAL_BOOL; \
_info->data.flag = true; \
opal_list_append(&(_ilist), &(_info)->super); \
if (OPAL_SUCCESS == ((r) = opal_pmix.get((p), (s), &(_ilist), &(_kv)))) { \
if (NULL == _kv) { \
(r) = OPAL_ERR_NOT_FOUND; \
} else { \
(r) = opal_value_unload(_kv, (void**)(d), (t)); \
OBJ_RELEASE(_kv); \
} \
} \
OPAL_LIST_DESTRUCT(&(_ilist)); \
} while(0);
/**
* Provide a simplified macro for retrieving modex data
* from another process:
*
* r - the integer return status from the modex op (int)
* s - string key (char*)
* p - pointer to the opal_process_name_t of the proc that posted
* the data (opal_process_name_t*)
* d - pointer to a location wherein the data object
* is to be returned
* t - the expected data type
*/
#define OPAL_MODEX_RECV_VALUE(r, s, p, d, t) \
do { \
opal_value_t *_kv; \
OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output, \
"%s[%s:%d] MODEX RECV VALUE FOR PROC %s KEY %s", \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, \
OPAL_NAME_PRINT(*(p)), (s))); \
if (OPAL_SUCCESS == ((r) = opal_pmix.get((p), (s), NULL, &(_kv)))) { \
if (NULL == _kv) { \
(r) = OPAL_ERR_NOT_FOUND; \
} else { \
(r) = opal_value_unload(_kv, (void**)(d), (t)); \
OBJ_RELEASE(_kv); \
} \
} \
} while(0);
/**
* Provide a simplified macro for retrieving modex data
* from another process:
*
* r - the integer return status from the modex op (int)
* s - string key (char*)
* p - pointer to the opal_process_name_t of the proc that posted
* the data (opal_process_name_t*)
* d - pointer to a location wherein the data object
* it to be returned (char**)
* sz - pointer to a location wherein the number of bytes
* in the data object can be returned (size_t)
*/
#define OPAL_MODEX_RECV_STRING(r, s, p, d, sz) \
do { \
opal_value_t *_kv; \
OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output, \
"%s[%s:%d] MODEX RECV STRING FOR PROC %s KEY %s", \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, \
OPAL_NAME_PRINT(*(p)), (s))); \
if (OPAL_SUCCESS == ((r) = opal_pmix.get((p), (s), NULL, &(_kv)))) { \
if (NULL == _kv) { \
*(sz) = 0; \
(r) = OPAL_ERR_NOT_FOUND; \
} else { \
*(d) = _kv->data.bo.bytes; \
*(sz) = _kv->data.bo.size; \
_kv->data.bo.bytes = NULL; /* protect the data */ \
OBJ_RELEASE(_kv); \
} \
} else { \
*(sz) = 0; \
(r) = OPAL_ERR_NOT_FOUND; \
} \
} while(0);
/**
* Provide a simplified macro for retrieving modex data
* from another process:
*
* r - the integer return status from the modex op (int)
* s - the MCA component that posted the data (mca_base_component_t*)
* p - pointer to the opal_process_name_t of the proc that posted
* the data (opal_process_name_t*)
* d - pointer to a location wherein the data object
* it to be returned (char**)
* sz - pointer to a location wherein the number of bytes
* in the data object can be returned (size_t)
*/
#define OPAL_MODEX_RECV(r, s, p, d, sz) \
do { \
char *_key; \
_key = mca_base_component_to_string((s)); \
OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output, \
"%s[%s:%d] MODEX RECV FOR PROC %s KEY %s", \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, \
OPAL_NAME_PRINT(*(p)), _key)); \
if (NULL == _key) { \
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE); \
(r) = OPAL_ERR_OUT_OF_RESOURCE; \
} else { \
OPAL_MODEX_RECV_STRING((r), _key, (p), (d), (sz)); \
free(_key); \
} \
} while(0);
/**
* Provide a macro for accessing a base function that exchanges
* data values between two procs using the PMIx Publish/Lookup
* APIs */
#define OPAL_PMIX_EXCHANGE(r, i, p, t) \
do { \
OPAL_OUTPUT_VERBOSE((1, opal_pmix_verbose_output, \
"%s[%s:%d] EXCHANGE %s WITH %s", \
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), \
__FILE__, __LINE__, \
(i)->key, (p)->value.key)); \
(r) = opal_pmix_base_exchange((i), (p), (t)); \
} while(0);
/************************************************************
* CLIENT APIs *
************************************************************/
/* Initialize the PMIx client
* When called the client will check for the required connection
* information of the local server and will establish the connection.
* If the information is not found, or the server connection fails, then
* an appropriate error constant will be returned.
*/
typedef int (*opal_pmix_base_module_init_fn_t)(opal_list_t *ilist);
/* Finalize the PMIx client, closing the connection to the local server.
* An error code will be returned if, for some reason, the connection
* cannot be closed. */
typedef int (*opal_pmix_base_module_fini_fn_t)(void);
/* Returns _true_ if the PMIx client has been successfully initialized,
* returns _false_ otherwise. Note that the function only reports the
* internal state of the PMIx client - it does not verify an active
* connection with the server, nor that the server is functional. */
typedef int (*opal_pmix_base_module_initialized_fn_t)(void);
/* Request that the provided list of opal_namelist_t procs be aborted, returning the
* provided _status_ and printing the provided message. A _NULL_
* for the proc list indicates that all processes in the caller's
* nspace are to be aborted.
*
* The response to this request is somewhat dependent on the specific resource
* manager and its configuration (e.g., some resource managers will
* not abort the application if the provided _status_ is zero unless
* specifically configured to do so), and thus lies outside the control
* of PMIx itself. However, the client will inform the RM of
* the request that the application be aborted, regardless of the
* value of the provided _status_.
*
* Passing a _NULL_ msg parameter is allowed. Note that race conditions
* caused by multiple processes calling PMIx_Abort are left to the
* server implementation to resolve with regard to which status is
* returned and what messages (if any) are printed.
*/
typedef int (*opal_pmix_base_module_abort_fn_t)(int status, const char *msg,
opal_list_t *procs);
/* Push all previously _PMIx_Put_ values to the local PMIx server.
* This is an asynchronous operation - the library will immediately
* return to the caller while the data is transmitted to the local
* server in the background */
typedef int (*opal_pmix_base_module_commit_fn_t)(void);
/* Execute a blocking barrier across the processes identified in the
* specified list of opal_namelist_t. Passing a _NULL_ pointer
* indicates that the barrier is to span all processes in the client's
* namespace. Each provided opal_namelist_t can pass PMIX_RANK_WILDCARD to
* indicate that all processes in the given jobid are
* participating.
*
* The _collect_data_ parameter is passed to the server to indicate whether
* or not the barrier operation is to return the _put_ data from all
* participating processes. A value of _false_ indicates that the callback
* is just used as a release and no data is to be returned at that time. A
* value of _true_ indicates that all _put_ data is to be collected by the
* barrier. Returned data is locally cached so that subsequent calls to _PMIx_Get_
* can be serviced without communicating to/from the server, but at the cost
* of increased memory footprint
*/
typedef int (*opal_pmix_base_module_fence_fn_t)(opal_list_t *procs, int collect_data);
/* Fence_nb */
/* Non-blocking version of PMIx_Fence. Note that the function will return
* an error if a _NULL_ callback function is given. */
typedef int (*opal_pmix_base_module_fence_nb_fn_t)(opal_list_t *procs, int collect_data,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* Push a value into the client's namespace. The client library will cache
* the information locally until _PMIx_Commit_ is called. The provided scope
* value is passed to the local PMIx server, which will distribute the data
* as directed. */
typedef int (*opal_pmix_base_module_put_fn_t)(opal_pmix_scope_t scope,
opal_value_t *val);
/* Retrieve information for the specified _key_ as published by the rank
* and jobid i the provided opal_process_name, and subject to any provided
* constraints, returning a pointer to the value in the given address.
*
* This is a blocking operation - the caller will block until
* the specified data has been _PMIx_Put_ by the specified rank. The caller is
* responsible for freeing all memory associated with the returned value when
* no longer required. */
typedef int (*opal_pmix_base_module_get_fn_t)(const opal_process_name_t *proc,
const char *key, opal_list_t *info,
opal_value_t **val);
/* Retrieve information for the specified _key_ as published by the given rank
* and jobid in the opal_process_name_t, and subject to any provided
* constraints. This is a non-blocking operation - the
* callback function will be executed once the specified data has been _PMIx_Put_
* by the specified proc and retrieved by the local server. */
typedef int (*opal_pmix_base_module_get_nb_fn_t)(const opal_process_name_t *proc,
const char *key, opal_list_t *info,
opal_pmix_value_cbfunc_t cbfunc, void *cbdata);
/* Publish the given data to the "universal" nspace
* for lookup by others subject to the provided scope.
* Note that the keys must be unique within the specified
* scope or else an error will be returned (first published
* wins). Attempts to access the data by procs outside of
* the provided scope will be rejected.
*
* Note: Some host environments may support user/group level
* access controls on the information in addition to the scope.
* These can be specified in the info array using the appropriately
* defined keys.
*
* The persistence parameter instructs the server as to how long
* the data is to be retained, within the context of the scope.
* For example, data published within _PMIX_NAMESPACE_ will be
* deleted along with the namespace regardless of the persistence.
* However, data published within PMIX_USER would be retained if
* the persistence was set to _PMIX_PERSIST_SESSION_ until the
* allocation terminates.
*
* The blocking form will block until the server confirms that the
* data has been posted and is available. The non-blocking form will
* return immediately, executing the callback when the server confirms
* availability of the data */
typedef int (*opal_pmix_base_module_publish_fn_t)(opal_list_t *info);
typedef int (*opal_pmix_base_module_publish_nb_fn_t)(opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* Lookup information published by another process within the
* specified scope. A scope of _PMIX_SCOPE_UNDEF_ requests that
* the search be conducted across _all_ namespaces. The "data"
* parameter consists of an array of pmix_pdata_t struct with the
* keys specifying the requested information. Data will be returned
* for each key in the associated info struct - any key that cannot
* be found will return with a data type of "PMIX_UNDEF". The function
* will return SUCCESS if _any_ values can be found, so the caller
* must check each data element to ensure it was returned.
*
* The proc field in each pmix_pdata_t struct will contain the
* nspace/rank of the process that published the data.
*
* Note: although this is a blocking function, it will _not_ wait
* for the requested data to be published. Instead, it will block
* for the time required by the server to lookup its current data
* and return any found items. Thus, the caller is responsible for
* ensuring that data is published prior to executing a lookup, or
* for retrying until the requested data is found */
typedef int (*opal_pmix_base_module_lookup_fn_t)(opal_list_t *data,
opal_list_t *info);
/* Non-blocking form of the _PMIx_Lookup_ function. Data for
* the provided NULL-terminated keys array will be returned
* in the provided callback function. The _wait_ parameter
* is used to indicate if the caller wishes the callback to
* wait for _all_ requested data before executing the callback
* (_true_), or to callback once the server returns whatever
* data is immediately available (_false_) */
typedef int (*opal_pmix_base_module_lookup_nb_fn_t)(char **keys, opal_list_t *info,
opal_pmix_lookup_cbfunc_t cbfunc, void *cbdata);
/* Unpublish data posted by this process using the given keys
* within the specified scope. The function will block until
* the data has been removed by the server. A value of _NULL_
* for the keys parameter instructs the server to remove
* _all_ data published by this process within the given scope */
typedef int (*opal_pmix_base_module_unpublish_fn_t)(char **keys, opal_list_t *info);
/* Non-blocking form of the _PMIx_Unpublish_ function. The
* callback function will be executed once the server confirms
* removal of the specified data. A value of _NULL_
* for the keys parameter instructs the server to remove
* _all_ data published by this process within the given scope */
typedef int (*opal_pmix_base_module_unpublish_nb_fn_t)(char **keys, opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* Spawn a new job. The spawned applications are automatically
* connected to the calling process, and their assigned namespace
* is returned in the nspace parameter - a _NULL_ value in that
* location indicates that the caller doesn't wish to have the
* namespace returned. Behavior of individual resource managers
* may differ, but it is expected that failure of any application
* process to start will result in termination/cleanup of _all_
* processes in the newly spawned job and return of an error
* code to the caller */
typedef int (*opal_pmix_base_module_spawn_fn_t)(opal_list_t *job_info,
opal_list_t *apps,
opal_jobid_t *jobid);
/* Non-blocking form of the _PMIx_Spawn_ function. The callback
* will be executed upon launch of the specified applications,
* or upon failure to launch any of them. */
typedef int (*opal_pmix_base_module_spawn_nb_fn_t)(opal_list_t *job_info,
opal_list_t *apps,
opal_pmix_spawn_cbfunc_t cbfunc,
void *cbdata);
/* Record the specified processes as "connected". Both blocking and non-blocking
* versions are provided. This means that the resource manager should treat the
* failure of any process in the specified group as a reportable event, and take
* appropriate action. Note that different resource managers may respond to
* failures in different manners.
*
* The list is to be provided as opal_namelist_t objects
*
* The callback function is to be called once all participating processes have
* called connect. The server is required to return any job-level info for the
* connecting processes that might not already have - i.e., if the connect
* request involves procs from different nspaces, then each proc shall receive
* the job-level info from those nspaces other than their own.
*
* Note: a process can only engage in _one_ connect operation involving the identical
* set of ranges at a time. However, a process _can_ be simultaneously engaged
* in multiple connect operations, each involving a different set of ranges */
typedef int (*opal_pmix_base_module_connect_fn_t)(opal_list_t *procs);
typedef int (*opal_pmix_base_module_connect_nb_fn_t)(opal_list_t *procs,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Disconnect a previously connected set of processes. An error will be returned
* if the specified set of procs was not previously "connected". As above, a process
* may be involved in multiple simultaneous disconnect operations. However, a process
* is not allowed to reconnect to a set of procs that has not fully completed
* disconnect - i.e., you have to fully disconnect before you can reconnect to the
* _same_ group of processes. */
typedef int (*opal_pmix_base_module_disconnect_fn_t)(opal_list_t *procs);
typedef int (*opal_pmix_base_module_disconnect_nb_fn_t)(opal_list_t *procs,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Given a node name, return an array of processes within the specified jobid
* on that node. If the jobid is OPAL_JOBID_WILDCARD, then all processes on the node will
* be returned. If the specified node does not currently host any processes,
* then the returned list will be empty.
*/
typedef int (*opal_pmix_base_module_resolve_peers_fn_t)(const char *nodename,
opal_jobid_t jobid,
opal_list_t *procs);
/* Given a jobid, return the list of nodes hosting processes within
* that jobid. The returned string will contain a comma-delimited list
* of nodenames. The caller is responsible for releasing the string
* when done with it */
typedef int (*opal_pmix_base_module_resolve_nodes_fn_t)(opal_jobid_t jobid, char **nodelist);
/************************************************************
* SERVER APIs *
* *
* These are calls that go down (or "south") from the ORTE *
* daemon into the PMIx server library *
************************************************************/
/* Initialize the server support library - must pass the callback
* module for the server to use, plus any attributes we want to
* pass down to it */
typedef int (*opal_pmix_base_module_server_init_fn_t)(opal_pmix_server_module_t *module,
opal_list_t *info);
/* Finalize the server support library */
typedef int (*opal_pmix_base_module_server_finalize_fn_t)(void);
/* given a semicolon-separated list of input values, generate
* a regex that can be passed down to the client for parsing.
* The caller is responsible for free'ing the resulting
* string
*
* If values have leading zero's, then that is preserved. You
* have to add back any prefix/suffix for node names
* odin[009-015,017-023,076-086]
*
* "pmix:odin[009-015,017-023,076-086]"
*
* Note that the "pmix" at the beginning of each regex indicates
* that the PMIx native parser is to be used by the client for
* parsing the provided regex. Other parsers may be supported - see
* the pmix_client.h header for a list.
*/
typedef int (*opal_pmix_base_module_generate_regex_fn_t)(const char *input, char **regex);
/* The input is expected to consist of a comma-separated list
* of ranges. Thus, an input of:
* "1-4;2-5;8,10,11,12;6,7,9"
* would generate a regex of
* "[pmix:2x(3);8,10-12;6-7,9]"
*
* Note that the "pmix" at the beginning of each regex indicates
* that the PMIx native parser is to be used by the client for
* parsing the provided regex. Other parsers may be supported - see
* the pmix_client.h header for a list.
*/
typedef int (*opal_pmix_base_module_generate_ppn_fn_t)(const char *input, char **ppn);
/* Setup the data about a particular nspace so it can
* be passed to any child process upon startup. The PMIx
* connection procedure provides an opportunity for the
* host PMIx server to pass job-related info down to a
* child process. This might include the number of
* processes in the job, relative local ranks of the
* processes within the job, and other information of
* use to the process. The server is free to determine
* which, if any, of the supported elements it will
* provide - defined values are provided in pmix_common.h.
*
* NOTE: the server must register ALL nspaces that will
* participate in collective operations with local processes.
* This means that the server must register an nspace even
* if it will not host any local procs from within that
* nspace IF any local proc might at some point perform
* a collective operation involving one or more procs from
* that nspace. This is necessary so that the collective
* operation can know when it is locally complete.
*
* The caller must also provide the number of local procs
* that will be launched within this nspace. This is required
* for the PMIx server library to correctly handle collectives
* as a collective operation call can occur before all the
* procs have been started */
typedef int (*opal_pmix_base_module_server_register_nspace_fn_t)(opal_jobid_t jobid,
int nlocalprocs,
opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Deregister an nspace. Instruct the PMIx server to purge
* all info relating to the provided jobid so that memory
* can be freed. Note that the server will automatically
* purge all info relating to any clients it has from
* this nspace */
typedef void (*opal_pmix_base_module_server_deregister_nspace_fn_t)(opal_jobid_t jobid,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Register a client process with the PMIx server library. The
* expected user ID and group ID of the child process helps the
* server library to properly authenticate clients as they connect
* by requiring the two values to match.
*
* The host server can also, if it desires, provide an object
* it wishes to be returned when a server function is called
* that relates to a specific process. For example, the host
* server may have an object that tracks the specific client.
* Passing the object to the library allows the library to
* return that object when the client calls "finalize", thus
* allowing the host server to access the object without
* performing a lookup. */
typedef int (*opal_pmix_base_module_server_register_client_fn_t)(const opal_process_name_t *proc,
uid_t uid, gid_t gid,
void *server_object,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Deregister a client. Instruct the PMIx server to purge
* all info relating to the provided client so that memory
* can be freed. As per above note, the server will automatically
* free all client-related data when the nspace is deregistered,
* so there is no need to call this function during normal
* finalize operations. Instead, this is provided for use
* during exception operations */
typedef void (*opal_pmix_base_module_server_deregister_client_fn_t)(const opal_process_name_t *proc,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Setup the environment of a child process to be forked
* by the host so it can correctly interact with the PMIx
* server. The PMIx client needs some setup information
* so it can properly connect back to the server. This function
* will set appropriate environmental variables for this purpose. */
typedef int (*opal_pmix_base_module_server_setup_fork_fn_t)(const opal_process_name_t *proc, char ***env);
/* Define a function by which the host server can request modex data
* from the local PMIx server. This is used to support the direct modex
* operation - i.e., where data is cached locally on each PMIx
* server for its own local clients, and is obtained on-demand
* for remote requests. Upon receiving a request from a remote
* server, the host server will call this function to pass the
* request into the PMIx server. The PMIx server will return a blob
* (once it becomes available) via the cbfunc - the host
* server shall send the blob back to the original requestor */
typedef int (*opal_pmix_base_module_server_dmodex_request_fn_t)(const opal_process_name_t *proc,
opal_pmix_modex_cbfunc_t cbfunc,
void *cbdata);
/* Report an event to a process for notification via any
* registered event handler. The handler registration can be
* called by both the server and the client application. On the
* server side, the handler is used to report events detected
* by PMIx to the host server for handling. On the client side,
* the handler is used to notify the process of events
* reported by the server - e.g., the failure of another process.
*
* This function allows the host server to direct the server
* convenience library to notify all registered local procs of
* an event. The event can be local, or anywhere in the cluster.
* The status indicates the event being reported.
*
* The source parameter informs the handler of the source that
* generated the event. This will be NULL if the event came
* from the external resource manager.
*
* The info array contains any further info the RM can and/or chooses
* to provide.
*
* The callback function will be called upon completion of the
* notify_event function's actions. Note that any messages will
* have been queued, but may not have been transmitted by this
* time. Note that the caller is required to maintain the input
* data until the callback function has been executed if this
* function returns OPAL_SUCCESS! */
typedef int (*opal_pmix_base_module_server_notify_event_fn_t)(int status,
const opal_process_name_t *source,
opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* push IO to local clients */
typedef int (*opal_pmix_base_module_server_push_io_fn_t)(const opal_process_name_t *source,
opal_pmix_iof_channel_t channel,
unsigned char *data, size_t nbytes);
/* define a callback function for the setup_application API. The returned info
* array is owned by the PMIx server library and will be free'd when the
* provided cbfunc is called. */
typedef void (*opal_pmix_setup_application_cbfunc_t)(int status,
opal_list_t *info,
void *provided_cbdata,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* Provide a function by which we can request
* any application-specific environmental variables prior to
* launch of an application. For example, network libraries may
* opt to provide security credentials for the application. This
* is defined as a non-blocking operation in case network
* libraries need to perform some action before responding. The
* returned env will be distributed along with the application */
typedef int (*opal_pmix_server_setup_application_fn_t)(opal_jobid_t jobid,
opal_list_t *info,
opal_pmix_setup_application_cbfunc_t cbfunc, void *cbdata);
/* Provide a function by which the local PMIx server can perform
* any application-specific operations prior to spawning local
* clients of a given application. For example, a network library
* might need to setup the local driver for "instant on" addressing.
*/
typedef int (*opal_pmix_server_setup_local_support_fn_t)(opal_jobid_t jobid,
opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/************************************************************
* TOOL APIs *
************************************************************/
/* Initialize the PMIx tool support
* When called the library will check for the required connection
* information of the local server and will establish the connection.
* The connection info can be provided either in the environment or
* in the list of attributes. If the information is not found, or the
* server connection fails, then an appropriate error constant will
* be returned.
*/
typedef int (*opal_pmix_base_module_tool_init_fn_t)(opal_list_t *ilist);
/* Finalize the PMIx tool support */
typedef int (*opal_pmix_base_module_tool_fini_fn_t)(void);
/************************************************************
* UTILITY APIs *
************************************************************/
/* get the version of the embedded library */
typedef const char* (*opal_pmix_base_module_get_version_fn_t)(void);
/* Register an event handler to report event. Three types of events
* can be reported:
*
* (a) those that occur within the client library, but are not
* reportable via the API itself (e.g., loss of connection to
* the server). These events typically occur during behind-the-scenes
* non-blocking operations.
*
* (b) job-related events such as the failure of another process in
* the job or in any connected job, impending failure of hardware
* within the job's usage footprint, etc.
*
* (c) system notifications that are made available by the local
* administrators
*
* By default, only events that directly affect the process and/or
* any process to which it is connected (via the PMIx_Connect call)
* will be reported. Options to modify that behavior can be provided
* in the info array
*
* Both the client application and the resource manager can register
* event handlers for specific events. PMIx client/server calls the registered
* event handler upon receiving event notify notification (via PMIx_Notify_event)
* from the other end (Resource Manager/Client application).
*
* Multiple event handlers can be registered for different events. PMIX returns
* a size_t reference to each register handler in the callback fn. The caller
* must retain the reference in order to deregister the evhandler.
* Modification of the notification behavior can be accomplished by
* deregistering the current evhandler, and then registering it
* using a new set of info values.
*
* A NULL for event_codes indicates registration as a default event handler
*
* See pmix_types.h for a description of the notification function */
typedef void (*opal_pmix_base_module_register_fn_t)(opal_list_t *event_codes,
opal_list_t *info,
opal_pmix_notification_fn_t evhandler,
opal_pmix_evhandler_reg_cbfunc_t cbfunc,
void *cbdata);
/* deregister the evhandler
* evhandler_ref is the reference returned by PMIx for the evhandler
* to pmix_evhandler_reg_cbfunc_t */
typedef void (*opal_pmix_base_module_deregister_fn_t)(size_t evhandler,
opal_pmix_op_cbfunc_t cbfunc,
void *cbdata);
/* Report an event for notification via any
* registered evhandler. On the PMIx
* server side, this is used to report events detected
* by PMIx to the host server for handling and/or distribution.
*
* The client application can also call this function to notify the
* resource manager of an event it detected. It can specify the
* range over which that notification should occur.
*
* The info array contains any further info the caller can and/or chooses
* to provide.
*
* The callback function will be called upon completion of the
* notify_event function's actions. Note that any messages will
* have been queued, but may not have been transmitted by this
* time. Note that the caller is required to maintain the input
* data until the callback function has been executed if it
* returns OPAL_SUCCESS!
*/
typedef int (*opal_pmix_base_module_notify_event_fn_t)(int status,
const opal_process_name_t *source,
opal_pmix_data_range_t range,
opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* store data internally, but don't push it out to be shared - this is
* intended solely for storage of info on other procs that comes thru
* a non-PMIx channel (e.g., may be computed locally) but is desired
* to be available via a PMIx_Get call */
typedef int (*opal_pmix_base_module_store_fn_t)(const opal_process_name_t *proc,
opal_value_t *val);
/* retrieve the nspace corresponding to a given jobid */
typedef const char* (*opal_pmix_base_module_get_nspace_fn_t)(opal_jobid_t jobid);
/* register a jobid-to-nspace pair */
typedef void (*opal_pmix_base_module_register_jobid_fn_t)(opal_jobid_t jobid, const char *nspace);
/* query information from the system */
typedef void (*opal_pmix_base_module_query_fn_t)(opal_list_t *queries,
opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
/* log data to the system */
typedef void (*opal_pmix_base_log_fn_t)(opal_list_t *info,
opal_pmix_op_cbfunc_t cbfunc, void *cbdata);
/* allocation */
typedef int (*opal_pmix_base_alloc_fn_t)(opal_pmix_alloc_directive_t directive,
opal_list_t *info,
opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
/* job control */
typedef int (*opal_pmix_base_job_control_fn_t)(opal_list_t *targets,
opal_list_t *directives,
opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
/* monitoring */
typedef int (*opal_pmix_base_process_monitor_fn_t)(opal_list_t *monitor,
opal_list_t *directives,
opal_pmix_info_cbfunc_t cbfunc, void *cbdata);
/* register cleanup */
typedef int (*opal_pmix_base_register_cleanup_fn_t)(char *path, bool directory, bool ignore, bool jobscope);
typedef bool (*opal_pmix_base_legacy_get_fn_t)(void);
/*
* the standard public API data structure
*/
typedef struct {
opal_pmix_base_legacy_get_fn_t legacy_get;
/* client APIs */
opal_pmix_base_module_init_fn_t init;
opal_pmix_base_module_fini_fn_t finalize;
opal_pmix_base_module_initialized_fn_t initialized;
opal_pmix_base_module_abort_fn_t abort;
opal_pmix_base_module_commit_fn_t commit;
opal_pmix_base_module_fence_fn_t fence;
opal_pmix_base_module_fence_nb_fn_t fence_nb;
opal_pmix_base_module_put_fn_t put;
opal_pmix_base_module_get_fn_t get;
opal_pmix_base_module_get_nb_fn_t get_nb;
opal_pmix_base_module_publish_fn_t publish;
opal_pmix_base_module_publish_nb_fn_t publish_nb;
opal_pmix_base_module_lookup_fn_t lookup;
opal_pmix_base_module_lookup_nb_fn_t lookup_nb;
opal_pmix_base_module_unpublish_fn_t unpublish;
opal_pmix_base_module_unpublish_nb_fn_t unpublish_nb;
opal_pmix_base_module_spawn_fn_t spawn;
opal_pmix_base_module_spawn_nb_fn_t spawn_nb;
opal_pmix_base_module_connect_fn_t connect;
opal_pmix_base_module_connect_nb_fn_t connect_nb;
opal_pmix_base_module_disconnect_fn_t disconnect;
opal_pmix_base_module_disconnect_nb_fn_t disconnect_nb;
opal_pmix_base_module_resolve_peers_fn_t resolve_peers;
opal_pmix_base_module_resolve_nodes_fn_t resolve_nodes;
opal_pmix_base_module_query_fn_t query;
opal_pmix_base_log_fn_t log;
opal_pmix_base_alloc_fn_t allocate;
opal_pmix_base_job_control_fn_t job_control;
opal_pmix_base_process_monitor_fn_t monitor;
opal_pmix_base_register_cleanup_fn_t register_cleanup;
/* server APIs */
opal_pmix_base_module_server_init_fn_t server_init;
opal_pmix_base_module_server_finalize_fn_t server_finalize;
opal_pmix_base_module_generate_regex_fn_t generate_regex;
opal_pmix_base_module_generate_ppn_fn_t generate_ppn;
opal_pmix_base_module_server_register_nspace_fn_t server_register_nspace;
opal_pmix_base_module_server_deregister_nspace_fn_t server_deregister_nspace;
opal_pmix_base_module_server_register_client_fn_t server_register_client;
opal_pmix_base_module_server_deregister_client_fn_t server_deregister_client;
opal_pmix_base_module_server_setup_fork_fn_t server_setup_fork;
opal_pmix_base_module_server_dmodex_request_fn_t server_dmodex_request;
opal_pmix_base_module_server_notify_event_fn_t server_notify_event;
opal_pmix_base_module_server_push_io_fn_t server_iof_push;
opal_pmix_server_setup_application_fn_t server_setup_application;
opal_pmix_server_setup_local_support_fn_t server_setup_local_support;
/* tool APIs */
opal_pmix_base_module_tool_init_fn_t tool_init;
opal_pmix_base_module_tool_fini_fn_t tool_finalize;
/* Utility APIs */
opal_pmix_base_module_get_version_fn_t get_version;
opal_pmix_base_module_register_fn_t register_evhandler;
opal_pmix_base_module_deregister_fn_t deregister_evhandler;
opal_pmix_base_module_notify_event_fn_t notify_event;
opal_pmix_base_module_store_fn_t store_local;
opal_pmix_base_module_get_nspace_fn_t get_nspace;
opal_pmix_base_module_register_jobid_fn_t register_jobid;
} opal_pmix_base_module_t;
typedef struct {
mca_base_component_t base_version;
mca_base_component_data_t base_data;
int priority;
} opal_pmix_base_component_t;
/*
* Macro for use in components that are of type pmix
*/
#define OPAL_PMIX_BASE_VERSION_2_0_0 \
OPAL_MCA_BASE_VERSION_2_1_0("pmix", 2, 0, 0)
/* Global structure for accessing store functions */
OPAL_DECLSPEC extern opal_pmix_base_module_t opal_pmix; /* holds base function pointers */
END_C_DECLS
#endif