/*
* Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved
* Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef OPAL_HWLOC_BASE_H
#define OPAL_HWLOC_BASE_H
#include "opal_config.h"
#include "opal/dss/dss_types.h"
#include "opal/mca/hwloc/hwloc-internal.h"
#if HWLOC_API_VERSION < 0x20000
#define HWLOC_OBJ_L3CACHE HWLOC_OBJ_CACHE
#define HWLOC_OBJ_L2CACHE HWLOC_OBJ_CACHE
#define HWLOC_OBJ_L1CACHE HWLOC_OBJ_CACHE
#endif
/*
* Global functions for MCA overall hwloc open and close
*/
BEGIN_C_DECLS
/* ******************************************************************** */
/**
* Note that the open function does NOT fill the global variable
* opal_hwloc_topology, nor does it set the process-wide memory
* affinity policy. Filling opal_hwloc_topology via
* hwloc_topology_load() can be expensive (and/or serialized by the
* OS); it may not be desireable to call this function in every MPI
* process on a machine. Hence, it is the responsibility for an upper
* layer to both fill opal_hwloc_topology in some scalable way, as
* well as to invoke opal_hwloc_base_set_process_membind_policy()
* (after opal_hwloc_topology has been loaded) to set the process-wide
* memory affinity policy.
*/
/**
* Debugging output stream
*/
OPAL_DECLSPEC extern bool opal_hwloc_base_inited;
OPAL_DECLSPEC extern bool opal_hwloc_topology_inited;
OPAL_DECLSPEC extern mca_base_framework_t opal_hwloc_base_framework;
/* we always must have some minimal locality support */
#define OPAL_HWLOC_PRINT_MAX_SIZE 50
#define OPAL_HWLOC_PRINT_NUM_BUFS 16
typedef struct {
char *buffers[OPAL_HWLOC_PRINT_NUM_BUFS];
int cntr;
} opal_hwloc_print_buffers_t;
opal_hwloc_print_buffers_t *opal_hwloc_get_print_buffer(void);
extern char* opal_hwloc_print_null;
OPAL_DECLSPEC char* opal_hwloc_base_print_locality(opal_hwloc_locality_t locality);
OPAL_DECLSPEC extern char *opal_hwloc_base_cpu_list;
OPAL_DECLSPEC extern hwloc_cpuset_t opal_hwloc_base_given_cpus;
OPAL_DECLSPEC extern char *opal_hwloc_base_topo_file;
/* convenience macro for debugging */
#define OPAL_HWLOC_SHOW_BINDING(n, v, t) \
do { \
char tmp1[1024]; \
hwloc_cpuset_t bind; \
bind = opal_hwloc_alloc(); \
if (hwloc_get_cpubind(t, bind, \
HWLOC_CPUBIND_PROCESS) < 0) { \
opal_output_verbose(n, v, \
"CANNOT DETERMINE BINDING AT %s:%d", \
__FILE__, __LINE__); \
} else { \
opal_hwloc_base_cset2mapstr(tmp1, sizeof(tmp1), t, bind); \
opal_output_verbose(n, v, \
"BINDINGS AT %s:%d: %s", \
__FILE__, __LINE__, tmp1); \
} \
hwloc_bitmap_free(bind); \
} while(0);
#if HWLOC_API_VERSION < 0x20000
#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level) \
do { \
obj = HWLOC_OBJ_CACHE; \
cache_level = level; \
} while(0)
#else
#define OPAL_HWLOC_MAKE_OBJ_CACHE(level, obj, cache_level) \
do { \
obj = HWLOC_OBJ_L##level##CACHE; \
cache_level = 0; \
} while(0)
#endif
OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_base_get_relative_locality(hwloc_topology_t topo,
char *cpuset1, char *cpuset2);
OPAL_DECLSPEC int opal_hwloc_base_set_binding_policy(opal_binding_policy_t *policy, char *spec);
/**
* Loads opal_hwloc_my_cpuset (global variable in
* opal/mca/hwloc/hwloc-internal.h) for this process. opal_hwloc_my_cpuset
* will be loaded with this process' binding, or, if the process is
* not bound, use the hwloc root object's (available and online)
* cpuset.
*/
OPAL_DECLSPEC void opal_hwloc_base_get_local_cpuset(void);
struct opal_rmaps_numa_node_t {
opal_list_item_t super;
int index;
float dist_from_closed;
};
typedef struct opal_rmaps_numa_node_t opal_rmaps_numa_node_t;
OBJ_CLASS_DECLARATION(opal_rmaps_numa_node_t);
/**
* Enum for what memory allocation policy we want for user allocations.
* MAP = memory allocation policy.
*/
typedef enum {
OPAL_HWLOC_BASE_MAP_NONE,
OPAL_HWLOC_BASE_MAP_LOCAL_ONLY
} opal_hwloc_base_map_t;
/**
* Global reflecting the MAP (set by MCA param).
*/
OPAL_DECLSPEC extern opal_hwloc_base_map_t opal_hwloc_base_map;
/**
* Enum for what to do if the hwloc framework tries to bind memory
* and fails. BFA = bind failure action.
*/
typedef enum {
OPAL_HWLOC_BASE_MBFA_SILENT,
OPAL_HWLOC_BASE_MBFA_WARN,
OPAL_HWLOC_BASE_MBFA_ERROR
} opal_hwloc_base_mbfa_t;
/**
* Global reflecting the BFA (set by MCA param).
*/
OPAL_DECLSPEC extern opal_hwloc_base_mbfa_t opal_hwloc_base_mbfa;
/**
* Discover / load the hwloc topology (i.e., call hwloc_topology_init() and
* hwloc_topology_load()).
*/
OPAL_DECLSPEC int opal_hwloc_base_get_topology(void);
/**
* Set the hwloc topology to that from the given topo file
*/
OPAL_DECLSPEC int opal_hwloc_base_set_topology(char *topofile);
OPAL_DECLSPEC int opal_hwloc_base_filter_cpus(hwloc_topology_t topo);
/**
* Free the hwloc topology.
*/
OPAL_DECLSPEC void opal_hwloc_base_free_topology(hwloc_topology_t topo);
OPAL_DECLSPEC unsigned int opal_hwloc_base_get_nbobjs_by_type(hwloc_topology_t topo,
hwloc_obj_type_t target,
unsigned cache_level,
opal_hwloc_resource_type_t rtype);
OPAL_DECLSPEC void opal_hwloc_base_clear_usage(hwloc_topology_t topo);
OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_obj_by_type(hwloc_topology_t topo,
hwloc_obj_type_t target,
unsigned cache_level,
unsigned int instance,
opal_hwloc_resource_type_t rtype);
OPAL_DECLSPEC unsigned int opal_hwloc_base_get_obj_idx(hwloc_topology_t topo,
hwloc_obj_t obj,
opal_hwloc_resource_type_t rtype);
OPAL_DECLSPEC int opal_hwloc_get_sorted_numa_list(hwloc_topology_t topo,
char* device_name,
opal_list_t *sorted_list);
/**
* Get the number of pu's under a given hwloc object.
*/
OPAL_DECLSPEC unsigned int opal_hwloc_base_get_npus(hwloc_topology_t topo,
hwloc_obj_t target);
OPAL_DECLSPEC char* opal_hwloc_base_print_binding(opal_binding_policy_t binding);
/**
* Determine if there is a single cpu in a bitmap.
*/
OPAL_DECLSPEC bool opal_hwloc_base_single_cpu(hwloc_cpuset_t cpuset);
/**
* Provide a utility to parse a slot list against the local
* cpus of given type, and produce a cpuset for the described binding
*/
OPAL_DECLSPEC int opal_hwloc_base_cpu_list_parse(const char *slot_str,
hwloc_topology_t topo,
opal_hwloc_resource_type_t rtype,
hwloc_cpuset_t cpumask);
OPAL_DECLSPEC char* opal_hwloc_base_find_coprocessors(hwloc_topology_t topo);
OPAL_DECLSPEC char* opal_hwloc_base_check_on_coprocessor(void);
/**
* Report a bind failure using the normal mechanisms if a component
* fails to bind memory -- according to the value of the
* hwloc_base_bind_failure_action MCA parameter.
*/
OPAL_DECLSPEC int opal_hwloc_base_report_bind_failure(const char *file,
int line,
const char *msg,
int rc);
/**
* This function sets the process-wide memory affinity policy
* according to opal_hwloc_base_map and opal_hwloc_base_mbfa. It needs
* to be a separate, standalone function (as opposed to being done
* during opal_hwloc_base_open()) because opal_hwloc_topology is not
* loaded by opal_hwloc_base_open(). Hence, an upper layer needs to
* invoke this function after opal_hwloc_topology has been loaded.
*/
OPAL_DECLSPEC int opal_hwloc_base_set_process_membind_policy(void);
OPAL_DECLSPEC int opal_hwloc_base_membind(opal_hwloc_base_memory_segment_t *segs,
size_t count, int node_id);
OPAL_DECLSPEC int opal_hwloc_base_node_name_to_id(char *node_name, int *id);
OPAL_DECLSPEC int opal_hwloc_base_memory_set(opal_hwloc_base_memory_segment_t *segments,
size_t num_segments);
/* datatype support */
OPAL_DECLSPEC int opal_hwloc_pack(opal_buffer_t *buffer, const void *src,
int32_t num_vals,
opal_data_type_t type);
OPAL_DECLSPEC int opal_hwloc_unpack(opal_buffer_t *buffer, void *dest,
int32_t *num_vals,
opal_data_type_t type);
OPAL_DECLSPEC int opal_hwloc_copy(hwloc_topology_t *dest,
hwloc_topology_t src,
opal_data_type_t type);
OPAL_DECLSPEC int opal_hwloc_compare(const hwloc_topology_t topo1,
const hwloc_topology_t topo2,
opal_data_type_t type);
OPAL_DECLSPEC int opal_hwloc_print(char **output, char *prefix,
hwloc_topology_t src,
opal_data_type_t type);
/**
* Make a prettyprint string for a hwloc_cpuset_t (e.g., "socket
* 2[core 3]").
*/
OPAL_DECLSPEC int opal_hwloc_base_cset2str(char *str, int len,
hwloc_topology_t topo,
hwloc_cpuset_t cpuset);
/**
* Make a prettyprint string for a cset in a map format.
* Example: [B./..]
* Key: [] - signifies socket
* / - divider between cores
* . - signifies PU a process not bound to
* B - signifies PU a process is bound to
*/
OPAL_DECLSPEC int opal_hwloc_base_cset2mapstr(char *str, int len,
hwloc_topology_t topo,
hwloc_cpuset_t cpuset);
/* get the hwloc object that corresponds to the given processor id and type */
OPAL_DECLSPEC hwloc_obj_t opal_hwloc_base_get_pu(hwloc_topology_t topo,
int lid,
opal_hwloc_resource_type_t rtype);
/* get the topology "signature" so we can check for differences - caller
* if responsible for freeing the returned string */
OPAL_DECLSPEC char* opal_hwloc_base_get_topo_signature(hwloc_topology_t topo);
/* get a string describing the locality of a given process */
OPAL_DECLSPEC char* opal_hwloc_base_get_locality_string(hwloc_topology_t topo, char *bitmap);
/* extract a location from the locality string */
OPAL_DECLSPEC char* opal_hwloc_base_get_location(char *locality,
hwloc_obj_type_t type,
unsigned index);
OPAL_DECLSPEC opal_hwloc_locality_t opal_hwloc_compute_relative_locality(char *loc1, char *loc2);
OPAL_DECLSPEC int opal_hwloc_base_topology_export_xmlbuffer(hwloc_topology_t topology, char **xmlpath, int *buflen);
OPAL_DECLSPEC int opal_hwloc_base_topology_set_flags (hwloc_topology_t topology, unsigned long flags, bool io);
END_C_DECLS
#endif /* OPAL_HWLOC_BASE_H */