Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move help text output regarding PSM2_CUDA environment variable #4323

Merged
merged 1 commit into from
Oct 27, 2017
Merged

Move help text output regarding PSM2_CUDA environment variable #4323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions ompi/mca/mtl/psm2/help-mtl-psm2.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# -*- text -*-
#
# Copyright (C) 2009. QLogic Corporation. All rights reserved.
# Copyright (c) 2013-2015 Intel, Inc. All rights reserved.
# Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
Expand Down Expand Up @@ -47,5 +47,17 @@ Unknown path record query mechanism %s. Supported mechanisms are %s.
Message size %llu bigger than supported by PSM2 API. Max = %llu
#
[no psm2 cuda env]
Using CUDA enabled OpenMPI but PSM2_CUDA environment variable is %s.
This is not a recommended combination. If the application uses %s.
Warning: Open MPI has detected that you are running in an environment with CUDA
devices present and that you are using Intel(r) Ompi-Path networking. However,
the environment variable PSM2_CUDA was not set, meaning that the PSM2 Omni-Path
networking library was not told how to handle CUDA support.

If your application uses CUDA buffers, you should set the environment variable
PSM2_CUDA to 1; otherwise, set it to 0. Setting the variable to the wrong value
can have performance implications on your application, or even cause it to
crash.

Since it was not set, Open MPI has defaulted to setting the PSM2_CUDA
environment variable to 1.

Local hostname: %s
72 changes: 40 additions & 32 deletions ompi/mca/mtl/psm2/mtl_psm2_component.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
* Copyright (c) 2006-2010 QLogic Corporation. All rights reserved.
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2013-2015 Intel, Inc. All rights reserved
* Copyright (c) 2013-2017 Intel, Inc. All rights reserved
* Copyright (c) 2017 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
Expand All @@ -28,6 +28,7 @@
#include "opal/mca/event/event.h"
#include "opal/util/output.h"
#include "opal/util/show_help.h"
#include "opal/util/opal_environ.h"
#include "ompi/proc/proc.h"

#include "mtl_psm2.h"
Expand All @@ -45,6 +46,10 @@ static int param_priority;
/* MPI_THREAD_MULTIPLE_SUPPORT */
opal_mutex_t mtl_psm2_mq_mutex = OPAL_MUTEX_STATIC_INIT;

#if OPAL_CUDA_SUPPORT
static bool cuda_envvar_set = false;
#endif

static int ompi_mtl_psm2_component_open(void);
static int ompi_mtl_psm2_component_close(void);
static int ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority);
Expand Down Expand Up @@ -201,9 +206,6 @@ static int
ompi_mtl_psm2_component_register(void)
{
int num_local_procs, num_total_procs;
#if OPAL_CUDA_SUPPORT
char *cuda_env;
#endif

ompi_mtl_psm2.connect_timeout = 180;
(void) mca_base_component_var_register(&mca_mtl_psm2_component.super.mtl_version,
Expand All @@ -228,30 +230,6 @@ ompi_mtl_psm2_component_register(void)
param_priority = 40;
}

#if OPAL_CUDA_SUPPORT
/*
* If using CUDA enabled OpenMPI, the user likely intends to
* run with CUDA buffers. So, force-set the envvar here if user failed
* to set it.
*/
cuda_env = getenv("PSM2_CUDA");
if (!cuda_env) {
opal_show_help("help-mtl-psm2.txt",
"no psm2 cuda env", true,
"not set",
"Host buffers,\nthere will be a performance penalty"
" due to OMPI force setting this variable now.\n"
"Set environment variable to 0 if using Host buffers" );
setenv("PSM2_CUDA", "1", 0);
} else if (strcmp(cuda_env, "0") == 0) {
opal_show_help("help-mtl-psm2.txt",
"no psm2 cuda env", true,
"set to 0",
"CUDA buffers,\nthe execution will SEGFAULT."
" Set environment variable to 1 if using CUDA buffers");
}
#endif

(void) mca_base_component_var_register (&mca_mtl_psm2_component.super.mtl_version,
"priority", "Priority of the PSM2 MTL component",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
Expand All @@ -272,17 +250,16 @@ static int
ompi_mtl_psm2_component_open(void)
{
int res;
glob_t globbuf;
globbuf.gl_offs = 0;
glob_t globbuf = {0};

/* Component available only if Omni-Path hardware is present */
res = glob("/dev/hfi1_[0-9]", GLOB_DOOFFS, NULL, &globbuf);
if (0 == res || GLOB_NOMATCH == res) {
if (globbuf.gl_pathc > 0) {
globfree(&globbuf);
}
if (0 != res) {
res = glob("/dev/hfi1_[0-9][0-9]", GLOB_APPEND, NULL, &globbuf);
if (0 == res || GLOB_NOMATCH == res) {
if (globbuf.gl_pathc > 0) {
globfree(&globbuf);
}
if (0 != res) {
Expand Down Expand Up @@ -336,6 +313,11 @@ ompi_mtl_psm2_component_query(mca_base_module_t **module, int *priority)
static int
ompi_mtl_psm2_component_close(void)
{
#if OPAL_CUDA_SUPPORT
if (cuda_envvar_set) {
opal_unsetenv("PSM2_CUDA", &environ);
}
#endif
return OMPI_SUCCESS;
}

Expand All @@ -362,6 +344,11 @@ ompi_mtl_psm2_component_init(bool enable_progress_threads,
int verno_major = PSM2_VERNO_MAJOR;
int verno_minor = PSM2_VERNO_MINOR;
int local_rank = -1, num_local_procs = 0;
#if OPAL_CUDA_SUPPORT
int ret;
char *cuda_env;
glob_t globbuf = {0};
#endif

/* Compute the total number of processes on this host and our local rank
* on that node. We need to provide PSM2 with these values so it can
Expand Down Expand Up @@ -389,6 +376,27 @@ ompi_mtl_psm2_component_init(bool enable_progress_threads,
ompi_mtl_psm2_set_shadow_env (ompi_mtl_psm2_shadow_variables + i);
}

#if OPAL_CUDA_SUPPORT
/*
* If using CUDA enabled Open MPI, the user likely intends to
* run with CUDA buffers. So, force-set the envvar here if user failed
* to set it.
*/
ret = glob("/sys/module/nvidia", GLOB_DOOFFS, NULL, &globbuf);
if (globbuf.gl_pathc > 0) {
globfree(&globbuf);
}

cuda_env = getenv("PSM2_CUDA");
if (!cuda_env && (0 == ret)) {
opal_show_help("help-mtl-psm2.txt",
"no psm2 cuda env", true,
ompi_process_info.nodename);
opal_setenv("PSM2_CUDA", "1", false, &environ);
cuda_envvar_set = true;
}
#endif

err = psm2_init(&verno_major, &verno_minor);
if (err) {
opal_show_help("help-mtl-psm2.txt",
Expand Down