Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature 1320 OMP_NUM_THREADS #1338

Merged
merged 2 commits into from
Jan 11, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docs/Users_Guide/glossary.rst
Original file line number Diff line number Diff line change
Expand Up @@ -8693,3 +8693,13 @@ METplus Configuration Glossary
:term:`VALID_TIME_FMT` or they will be skipped.

| *Used by:* All

OMP_NUM_THREADS
Sets environment variable of the same name that determines the number
of threads to use in the MET executables. Defaults to 1 thread.
If the environment variable of the same name is already set in the
user's environment, then that value will be used instead of the value
set in the METplus configuration. A warning will be output if this is the
case and the values differ between them.

| *Used by:* All
7 changes: 7 additions & 0 deletions docs/Users_Guide/systemconfiguration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,13 @@ By default this is a directory called **stage** inside the

This value is rarely changed, but it can be if desired.

OMP_NUM_THREADS
^^^^^^^^^^^^^^^

If the MET executables were installed with threading support, then the number
of threads used by the tools can be configured with this variable. See
the glossary entry for :term:`OMP_NUM_THREADS` for more information.

CONVERT
^^^^^^^

Expand Down
43 changes: 30 additions & 13 deletions metplus/util/met_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ def pre_run_setup(config_inputs):
# handle dir to write temporary files
handle_tmp_dir(config)

# handle OMP_NUM_THREADS environment variable
handle_env_var_config(config,
env_var_name='OMP_NUM_THREADS',
config_name='OMP_NUM_THREADS')

config.env = os.environ.copy()

return config
Expand Down Expand Up @@ -230,19 +235,7 @@ def handle_tmp_dir(config):
get config temp dir using getdir_nocheck to bypass check for /path/to
this is done so the user can set env MET_TMP_DIR instead of config TMP_DIR
and config TMP_DIR will be set automatically"""
met_tmp_dir = os.environ.get('MET_TMP_DIR', '')
conf_tmp_dir = config.getdir_nocheck('TMP_DIR', '')

# if env MET_TMP_DIR is set
if met_tmp_dir:
# override config TMP_DIR to env MET_TMP_DIR value
config.set('config', 'TMP_DIR', met_tmp_dir)

# if config TMP_DIR differed from env MET_TMP_DIR, warn
if conf_tmp_dir != met_tmp_dir:
msg = 'TMP_DIR in config will be overridden by the ' +\
'environment variable MET_TMP_DIR ({})'.format(met_tmp_dir)
config.logger.warning(msg)
handle_env_var_config(config, 'MET_TMP_DIR', 'TMP_DIR')

# create temp dir if it doesn't exist already
# this will fail if TMP_DIR is not set correctly and
Expand All @@ -251,6 +244,30 @@ def handle_tmp_dir(config):
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)

def handle_env_var_config(config, env_var_name, config_name):
"""! If environment variable is set, use that value
for the config variable and warn if the previous config value differs

@param config METplusConfig object to read
@param env_var_name name of environment variable to read
@param config_name name of METplus config variable to check
"""
env_var_value = os.environ.get(env_var_name, '')
config_value = config.getdir_nocheck(config_name, '')

# do nothing if environment variable is not set
if not env_var_value:
return

# override config config variable to environment variable value
config.set('config', config_name, env_var_value)

# if config config value differed from environment variable value, warn
if config_value != env_var_value:
config.logger.warning(f'Config variable {config_name} ({config_value}) '
'will be overridden by the environment variable '
f'{env_var_name} ({env_var_value})')

def get_skip_times(config, wrapper_name=None):
"""! Read SKIP_TIMES config variable and populate dictionary of times that should be skipped.
SKIP_TIMES should be in the format: "%m:begin_end_incr(3,11,1)", "%d:30,31", "%Y%m%d:20201031"
Expand Down
6 changes: 6 additions & 0 deletions metplus/wrappers/command_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ def __init__(self, config, instance=None, config_overrides=None):
# list of environment variables to set before running command
self.env_var_keys = [
'MET_TMP_DIR',
'OMP_NUM_THREADS',
]
if hasattr(self, 'WRAPPER_ENV_VAR_KEYS'):
self.env_var_keys.extend(self.WRAPPER_ENV_VAR_KEYS)
Expand Down Expand Up @@ -117,6 +118,11 @@ def __init__(self, config, instance=None, config_overrides=None):
# where the MET tools write temporary files
self.env_var_dict['MET_TMP_DIR'] = self.config.getdir('TMP_DIR')

# set OMP_NUM_THREADS environment variable
self.env_var_dict['OMP_NUM_THREADS'] = (
self.config.getstr('config', 'OMP_NUM_THREADS')
)

self.check_for_externals()

self.cmdrunner = CommandRunner(
Expand Down
5 changes: 5 additions & 0 deletions parm/metplus_config/defaults.conf
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,18 @@ GFDL_TRACKER_EXEC = /path/to/standalone_gfdl-vortextracker_v3.9a/trk_exec

###############################################################################
# Runtime Configuration #
# * OMP_NUM_THREADS sets an environment variable of the same name that #
# determines the number of threads to use in the MET executables. If the #
# environment variable is already set in the user's environment, then #
# that value will be used instead of the value set in this file. #
###############################################################################


LOOP_ORDER = processes

PROCESS_LIST = Usage

OMP_NUM_THREADS = 1

###############################################################################
# Log File Information (Where to write logs files) #
Expand Down