Skip to content

Commit

Permalink
Merge pull request #436 from climbfuji/ccpp_framework_stub_merged_202…
Browse files Browse the repository at this point in the history 
…20202

ccpp_prebuild: ccpp stub and various bugfixes
  • Loading branch information
@climbfuji
climbfuji authored Jun 9, 2022
2 parents 32afb97 + e107542 commit 167313e
Show file tree
Hide file tree
Showing 12 changed files with 443 additions and 52 deletions.
82 changes: 67 additions & 15 deletions scripts/ccpp_prebuild.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# Standard modules
import argparse
import collections
import copy
import filecmp
import importlib
import itertools
Expand All @@ -23,6 +24,7 @@
TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile
from mkdoc import metadata_to_html, metadata_to_latex
from mkstatic import API, Suite, Group
from mkstatic import CCPP_SUITE_VARIABLES

###############################################################################
# Set up the command line argument parser and other global variables #
Expand All @@ -35,6 +37,7 @@
parser.add_argument('--debug', action='store_true', help='enable debugging features in auto-generated code', default=False)
parser.add_argument('--suites', action='store', help='suite definition files to use (comma-separated, without path)', default='')
parser.add_argument('--builddir', action='store', help='relative path to CCPP build directory', required=False, default=None)
parser.add_argument('--namespace', action='store', help='namespace suffix to be added to the name of static api module', required=False, default='')

# BASEDIR is the current directory where this script is executed
BASEDIR = os.getcwd()
Expand All @@ -56,7 +59,8 @@ def parse_arguments():
else:
sdfs = None
builddir = args.builddir
return (success, configfile, clean, verbose, debug, sdfs, builddir)
namespace = args.namespace
return (success, configfile, clean, verbose, debug, sdfs, builddir, namespace)

def import_config(configfile, builddir):
"""Import the configuration from a given configuration file"""
Expand Down Expand Up @@ -95,13 +99,14 @@ def import_config(configfile, builddir):
config['caps_cmakefile'] = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir)
config['caps_sourcefile'] = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir)
config['caps_dir'] = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir)
config['suites_dir'] = ccpp_prebuild_config.SUITES_DIR
config['suites_dir'] = ccpp_prebuild_config.SUITES_DIR.format(build_dir=builddir)
config['host_model'] = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER
config['html_vartable_file'] = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir)
config['latex_vartable_file'] = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir)
# Location of static API file, and shell script to source
# Location of static API file, shell script to source, cmake include file
config['static_api_dir'] = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir)
config['static_api_srcfile'] = ccpp_prebuild_config.STATIC_API_SRCFILE.format(build_dir=builddir)
config['static_api_sourcefile'] = ccpp_prebuild_config.STATIC_API_SOURCEFILE.format(build_dir=builddir)
config['static_api_cmakefile'] = ccpp_prebuild_config.STATIC_API_CMAKEFILE.format(build_dir=builddir)

# To handle new metadata: import DDT references (if exist)
try:
Expand All @@ -127,10 +132,14 @@ def setup_logging(verbose):
logging.info('Logging level set to INFO')
return success

def clean_files(config):
def clean_files(config, namespace):
"""Clean files created by ccpp_prebuild.py"""
success = True
logging.info('Performing clean ....')
if namespace:
static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE+'_'+namespace)
else:
static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)
# Create list of files to remove, use wildcards where necessary
files_to_remove = [
config['typedefs_makefile'],
Expand All @@ -145,8 +154,8 @@ def clean_files(config):
config['html_vartable_file'],
config['latex_vartable_file'],
os.path.join(config['caps_dir'], 'ccpp_*_cap.F90'),
os.path.join(config['static_api_dir'], '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)),
config['static_api_srcfile'],
os.path.join(config['static_api_dir'], static_api_file),
config['static_api_sourcefile'],
]
# Not very pythonic, but the easiest way w/o importing another Python module
cmd = 'rm -vf {0}'.format(' '.join(files_to_remove))
Expand Down Expand Up @@ -329,6 +338,19 @@ def collect_physics_subroutines(scheme_files):
os.chdir(BASEDIR)
return (success, metadata_request, arguments_request, dependencies_request, schemes_in_files)

def check_schemes_in_suites(arguments, suites):
"""Check that all schemes that are requested in the suites exist"""
success = True
logging.info("Checking for existence of schemes in suites ...")
for suite in suites:
for group in suite.groups:
for subcycle in group.subcycles:
for scheme_name in subcycle.schemes:
if not scheme_name in arguments.keys():
success = False
logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name))
return success

def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites):
"""Remove all variables from metadata that are not used in the given suite;
also remove information on argument lists, dependencies and schemes in files"""
Expand Down Expand Up @@ -376,6 +398,16 @@ def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites)
schemes_in_files_filtered[scheme] = schemes_in_files[scheme]
return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered)

def add_ccpp_suite_variables(metadata):
""" Add variables that are required to construct CCPP suites to the list of requested variables"""
success = True
logging.info("Adding CCPP suite variables to list of requested variables")
for var_name in CCPP_SUITE_VARIABLES.keys():
if not var_name in metadata.keys():
metadata[var_name] = [copy.deepcopy(CCPP_SUITE_VARIABLES[var_name])]
logging.debug("Adding CCPP suite variable {0} to list of requested variables".format(var_name))
return (success, metadata)

def generate_list_of_schemes_and_dependencies_to_compile(schemes_in_files, dependencies1, dependencies2):
"""Generate a flat list of schemes and dependencies in two dependency dictionaries to compile"""
success = True
Expand Down Expand Up @@ -489,14 +521,20 @@ def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arg
success = False
return (success, suite_and_group_caps)

def generate_static_api(suites, static_api_dir):
def generate_static_api(suites, static_api_dir, namespace):
"""Generate static API for given suite(s)"""
success = True
# Change to caps directory, create if necessary
if not os.path.isdir(static_api_dir):
os.makedirs(static_api_dir)
os.chdir(static_api_dir)
api = API(suites=suites, directory=static_api_dir)
if namespace:
base = os.path.splitext(os.path.basename(api.filename))[0]
logging.info('Static API file name is ''{}'''.format(api.filename))
api.filename = base+'_'+namespace+'.F90'
api.module = base+'_'+namespace
logging.info('Static API file name is changed to ''{}'''.format(api.filename))
logging.info('Generating static API {0} in {1} ...'.format(api.filename, static_api_dir))
api.write()
os.chdir(BASEDIR)
Expand Down Expand Up @@ -655,7 +693,7 @@ def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile,
def main():
"""Main routine that handles the CCPP prebuild for different host models."""
# Parse command line arguments
(success, configfile, clean, verbose, debug, sdfs, builddir) = parse_arguments()
(success, configfile, clean, verbose, debug, sdfs, builddir, namespace) = parse_arguments()
if not success:
raise Exception('Call to parse_arguments failed.')

Expand All @@ -669,7 +707,7 @@ def main():

# Perform clean if requested, then exit
if clean:
success = clean_files(config)
success = clean_files(config, namespace)
logging.info('CCPP prebuild clean completed successfully, exiting.')
sys.exit(0)

Expand Down Expand Up @@ -699,12 +737,22 @@ def main():
if not success:
raise Exception('Call to collect_physics_subroutines failed.')

# Check that the schemes requested in the suites exist
success = check_schemes_in_suites(arguments_request, suites)
if not success:
raise Exception('Call to check_schemes_in_suites failed.')

# Filter metadata/arguments - remove whatever is not included in suite definition files
(success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = filter_metadata(
metadata_request, arguments_request, dependencies_request, schemes_in_files, suites)
if not success:
raise Exception('Call to filter_metadata failed.')

# Add variables that are required to construct CCPP suites to the list of requested variables
(success, metadata_request) = add_ccpp_suite_variables(metadata_request)
if not success:
raise Exception('Call to add_ccpp_suite_variables failed.')

(success, schemes_and_dependencies_to_compile) = generate_list_of_schemes_and_dependencies_to_compile(
schemes_in_files, dependencies_request, dependencies_define)
if not success:
Expand Down Expand Up @@ -739,13 +787,17 @@ def main():
if not success:
raise Exception('Call to generate_suite_and_group_caps failed.')

(success, api) = generate_static_api(suites, config['static_api_dir'])
if not success:
(success, api) = generate_static_api(suites, config['static_api_dir'], namespace)
if not success:
raise Exception('Call to generate_static_api failed.')

success = api.write_sourcefile(config['static_api_srcfile'])
if not success:
raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_srcfile']))
success = api.write_includefile(config['static_api_sourcefile'], type='shell')
if not success:
raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_sourcefile']))

success = api.write_includefile(config['static_api_cmakefile'], type='cmake')
if not success:
raise Exception("Writing API cmakefile {cmakefile} failed".format(cmakefile=config['static_api_cmakefile']))

# Add filenames of caps to makefile/cmakefile/shell script
all_caps = suite_and_group_caps
Expand Down
45 changes: 23 additions & 22 deletions scripts/metadata_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from xml.etree import ElementTree as ET

from common import encode_container, CCPP_STAGES
from common import CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE
from mkcap import Var

sys.path.append(os.path.join(os.path.split(__file__)[0], 'fortran_tools'))
Expand Down Expand Up @@ -38,28 +39,28 @@

# Mandatory variables that every scheme needs to have
CCPP_MANDATORY_VARIABLES = {
'ccpp_error_message' : Var(local_name = 'errmsg',
standard_name = 'ccpp_error_message',
long_name = 'error message for error handling in CCPP',
units = 'none',
type = 'character',
dimensions = [],
rank = '',
kind = 'len=*',
intent = 'out',
active = 'T',
),
'ccpp_error_code' : Var(local_name = 'ierr',
standard_name = 'ccpp_error_code',
long_name = 'error code for error handling in CCPP',
units = '1',
type = 'integer',
dimensions = [],
rank = '',
kind = '',
intent = 'out',
active = 'T',
),
CCPP_ERROR_MSG_VARIABLE : Var(local_name = 'errmsg',
standard_name = CCPP_ERROR_MSG_VARIABLE,
long_name = 'error message for error handling in CCPP',
units = 'none',
type = 'character',
dimensions = [],
rank = '',
kind = 'len=*',
intent = 'out',
active = 'T',
),
CCPP_ERROR_CODE_VARIABLE : Var(local_name = 'ierr',
standard_name = CCPP_ERROR_CODE_VARIABLE,
long_name = 'error code for error handling in CCPP',
units = '1',
type = 'integer',
dimensions = [],
rank = '',
kind = '',
intent = 'out',
active = 'T',
),
}

# Save metadata to avoid repeated parsing of type/variable definition files
Expand Down
2 changes: 1 addition & 1 deletion scripts/mkdoc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def metadata_to_latex(metadata_define, metadata_request, model, filename):
target = 'MISSING'
local_name = 'MISSING'
if var_name in metadata_request.keys():
requested_list = [ escape_tex(decode_container(v.container)) for v in metadata_request[var_name] ]
requested_list = [ escape_tex(decode_container(v.container)) if v.container else 'none' for v in metadata_request[var_name] ]
# for the purpose of the table, just output the name of the subroutine
for i in range(len(requested_list)):
entry = requested_list[i]
Expand Down
Loading

0 comments on commit 167313e

Please sign in to comment.