ccpp_prebuild: ccpp stub and various bugfixes

scripts/ccpp_prebuild.py

@@ -3,6 +3,7 @@
 # Standard modules
 import argparse
 import collections
+import copy
 import filecmp
 import importlib
 import itertools
@@ -23,6 +24,7 @@
                   TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile
 from mkdoc import metadata_to_html, metadata_to_latex
 from mkstatic import API, Suite, Group
+from mkstatic import CCPP_SUITE_VARIABLES
 
 ###############################################################################
 # Set up the command line argument parser and other global variables          #
@@ -35,6 +37,7 @@
 parser.add_argument('--debug',      action='store_true', help='enable debugging features in auto-generated code', default=False)
 parser.add_argument('--suites',     action='store', help='suite definition files to use (comma-separated, without path)', default='')
 parser.add_argument('--builddir',   action='store', help='relative path to CCPP build directory', required=False, default=None)
+parser.add_argument('--namespace',  action='store', help='namespace suffix to be added to the name of static api module', required=False, default='')
 
 # BASEDIR is the current directory where this script is executed
 BASEDIR = os.getcwd()
@@ -56,7 +59,8 @@ def parse_arguments():
     else:
         sdfs = None
     builddir = args.builddir
-    return (success, configfile, clean, verbose, debug, sdfs, builddir)
+    namespace = args.namespace
+    return (success, configfile, clean, verbose, debug, sdfs, builddir, namespace)
 
 def import_config(configfile, builddir):
     """Import the configuration from a given configuration file"""
@@ -95,13 +99,14 @@ def import_config(configfile, builddir):
     config['caps_cmakefile']            = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir)
     config['caps_sourcefile']           = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir)
     config['caps_dir']                  = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir)
-    config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR
+    config['suites_dir']                = ccpp_prebuild_config.SUITES_DIR.format(build_dir=builddir)
     config['host_model']                = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER
     config['html_vartable_file']        = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir)
     config['latex_vartable_file']       = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir)
-    # Location of static API file, and shell script to source
+    # Location of static API file, shell script to source, cmake include file
     config['static_api_dir']            = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir)
-    config['static_api_srcfile']        = ccpp_prebuild_config.STATIC_API_SRCFILE.format(build_dir=builddir)
+    config['static_api_sourcefile']     = ccpp_prebuild_config.STATIC_API_SOURCEFILE.format(build_dir=builddir)
+    config['static_api_cmakefile']      = ccpp_prebuild_config.STATIC_API_CMAKEFILE.format(build_dir=builddir)
 
     # To handle new metadata: import DDT references (if exist)
     try:
@@ -127,10 +132,14 @@ def setup_logging(verbose):
         logging.info('Logging level set to INFO')
     return success
 
-def clean_files(config):
+def clean_files(config, namespace):
     """Clean files created by ccpp_prebuild.py"""
     success = True
     logging.info('Performing clean ....')
+    if namespace:
+        static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE+'_'+namespace)
+    else:
+        static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)
     # Create list of files to remove, use wildcards where necessary
     files_to_remove = [
         config['typedefs_makefile'],
@@ -145,8 +154,8 @@ def clean_files(config):
         config['html_vartable_file'],
         config['latex_vartable_file'],
         os.path.join(config['caps_dir'], 'ccpp_*_cap.F90'),
-        os.path.join(config['static_api_dir'], '{api}.F90'.format(api=CCPP_STATIC_API_MODULE)),
-        config['static_api_srcfile'],
+        os.path.join(config['static_api_dir'], static_api_file),
+        config['static_api_sourcefile'],
         ]
     # Not very pythonic, but the easiest way w/o importing another Python module
     cmd = 'rm -vf {0}'.format(' '.join(files_to_remove))
@@ -329,6 +338,19 @@ def collect_physics_subroutines(scheme_files):
     os.chdir(BASEDIR)
     return (success, metadata_request, arguments_request, dependencies_request, schemes_in_files)
 
+def check_schemes_in_suites(arguments, suites):
+    """Check that all schemes that are requested in the suites exist"""
+    success = True
+    logging.info("Checking for existence of schemes in suites ...")
+    for suite in suites:
+        for group in suite.groups:
+            for subcycle in group.subcycles:
+                for scheme_name in subcycle.schemes:
+                    if not scheme_name in arguments.keys():
+                        success = False
+                        logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name))
+    return success
+
 def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites):
     """Remove all variables from metadata that are not used in the given suite;
     also remove information on argument lists, dependencies and schemes in files"""
@@ -376,6 +398,16 @@ def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites)
                 schemes_in_files_filtered[scheme] = schemes_in_files[scheme]
     return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered)
 
+def add_ccpp_suite_variables(metadata):
+    """ Add variables that are required to construct CCPP suites to the list of requested variables"""
+    success = True
+    logging.info("Adding CCPP suite variables to list of requested variables")
+    for var_name in CCPP_SUITE_VARIABLES.keys():
+        if not var_name in metadata.keys():
+            metadata[var_name] = [copy.deepcopy(CCPP_SUITE_VARIABLES[var_name])]
+            logging.debug("Adding CCPP suite variable {0} to list of requested variables".format(var_name))
+    return (success, metadata)
+
 def generate_list_of_schemes_and_dependencies_to_compile(schemes_in_files, dependencies1, dependencies2):
     """Generate a flat list of schemes and dependencies in two dependency dictionaries to compile"""
     success = True
@@ -489,14 +521,20 @@ def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arg
         success = False
     return (success, suite_and_group_caps)
 
-def generate_static_api(suites, static_api_dir):
+def generate_static_api(suites, static_api_dir, namespace):
     """Generate static API for given suite(s)"""
     success = True
     # Change to caps directory, create if necessary
     if not os.path.isdir(static_api_dir):
         os.makedirs(static_api_dir)
     os.chdir(static_api_dir)
     api = API(suites=suites, directory=static_api_dir)
+    if namespace:
+        base = os.path.splitext(os.path.basename(api.filename))[0]
+        logging.info('Static API file name is ''{}'''.format(api.filename))
+        api.filename = base+'_'+namespace+'.F90'
+        api.module = base+'_'+namespace
+        logging.info('Static API file name is changed to ''{}'''.format(api.filename))
     logging.info('Generating static API {0} in {1} ...'.format(api.filename, static_api_dir))
     api.write()
     os.chdir(BASEDIR)
@@ -655,7 +693,7 @@ def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile,
 def main():
     """Main routine that handles the CCPP prebuild for different host models."""
     # Parse command line arguments
-    (success, configfile, clean, verbose, debug, sdfs, builddir) = parse_arguments()
+    (success, configfile, clean, verbose, debug, sdfs, builddir, namespace) = parse_arguments()
     if not success:
         raise Exception('Call to parse_arguments failed.')
 
@@ -669,7 +707,7 @@ def main():
 
     # Perform clean if requested, then exit
     if clean:
-        success = clean_files(config)
+        success = clean_files(config, namespace)
         logging.info('CCPP prebuild clean completed successfully, exiting.')
         sys.exit(0)
 
@@ -699,12 +737,22 @@ def main():
     if not success:
         raise Exception('Call to collect_physics_subroutines failed.')
 
+    # Check that the schemes requested in the suites exist
+    success = check_schemes_in_suites(arguments_request, suites)
+    if not success:
+        raise Exception('Call to check_schemes_in_suites failed.')
+
     # Filter metadata/arguments - remove whatever is not included in suite definition files
     (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = filter_metadata(
                          metadata_request, arguments_request, dependencies_request, schemes_in_files, suites)
     if not success:
         raise Exception('Call to filter_metadata failed.')
 
+    # Add variables that are required to construct CCPP suites to the list of requested variables
+    (success, metadata_request) = add_ccpp_suite_variables(metadata_request)
+    if not success:
+        raise Exception('Call to add_ccpp_suite_variables failed.')
+
     (success, schemes_and_dependencies_to_compile) = generate_list_of_schemes_and_dependencies_to_compile(
                                               schemes_in_files, dependencies_request, dependencies_define)
     if not success:
@@ -739,13 +787,17 @@ def main():
     if not success:
         raise Exception('Call to generate_suite_and_group_caps failed.')
 
-    (success, api) = generate_static_api(suites, config['static_api_dir'])
-    if not success: 
+    (success, api) = generate_static_api(suites, config['static_api_dir'], namespace)
+    if not success:
         raise Exception('Call to generate_static_api failed.')
 
-    success = api.write_sourcefile(config['static_api_srcfile'])
-    if not success: 
-        raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_srcfile']))
+    success = api.write_includefile(config['static_api_sourcefile'], type='shell')
+    if not success:
+        raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_sourcefile']))
+
+    success = api.write_includefile(config['static_api_cmakefile'], type='cmake')
+    if not success:
+        raise Exception("Writing API cmakefile {cmakefile} failed".format(cmakefile=config['static_api_cmakefile']))
 
     # Add filenames of caps to makefile/cmakefile/shell script
     all_caps = suite_and_group_caps

scripts/metadata_parser.py

@@ -9,6 +9,7 @@
 from xml.etree import ElementTree as ET
 
 from common import encode_container, CCPP_STAGES
+from common import CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE
 from mkcap import Var
 
 sys.path.append(os.path.join(os.path.split(__file__)[0], 'fortran_tools'))
@@ -38,28 +39,28 @@
 
 # Mandatory variables that every scheme needs to have
 CCPP_MANDATORY_VARIABLES = {
-    'ccpp_error_message' : Var(local_name    = 'errmsg',
-                               standard_name = 'ccpp_error_message',
-                               long_name     = 'error message for error handling in CCPP',
-                               units         = 'none',
-                               type          = 'character',
-                               dimensions    = [],
-                               rank          = '',
-                               kind          = 'len=*',
-                               intent        = 'out',
-                               active        = 'T',
-                               ),
-    'ccpp_error_code' : Var(local_name    = 'ierr',
-                            standard_name = 'ccpp_error_code',
-                            long_name     = 'error code for error handling in CCPP',
-                            units         = '1',
-                            type          = 'integer',
-                            dimensions    = [],
-                            rank          = '',
-                            kind          = '',
-                            intent        = 'out',
-                            active        = 'T',
-                            ),
+    CCPP_ERROR_MSG_VARIABLE : Var(local_name    = 'errmsg',
+                                  standard_name = CCPP_ERROR_MSG_VARIABLE,
+                                  long_name     = 'error message for error handling in CCPP',
+                                  units         = 'none',
+                                  type          = 'character',
+                                  dimensions    = [],
+                                  rank          = '',
+                                  kind          = 'len=*',
+                                  intent        = 'out',
+                                  active        = 'T',
+                                  ),
+    CCPP_ERROR_CODE_VARIABLE : Var(local_name    = 'ierr',
+                                   standard_name = CCPP_ERROR_CODE_VARIABLE,
+                                   long_name     = 'error flag for error handling in CCPP',
+                                   units         = '1',
+                                   type          = 'integer',
+                                   dimensions    = [],
+                                   rank          = '',
+                                   kind          = '',
+                                   intent        = 'out',
+                                   active        = 'T',
+                                   ),
     }
 
 # Save metadata to avoid repeated parsing of type/variable definition files

scripts/mkdoc.py

@@ -127,7 +127,7 @@ def metadata_to_latex(metadata_define, metadata_request, model, filename):
             target = 'MISSING'
             local_name = 'MISSING'
         if var_name in metadata_request.keys():
-            requested_list = [ escape_tex(decode_container(v.container)) for v in metadata_request[var_name] ]
+            requested_list = [ escape_tex(decode_container(v.container)) if v.container else 'none' for v in metadata_request[var_name] ]
             # for the purpose of the table, just output the name of the subroutine
             for i in range(len(requested_list)):
                 entry = requested_list[i]