diff --git a/.github/workflows/fleximod-test.yml b/.github/workflows/fleximod-test.yml new file mode 100644 index 0000000000..059aba2d8b --- /dev/null +++ b/.github/workflows/fleximod-test.yml @@ -0,0 +1,20 @@ +on: + push: + branches: + - master + - remove_manage_externals + pull_request: + branches: + - master + - remove_manage_externals +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.11"] + steps: + - uses: actions/checkout@v4 + - name: git-fleximod test + run: | + $GITHUB_WORKSPACE/bin/git-fleximod test diff --git a/.gitmodules b/.gitmodules index 3570f336d9..02d5e67737 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,58 +1,70 @@ [submodule "fates"] path = src/fates url = https://github.com/NGEET/fates + fxurl = https://github.com/NGEET/fates fxrequired = I:T fxtag = sci.1.67.2_api.27.0.0 [submodule "rtm"] path = components/rtm url = https://github.com/ESCOMP/RTM + fxurl = https://github.com/ESCOMP/RTM fxrequired = T:T fxtag = rtm1_0_78 [submodule "mosart"] path = components/mosart url = https://github.com/ESCOMP/MOSART + fxurl = https://github.com/ESCOMP/MOSART fxrequired = T:T fxtag = mosart1_0_48 [submodule "mizuRoute"] path = components/mizuRoute - url = https://github.com/nmizukami/mizuRoute + url = https://github.com/ESCOMP/mizuRoute + fxurl = https://github.com/ESCOMP/mizuRoute + fxtag = cesm-coupling.n02_v2.1.2 fxrequired = T:T [submodule "ccs_config"] path = ccs_config url = https://github.com/ESMCI/ccs_config_cesm.git + fxurl = https://github.com/ESMCI/ccs_config_cesm.git fxrequired = T:T fxtag = ccs_config_cesm0.0.84 [submodule "cime"] path = cime url = https://github.com/jedwards4b/cime + fxurl = https://github.com/ESMCI/cime fxrequired = T:T fxtag = cime6.0.198_rme01 [submodule "share"] path = share url = https://github.com/ESCOMP/CESM_share + fxurl = https://github.com/ESCOMP/CESM_share fxrequired = T:T fxtag = share1.0.17 [submodule "mct"] path = libraries/mct url = https://github.com/MCSclimate/MCT + fxurl = https://github.com/MCSclimate/MCT fxrequired = T:T fxtag = MCT_2.11.0 [submodule "parallelio"] path = libraries/parallelio url = https://github.com/NCAR/ParallelIO + fxurl = https://github.com/NCAR/ParallelIO fxtag = pio2_6_2 fxrequired = T:T [submodule "doc-builder"] path = doc/doc-builder url = https://github.com/ESMCI/doc-builder + fxurl = https://github.com/ESMCI/doc-builder + fxtag = v1.0.8 fxrequired = T:T diff --git a/Externals.cfg b/Externals.cfg deleted file mode 100644 index 9e7b4acc31..0000000000 --- a/Externals.cfg +++ /dev/null @@ -1,101 +0,0 @@ -[clm] -local_path = . -protocol = externals_only -externals = Externals_CLM.cfg -required = True - -[cism] -local_path = components/cism -protocol = git -repo_url = https://github.com/ESCOMP/CISM-wrapper -tag = cismwrap_2_1_96 -externals = Externals_CISM.cfg -required = True - -[rtm] -local_path = components/rtm -protocol = git -repo_url = https://github.com/ESCOMP/RTM -tag = rtm1_0_78 -required = True - -[mosart] -local_path = components/mosart -protocol = git -repo_url = https://github.com/ESCOMP/MOSART -tag = mosart1_0_48 -required = True - -[mizuRoute] -local_path = components/mizuRoute -protocol = git -repo_url = https://github.com/nmizukami/mizuRoute -hash = 34723c2 -required = True - -[ccs_config] -tag = ccs_config_cesm0.0.82 -protocol = git -repo_url = https://github.com/ESMCI/ccs_config_cesm.git -local_path = ccs_config -required = True - -[cime] -local_path = cime -protocol = git -repo_url = https://github.com/ESMCI/cime -tag = cime6.0.175 -required = True - -[cmeps] -tag = cmeps0.14.43 -protocol = git -repo_url = https://github.com/ESCOMP/CMEPS.git -local_path = components/cmeps -required = True - -[cdeps] -tag = cdeps1.0.23 -protocol = git -repo_url = https://github.com/ESCOMP/CDEPS.git -local_path = components/cdeps -externals = Externals_CDEPS.cfg -required = True - -[cpl7] -tag = cpl77.0.7 -protocol = git -repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps -local_path = components/cpl7 -required = True - -[share] -tag = share1.0.17 -protocol = git -repo_url = https://github.com/ESCOMP/CESM_share -local_path = share -required = True - -[mct] -tag = MCT_2.11.0 -protocol = git -repo_url = https://github.com/MCSclimate/MCT -local_path = libraries/mct -required = True - -[parallelio] -tag = pio2_6_2 -protocol = git -repo_url = https://github.com/NCAR/ParallelIO -local_path = libraries/parallelio -required = True - -[doc-builder] -local_path = doc/doc-builder -protocol = git -repo_url = https://github.com/ESMCI/doc-builder -tag = v1.0.8 -required = False - -[externals_description] -schema_version = 1.0.0 diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg deleted file mode 100644 index 14ba14d8b4..0000000000 --- a/Externals_CLM.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[fates] -local_path = src/fates -protocol = git -repo_url = https://github.com/NGEET/fates -tag = sci.1.67.2_api.27.0.0 -required = True - -[externals_description] -schema_version = 1.0.0 diff --git a/bin/git-fleximod b/bin/git-fleximod new file mode 100755 index 0000000000..c58c7bf14e --- /dev/null +++ b/bin/git-fleximod @@ -0,0 +1,434 @@ +#!/usr/bin/env python +import sys +import os +import shutil +import logging +import argparse + +sys.path.append(os.path.join(os.path.dirname(__file__),"..","lib","python","site-packages")) + +from fleximod import utils +from fleximod.gitinterface import GitInterface +from fleximod.gitmodules import GitModules +from fleximod.version import __version__ +from configparser import NoOptionError +# logger variable is global +logger = None + +def commandline_arguments(args=None): + description = """ + %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models + """ + parser = argparse.ArgumentParser( + description=description, formatter_class=argparse.RawDescriptionHelpFormatter + ) + + # + # user options + # + choices = ["update", "install", "status", "test"] + parser.add_argument( + "action", + choices=choices, + default="install", + help=f"Subcommand of fleximod, choices are {choices}", + ) + + parser.add_argument( + "components", + nargs="*", + help="Specific component(s) to checkout. By default, " + "all required submodules are checked out.", + ) + + parser.add_argument( + "-C", + "--path", + default=os.getcwd(), + help="Toplevel repository directory. Defaults to current directory.", + ) + + parser.add_argument( + "-g", + "--gitmodules", + nargs="?", + default=".gitmodules", + help="The submodule description filename. " "Default: %(default)s.", + ) + + parser.add_argument( + "-x", + "--exclude", + nargs="*", + help="Component(s) listed in the gitmodules file which should be ignored.", + ) + + parser.add_argument( + "-o", + "--optional", + action="store_true", + default=False, + help="By default only the required submodules " + "are checked out. This flag will also checkout the " + "optional submodules relative to the toplevel directory.", + ) + + parser.add_argument( + "-v", + "--verbose", + action="count", + default=0, + help="Output additional information to " + "the screen and log file. This flag can be " + "used up to two times, increasing the " + "verbosity level each time.", + ) + + parser.add_argument( + "-V", + "--version", + action="version", + version=f"%(prog)s {__version__}", + help="Print version and exit.", + ) + + # + # developer options + # + parser.add_argument( + "--backtrace", + action="store_true", + help="DEVELOPER: show exception backtraces as extra " "debugging output", + ) + + parser.add_argument( + "-d", + "--debug", + action="store_true", + default=False, + help="DEVELOPER: output additional debugging " + "information to the screen and log file.", + ) + + if args: + options = parser.parse_args(args) + else: + options = parser.parse_args() + +# explicitly listing a component overrides the optional flag + if options.optional or options.components: + fxrequired = ["T:T", "T:F", "I:T"] + else: + fxrequired = ["T:T", "I:T"] + + action = options.action + if not action: + action = "install" + + if options.debug: + level = logging.DEBUG + elif options.verbose: + level = logging.INFO + else: + level = logging.WARNING + # Configure the root logger + logging.basicConfig( + level=level, + format="%(name)s - %(levelname)s - %(message)s", + handlers=[logging.FileHandler("fleximod.log"), logging.StreamHandler()], + ) + if hasattr(options, 'version'): + exit() + + return ( + options.path, + options.gitmodules, + fxrequired, + options.components, + options.exclude, + options.verbose, + action, + ) + + +def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master"): + # first create the module directory + if not os.path.isdir(path): + os.makedirs(path) + # Check first if the module is already defined + # and the sparse-checkout file exists + git = GitInterface(root_dir, logger) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(root_dir, path) + sprepo_git = GitInterface(sprep_repo, logger) + if os.path.exists(os.path.join(sprep_repo,".git")): + try: + logger.info("Submodule {} found".format(name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + logger.info("Sparse submodule {} already checked out".format(name)) + return + except NoOptionError: + logger.debug("Sparse submodule {} not present".format(name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + sprepo_git.git_operation("remote", "add", "origin", url) + + superroot = git.git_operation("rev-parse", "--show-superproject-working-tree") + if os.path.isfile(os.path.join(root_dir, ".git")): + with open(os.path.join(root_dir, ".git")) as f: + gitpath = os.path.abspath(os.path.join(root_dir,f.read().split()[1])) + topgit = os.path.abspath(os.path.join(gitpath, "modules")) + else: + topgit = os.path.abspath(os.path.join(root_dir, ".git", "modules")) + + if not os.path.isdir(topgit): + os.makedirs(topgit) + topgit = os.path.join(topgit, name) + logger.debug(f"root_dir is {root_dir} topgit is {topgit} superroot is {superroot}") + + if os.path.isdir(os.path.join(root_dir,path,".git")): + shutil.move(os.path.join(root_dir,path, ".git"), topgit) + with open(os.path.join(root_dir,path, ".git"), "w") as f: + f.write("gitdir: " + os.path.relpath(topgit, os.path.join(root_dir,path))) + + gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + if os.path.isfile(gitsparse): + logger.warning("submodule {} is already initialized".format(name)) + return + + + shutil.copy(os.path.join(root_dir,path, sparsefile), gitsparse) + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags") + sprepo_git.git_operation("checkout", tag) + print(f"Successfully checked out {name}") + + +def submodule_checkout(root, name, path, url=None, tag=None): + git = GitInterface(root, logger) + repodir = os.path.join(root, path) + if os.path.exists(os.path.join(repodir, ".git")): + logger.info("Submodule {} already checked out".format(name)) + return + # if url is provided update to the new url + tmpurl = None + + # Look for a .gitmodules file in the newly checkedout repo + if url: + # ssh urls cause problems for those who dont have git accounts with ssh keys defined + # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was + # opened with a GitModules object we don't need to worry about restoring the file here + # it will be done by the GitModules class + if url.startswith("git@"): + tmpurl = url + url = url.replace("git@github.com:", "https://github.com") + git.git_operation("clone", "-b", tag, url, path) + # Now need to move the .git dir to the submodule location + + + if not tmpurl: + logger.debug(git.git_operation("submodule", "update", "--init", "--", path)) + + if os.path.exists(os.path.join(repodir, ".gitmodules")): + # recursively handle this checkout + print(f"Recursively checking out submodules of {name} {repodir} {url}") + gitmodules = GitModules(logger,confpath=repodir) + submodules_install(gitmodules, repodir, ["I:T"]) + if os.path.exists(os.path.join(repodir, ".git")): + print(f"Successfully checked out {name}") + else: + utils.fatal_error(f"Failed to checkout {name}") + + if tmpurl: + print(git.git_operation("restore", ".gitmodules")) + + return + + +def submodules_status(gitmodules, root_dir): + testfails = 0 + for name in gitmodules.sections(): + path = gitmodules.get(name, "path") + tag = gitmodules.get(name, "fxtag") + if not path: + utils.fatal_error("No path found in .gitmodules for {}".format(name)) + newpath = os.path.join(root_dir, path) + logger.debug("newpath is {}".format(newpath)) + if not os.path.exists(os.path.join(newpath, ".git")): + rootgit = GitInterface(root_dir, logger) + # submodule commands use path, not name + nhash = (rootgit.git_operation("submodule","status",path).split()[0])[1:] + url = gitmodules.get(name, "url") + tags = rootgit.git_operation("ls-remote","--tags",url) + atag = None + for htag in tags.split('\n'): + if tag in htag: + atag = (htag.split()[1])[10:] + break + if tag == atag: + print(f"Submodule {name} not checked out, aligned at tag {tag}") + else: + print(f"Submodule {name} not checked out, out of sync at tag {atag}, expected tag is {tag}") + testfails += 1 + else: + with utils.pushd(newpath): + git = GitInterface(newpath, logger) + atag = git.git_operation("describe", "--tags", "--always").rstrip() + if tag and atag != tag: + print(f"Submodule {name} {atag} is out of sync with .gitmodules {tag}") + testfails += 1 + elif tag: + print(f"Submodule {name} at tag {tag}") + else: + print( + f"Submodule {name} has no tag defined in .gitmodules, module at {atag}" + ) + testfails += 1 + + status = git.git_operation("status","--ignore-submodules","untracked") + if "nothing to commit" not in status: + print(status) + + return testfails + +def submodules_update(gitmodules, root_dir): + for name in gitmodules.sections(): + fxtag = gitmodules.get(name, "fxtag") + path = gitmodules.get(name, "path") + url = gitmodules.get(name, "url") + logger.info(f"name={name} path={path} url={url} fxtag={fxtag}") + if os.path.exists(os.path.join(path, ".git")): + submoddir = os.path.join(root_dir, path) + with utils.pushd(submoddir): + git = GitInterface(submoddir, logger) + # first make sure the url is correct + upstream = git.git_operation("ls-remote", "--get-url").rstrip() + newremote = "origin" + if upstream != url: + # TODO - this needs to be a unique name + remotes = git.git_operation("remote", "-v") + if url in remotes: + for line in remotes: + if url in line and "fetch" in line: + newremote = line.split()[0] + break + else: + i = 0 + while newremote in remotes: + i = i + 1 + newremote = f"newremote.{i:02d}" + git.git_operation("remote", "add", newremote, url) + + tags = git.git_operation("tag", "-l") + if fxtag and fxtag not in tags: + git.git_operation("fetch", newremote, "--tags") + atag = git.git_operation("describe", "--tags", "--always").rstrip() + if fxtag and fxtag != atag: + print(f"Updating {name} to {fxtag}") + git.git_operation("checkout", fxtag) + elif not fxtag: + print(f"No fxtag found for submodule {name}") + else: + print(f"submodule {name} up to date.") + + +def submodules_install(gitmodules, root_dir, requiredlist): + for name in gitmodules.sections(): + fxrequired = gitmodules.get(name, "fxrequired") + fxsparse = gitmodules.get(name, "fxsparse") + fxtag = gitmodules.get(name, "fxtag") + path = gitmodules.get(name, "path") + url = gitmodules.get(name, "url") + + if fxrequired and fxrequired not in requiredlist: + if "T:F" == fxrequired: + print("Skipping optional component {}".format(name)) + continue + + if fxsparse: + logger.debug( + f"Callng submodule_sparse_checkout({root_dir}, {name}, {url}, {path}, {fxsparse}, {fxtag}" + ) + submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag) + else: + logger.debug( + "Calling submodule_checkout({},{},{})".format(root_dir, name, path) + ) + + submodule_checkout(root_dir, name, path, url=url, tag=fxtag) + +def submodules_test(gitmodules, root_dir): + # First check that fxtags are present and in sync with submodule hashes + testfails = submodules_status(gitmodules, root_dir) + # Then make sure that urls are consistant with fxurls (not forks and not ssh) + # and that sparse checkout files exist + for name in gitmodules.sections(): + url = gitmodules.get(name, "url") + fxurl = gitmodules.get(name, "fxurl") + fxsparse = gitmodules.get(name, "fxsparse") + path = gitmodules.get(name, "path") + if not fxurl or url != fxurl: + print(f"submodule {name} url {url} not in sync with required {fxurl}") + testfails += 1 + if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)): + print(f"sparse submodule {name} sparse checkout file {fxsparse} not found") + testfails += 1 + return testfails + + + +def _main_func(): + ( + root_dir, + file_name, + fxrequired, + includelist, + excludelist, + verbose, + action, + ) = commandline_arguments() + # Get a logger for the package + global logger + logger = logging.getLogger(__name__) + + logger.info(f"action is {action}") + + if not os.path.isfile(os.path.join(root_dir, file_name)): + file_path = utils.find_upwards(root_dir, file_name) + + if file_path is None: + utils.fatal_error( + "No {} found in {} or any of it's parents".format(file_name, root_dir) + ) + root_dir = os.path.dirname(file_path) + logger.info(f"root_dir is {root_dir}") + gitmodules = GitModules( + logger, + confpath=root_dir, + conffile=file_name, + includelist=includelist, + excludelist=excludelist, + ) + retval = 0 + if action == "update": + submodules_update(gitmodules, root_dir) + elif action == "install": + submodules_install(gitmodules, root_dir, fxrequired) + elif action == "status": + submodules_status(gitmodules, root_dir) + elif action == "test": + retval = submodules_test(gitmodules, root_dir) + else: + utils.fatal_error(f"unrecognized action request {action}") + return(retval) + +if __name__ == "__main__": + sys.exit(_main_func()) diff --git a/install b/install deleted file mode 100644 index c51dfd1397..0000000000 --- a/install +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -./install_CLM -submodules('cime' 'components/rtm' 'components/mosart' \ - 'components/mizuRoute' 'ccs_config' 'components/cmeps' \ - 'components/cdeps' 'components/cpl7' 'share' 'libraries/mct' \ - 'libraries/parallelio' 'doc/doc-builder') - -for mod in "${submodules[@]}" -do - echo "Initializing $mod" - git submodule update --init $mod -done - - \ No newline at end of file diff --git a/install_CLM b/install_CLM deleted file mode 100644 index df0e96f99f..0000000000 --- a/install_CLM +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -submodules('src/fates') - -for mod in "${submodules[@]}" -do - echo "Initializing $mod" - git submodule update --init $mod -done \ No newline at end of file diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/INSTALLER b/lib/python/site-packages/fleximod-0.1.8.dist-info/INSTALLER new file mode 100644 index 0000000000..a1b589e38a --- /dev/null +++ b/lib/python/site-packages/fleximod-0.1.8.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/License b/lib/python/site-packages/fleximod-0.1.8.dist-info/License new file mode 100644 index 0000000000..2c6fe768c2 --- /dev/null +++ b/lib/python/site-packages/fleximod-0.1.8.dist-info/License @@ -0,0 +1,20 @@ +Copyright 2024 National Center for Atmospheric Sciences (NCAR) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/METADATA b/lib/python/site-packages/fleximod-0.1.8.dist-info/METADATA new file mode 100644 index 0000000000..ea0ff77e5f --- /dev/null +++ b/lib/python/site-packages/fleximod-0.1.8.dist-info/METADATA @@ -0,0 +1,112 @@ +Metadata-Version: 2.1 +Name: fleximod +Version: 0.1.8 +Summary: Extended support for git-submodule and git-sparse-checkout +Author: Jim Edwards +Maintainer: jedwards4b +License: MIT License +Classifier: Programming Language :: Python :: 3 +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Requires-Python: >=3.6 +Description-Content-Type: text/markdown +License-File: License +Requires-Dist: GitPython + +# git-fleximod + +Flexible Submodule Management for Git + +## Overview + +Git-fleximod is a Python-based tool that extends Git's submodule capabilities, offering additional features for managing submodules in a more flexible and efficient way. + +## Installation + +#TODO Install using pip: +# pip install git-fleximod + If you choose to locate git-fleximod in your path you can access it via command: git fleximod + +## Usage + + Basic Usage: + git fleximod [options] + Available Commands: + install: Install submodules according to configuration. + status: Display the status of submodules. + update: Update submodules to the tag indicated in .gitmodules variable fxtag. + Additional Options: + See git fleximod --help for more details. + +## Supported .gitmodules Variables + + fxtag: Specify a specific tag or branch to checkout for a submodule. + fxrequired: Mark a submodule's checkout behavior, with allowed values: + - T:T: Top-level and required (checked out only when this is the Toplevel module). + - T:F: Top-level and optional (checked out with --optional flag if this is the Toplevel module). + - I:T: Internal and required (always checked out). + - I:F: Internal and optional (checked out with --optional flag). + fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. + +## Sparse Checkouts + + To enable sparse checkout for a submodule, set the fxsparse variable + in the .gitmodules file to the path of a file containing the desired + sparse checkout paths. Git-fleximod will automatically configure + sparse checkout based on this file when applicable commands are run. + See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set) for details on the format of this file. + +## Examples + +Here are some common usage examples: + +Installing submodules, including optional ones: +```bash + git fleximod install --optional +``` + +Updating a specific submodule to the fxtag indicated in .gitmodules: + +```bash + git fleximod update submodule-name +``` +Example .gitmodules entry: +```ini, toml + [submodule "cosp2"] + path = src/physics/cosp2/src + url = https://github.com/CFMIP/COSPv2.0 + fxsparse = ../.cosp_sparse_checkout + fxtag = v2.1.4cesm +``` +Explanation: + +This entry indicates that the submodule named cosp2 at tag v2.1.4cesm +should be checked out into the directory src/physics/cosp2/src +relative to the .gitmodules directory. It should be checked out from +the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as +described in the file ../.cosp_sparse_checkout relative to the path +directory. + +Additional example: +```ini, toml + [submodule "cime"] + path = cime + url = https://github.com/jedwards4b/cime + fxrequired = T:T + fxtag = cime6.0.198_rme01 +``` + +Explanation: + +This entry indicates that the submodule cime should be checked out +into a directory named cime at tag cime6.0.198_rme01 from the URL +https://github.com/jedwards4b/cime. This should only be done if +the .gitmodules file is at the top level of the repository clone. + +## Contributing + +We welcome contributions! Please see the CONTRIBUTING.md file for guidelines. + +## License + +Git-fleximod is released under the MIT License. diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/RECORD b/lib/python/site-packages/fleximod-0.1.8.dist-info/RECORD new file mode 100644 index 0000000000..3b6ab1837b --- /dev/null +++ b/lib/python/site-packages/fleximod-0.1.8.dist-info/RECORD @@ -0,0 +1,20 @@ +../../../bin/git-fleximod,sha256=gjA9gbSciH4XF893iHYvNSjtoF_26tkHm95L-M7liNQ,14683 +fleximod-0.1.8.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +fleximod-0.1.8.dist-info/License,sha256=DpInTZ-XtM3SVeiP-4FH8-NEQKXp2RkP1x_0Cubo3-c,1095 +fleximod-0.1.8.dist-info/METADATA,sha256=NfVSN5uyWGXb04AHyz4svVlA2NHKodRk2pnRlCJKwYM,3769 +fleximod-0.1.8.dist-info/RECORD,, +fleximod-0.1.8.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fleximod-0.1.8.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92 +fleximod-0.1.8.dist-info/top_level.txt,sha256=X-eC_MHIg4MiGH0jLABGFcHAC2wpEPMaKGtcQlr9F-M,9 +fleximod/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +fleximod/__pycache__/__init__.cpython-312.pyc,, +fleximod/__pycache__/gitinterface.cpython-312.pyc,, +fleximod/__pycache__/gitmodules.cpython-312.pyc,, +fleximod/__pycache__/lstripreader.cpython-312.pyc,, +fleximod/__pycache__/utils.cpython-312.pyc,, +fleximod/__pycache__/version.cpython-312.pyc,, +fleximod/gitinterface.py,sha256=jmcWRZvIvSiPLPSzr1DS84HLB2DvaTbNmMh4bOm29mY,2516 +fleximod/gitmodules.py,sha256=xhNofIdgIBiaOWPqNVU29z-xOOXhUhs01ieMQ2ZpTnY,3381 +fleximod/lstripreader.py,sha256=KJosD1B5LiASEaStrSTu_DVqgLnPqS3T-mH9fRzV5LE,1191 +fleximod/utils.py,sha256=pYo-Lwd5pCcCmL5AmE9pwVuDd0SlrD39_hEmYoOhodc,10271 +fleximod/version.py,sha256=rq5OMW-khxQo_5FAvSrPaAS1lRfG9aG2hSuo-KK2Yfk,22 diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/REQUESTED b/lib/python/site-packages/fleximod-0.1.8.dist-info/REQUESTED new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/WHEEL b/lib/python/site-packages/fleximod-0.1.8.dist-info/WHEEL new file mode 100644 index 0000000000..98c0d20b7a --- /dev/null +++ b/lib/python/site-packages/fleximod-0.1.8.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.42.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python/site-packages/fleximod-0.1.8.dist-info/top_level.txt b/lib/python/site-packages/fleximod-0.1.8.dist-info/top_level.txt new file mode 100644 index 0000000000..52d276b037 --- /dev/null +++ b/lib/python/site-packages/fleximod-0.1.8.dist-info/top_level.txt @@ -0,0 +1 @@ +fleximod diff --git a/lib/python/site-packages/fleximod/__init__.py b/lib/python/site-packages/fleximod/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lib/python/site-packages/fleximod/gitinterface.py b/lib/python/site-packages/fleximod/gitinterface.py new file mode 100644 index 0000000000..c127163dc6 --- /dev/null +++ b/lib/python/site-packages/fleximod/gitinterface.py @@ -0,0 +1,66 @@ +import os +import logging +from fleximod import utils + +class GitInterface: + def __init__(self, repo_path, logger): + logger.debug("Initialize GitInterface for {}".format(repo_path)) + self.repo_path = repo_path + self.logger = logger + try: + import git + self._use_module = True + try: + self.repo = git.Repo(repo_path) # Initialize GitPython repo + except git.exc.InvalidGitRepositoryError: + self.git = git + self._init_git_repo() + msg = "Using GitPython interface to git" + except ImportError: + self._use_module = False + if not os.path.exists(os.path.join(repo_path,".git")): + self._init_git_repo() + msg = "Using shell interface to git" + self.logger.info(msg) + + def _git_command(self, operation, *args): + self.logger.info(operation) + if self._use_module and operation != "submodule": + return getattr(self.repo.git, operation)(*args) + else: + return ["git", "-C",self.repo_path, operation] + list(args) + + def _init_git_repo(self): + if self._use_module: + self.repo = self.git.Repo.init(self.repo_path) + else: + command = ("git", "-C", self.repo_path, "init") + utils.execute_subprocess(command) + + + def git_operation(self, operation, *args, **kwargs): + command = self._git_command(operation, *args) + self.logger.info(command) + if isinstance(command, list): + return utils.execute_subprocess(command, output_to_caller=True) + else: + return command + + def config_get_value(self, section, name): + if self._use_module: + config = self.repo.config_reader() + return config.get_value(section, name) + else: + cmd = ("git","-C",self.repo_path,"config", "--get", f"{section}.{name}") + output = utils.execute_subprocess(cmd, output_to_caller=True) + return output.strip() + + def config_set_value(self, section, name, value): + if self._use_module: + with self.repo.config_writer() as writer: + writer.set_value(section, name, value) + writer.release() # Ensure changes are saved + else: + cmd = ("git","-C",self.repo_path,"config", f"{section}.{name}", value) + self.logger.info(cmd) + utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/lib/python/site-packages/fleximod/gitmodules.py b/lib/python/site-packages/fleximod/gitmodules.py new file mode 100644 index 0000000000..a6f7589319 --- /dev/null +++ b/lib/python/site-packages/fleximod/gitmodules.py @@ -0,0 +1,83 @@ +import os +import shutil +from configparser import ConfigParser +from fleximod.lstripreader import LstripReader + +class GitModules(ConfigParser): + def __init__( + self, + logger, + confpath=os.getcwd(), + conffile=".gitmodules", + includelist=None, + excludelist=None, + ): + """ + confpath: Path to the directory containing the .gitmodules file (defaults to the current working directory). + conffile: Name of the configuration file (defaults to .gitmodules). + includelist: Optional list of submodules to include. + excludelist: Optional list of submodules to exclude. + """ + self.logger = logger + self.logger.debug("Creating a GitModules object {} {} {} {}".format(confpath,conffile,includelist,excludelist)) + ConfigParser.__init__(self) + self.conf_file = os.path.join(confpath, conffile) + # first create a backup of this file to be restored on deletion of the object + shutil.copy(self.conf_file, self.conf_file+".save") + self.read_file(LstripReader(self.conf_file), source=conffile) + self.includelist = includelist + self.excludelist = excludelist + + def set(self, name, option, value): + """ + Sets a configuration value for a specific submodule: + Ensures the appropriate section exists for the submodule. + Calls the parent class's set method to store the value. + """ + self.logger.debug("set called {} {} {}".format(name,option,value)) + section = f'submodule "{name}"' + if not self.has_section(section): + self.add_section(section) + ConfigParser.set(self, section, option, str(value)) + + # pylint: disable=redefined-builtin, arguments-differ + def get(self, name, option, raw=False, vars=None, fallback=None): + """ + Retrieves a configuration value for a specific submodule: + Uses the parent class's get method to access the value. + Handles potential errors if the section or option doesn't exist. + """ + self.logger.debug("get called {} {}".format(name,option)) + section = f'submodule "{name}"' + try: + return ConfigParser.get( + self, section, option, raw=raw, vars=vars, fallback=fallback + ) + except ConfigParser.NoOptionError: + return None + + def save(self): + print("Called gitmodules save, not expected") + # self.write(open(self.conf_file, "w")) + + def __del__(self): + self.logger.debug("Destroying GitModules object") + shutil.move(self.conf_file+".save", self.conf_file) + + def sections(self): + """Strip the submodule part out of section and just use the name""" + self.logger.debug("calling GitModules sections iterator") + names = [] + for section in ConfigParser.sections(self): + name = section[11:-1] + if self.includelist and name not in self.includelist: + continue + if self.excludelist and name in self.excludelist: + continue + names.append(name) + return names + + def items(self, name, raw=False, vars=None): + self.logger.debug("calling GitModules items for {}".format(name)) + section = f'submodule "{name}"' + return ConfigParser.items(section, raw=raw, vars=vars) diff --git a/lib/python/site-packages/fleximod/lstripreader.py b/lib/python/site-packages/fleximod/lstripreader.py new file mode 100644 index 0000000000..530abd297e --- /dev/null +++ b/lib/python/site-packages/fleximod/lstripreader.py @@ -0,0 +1,44 @@ + +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + def __init__(self, filename): + with open(filename, 'r') as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = '' + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() + diff --git a/lib/python/site-packages/fleximod/utils.py b/lib/python/site-packages/fleximod/utils.py new file mode 100644 index 0000000000..f0753367e5 --- /dev/null +++ b/lib/python/site-packages/fleximod/utils.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +""" +Common public utilities for manic package + +""" + +import logging +import os +import subprocess +import sys +from threading import Timer +from pathlib import Path + +LOCAL_PATH_INDICATOR = "." +# --------------------------------------------------------------------- +# +# functions to massage text for output and other useful utilities +# +# --------------------------------------------------------------------- +from contextlib import contextmanager + + +@contextmanager +def pushd(new_dir): + """context for chdir. usage: with pushd(new_dir)""" + previous_dir = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(previous_dir) + + +def log_process_output(output): + """Log each line of process output at debug level so it can be + filtered if necessary. By default, output is a single string, and + logging.debug(output) will only put log info heading on the first + line. This makes it hard to filter with grep. + + """ + output = output.split("\n") + for line in output: + logging.debug(line) + + +def printlog(msg, **kwargs): + """Wrapper script around print to ensure that everything printed to + the screen also gets logged. + + """ + logging.info(msg) + if kwargs: + print(msg, **kwargs) + else: + print(msg) + sys.stdout.flush() + + +def find_upwards(root_dir, filename): + """Find a file in root dir or any of it's parents""" + d = Path(root_dir) + root = Path(d.root) + while d != root: + attempt = d / filename + if attempt.exists(): + return attempt + d = d.parent + return None + + +def last_n_lines(the_string, n_lines, truncation_message=None): + """Returns the last n lines of the given string + + Args: + the_string: str + n_lines: int + truncation_message: str, optional + + Returns a string containing the last n lines of the_string + + If truncation_message is provided, the returned string begins with + the given message if and only if the string is greater than n lines + to begin with. + """ + + lines = the_string.splitlines(True) + if len(lines) <= n_lines: + return_val = the_string + else: + lines_subset = lines[-n_lines:] + str_truncated = "".join(lines_subset) + if truncation_message: + str_truncated = truncation_message + "\n" + str_truncated + return_val = str_truncated + + return return_val + + +def indent_string(the_string, indent_level): + """Indents the given string by a given number of spaces + + Args: + the_string: str + indent_level: int + + Returns a new string that is the same as the_string, except that + each line is indented by 'indent_level' spaces. + + In python3, this can be done with textwrap.indent. + """ + + lines = the_string.splitlines(True) + padding = " " * indent_level + lines_indented = [padding + line for line in lines] + return "".join(lines_indented) + + +# --------------------------------------------------------------------- +# +# error handling +# +# --------------------------------------------------------------------- + + +def fatal_error(message): + """ + Error output function + """ + logging.error(message) + raise RuntimeError("{0}ERROR: {1}".format(os.linesep, message)) + + +# --------------------------------------------------------------------- +# +# Data conversion / manipulation +# +# --------------------------------------------------------------------- +def str_to_bool(bool_str): + """Convert a sting representation of as boolean into a true boolean. + + Conversion should be case insensitive. + """ + value = None + str_lower = bool_str.lower() + if str_lower in ("true", "t"): + value = True + elif str_lower in ("false", "f"): + value = False + if value is None: + msg = ( + 'ERROR: invalid boolean string value "{0}". ' + 'Must be "true" or "false"'.format(bool_str) + ) + fatal_error(msg) + return value + + +REMOTE_PREFIXES = ["http://", "https://", "ssh://", "git@"] + + +def is_remote_url(url): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + """ + remote_url = False + for prefix in REMOTE_PREFIXES: + if url.startswith(prefix): + remote_url = True + return remote_url + + +def split_remote_url(url): + """check if the user provided a local file path or a + remote. If remote, try to strip off protocol info. + + """ + remote_url = is_remote_url(url) + if not remote_url: + return url + + for prefix in REMOTE_PREFIXES: + url = url.replace(prefix, "") + + if "@" in url: + url = url.split("@")[1] + + if ":" in url: + url = url.split(":")[1] + + return url + + +def expand_local_url(url, field): + """check if the user provided a local file path instead of a + remote. If so, it must be expanded to an absolute + path. + + Note: local paths of LOCAL_PATH_INDICATOR have special meaning and + represent local copy only, don't work with the remotes. + + """ + remote_url = is_remote_url(url) + if not remote_url: + if url.strip() == LOCAL_PATH_INDICATOR: + pass + else: + url = os.path.expandvars(url) + url = os.path.expanduser(url) + if not os.path.isabs(url): + msg = ( + 'WARNING: Externals description for "{0}" contains a ' + "url that is not remote and does not expand to an " + "absolute path. Version control operations may " + "fail.\n\nurl={1}".format(field, url) + ) + printlog(msg) + else: + url = os.path.normpath(url) + return url + + +# --------------------------------------------------------------------- +# +# subprocess +# +# --------------------------------------------------------------------- + +# Give the user a helpful message if we detect that a command seems to +# be hanging. +_HANGING_SEC = 300 + + +def _hanging_msg(working_directory, command): + print( + """ + +Command '{command}' +from directory {working_directory} +has taken {hanging_sec} seconds. It may be hanging. + +The command will continue to run, but you may want to abort +manage_externals with ^C and investigate. A possible cause of hangs is +when svn or git require authentication to access a private +repository. On some systems, svn and git requests for authentication +information will not be displayed to the user. In this case, the program +will appear to hang. Ensure you can run svn and git manually and access +all repositories without entering your authentication information. + +""".format( + command=command, + working_directory=working_directory, + hanging_sec=_HANGING_SEC, + ) + ) + + +def execute_subprocess(commands, status_to_caller=False, output_to_caller=False): + """Wrapper around subprocess.check_output to handle common + exceptions. + + check_output runs a command with arguments and waits + for it to complete. + + check_output raises an exception on a nonzero return code. if + status_to_caller is true, execute_subprocess returns the subprocess + return code, otherwise execute_subprocess treats non-zero return + status as an error and raises an exception. + + """ + cwd = os.getcwd() + msg = "In directory: {0}\nexecute_subprocess running command:".format(cwd) + logging.info(msg) + commands_str = " ".join(commands) + logging.info(commands_str) + return_to_caller = status_to_caller or output_to_caller + status = -1 + output = "" + hanging_timer = Timer( + _HANGING_SEC, + _hanging_msg, + kwargs={"working_directory": cwd, "command": commands_str}, + ) + hanging_timer.start() + try: + output = subprocess.check_output( + commands, stderr=subprocess.STDOUT, universal_newlines=True + ) + log_process_output(output) + status = 0 + except OSError as error: + msg = failed_command_msg( + "Command execution failed. Does the executable exist?", commands + ) + logging.error(error) + fatal_error(msg) + except ValueError as error: + msg = failed_command_msg( + "DEV_ERROR: Invalid arguments trying to run subprocess", commands + ) + logging.error(error) + fatal_error(msg) + except subprocess.CalledProcessError as error: + # Only report the error if we are NOT returning to the + # caller. If we are returning to the caller, then it may be a + # simple status check. If returning, it is the callers + # responsibility determine if an error occurred and handle it + # appropriately. + if not return_to_caller: + msg_context = ( + "Process did not run successfully; " + "returned status {0}".format(error.returncode) + ) + msg = failed_command_msg(msg_context, commands, output=error.output) + logging.error(error) + logging.error(msg) + log_process_output(error.output) + fatal_error(msg) + status = error.returncode + finally: + hanging_timer.cancel() + + if status_to_caller and output_to_caller: + ret_value = (status, output) + elif status_to_caller: + ret_value = status + elif output_to_caller: + ret_value = output + else: + ret_value = None + + return ret_value + + +def failed_command_msg(msg_context, command, output=None): + """Template for consistent error messages from subprocess calls. + + If 'output' is given, it should provide the output from the failed + command + """ + + if output: + output_truncated = last_n_lines( + output, 20, truncation_message="[... Output truncated for brevity ...]" + ) + errmsg = ( + "Failed with output:\n" + indent_string(output_truncated, 4) + "\nERROR: " + ) + else: + errmsg = "" + + command_str = " ".join(command) + errmsg += """In directory + {cwd} +{context}: + {command} +""".format( + cwd=os.getcwd(), context=msg_context, command=command_str + ) + + if output: + errmsg += "See above for output from failed command.\n" + + return errmsg diff --git a/lib/python/site-packages/fleximod/version.py b/lib/python/site-packages/fleximod/version.py new file mode 100644 index 0000000000..1c98a23a89 --- /dev/null +++ b/lib/python/site-packages/fleximod/version.py @@ -0,0 +1 @@ +__version__ = '0.1.9'