diff --git a/.evergreen/auth_oidc/azure/remote-scripts/start-mongodb.sh b/.evergreen/auth_oidc/azure/remote-scripts/start-mongodb.sh index 92c06d37..6cf454de 100755 --- a/.evergreen/auth_oidc/azure/remote-scripts/start-mongodb.sh +++ b/.evergreen/auth_oidc/azure/remote-scripts/start-mongodb.sh @@ -11,7 +11,7 @@ export TOPOLOGY=server export ORCHESTRATION_FILE=auth-oidc.json export DRIVERS_TOOLS=$HOME/drivers-evergreen-tools export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration -export MONGO_ORCHESTRATION_HOME=$HOME +export MONGO_ORCHESTRATION_HOME=$PROJECT_ORCHESTRATION_HOME export SKIP_LEGACY_SHELL=true export NO_IPV6=${NO_IPV6:-""} diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 49db2070..5eb3fca9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -295,6 +295,7 @@ functions: MONGODB_VERSION: ${VERSION} TOPOLOGY: ${TOPOLOGY} SSL: ${SSL} + AUTH: ${AUTH} STORAGE_ENGINE: ${STORAGE_ENGINE} LOAD_BALANCER: ${LOAD_BALANCER} REQUIRE_API_VERSION: ${REQUIRE_API_VERSION} diff --git a/.evergreen/docker/overwrite_orchestration.py b/.evergreen/docker/overwrite_orchestration.py deleted file mode 100644 index 9d639892..00000000 --- a/.evergreen/docker/overwrite_orchestration.py +++ /dev/null @@ -1,46 +0,0 @@ -import json -import os - -orch_file = os.environ["ORCHESTRATION_FILE"] - -with open(orch_file) as fid: - data = json.load(fid) - -items = [] - - -# Gather all the items that have process settings. -def traverse(root): - if isinstance(root, list): - [traverse(i) for i in root] - return - if "ipv6" in root: - items.append(root) - return - for key, value in root.items(): - if key == "routers": - continue - if isinstance(value, (dict, list)): - traverse(value) - - -traverse(data) - -# Docker does not enable ipv6 by default. -# https://docs.docker.com/config/daemon/ipv6/ -# We also need to use 0.0.0.0 instead of 127.0.0.1 -for item in items: - item["ipv6"] = False - item["bind_ip"] = "0.0.0.0,::1" - item["dbpath"] = f"/tmp/mongo-{item['port']}" - -if "routers" in data: - for router in data["routers"]: - router["ipv6"] = False - router["bind_ip"] = "0.0.0.0,::1" - router["logpath"] = f"/tmp/mongodb-{item['port']}.log" - -print(json.dumps(data, indent=2)) - -with open(orch_file, "w") as fid: - json.dump(data, fid) diff --git a/.evergreen/docker/rhel8/base-entrypoint.sh b/.evergreen/docker/rhel8/base-entrypoint.sh index 6008e277..ae17993a 100755 --- a/.evergreen/docker/rhel8/base-entrypoint.sh +++ b/.evergreen/docker/rhel8/base-entrypoint.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -eu +# Remove the virtual env. +rm -rf $DRIVERS_TOOLS/.evergreen/venv || true + # Start the server. cd $DRIVERS_TOOLS make run-server diff --git a/.evergreen/docker/run-server.sh b/.evergreen/docker/run-server.sh index 6c78e686..a7a3514a 100755 --- a/.evergreen/docker/run-server.sh +++ b/.evergreen/docker/run-server.sh @@ -76,3 +76,6 @@ test -t 1 && ARGS+=" -t" # Launch server container. $DOCKER run $ARGS $NAME $ENTRYPOINT + +# Restore the clis +bash $DRIVERS_TOOLS/.evergreen/orchestration/setup.sh diff --git a/.evergreen/docker/ubuntu20.04/base-entrypoint.sh b/.evergreen/docker/ubuntu20.04/base-entrypoint.sh index 6008e277..966c5cb0 100755 --- a/.evergreen/docker/ubuntu20.04/base-entrypoint.sh +++ b/.evergreen/docker/ubuntu20.04/base-entrypoint.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -set -eu +set -eux + +# Remove the virtual env. +rm -rf $DRIVERS_TOOLS/.evergreen/venv || true # Start the server. cd $DRIVERS_TOOLS diff --git a/.evergreen/download-mongodb.sh b/.evergreen/download-mongodb.sh index 1887ea6f..366fc7a1 100755 --- a/.evergreen/download-mongodb.sh +++ b/.evergreen/download-mongodb.sh @@ -1,49 +1,16 @@ #!/usr/bin/env bash # shellcheck shell=sh -# For future use the feed to get full list of distros : http://downloads.mongodb.org/full.json - +# This file is no longer used directly by drivers-evergreen-tools. +# If using this file to download mongodb binaries, you should consider instead using `mongodl.py` and `mongosh-dl.py`. +# If using this file for get_distro, use `get-distro.sh`. set -o errexit # Exit the script with error if any of the commands fail get_distro () { - if [ -f /etc/os-release ]; then - . /etc/os-release - DISTRO="${ID}-${VERSION_ID}" - elif [ -f /etc/centos-release ]; then - version=$(cat /etc/centos-release | tr -dc '0-9.' | cut -d '.' -f1) - DISTRO="centos-${version}" - elif command -v lsb_release >/dev/null 2>&1; then - name=$(lsb_release -s -i) - if [ "$name" = "RedHatEnterpriseServer" ]; then # RHEL 6.2 at least - name="rhel" - fi - version=$(lsb_release -s -r) - DISTRO="${name}-${version}" - elif [ -f /etc/redhat-release ]; then - release=$(cat /etc/redhat-release) - case $release in - *Red\ Hat*) - name="rhel" - ;; - Fedora*) - name="fedora" - ;; - esac - version=$(echo $release | sed 's/.*\([[:digit:]]\).*/\1/g') - DISTRO="${name}-${version}" - elif [ -f /etc/lsb-release ]; then - . /etc/lsb-release - DISTRO="${DISTRIB_ID}-${DISTRIB_RELEASE}" - elif grep -R "Amazon Linux" "/etc/system-release" >/dev/null 2>&1; then - DISTRO="amzn64" - fi - - OS_NAME=$(uname -s) - MARCH=$(uname -m) - DISTRO=$(echo "$OS_NAME-$DISTRO-$MARCH" | tr '[:upper:]' '[:lower:]') - - echo $DISTRO + # shellcheck disable=SC3028 + _script_dir="$(dirname ${BASH_SOURCE:-$0})" + . ${_script_dir}/get-distro.sh } # get_mongodb_download_url_for "linux-distro-version-architecture" "latest|44|42|40|36|34|32|30|28|26|24" "true|false" diff --git a/.evergreen/get-distro.sh b/.evergreen/get-distro.sh new file mode 100755 index 00000000..1ebc5a54 --- /dev/null +++ b/.evergreen/get-distro.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +get_distro () +{ + if [ -f /etc/os-release ]; then + . /etc/os-release + DISTRO="${ID}-${VERSION_ID}" + elif [ -f /etc/centos-release ]; then + version=$(cat /etc/centos-release | tr -dc '0-9.' | cut -d '.' -f1) + DISTRO="centos-${version}" + elif command -v lsb_release >/dev/null 2>&1; then + name=$(lsb_release -s -i) + if [ "$name" = "RedHatEnterpriseServer" ]; then # RHEL 6.2 at least + name="rhel" + fi + version=$(lsb_release -s -r) + DISTRO="${name}-${version}" + elif [ -f /etc/redhat-release ]; then + release=$(cat /etc/redhat-release) + case $release in + *Red\ Hat*) + name="rhel" + ;; + Fedora*) + name="fedora" + ;; + esac + version=$(echo $release | sed 's/.*\([[:digit:]]\).*/\1/g') + DISTRO="${name}-${version}" + elif [ -f /etc/lsb-release ]; then + . /etc/lsb-release + DISTRO="${DISTRIB_ID}-${DISTRIB_RELEASE}" + elif grep -R "Amazon Linux" "/etc/system-release" >/dev/null 2>&1; then + DISTRO="amzn64" + fi + + OS_NAME=$(uname -s) + MARCH=$(uname -m) + DISTRO=$(echo "$OS_NAME-$DISTRO-$MARCH" | tr '[:upper:]' '[:lower:]') + + echo $DISTRO +} + +get_distro diff --git a/.evergreen/install-cli.sh b/.evergreen/install-cli.sh index ed441640..0fa9f560 100755 --- a/.evergreen/install-cli.sh +++ b/.evergreen/install-cli.sh @@ -15,12 +15,18 @@ fi SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/handle-paths.sh -pushd $SCRIPT_DIR +pushd $SCRIPT_DIR > /dev/null # Ensure pipx is writing assets to a contained location. export UV_CACHE_DIR=${DRIVERS_TOOLS}/.local/uv-cache export UV_TOOL_DIR=${DRIVERS_TOOLS}/.local/uv-tool +if [ "${DOCKER_RUNNING:-}" == "true" ]; then + _root_dir=$(mktemp -d) + UV_CACHE_DIR=$_root_dir/uv-cache + UV_TOOL_DIR=$_root_dir/uv-tool +fi + . ./venv-utils.sh if [ ! -d $SCRIPT_DIR/venv ]; then @@ -34,13 +40,12 @@ if [ ! -d $SCRIPT_DIR/venv ]; then echo "Creating virtual environment 'venv'..." venvcreate "${PYTHON:?}" venv echo "Creating virtual environment 'venv'... done." - - python -m pip install uv + python -m pip install -q uv else venvactivate venv fi -pushd $1 +pushd $1 > /dev/null # On Windows, we have to do a bit of path manipulation. if [ "Windows_NT" == "${OS:-}" ]; then @@ -53,8 +58,8 @@ if [ "Windows_NT" == "${OS:-}" ]; then done rm -rf $TMP_DIR else - UV_TOOL_BIN_DIR=$(pwd) uv tool install --python "$(which python)" --force --editable . + UV_TOOL_BIN_DIR=$(pwd) uv tool install -q --python "$(which python)" --force --editable . fi -popd -popd +popd > /dev/null +popd > /dev/null diff --git a/.evergreen/mongosh_dl.py b/.evergreen/mongosh_dl.py index 0d53d6be..fbeefba1 100755 --- a/.evergreen/mongosh_dl.py +++ b/.evergreen/mongosh_dl.py @@ -61,7 +61,7 @@ def _get_latest_version_git(): shlex.split(cmd), cwd=path, stderr=subprocess.PIPE ) for line in reversed(output.decode("utf-8").splitlines()): - if re.match("^v\d+\.\d+\.\d+$", line): + if re.match(r"^v\d+\.\d+\.\d+$", line): LOGGER.debug("Found version %s", line) return line.replace("v", "").strip() diff --git a/.evergreen/orchestration/drivers_orchestration.py b/.evergreen/orchestration/drivers_orchestration.py new file mode 100644 index 00000000..e0c6f2ad --- /dev/null +++ b/.evergreen/orchestration/drivers_orchestration.py @@ -0,0 +1,429 @@ +""" +Run mongo-orchestration and launch a deployment. + +Use '--help' for more information. +""" + +import argparse +import json +import logging +import os +import shlex +import shutil +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime +from pathlib import Path + +from mongodl import main as mongodl +from mongosh_dl import main as mongosh_dl + +# Get global values. +HERE = Path(__file__).absolute().parent +EVG_PATH = HERE.parent +DRIVERS_TOOLS = EVG_PATH.parent +LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s") + + +def get_options(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("command", choices=["run", "start", "stop"]) + parser.add_argument( + "--verbose", "-v", action="store_true", help="Whether to log at the DEBUG level" + ) + parser.add_argument( + "--quiet", "-q", action="store_true", help="Whether to log at the WARNING level" + ) + parser.add_argument( + "--version", + default="latest", + help='The version to download (Required). Use "latest" to download ' + "the newest available version (including release candidates).", + ) + parser.add_argument( + "--topology", + choices=["standalone", "replica_set", "sharded_cluster"], + default="standalone", + help="The topology of the server deployment", + ) + parser.add_argument( + "--auth", action="store_true", help="Whether to add authentication" + ) + parser.add_argument( + "--ssl", action="store_true", help="Whether to add TLS configuration" + ) + parser.add_argument( + "--orchestration-file", help="The name of the orchestration config file" + ) + + other_group = parser.add_argument_group("Other options") + parser.add_argument( + "--load-balancer", action="store_true", help="Whether to use a load balancer" + ) + parser.add_argument( + "--skip-crypt-shared", + action="store_true", + help="Whether to skip installing crypt_shared lib", + ) + other_group.add_argument( + "--install-legacy-shell", + action="store_true", + help="Whether to install the legacy shell", + ) + other_group.add_argument( + "--disable-test-commands", + action="store_true", + help="Whether to disable test commands", + ) + other_group.add_argument( + "--storage-engine", + choices=["", "mmapv1", "wiredtiger", "inmemory"], + help="The storage engine to use", + ) + other_group.add_argument( + "--require-api-version", + action="store_true", + help="Whether to set requireApiVersion", + ) + other_group.add_argument( + "--mongo-orchestration-home", help="The path to mongo-orchestration home" + ) + other_group.add_argument( + "--mongodb-binaries", help="The path to store the MongoDB binaries" + ) + + # Get the options, and then allow environment variable overrides. + opts = parser.parse_args() + for key in vars(opts).keys(): + env_var = key.upper() + if env_var == "VERSION": + env_var = "MONGODB_VERSION" + if env_var in os.environ: + if env_var == "AUTH": + opts.auth = os.environ.get("AUTH") == "auth" + elif env_var == "SSL": + opts.ssl = os.environ.get("SSL") == "ssl" + elif isinstance(getattr(opts, key), bool): + if os.environ[env_var]: + setattr(opts, key, True) + else: + setattr(opts, key, os.environ[env_var]) + + if opts.mongo_orchestration_home is None: + opts.mongo_orchestration_home = DRIVERS_TOOLS / ".evergreen/orchestration" + if opts.mongodb_binaries is None: + opts.mongodb_binaries = DRIVERS_TOOLS / "mongodb/bin" + if opts.topology == "standalone": + opts.topology = "server" + + if opts.verbose: + LOGGER.setLevel(logging.DEBUG) + elif opts.quiet: + LOGGER.setLevel(logging.WARNING) + return opts + + +def handle_docker_config(data): + """Modify config to when running in a docker container.""" + items = [] + + # Gather all the items that have process settings. + def traverse(root): + if isinstance(root, list): + [traverse(i) for i in root] + return + if "ipv6" in root: + items.append(root) + return + for key, value in root.items(): + if key == "routers": + continue + if isinstance(value, (dict, list)): + traverse(value) + + traverse(data) + + # Docker does not enable ipv6 by default. + # https://docs.docker.com/config/daemon/ipv6/ + # We also need to use 0.0.0.0 instead of 127.0.0.1 + for item in items: + item["ipv6"] = False + item["bind_ip"] = "0.0.0.0,::1" + item["dbpath"] = f"/tmp/mongo-{item['port']}" + + if "routers" in data: + for router in data["routers"]: + router["ipv6"] = False + router["bind_ip"] = "0.0.0.0,::1" + router["logpath"] = f"/tmp/mongodb-{item['port']}.log" + + +def run(opts): + LOGGER.info("Running orchestration...") + + # Clean up previous files. + mdb_binaries = Path(opts.mongodb_binaries) + # NOTE: in general, we need to use posix strings to avoid path escapes on cygwin. + mdb_binaries_str = mdb_binaries.as_posix() + shutil.rmtree(mdb_binaries, ignore_errors=True) + expansion_yaml = Path("mo-expansion.yml") + expansion_yaml.unlink(missing_ok=True) + expansion_sh = Path("mo-expansion.sh") + expansion_sh.unlink(missing_ok=True) + + # The evergreen directory to path. + os.environ["PATH"] = f"{EVG_PATH}:{os.environ['PATH']}" + + # Download the archive. + dl_start = datetime.now() + version = opts.version + cache_dir = DRIVERS_TOOLS / ".local/cache" + cache_dir_str = cache_dir.as_posix() + default_args = f"--out {mdb_binaries_str} --cache-dir {cache_dir_str}" + if opts.quiet: + default_args += " -q" + elif opts.verbose: + default_args += " -v" + args = f"{default_args} --version {version}" + args += " --strip-path-components 2 --component archive" + LOGGER.info(f"Downloading mongodb {version}...") + mongodl(shlex.split(args)) + LOGGER.info(f"Downloading mongodb {version}... done.") + + # Download legacy shell. + if opts.install_legacy_shell: + args = f"{default_args} --version 5.0" + args += " --strip-path-components 2 --component shell" + LOGGER.INFO("Downloading legacy shell...") + mongodl(shlex.split(args)) + LOGGER.INFO("Downloading legacy shell... done.") + + # Download crypt shared. + if not opts.skip_crypt_shared: + # Get the download URL for crypt_shared. + args = f"{default_args} --version {version}" + args += " --strip-path-components 1 --component crypt_shared" + LOGGER.info("Downloading crypt_shared...") + mongodl(shlex.split(args)) + LOGGER.info("Downloading crypt_shared... done.") + crypt_shared_path = None + for fname in os.listdir(mdb_binaries_str): + if fname.startswith("mongo_crypt_v1"): + crypt_shared_path = (mdb_binaries / fname).as_posix() + assert crypt_shared_path is not None + crypt_text = f'CRYPT_SHARED_LIB_PATH: "{crypt_shared_path}"' + expansion_yaml.write_text(crypt_text) + expansion_sh.write_text(crypt_text.replace(": ", "=")) + + # Download mongosh + args = f"--out {mdb_binaries_str} --strip-path-components 2" + if opts.verbose: + args += " -v" + elif opts.quiet: + args += " -q" + LOGGER.info("Downloading mongosh...") + mongosh_dl(shlex.split(args)) + LOGGER.info("Downloading mongosh... done.") + + dl_end = datetime.now() + + # Handle orchestration file - explicit or implicit. + orchestration_file = opts.orchestration_file + if not orchestration_file: + fname = "basic" + if opts.auth: + fname = "auth" + if opts.ssl: + fname += "-ssl" + if opts.load_balancer: + fname += "-load-balancer" + elif opts.disable_test_commands: + fname = "disableTestCommands" + elif opts.storage_engine: + fname = opts.storage_engine + orchestration_file = f"{fname}.json" + + # Get the orchestration config data. + topology = opts.topology + mo_home = Path(opts.mongo_orchestration_home) + orch_path = mo_home / f"configs/{topology}s/{orchestration_file}" + LOGGER.info(f"Using orchestration file: {orch_path}") + text = orch_path.read_text() + text = text.replace("ABSOLUTE_PATH_REPLACEMENT_TOKEN", DRIVERS_TOOLS.as_posix()) + data = json.loads(text) + + if opts.require_api_version: + if opts.topology == "replica_set": + raise ValueError( + "requireApiVersion is not supported with replica_sets, see SERVER-97010" + ) + data["requireApiVersion"] = "1" + + # If running on Docker, update the orchestration file to be docker-friendly. + if os.environ.get("DOCKER_RUNNING"): + handle_docker_config(data) + + # Write the config file. + orch_file = Path(mo_home / "config.json") + orch_file.write_text(json.dumps(data, indent=2)) + + # Start the orchestration. + mo_start = datetime.now() + start(opts) + + # Configure the server. + LOGGER.info("Starting deployment...") + url = f"http://localhost:8889/v1/{topology}s" + req = urllib.request.Request( + url, data=json.dumps(data).encode("utf-8"), method="POST" + ) + try: + resp = urllib.request.urlopen(req) + except urllib.error.HTTPError as e: + stop() + raise e + resp = json.loads(resp.read().decode("utf-8")) + LOGGER.debug(resp) + LOGGER.info("Starting deployment... done.") + + # Handle the cluster uri. + uri = resp.get("mongodb_auth_uri", resp["mongodb_uri"]) + expansion_yaml.write_text(expansion_yaml.read_text() + f"\nMONGODB_URI: {uri}") + LOGGER.info(f"Cluster URI: {uri}") + + # Write the results file. + mo_end = datetime.now() + data = dict( + results=[ + dict( + status="PASS", + test_file="Orchestration", + start=int(mo_start.timestamp()), + end=int(mo_end.timestamp()), + elapsed=(mo_end - mo_start).total_seconds(), + ), + dict( + status="PASS", + test_file="Download MongoDB", + start=int(dl_start.timestamp()), + end=int(dl_end.timestamp()), + elapsed=(dl_end - dl_start).total_seconds(), + ), + ] + ) + Path(DRIVERS_TOOLS / "results.json").write_text(json.dumps(data, indent=2)) + + LOGGER.info("Running orchestration... done.") + + +def start(opts): + # Start mongo-orchestration + + # Stop a running server. + mo_home = Path(opts.mongo_orchestration_home) + if (mo_home / "server.pid").exists(): + stop() + + # Clean up previous files. + for fname in ["out.log", "server.log", "orchestration.config", "config.json"]: + if (mo_home / fname).exists(): + (mo_home / fname).unlink() + + # Set up the mongo orchestration config. + os.makedirs(mo_home / "lib", exist_ok=True) + mo_config = mo_home / "orchestration.config" + mdb_binaries = Path(opts.mongodb_binaries) + config = dict(releases=dict(default=mdb_binaries.as_posix())) + mo_config.write_text(json.dumps(config, indent=2)) + mo_config_str = mo_config.as_posix() + command = f"{sys.executable} -m mongo_orchestration.server" + + # Handle Windows-specific concerns. + if os.name == "nt": + # Copy client certificates. + src = DRIVERS_TOOLS / ".evergreen/x509gen/client.pem" + dst = mo_home / "lib/client.pem" + try: + shutil.copy2(src, dst) + except (shutil.SameFileError, PermissionError): + pass + + # We need to use the CLI executable, and add it to our path. + os.environ["PATH"] = ( + f"{Path(sys.executable).parent}{os.pathsep}{os.environ['PATH']}" + ) + command = "mongo-orchestration -s wsgiref" + + mo_start = datetime.now() + + # Start the process. + args = f"{command} start -e default -f {mo_config_str}" + args += " --socket-timeout-ms=60000 --bind=127.0.0.1 --enable-majority-read-concern" + + LOGGER.info("Starting mongo-orchestration...") + output_file = mo_home / "out.log" + server_file = mo_home / "server.log" + + # NOTE: we need to use a separate file id for stdout and close it so Evergreen does not hang. + output_fid = output_file.open("w") + try: + subprocess.run( + shlex.split(args), check=True, stderr=subprocess.STDOUT, stdout=output_fid + ) + except subprocess.CalledProcessError: + LOGGER.error("Orchestration failed!") + LOGGER.error(f"server.log:\n{server_file.read_text().strip()}") + raise + finally: + output_fid.close() + LOGGER.info(f"out.log:\n{output_file.read_text().strip()}") + + # Wait for the server to be available. + attempt = 0 + while True: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.connect(("localhost", 8889)) + break + except ConnectionRefusedError: + if (datetime.now() - mo_start).seconds > 120: + stop() + LOGGER.error("Orchestration failed!") + LOGGER.error(f"server.log: {server_file.read_text()}") + raise TimeoutError("Server failed to start") from None + attempt += 1 + time.sleep(attempt * 1000) + + LOGGER.info("Starting mongo-orchestration... done.") + + +def stop(): + LOGGER.info("Stopping mongo-orchestration...") + py_exe = Path(sys.executable).as_posix() + args = f"{py_exe} -m mongo_orchestration.server stop" + proc = subprocess.run( + shlex.split(args), check=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE + ) + LOGGER.debug(proc.stdout.decode("utf-8")) + LOGGER.info("Stopping mongo-orchestration... done.") + + +def main(): + opts = get_options() + if opts.command == "run": + run(opts) + elif opts.command == "start": + start(opts) + elif opts.command == "stop": + stop() + + +if __name__ == "__main__": + main() diff --git a/.evergreen/orchestration/pyproject.toml b/.evergreen/orchestration/pyproject.toml new file mode 100644 index 00000000..8f0fb4a9 --- /dev/null +++ b/.evergreen/orchestration/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[project] +name = "drivers-orchestration" +version = "0.1.0" +description = 'Scripts for drivers orchestration' +requires-python = ">=3.8" +license = "MIT" +keywords = [] +authors = [ + { name = "Steven Silvester", email = "steve.silvester@mongodb.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python", + "Programming Language :: Python :: Implementation :: CPython", +] +dependencies = ["drivers-evergreen-tools @ {root:parent:uri}", "mongo-orchestration @ https://github.com/mongodb/mongo-orchestration/archive/master.tar.gz"] + +[project.scripts] +drivers-orchestration = "drivers_orchestration:main" + +[tool.hatch.build] +include = ["drivers_orchestration.py"] + +[tool.hatch.metadata] +allow-direct-references=true diff --git a/.evergreen/orchestration/require-api-version.js b/.evergreen/orchestration/require-api-version.js deleted file mode 100644 index 9b6de247..00000000 --- a/.evergreen/orchestration/require-api-version.js +++ /dev/null @@ -1 +0,0 @@ -db.adminCommand({ "setParameter": 1, "requireApiVersion": 1 }); diff --git a/.evergreen/orchestration/setup.sh b/.evergreen/orchestration/setup.sh new file mode 100755 index 00000000..93ddc63d --- /dev/null +++ b/.evergreen/orchestration/setup.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Handle setup for orchestration + +set -o errexit + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +. $SCRIPT_DIR/../handle-paths.sh + +# Install the clis in this folder. +bash $SCRIPT_DIR/../install-cli.sh $SCRIPT_DIR diff --git a/.evergreen/run-orchestration.sh b/.evergreen/run-orchestration.sh index 380b6883..7481676c 100755 --- a/.evergreen/run-orchestration.sh +++ b/.evergreen/run-orchestration.sh @@ -1,179 +1,12 @@ #!/usr/bin/env bash # shellcheck shell=sh set -o errexit # Exit the script with error if any of the commands fail - -# Supported environment variables: -# AUTH Set to "auth" to enable authentication. Defaults to "noauth" -# SSL Set to "yes" to enable SSL. Defaults to "nossl" -# TOPOLOGY Set to "server", "replica_set", or "sharded_cluster". Defaults to "server" (i.e. standalone). -# LOAD_BALANCER Set to a non-empty string to enable load balancer. Only supported for sharded clusters. -# STORAGE_ENGINE Set to a non-empty string to use the /.json configuration (e.g. STORAGE_ENGINE=inmemory). -# REQUIRE_API_VERSION Set to a non-empty string to set the requireApiVersion parameter. Currently only supported for standalone servers. -# DISABLE_TEST_COMMANDS Set to a non-empty string to use the /disableTestCommands.json configuration (e.g. DISABLE_TEST_COMMANDS=1). -# MONGODB_VERSION Set to a MongoDB version to use for download-mongodb.sh. Defaults to "latest". -# MONGODB_DOWNLOAD_URL Set to a MongoDB download URL to use for download-mongodb.sh. -# ORCHESTRATION_FILE Set to a non-empty string to use the /.json configuration. -# SKIP_CRYPT_SHARED Set to a non-empty string to skip downloading crypt_shared -# MONGODB_BINARIES Set to a non-empty string to set the path to the MONGODB_BINARIES for mongo orchestration. -# PYTHON Set to a non-empty string to set the Python binary to use. -# INSTALL_LEGACY_SHELL Set to a non-empty string to install the legacy mongo shell. - +set -x # See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 # Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) # shellcheck disable=SC3028 SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) . $SCRIPT_DIR/handle-paths.sh -AUTH=${AUTH:-noauth} -SSL=${SSL:-nossl} -TOPOLOGY=${TOPOLOGY:-server} -LOAD_BALANCER=${LOAD_BALANCER} -STORAGE_ENGINE=${STORAGE_ENGINE} -REQUIRE_API_VERSION=${REQUIRE_API_VERSION} -DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS} -MONGODB_VERSION=${MONGODB_VERSION:-latest} -MONGODB_DOWNLOAD_URL=${MONGODB_DOWNLOAD_URL} -ORCHESTRATION_FILE=${ORCHESTRATION_FILE} -INSTALL_LEGACY_SHELL=${INSTALL_LEGACY_SHELL:-} -PYTHON=${PYTHON:-} -# Note: MONGO_ORCHESTRATION_HOME and MONGODB_BINARIES defaults are handled in handle-paths.sh. - -DL_START=$(date +%s) - -# Functions to fetch MongoDB binaries. -. $SCRIPT_DIR/download-mongodb.sh - -# To continue supporting `sh run-orchestration.sh` for backwards-compatibility, -# explicitly invoke Bash as a subshell here when running `ensure_python3`. -PYTHON=$(bash -c ". $SCRIPT_DIR/find-python3.sh && ensure_python3 2>/dev/null") - -# Set up the mongo orchestration config. -mkdir -p $MONGO_ORCHESTRATION_HOME -printf '%s' $MONGODB_BINARIES | $PYTHON -c 'import json,sys; print(json.dumps({"releases": {"default": sys.stdin.read() }}))' > $MONGO_ORCHESTRATION_HOME/orchestration.config - -# Copy client certificate because symlinks do not work on Windows. -mkdir -p ${MONGO_ORCHESTRATION_HOME}/lib -cp ${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem ${MONGO_ORCHESTRATION_HOME}/lib/client.pem 2> /dev/null || true - -get_distro -if [ -z "$MONGODB_DOWNLOAD_URL" ]; then - get_mongodb_download_url_for "$DISTRO" "$MONGODB_VERSION" -else - # Even though we have the MONGODB_DOWNLOAD_URL, we still call this to get the proper EXTRACT variable - get_mongodb_download_url_for "$DISTRO" -fi -download_and_extract "$MONGODB_DOWNLOAD_URL" "$EXTRACT" "$MONGOSH_DOWNLOAD_URL" "$EXTRACT_MONGOSH" - -# Write the crypt shared path to the expansion file if given. -if [ -n "$CRYPT_SHARED_LIB_PATH" ]; then - cat <> mo-expansion.yml -CRYPT_SHARED_LIB_PATH: "$CRYPT_SHARED_LIB_PATH" -EOT - - cat <> mo-expansion.sh -export CRYPT_SHARED_LIB_PATH="$CRYPT_SHARED_LIB_PATH" -EOT -fi - -DL_END=$(date +%s) -MO_START=$(date +%s) - -# If no orchestration file was specified, build up the name based on configuration parameters. -if [ -z "$ORCHESTRATION_FILE" ]; then - ORCHESTRATION_FILE="basic" - if [ "$AUTH" = "auth" ]; then - ORCHESTRATION_FILE="auth" - fi - - if [ "$SSL" != "nossl" ]; then - ORCHESTRATION_FILE="${ORCHESTRATION_FILE}-ssl" - fi - - if [ -n "$LOAD_BALANCER" ]; then - ORCHESTRATION_FILE="${ORCHESTRATION_FILE}-load-balancer" - fi - - # disableTestCommands files do not exist for different auth or ssl modes. - if [ ! -z "$DISABLE_TEST_COMMANDS" ]; then - ORCHESTRATION_FILE="disableTestCommands" - fi - - # Storage engine config files do not exist for different auth or ssl modes. - if [ ! -z "$STORAGE_ENGINE" ]; then - ORCHESTRATION_FILE="$STORAGE_ENGINE" - fi - - ORCHESTRATION_FILE="${ORCHESTRATION_FILE}.json" -fi - -# Allow projects to override orchestration configs -ORCHESTRATION_FILE="configs/${TOPOLOGY}s/${ORCHESTRATION_FILE}" - -if [ -f "$PROJECT_ORCHESTRATION_HOME/$ORCHESTRATION_FILE" ]; then - export ORCHESTRATION_FILE="$PROJECT_ORCHESTRATION_HOME/$ORCHESTRATION_FILE" -elif [ -f "$MONGO_ORCHESTRATION_HOME/$ORCHESTRATION_FILE" ]; then - export ORCHESTRATION_FILE="$MONGO_ORCHESTRATION_HOME/$ORCHESTRATION_FILE" -else - echo "Could not find orchestration file $ORCHESTRATION_FILE (checked in $PROJECT_ORCHESTRATION_HOME and $MONGO_ORCHESTRATION_HOME)" - exit 1 -fi -echo "ORCHESTRATION_FILE=$ORCHESTRATION_FILE" - -# Copy the orchestration file so we can override it. -cp -f "$ORCHESTRATION_FILE" "$MONGO_ORCHESTRATION_HOME/config.json" -ORCHESTRATION_FILE="$MONGO_ORCHESTRATION_HOME/config.json" - -# Handle absolute path. -perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$(echo $DRIVERS_TOOLS | sed 's/\\/\\\\\\\\/g')|g" $ORCHESTRATION_FILE - -# If running on Docker, update the orchestration file to be docker-friendly. -if [ -n "$DOCKER_RUNNING" ]; then - $PYTHON $SCRIPT_DIR/docker/overwrite_orchestration.py -fi - -export ORCHESTRATION_URL="http://localhost:8889/v1/${TOPOLOGY}s" - -# Start mongo-orchestration -PYTHON="${PYTHON:?}" bash $SCRIPT_DIR/start-orchestration.sh "$MONGO_ORCHESTRATION_HOME" - -if ! curl --silent --show-error --data @"$ORCHESTRATION_FILE" "$ORCHESTRATION_URL" --max-time 600 --fail -o tmp.json; then - echo Failed to start cluster, see $MONGO_ORCHESTRATION_HOME/out.log: - cat $MONGO_ORCHESTRATION_HOME/out.log - echo Failed to start cluster, see $MONGO_ORCHESTRATION_HOME/server.log: - cat $MONGO_ORCHESTRATION_HOME/server.log - exit 1 -fi -cat tmp.json - -URI=$(${PYTHON:?} -c 'import json; j=json.load(open("tmp.json")); print(j["mongodb_auth_uri" if "mongodb_auth_uri" in j else "mongodb_uri"])' | tr -d '\r') -echo 'MONGODB_URI: "'$URI'"' >> mo-expansion.yml -echo $URI > $DRIVERS_TOOLS/uri.txt -printf "\nCluster URI: %s\n" "$URI" - -MO_END=$(date +%s) -MO_ELAPSED=$(expr $MO_END - $MO_START) -DL_ELAPSED=$(expr $DL_END - $DL_START) -cat <$DRIVERS_TOOLS/results.json -{"results": [ - { - "status": "PASS", - "test_file": "Orchestration", - "start": $MO_START, - "end": $MO_END, - "elapsed": $MO_ELAPSED - }, - { - "status": "PASS", - "test_file": "Download MongoDB", - "start": $DL_START, - "end": $DL_END, - "elapsed": $DL_ELAPSED - } -]} - -EOT - -# Set the requireApiVersion parameter -if [ ! -z "$REQUIRE_API_VERSION" ]; then - $MONGODB_BINARIES/mongosh $URI $MONGO_ORCHESTRATION_HOME/require-api-version.js -fi +bash $SCRIPT_DIR/orchestration/setup.sh +$SCRIPT_DIR/orchestration/drivers-orchestration run "$@" diff --git a/.evergreen/setup.sh b/.evergreen/setup.sh index 473ff52d..5d712ff7 100755 --- a/.evergreen/setup.sh +++ b/.evergreen/setup.sh @@ -33,3 +33,6 @@ echo "DRIVERS_TOOLS_PYTHON=$DRIVERS_TOOLS_PYTHON" >> $DRIVERS_TOOLS/.env # Install the clis in this folder. bash $SCRIPT_DIR/install-cli.sh $SCRIPT_DIR + +# Set up the orchestration folder. +bash $SCRIPT_DIR/orchestration/setup.sh diff --git a/.evergreen/start-orchestration.sh b/.evergreen/start-orchestration.sh index f774f015..6521bf11 100755 --- a/.evergreen/start-orchestration.sh +++ b/.evergreen/start-orchestration.sh @@ -1,77 +1,13 @@ #!/usr/bin/env bash - -if [ -z "$BASH" ]; then - echo "start-orchestration.sh must be run in a Bash shell!" 1>&2 - exit 1 -fi - -if [ "$#" -ne 1 ]; then - echo "$0 requires one argument: " - echo "For example: $0 /tmp/mongo-orchestration-home" - exit 1 -fi +# shellcheck shell=sh set -o errexit # Exit the script with error if any of the commands fail - -MONGO_ORCHESTRATION_HOME="$1" - -echo "From shell `date`" > $MONGO_ORCHESTRATION_HOME/server.log - -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +# See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 +# Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) +# shellcheck disable=SC3028 +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) . $SCRIPT_DIR/handle-paths.sh -declare det_evergreen_dir -det_evergreen_dir=$SCRIPT_DIR -. "$det_evergreen_dir/find-python3.sh" -. "$det_evergreen_dir/venv-utils.sh" - -cd "$MONGO_ORCHESTRATION_HOME" - -PYTHON=$(ensure_python3) - -echo "Creating virtual environment 'venv'..." -venvcreate "${PYTHON:?}" venv -echo "Creating virtual environment 'venv'... done." - -# Install from github to get the latest mongo-orchestration, fall back on published wheel. -# The fallback was added to accommodate versions of Python 3 for which there is no compatible version -# of the hatchling backend used by mongo-orchestration. -python -m pip install -q --upgrade 'https://github.com/mongodb/mongo-orchestration/archive/master.tar.gz' || python -m pip install -q --upgrade mongo-orchestration -python -m pip list -cd $DRIVERS_TOOLS - -# Create default config file if it doesn't exist -if [ ! -f $MONGO_ORCHESTRATION_HOME/orchestration.config ]; then - printf '%s' $MONGODB_BINARIES | python -c 'import json,sys; print(json.dumps({"releases": {"default": sys.stdin.read() }}))' > $MONGO_ORCHESTRATION_HOME/orchestration.config -fi - -ORCHESTRATION_ARGUMENTS="-e default -f $MONGO_ORCHESTRATION_HOME/orchestration.config --socket-timeout-ms=60000 --bind=127.0.0.1 --enable-majority-read-concern" -if [[ "${OSTYPE:?}" == cygwin ]]; then - ORCHESTRATION_ARGUMENTS="$ORCHESTRATION_ARGUMENTS -s wsgiref" -fi - -# Forcibly kill the process listening on port 8889, most likely a wild -# mongo-orchestration left running from a previous task. -# NOTE: Killing mongo-orchestration with SIGTERM gives it a chance to -# cleanup any running mongo servers. -# Also kill any leftover mongo processes on common ports as a backup. -. "$det_evergreen_dir/process-utils.sh" -for port in 8889 27017 27018 27019 27217 27218 27219 1026; do - killport $port -done - - -mongo-orchestration $ORCHESTRATION_ARGUMENTS start > $MONGO_ORCHESTRATION_HOME/out.log 2>&1 < /dev/null & - -ls -la $MONGO_ORCHESTRATION_HOME - -sleep 5 -if ! curl http://localhost:8889/ --silent --show-error --max-time 120 --fail; then - echo Failed to start mongo-orchestration, see $MONGO_ORCHESTRATION_HOME/out.log: - cat $MONGO_ORCHESTRATION_HOME/out.log - echo Failed to start mongo-orchestration, see $MONGO_ORCHESTRATION_HOME/server.log: - cat $MONGO_ORCHESTRATION_HOME/server.log - exit 1 -fi -sleep 5 +bash $SCRIPT_DIR/orchestration/setup.sh +$SCRIPT_DIR/orchestration/drivers-orchestration start diff --git a/.evergreen/stop-orchestration.sh b/.evergreen/stop-orchestration.sh index 75356f11..e173a580 100755 --- a/.evergreen/stop-orchestration.sh +++ b/.evergreen/stop-orchestration.sh @@ -3,24 +3,11 @@ set -o errexit # Exit the script with error if any of the commands fail +# See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 +# Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) # shellcheck disable=SC3028 -SCRIPT_DIR=$(dirname "${BASH_SOURCE:-"$0"}") -. "$SCRIPT_DIR/handle-paths.sh" +SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) +. $SCRIPT_DIR/handle-paths.sh -cd ${DRIVERS_TOOLS} - -# source the mongo-orchestration virtualenv if it exists -VENV="$MONGO_ORCHESTRATION_HOME/venv" -if [ -x "$(command -v mongo-orchestration)" ]; then - mongo-orchestration stop -elif [ -f "$VENV/bin/activate" ]; then - . "$VENV/bin/activate" - mongo-orchestration stop -elif [ -f "$VENV/Scripts/activate" ]; then - . "$VENV/Scripts/activate" - mongo-orchestration stop -else - echo "No virtualenv found!" -fi - -cd - +bash $SCRIPT_DIR/orchestration/setup.sh +$SCRIPT_DIR/orchestration/drivers-orchestration stop diff --git a/.gitignore b/.gitignore index 26c4a54c..8479ac01 100644 --- a/.gitignore +++ b/.gitignore @@ -120,7 +120,9 @@ mo-expansion.sh .kube .cargo .rustup +.local .evergreen/orchestration/config.json +.evergreen/orchestration/drivers-orchestration .evergreen/socks5srv .evergreen/mongodl .evergreen/mongosh-dl