diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f331ad9aaf..b5f783323c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: mypy stages: [commit] name: mypy - entry: bash -c 'pipenv run mypy src/manifests tests/tests_manifests' + entry: bash -c 'pipenv run mypy src/run_assemble.py tests/test_run_assemble.py src/assemble_workflow tests/tests_assemble_workflow src/manifests tests/tests_manifests' language: system - id: pytest stages: [commit] diff --git a/src/assemble_workflow/assemble_args.py b/src/assemble_workflow/assemble_args.py index 466f88a81f..21c3cf794c 100644 --- a/src/assemble_workflow/assemble_args.py +++ b/src/assemble_workflow/assemble_args.py @@ -6,13 +6,14 @@ import argparse import logging +from typing import IO class AssembleArgs: - manifest: str + manifest: IO keep: bool - def __init__(self): + def __init__(self) -> None: parser = argparse.ArgumentParser(description="Assemble an OpenSearch Distribution") parser.add_argument("manifest", type=argparse.FileType("r"), help="Manifest file.") parser.add_argument("-b", "--base-url", dest="base_url", help="The base url to download the artifacts.") diff --git a/src/assemble_workflow/bundle.py b/src/assemble_workflow/bundle.py index 0e5da64cdf..424f694c7f 100644 --- a/src/assemble_workflow/bundle.py +++ b/src/assemble_workflow/bundle.py @@ -10,8 +10,11 @@ import shutil import subprocess from abc import ABC, abstractmethod +from typing import Any, List +from assemble_workflow.bundle_recorder import BundleRecorder from assemble_workflow.dist import Dist +from manifests.build_manifest import BuildComponent, BuildComponents, BuildManifest from paths.script_finder import ScriptFinder from system.temporary_directory import TemporaryDirectory @@ -23,28 +26,28 @@ class Bundle(ABC): - def __enter__(self): + def __enter__(self) -> 'Bundle': return self - def __exit__(self, exc_type, exc_value, exc_traceback): + def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> None: self.tmp_dir.__exit__(exc_type, exc_value, exc_traceback) - def __init__(self, build_manifest, artifacts_dir, bundle_recorder, keep=False): + def __init__(self, build_manifest: BuildManifest, artifacts_dir: str, bundle_recorder: BundleRecorder, keep: bool = False) -> None: """ Construct a new Bundle instance. :param build_manifest: A BuildManifest created from the build workflow. :param artifacts_dir: Dir location where build artifacts can be found locally :param bundle_recorder: The bundle recorder that will capture and build a BundleManifest """ - self.plugins = self.__get_plugins(build_manifest.components.values()) + self.plugins = self.__get_plugins(build_manifest.components) self.artifacts_dir = artifacts_dir self.bundle_recorder = bundle_recorder self.tmp_dir = TemporaryDirectory(keep=keep) - self.min_dist = self.__get_min_dist(build_manifest.components.values()) - self.installed_plugins = [] + self.min_dist = self.__get_min_dist(build_manifest.components) + self.installed_plugins: List[str] = [] self.build = build_manifest.build - def install_min(self): + def install_min(self) -> None: install_script = ScriptFinder.find_install_script(self.min_dist.name) install_command = " ".join( [ @@ -64,7 +67,7 @@ def install_min(self): ) self._execute(install_command) - def install_plugins(self): + def install_plugins(self) -> None: for plugin in self.plugins: logging.info(f"Installing {plugin.name}") self.install_plugin(plugin) @@ -73,7 +76,7 @@ def install_plugins(self): self.installed_plugins = os.listdir(plugins_path) @abstractmethod - def install_plugin(self, plugin): + def install_plugin(self, plugin: BuildComponent) -> None: install_script = ScriptFinder.find_install_script(plugin.name) install_command = " ".join( [ @@ -93,23 +96,23 @@ def install_plugin(self, plugin): ) self._execute(install_command) - def package(self, dest): + def package(self, dest: str) -> None: self.min_dist.build(self.bundle_recorder.package_name, dest) - def _execute(self, command): + def _execute(self, command: str) -> None: logging.info(f'Executing "{command}" in {self.min_dist.archive_path}') subprocess.check_call(command, cwd=self.min_dist.archive_path, shell=True) - def _copy_component(self, component, component_type): + def _copy_component(self, component: BuildComponent, component_type: str) -> str: rel_path = self.__get_rel_path(component, component_type) tmp_path = self.__copy_component_files(rel_path, self.tmp_dir.name) self.bundle_recorder.record_component(component, rel_path) return tmp_path - def __get_rel_path(self, component, component_type): + def __get_rel_path(self, component: BuildComponent, component_type: str) -> str: return next(iter(component.artifacts.get(component_type, [])), None) - def __copy_component_files(self, rel_path, dest): + def __copy_component_files(self, rel_path: str, dest: str) -> str: local_path = os.path.join(self.artifacts_dir, rel_path) dest_path = os.path.join(dest, os.path.basename(local_path)) if os.path.isfile(local_path): @@ -119,11 +122,11 @@ def __copy_component_files(self, rel_path, dest): else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), local_path) - def __get_plugins(self, build_components): - return [c for c in build_components if "plugins" in c.artifacts] + def __get_plugins(self, build_components: BuildComponents) -> List[BuildComponent]: + return [c for c in build_components.values() if "plugins" in c.artifacts] - def __get_min_dist(self, build_components): - min_bundle = next(iter([c for c in build_components if "dist" in c.artifacts]), None) + def __get_min_dist(self, build_components: BuildComponents) -> Dist: + min_bundle = next(iter([c for c in build_components.values() if "dist" in c.artifacts]), None) if min_bundle is None: raise ValueError('Missing min "dist" in input artifacts.') min_dist_path = self._copy_component(min_bundle, "dist") diff --git a/src/assemble_workflow/bundle_opensearch.py b/src/assemble_workflow/bundle_opensearch.py index f9e6297821..d0a2c0f7b0 100644 --- a/src/assemble_workflow/bundle_opensearch.py +++ b/src/assemble_workflow/bundle_opensearch.py @@ -7,15 +7,16 @@ import os from assemble_workflow.bundle import Bundle +from manifests.build_manifest import BuildComponent from system.os import current_platform class BundleOpenSearch(Bundle): @property - def install_plugin_script(self): + def install_plugin_script(self) -> str: return "opensearch-plugin.bat" if current_platform() == "windows" else "opensearch-plugin" - def install_plugin(self, plugin): + def install_plugin(self, plugin: BuildComponent) -> None: tmp_path = self._copy_component(plugin, "plugins") cli_path = os.path.join(self.min_dist.archive_path, "bin", self.install_plugin_script) self._execute(f"{cli_path} install --batch file:{tmp_path}") diff --git a/src/assemble_workflow/bundle_opensearch_dashboards.py b/src/assemble_workflow/bundle_opensearch_dashboards.py index ead515de64..6a819a4bab 100644 --- a/src/assemble_workflow/bundle_opensearch_dashboards.py +++ b/src/assemble_workflow/bundle_opensearch_dashboards.py @@ -7,10 +7,11 @@ import os from assemble_workflow.bundle import Bundle +from manifests.build_manifest import BuildComponent class BundleOpenSearchDashboards(Bundle): - def install_plugin(self, plugin): + def install_plugin(self, plugin: BuildComponent) -> None: tmp_path = self._copy_component(plugin, "plugins") cli_path = os.path.join(self.min_dist.archive_path, "bin", "opensearch-dashboards-plugin") self._execute(f"{cli_path} --allow-root install file:{tmp_path}") diff --git a/src/assemble_workflow/bundle_recorder.py b/src/assemble_workflow/bundle_recorder.py index e824eba42d..f7d4623674 100644 --- a/src/assemble_workflow/bundle_recorder.py +++ b/src/assemble_workflow/bundle_recorder.py @@ -5,13 +5,15 @@ # compatible open source license. import os +from typing import Any, Dict from urllib.parse import urljoin +from manifests.build_manifest import BuildComponent, BuildManifest from manifests.bundle_manifest import BundleManifest class BundleRecorder: - def __init__(self, build, output_dir, artifacts_dir, base_url): + def __init__(self, build: BuildManifest.Build, output_dir: str, artifacts_dir: str, base_url: str) -> None: self.output_dir = output_dir self.build_id = build.id self.base_url = base_url @@ -28,7 +30,7 @@ def __init__(self, build, output_dir, artifacts_dir, base_url): self.__get_package_location(), ) - def __get_package_name(self, build): + def __get_package_name(self, build: BuildManifest.Build) -> str: parts = [ build.name.lower().replace(" ", "-"), build.version, @@ -37,27 +39,27 @@ def __get_package_name(self, build): ] return "-".join(parts) + (".zip" if build.platform == "windows" else ".tar.gz") - def __get_public_url_path(self, folder, rel_path): + def __get_public_url_path(self, folder: str, rel_path: str) -> str: path = "/".join((folder, rel_path)) return urljoin(self.base_url + "/", path) - def __get_location(self, folder_name, rel_path, abs_path): + def __get_location(self, folder_name: str, rel_path: str, abs_path: str) -> str: if self.base_url: return self.__get_public_url_path(folder_name, rel_path) return abs_path # Assembled output are expected to be served from a separate "dist" folder # Example: https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/build-id/linux/x64/dist/ - def __get_package_location(self): + def __get_package_location(self) -> str: return self.__get_location("dist", self.package_name, os.path.join(self.output_dir, self.package_name)) # Build artifacts are expected to be served from a "builds" folder # Example: https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/build-id/linux/x64/builds/ - def __get_component_location(self, component_rel_path): + def __get_component_location(self, component_rel_path: str) -> str: abs_path = os.path.join(self.artifacts_dir, component_rel_path) return self.__get_location("builds", component_rel_path, abs_path) - def record_component(self, component, rel_path): + def record_component(self, component: BuildComponent, rel_path: str) -> None: self.bundle_manifest.append_component( component.name, component.repository, @@ -66,16 +68,16 @@ def record_component(self, component, rel_path): self.__get_component_location(rel_path), ) - def get_manifest(self): + def get_manifest(self) -> BundleManifest: return self.bundle_manifest.to_manifest() - def write_manifest(self, folder): + def write_manifest(self, folder: str) -> None: manifest_path = os.path.join(folder, "manifest.yml") self.get_manifest().to_file(manifest_path) class BundleManifestBuilder: - def __init__(self, build_id, name, version, platform, architecture, location): - self.data = {} + def __init__(self, build_id: str, name: str, version: str, platform: str, architecture: str, location: str) -> None: + self.data: Dict[str, Any] = {} self.data["build"] = {} self.data["build"]["id"] = build_id self.data["build"]["name"] = name @@ -88,7 +90,7 @@ def __init__(self, build_id, name, version, platform, architecture, location): # When we convert to a BundleManifest this will get converted back into a list self.data["components"] = [] - def append_component(self, name, repository_url, ref, commit_id, location): + def append_component(self, name: str, repository_url: str, ref: str, commit_id: str, location: str) -> None: component = { "name": name, "repository": repository_url, @@ -98,5 +100,5 @@ def append_component(self, name, repository_url, ref, commit_id, location): } self.data["components"].append(component) - def to_manifest(self): + def to_manifest(self) -> BundleManifest: return BundleManifest(self.data) diff --git a/src/assemble_workflow/bundles.py b/src/assemble_workflow/bundles.py index a71997969d..2d06f14b60 100644 --- a/src/assemble_workflow/bundles.py +++ b/src/assemble_workflow/bundles.py @@ -4,8 +4,12 @@ # this file be licensed under the Apache-2.0 license or a # compatible open source license. + +from assemble_workflow.bundle import Bundle from assemble_workflow.bundle_opensearch import BundleOpenSearch from assemble_workflow.bundle_opensearch_dashboards import BundleOpenSearchDashboards +from assemble_workflow.bundle_recorder import BundleRecorder +from manifests.build_manifest import BuildManifest class Bundles: @@ -15,13 +19,13 @@ class Bundles: } @classmethod - def from_name(cls, name): + def from_name(cls, name: str) -> Bundle: klass = cls.TYPES.get(name, None) if not klass: raise ValueError(f"Unsupported bundle: {name}") - return klass + return klass # type: ignore[return-value] @classmethod - def create(cls, build_manifest, artifacts_dir, bundle_recorder, keep): + def create(cls, build_manifest: BuildManifest, artifacts_dir: str, bundle_recorder: BundleRecorder, keep: bool) -> Bundle: klass = cls.from_name(build_manifest.build.name) - return klass(build_manifest, artifacts_dir, bundle_recorder, keep) + return klass(build_manifest, artifacts_dir, bundle_recorder, keep) # type: ignore[no-any-return, operator] diff --git a/src/assemble_workflow/dist.py b/src/assemble_workflow/dist.py index ea1f8ed7a1..4112c3be91 100644 --- a/src/assemble_workflow/dist.py +++ b/src/assemble_workflow/dist.py @@ -16,15 +16,19 @@ class Dist(ABC): - def __init__(self, name, path): + def __init__(self, name: str, path: str) -> None: self.name = name self.path = path @abstractmethod - def __extract__(self, dest): + def __extract__(self, dest: str) -> None: pass - def extract(self, dest): + @abstractmethod + def __build__(self, name: str, dest: str) -> None: + pass + + def extract(self, dest: str) -> str: self.__extract__(dest) # OpenSearch & Dashboard tars will include only a single folder at the top level of the tar. @@ -36,14 +40,14 @@ def extract(self, dest): raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(dest, "*")) - def build(self, name, dest): + def build(self, name: str, dest: str) -> None: self.__build__(name, dest) path = os.path.join(dest, name) shutil.copyfile(name, path) logging.info(f"Published {path}.") @classmethod - def from_path(cls, name, path): + def from_path(cls, name: str, path: str) -> 'Dist': ext = os.path.splitext(path)[1] if ext == ".gz": return DistTar(name, path) @@ -54,24 +58,24 @@ def from_path(cls, name, path): class DistZip(Dist): - def __extract__(self, dest): + def __extract__(self, dest: str) -> None: with ZipFile(self.path, "r") as zip: zip.extractall(dest) - def __build__(self, name, dest): + def __build__(self, name: str, dest: str) -> None: with ZipFile(name, "w", zipfile.ZIP_DEFLATED) as zip: rootlen = len(self.archive_path) + 1 - for base, dirs, files in os.walk(self.archive_path): + for base, _, files in os.walk(self.archive_path): for file in files: fn = os.path.join(base, file) zip.write(fn, fn[rootlen:]) class DistTar(Dist): - def __extract__(self, dest): + def __extract__(self, dest: str) -> None: with tarfile.open(self.path, "r:gz") as tar: tar.extractall(dest) - def __build__(self, name, dest): + def __build__(self, name: str, dest: str) -> None: with tarfile.open(name, "w:gz") as tar: tar.add(self.archive_path, arcname=os.path.basename(self.archive_path)) diff --git a/src/manifests/build_manifest.py b/src/manifests/build_manifest.py index 9eac84c1a8..d5030371cb 100644 --- a/src/manifests/build_manifest.py +++ b/src/manifests/build_manifest.py @@ -100,11 +100,11 @@ def __to_dict__(self) -> dict: class Build: def __init__(self, data: Any): - self.name = data["name"] - self.version = data["version"] - self.platform = data["platform"] - self.architecture = data["architecture"] - self.id = data["id"] + self.name: str = data["name"] + self.version: str = data["version"] + self.platform: str = data["platform"] + self.architecture: str = data["architecture"] + self.id: str = data["id"] def __to_dict__(self) -> dict: return { diff --git a/src/paths/assemble_output_dir.py b/src/paths/assemble_output_dir.py index 7c8daff945..3c349a426d 100644 --- a/src/paths/assemble_output_dir.py +++ b/src/paths/assemble_output_dir.py @@ -8,5 +8,5 @@ class AssembleOutputDir(OutputDir): - def __init__(cls, name, cwd=None, makedirs=True): + def __init__(cls, name: str, cwd: str = None, makedirs: bool = True) -> None: super().__init__("dist", name, cwd, makedirs) diff --git a/src/paths/output_dir.py b/src/paths/output_dir.py index 12218bf4c6..d5a42bb732 100644 --- a/src/paths/output_dir.py +++ b/src/paths/output_dir.py @@ -9,7 +9,7 @@ class OutputDir(abc.ABC): - def __init__(cls, parent_dir, name, cwd=None, makedirs=True): + def __init__(cls, parent_dir: str, name: str, cwd: str = None, makedirs: bool = True) -> None: cls.dir = os.path.join( cwd or os.getcwd(), parent_dir, diff --git a/src/paths/script_finder.py b/src/paths/script_finder.py index a8c7b5865a..032d6637d0 100644 --- a/src/paths/script_finder.py +++ b/src/paths/script_finder.py @@ -5,11 +5,12 @@ # compatible open source license. import os +from typing import Callable, List class ScriptFinder: class ScriptNotFoundError(Exception): - def __init__(self, kind, paths): + def __init__(self, kind: str, paths: List[str]) -> None: self.kind = kind self.paths = paths super().__init__(f"Could not find {kind} script. Looked in {paths}.") @@ -34,15 +35,16 @@ def __init__(self, kind, paths): """ @classmethod - def __find_script(cls, name, paths): - script = next(filter(lambda path: os.path.exists(path), paths), None) + def __find_script(cls, name: str, paths: List[str]) -> str: + exists: Callable[[str], bool] = lambda path: os.path.exists(path) + script = next(filter(exists, paths), None) if script is None: raise ScriptFinder.ScriptNotFoundError(name, paths) return script @classmethod - def __find_named_script(cls, script_name, component_name, git_dir): + def __find_named_script(cls, script_name: str, component_name: str, git_dir: str) -> str: paths = [ os.path.realpath(os.path.join(cls.component_scripts_path, component_name, script_name)), os.path.realpath(os.path.join(git_dir, script_name)), @@ -53,7 +55,7 @@ def __find_named_script(cls, script_name, component_name, git_dir): return cls.__find_script(script_name, paths) @classmethod - def find_build_script(cls, project, component_name, git_dir): + def find_build_script(cls, project: str, component_name: str, git_dir: str) -> str: paths = [ os.path.realpath(os.path.join(cls.component_scripts_path, component_name, "build.sh")), os.path.realpath(os.path.join(git_dir, "build.sh")), @@ -70,7 +72,7 @@ def find_build_script(cls, project, component_name, git_dir): return cls.__find_script("build.sh", paths) @classmethod - def find_install_script(cls, component_name): + def find_install_script(cls, component_name: str) -> str: paths = [ os.path.realpath(os.path.join(cls.component_scripts_path, component_name, "install.sh")), os.path.realpath(os.path.join(cls.default_scripts_path, "install.sh")), @@ -79,9 +81,9 @@ def find_install_script(cls, component_name): return cls.__find_script("install.sh", paths) @classmethod - def find_integ_test_script(cls, component_name, git_dir): + def find_integ_test_script(cls, component_name: str, git_dir: str) -> str: return cls.__find_named_script("integtest.sh", component_name, git_dir) @classmethod - def find_bwc_test_script(cls, component_name, git_dir): + def find_bwc_test_script(cls, component_name: str, git_dir: str) -> str: return cls.__find_named_script("bwctest.sh", component_name, git_dir) diff --git a/src/run_assemble.py b/src/run_assemble.py index 7dcdfe2bd6..4d8dba2661 100755 --- a/src/run_assemble.py +++ b/src/run_assemble.py @@ -18,7 +18,7 @@ from system import console -def main(): +def main() -> int: args = AssembleArgs() console.configure(level=args.logging_level) @@ -45,6 +45,7 @@ def main(): bundle_recorder.write_manifest(output_dir) logging.info("Done.") + return 0 if __name__ == "__main__": diff --git a/src/system/console.py b/src/system/console.py index f0cb5a8e46..01ee75c5de 100644 --- a/src/system/console.py +++ b/src/system/console.py @@ -8,7 +8,7 @@ import sys -def configure(level): +def configure(level: int) -> None: logging.basicConfig( stream=sys.stdout, level=level, diff --git a/src/system/os.py b/src/system/os.py index e2dcf132a5..26bddf701a 100644 --- a/src/system/os.py +++ b/src/system/os.py @@ -8,7 +8,7 @@ import subprocess -def current_architecture(): +def current_architecture() -> str: architecture = subprocess.check_output(["uname", "-m"]).decode().strip() if architecture == "x86_64": return "x64" @@ -18,7 +18,7 @@ def current_architecture(): raise ValueError(f"Unsupported architecture: {architecture}") -def current_platform(): +def current_platform() -> str: if os.name == "nt": return "windows" else: diff --git a/src/system/zip_file.py b/src/system/zip_file.py index 5033d3b952..1682a5c9b0 100644 --- a/src/system/zip_file.py +++ b/src/system/zip_file.py @@ -11,11 +11,11 @@ class ZipFile(zipfile.ZipFile): - def _extract_member(self, member, targetpath, pwd): + def _extract_member(self, member: zipfile.ZipInfo, targetpath: str, pwd: str) -> str: if not isinstance(member, zipfile.ZipInfo): member = self.getinfo(member) - targetpath = super()._extract_member(member, targetpath, pwd) + targetpath = super()._extract_member(member, targetpath, pwd) # type: ignore[misc] attr = member.external_attr >> 16 if attr != 0: diff --git a/tests/test_run_assemble.py b/tests/test_run_assemble.py index 3bbb0c0e65..2efbe85ecf 100644 --- a/tests/test_run_assemble.py +++ b/tests/test_run_assemble.py @@ -6,20 +6,21 @@ import os import unittest -from unittest.mock import MagicMock, call, patch +from unittest.mock import ANY, MagicMock, Mock, call, patch import pytest +from pytest import CaptureFixture from run_assemble import main class TestRunAssemble(unittest.TestCase): @pytest.fixture(autouse=True) - def capfd(self, capfd): + def getCapfd(self, capfd: CaptureFixture) -> None: self.capfd = capfd @patch("argparse._sys.argv", ["run_assemble.py", "--help"]) - def test_usage(self, *mocks): + def test_usage(self) -> None: with self.assertRaises(SystemExit): main() @@ -28,14 +29,12 @@ def test_usage(self, *mocks): BUILD_MANIFEST = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-1.1.0.yml") - @patch("os.chdir") @patch("os.makedirs") - @patch("shutil.copy2") @patch("os.getcwd", return_value="curdir") @patch("argparse._sys.argv", ["run_assemble.py", BUILD_MANIFEST]) @patch("run_assemble.Bundles.create") @patch("run_assemble.BundleRecorder", return_value=MagicMock()) - def test_main(self, mock_recorder, mock_bundles, *mocks): + def test_main(self, mock_recorder: Mock, mock_bundles: Mock, getcwd: Mock, makeDirs: Mock) -> None: mock_bundle = MagicMock(min_dist=MagicMock(archive_path="path")) mock_bundles.return_value.__enter__.return_value = mock_bundle @@ -50,3 +49,6 @@ def test_main(self, mock_recorder, mock_bundles, *mocks): call("path"), call(os.path.join("curdir", "dist", "opensearch")) ]) # manifest included in package + + getcwd.assert_called_once_with() + makeDirs.assert_called_once_with(ANY, exist_ok=True) diff --git a/tests/tests_assemble_workflow/test_assemble_args.py b/tests/tests_assemble_workflow/test_assemble_args.py index 8bb59a4817..fe6abfecbc 100644 --- a/tests/tests_assemble_workflow/test_assemble_args.py +++ b/tests/tests_assemble_workflow/test_assemble_args.py @@ -19,25 +19,25 @@ class TestAssembleArgs(unittest.TestCase): OPENSEARCH_MANIFEST = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests", "1.1.0", "opensearch-1.1.0.yml")) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST]) - def test_manifest(self): + def test_manifest(self) -> None: self.assertEqual(AssembleArgs().manifest.name, TestAssembleArgs.OPENSEARCH_MANIFEST) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST]) - def test_keep_default(self): + def test_keep_default(self) -> None: self.assertFalse(AssembleArgs().keep) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST, "--keep"]) - def test_keep_true(self): + def test_keep_true(self) -> None: self.assertTrue(AssembleArgs().keep) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST]) - def test_verbose_default(self): + def test_verbose_default(self) -> None: self.assertEqual(AssembleArgs().logging_level, logging.INFO) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST, "--verbose"]) - def test_verbose_true(self): + def test_verbose_true(self) -> None: self.assertTrue(AssembleArgs().logging_level, logging.DEBUG) @patch("argparse._sys.argv", [ASSEMBLE_PY, OPENSEARCH_MANIFEST, "--base-url", "url"]) - def test_base_url(self): + def test_base_url(self) -> None: self.assertEqual(AssembleArgs().base_url, "url") diff --git a/tests/tests_assemble_workflow/test_bundle.py b/tests/tests_assemble_workflow/test_bundle.py index 24fd87b362..bd37fdc062 100644 --- a/tests/tests_assemble_workflow/test_bundle.py +++ b/tests/tests_assemble_workflow/test_bundle.py @@ -6,19 +6,19 @@ import os import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, Mock, patch from assemble_workflow.bundle import Bundle -from manifests.build_manifest import BuildManifest +from manifests.build_manifest import BuildComponent, BuildManifest class TestBundle(unittest.TestCase): class DummyBundle(Bundle): - def install_plugin(self, plugin): + def install_plugin(self, plugin: BuildComponent) -> None: pass @patch("assemble_workflow.dist.Dist.extract") - def test_bundle(self, *mocks): + def test_bundle(self, dist_extract: Mock) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data/opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = self.DummyBundle(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -28,8 +28,9 @@ def test_bundle(self, *mocks): self.assertIsNotNone(bundle.bundle_recorder) self.assertEqual(bundle.installed_plugins, []) self.assertTrue(bundle.min_dist.path.endswith("opensearch-min-1.1.0-linux-x64.tar.gz")) + dist_extract.assert_called_once() - def test_bundle_does_not_exist_raises_error(self): + def test_bundle_does_not_exist_raises_error(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data/opensearch-build-linux-1.1.0.yml") with self.assertRaises(FileNotFoundError) as ctx: self.DummyBundle( diff --git a/tests/tests_assemble_workflow/test_bundle_opensearch.py b/tests/tests_assemble_workflow/test_bundle_opensearch.py index f843d924cb..ebf2ef6f8f 100644 --- a/tests/tests_assemble_workflow/test_bundle_opensearch.py +++ b/tests/tests_assemble_workflow/test_bundle_opensearch.py @@ -7,7 +7,7 @@ import os import unittest import zipfile -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, Mock, call, patch from assemble_workflow.bundle_opensearch import BundleOpenSearch from manifests.build_manifest import BuildManifest @@ -16,7 +16,7 @@ class TestBundleOpenSearch(unittest.TestCase): - def test_bundle_opensearch(self): + def test_bundle_opensearch(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -28,7 +28,7 @@ def test_bundle_opensearch(self): self.assertTrue(bundle.min_dist.path.endswith("opensearch-min-1.1.0-linux-x64.tar.gz")) self.assertIsNotNone(bundle.min_dist.archive_path) - def test_bundle_install_min(self): + def test_bundle_install_min(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -61,7 +61,7 @@ def test_bundle_install_min(self): ) @patch.object(BundleOpenSearch, "install_plugin") - def test_bundle_install_plugins(self, mocks_bundle): + def test_bundle_install_plugins(self, bundle_install_plugin: Mock) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") bundle = BundleOpenSearch( BuildManifest.from_path(manifest_path), @@ -70,10 +70,10 @@ def test_bundle_install_plugins(self, mocks_bundle): ) bundle.install_plugins() - self.assertEqual(mocks_bundle.call_count, 12) + self.assertEqual(bundle_install_plugin.call_count, 12) @patch("os.path.isfile", return_value=True) - def test_bundle_install_plugin(self, *mocks): + def test_bundle_install_plugin(self, path_isfile: Mock) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearch(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -115,8 +115,9 @@ def test_bundle_install_plugin(self, *mocks): ), ] ) + self.assertEqual(path_isfile.call_count, 2) - def test_bundle_package_tar(self): + def test_bundle_package_tar(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearch( @@ -134,7 +135,7 @@ def test_bundle_package_tar(self): mock_tarfile_add.assert_called_with(os.path.join(bundle.tmp_dir.name, "bundle"), arcname="bundle") self.assertEqual(mock_copyfile.call_count, 1) - def test_bundle_package_zip(self): + def test_bundle_package_zip(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-windows-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearch( diff --git a/tests/tests_assemble_workflow/test_bundle_opensearch_dashboards.py b/tests/tests_assemble_workflow/test_bundle_opensearch_dashboards.py index c2a1fe5f98..47756166ec 100644 --- a/tests/tests_assemble_workflow/test_bundle_opensearch_dashboards.py +++ b/tests/tests_assemble_workflow/test_bundle_opensearch_dashboards.py @@ -6,7 +6,7 @@ import os import unittest -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, Mock, call, patch from assemble_workflow.bundle_opensearch_dashboards import BundleOpenSearchDashboards from manifests.build_manifest import BuildManifest @@ -14,7 +14,7 @@ class TestBundleOpenSearchDashboards(unittest.TestCase): - def test_bundle_opensearch_dashboards(self): + def test_bundle_opensearch_dashboards(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data/opensearch-dashboards-build-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearchDashboards(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -25,7 +25,7 @@ def test_bundle_opensearch_dashboards(self): self.assertEqual(bundle.installed_plugins, []) self.assertTrue(bundle.min_dist.path.endswith("opensearch-dashboards-min-1.1.0-linux-x64.tar.gz")) - def test_bundle_install_min(self): + def test_bundle_install_min(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data/opensearch-dashboards-build-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data/artifacts") bundle = BundleOpenSearchDashboards(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -58,7 +58,7 @@ def test_bundle_install_min(self): ) @patch("os.path.isfile", return_value=True) - def test_bundle_install_plugin(self, *mocks): + def test_bundle_install_plugin(self, path_isfile: Mock) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data/opensearch-dashboards-build-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = BundleOpenSearchDashboards(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock()) @@ -99,3 +99,4 @@ def test_bundle_install_plugin(self, *mocks): ), ] ) + self.assertEqual(path_isfile.call_count, 2) diff --git a/tests/tests_assemble_workflow/test_bundle_recorder.py b/tests/tests_assemble_workflow/test_bundle_recorder.py index a62066a3e5..c01a75840e 100644 --- a/tests/tests_assemble_workflow/test_bundle_recorder.py +++ b/tests/tests_assemble_workflow/test_bundle_recorder.py @@ -6,6 +6,7 @@ import os import unittest +from typing import Optional import yaml @@ -16,13 +17,13 @@ class TestBundleRecorder(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.maxDiff = None manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") manifest = BuildManifest.from_path(manifest_path) self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir", None) - def test_record_component(self): + def test_record_component(self) -> None: component = BuildComponent( { "name": "job_scheduler", @@ -59,7 +60,7 @@ def test_record_component(self): }, ) - def test_get_manifest(self): + def test_get_manifest(self) -> None: manifest = self.bundle_recorder.get_manifest() self.assertIs(type(manifest), BundleManifest) self.assertEqual( @@ -77,7 +78,7 @@ def test_get_manifest(self): }, ) - def test_write_manifest(self): + def test_write_manifest(self) -> None: with TemporaryDirectory() as dest_dir: self.bundle_recorder.write_manifest(dest_dir.name) manifest_path = os.path.join(dest_dir.name, "manifest.yml") @@ -86,7 +87,7 @@ def test_write_manifest(self): with open(manifest_path) as f: self.assertEqual(yaml.safe_load(f), data) - def test_record_component_public(self): + def test_record_component_public(self) -> None: self.bundle_recorder.base_url = "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/" component = BuildComponent( { @@ -124,10 +125,12 @@ def test_record_component_public(self): }, ) - def test_get_location_scenarios(self): - def get_location(base_url): + def test_get_location_scenarios(self) -> None: + def get_location(base_url: Optional[str]) -> str: self.bundle_recorder.base_url = base_url - return self.bundle_recorder._BundleRecorder__get_location("builds", "dir1/dir2/file", "/tmp/builds/foo/dir1/dir2/file") + return self.bundle_recorder._BundleRecorder__get_location("builds", # type: ignore[no-any-return, attr-defined] + "dir1/dir2/file", + "/tmp/builds/foo/dir1/dir2/file") # No public URL - Fallback to ABS Path self.assertEqual(get_location(None), "/tmp/builds/foo/dir1/dir2/file") @@ -144,17 +147,17 @@ def get_location(base_url): "https://ci.opensearch.org/ci/ci-env-prod/job-name-opensearch/1.2.0/build-123/platform-mac/arch-amd64/builds/dir1/dir2/file", ) - def test_package_name(self): + def test_package_name(self) -> None: self.assertEqual(self.bundle_recorder.package_name, "opensearch-1.1.0-linux-x64.tar.gz") class TestBundleRecorderDashboards(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-build-1.1.0.yml") manifest = BuildManifest.from_path(manifest_path) self.bundle_recorder = BundleRecorder(manifest.build, "output_dir", "artifacts_dir", None) - def test_record_component(self): + def test_record_component(self) -> None: component = BuildComponent( { "name": "alertingDashboards", @@ -191,7 +194,7 @@ def test_record_component(self): }, ) - def test_get_manifest(self): + def test_get_manifest(self) -> None: manifest = self.bundle_recorder.get_manifest() self.assertIs(type(manifest), BundleManifest) self.assertEqual( @@ -209,7 +212,7 @@ def test_get_manifest(self): }, ) - def test_write_manifest(self): + def test_write_manifest(self) -> None: with TemporaryDirectory() as dest_dir: self.bundle_recorder.write_manifest(dest_dir.name) manifest_path = os.path.join(dest_dir.name, "manifest.yml") @@ -218,7 +221,7 @@ def test_write_manifest(self): with open(manifest_path) as f: self.assertEqual(yaml.safe_load(f), data) - def test_record_component_public(self): + def test_record_component_public(self) -> None: self.bundle_recorder.base_url = "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/" component = BuildComponent( { @@ -256,10 +259,12 @@ def test_record_component_public(self): }, ) - def test_get_location_scenarios(self): - def get_location(base_url): + def test_get_location_scenarios(self) -> None: + def get_location(base_url: Optional[str]) -> str: self.bundle_recorder.base_url = base_url - return self.bundle_recorder._BundleRecorder__get_location("builds", "dir1/dir2/file", "/tmp/builds/foo/dir1/dir2/file") + return self.bundle_recorder._BundleRecorder__get_location("builds", # type: ignore[no-any-return, attr-defined] + "dir1/dir2/file", + "/tmp/builds/foo/dir1/dir2/file") # No public URL - Fallback to ABS Path self.assertEqual(get_location(None), "/tmp/builds/foo/dir1/dir2/file") @@ -276,7 +281,7 @@ def get_location(base_url): "https://ci.opensearch.org/ci/ci-env-prod/job-name-dashboards/1.2.0/build-123/platform-mac/arch-amd64/builds/dir1/dir2/file", ) - def test_package_name(self): + def test_package_name(self) -> None: self.assertEqual( self.bundle_recorder.package_name, "opensearch-dashboards-1.1.0-linux-x64.tar.gz", diff --git a/tests/tests_assemble_workflow/test_bundles.py b/tests/tests_assemble_workflow/test_bundles.py index c9000ad334..4f8ee58bff 100644 --- a/tests/tests_assemble_workflow/test_bundles.py +++ b/tests/tests_assemble_workflow/test_bundles.py @@ -15,26 +15,26 @@ class TestBundles(unittest.TestCase): - def test_bundle_opensearch(self): + def test_bundle_opensearch(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = Bundles.create(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock(), False) self.assertIs(type(bundle), BundleOpenSearch) - def test_bundle_opensearch_dashboards(self): + def test_bundle_opensearch_dashboards(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-build-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = Bundles.create(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock(), False) self.assertIs(type(bundle), BundleOpenSearchDashboards) self.assertFalse(bundle.tmp_dir.keep) - def test_bundle_keep(self): + def test_bundle_keep(self) -> None: manifest_path = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-linux-1.1.0.yml") artifacts_path = os.path.join(os.path.dirname(__file__), "data", "artifacts") bundle = Bundles.create(BuildManifest.from_path(manifest_path), artifacts_path, MagicMock(), True) self.assertTrue(bundle.tmp_dir.keep) - def test_bundle_opensearch_invalid(self): + def test_bundle_opensearch_invalid(self) -> None: manifest = BuildManifest( { "schema-version": "1.2", diff --git a/tests/tests_assemble_workflow/test_dist.py b/tests/tests_assemble_workflow/test_dist.py index e73055130e..6ab527aba2 100644 --- a/tests/tests_assemble_workflow/test_dist.py +++ b/tests/tests_assemble_workflow/test_dist.py @@ -10,15 +10,15 @@ class TestDist(unittest.TestCase): - def test_from_path_tar_gz(self): + def test_from_path_tar_gz(self) -> None: dist = Dist.from_path("opensearch", "filename.tar.gz") self.assertIs(type(dist), DistTar) - def test_from_path_zip(self): + def test_from_path_zip(self) -> None: dist = Dist.from_path("opensearch", "filename.zip") self.assertIs(type(dist), DistZip) - def test_from_path_invalid(self): + def test_from_path_invalid(self) -> None: with self.assertRaises(ValueError) as ctx: Dist.from_path("opensearch", "filename.invalid") self.assertEqual(str(ctx.exception), 'Invalid min "dist" extension in input artifacts: .invalid (filename.invalid).')