From 1192e128eb9382b956589a3644abb106745805a8 Mon Sep 17 00:00:00 2001 From: Wenjie Du Date: Fri, 21 Apr 2023 12:36:28 +0800 Subject: [PATCH] feat: add pypots-cli env; --- pypots/utils/commands/base.py | 18 ++-- pypots/utils/commands/dev.py | 19 ++-- pypots/utils/commands/doc.py | 24 +++-- pypots/utils/commands/env.py | 136 ++++++++++++++++++++++++++++ pypots/utils/commands/pypots_cli.py | 2 + 5 files changed, 175 insertions(+), 24 deletions(-) create mode 100644 pypots/utils/commands/env.py diff --git a/pypots/utils/commands/base.py b/pypots/utils/commands/base.py index 9dd643bc..648460e3 100644 --- a/pypots/utils/commands/base.py +++ b/pypots/utils/commands/base.py @@ -23,6 +23,7 @@ def register_subcommand(parser: ArgumentParser): @staticmethod def execute_command(command: str, verbose: bool = True): + logger.info(f"Executing '{command}'...") if verbose: exec_result = subprocess.Popen( command, @@ -43,17 +44,14 @@ def execute_command(command: str, verbose: bool = True): stderr=subprocess.PIPE, shell=True, ) - if exec_result.returncode != 0: - if len(exec_result.stderr) > 0: - logger.error(exec_result.stderr) - if len(exec_result.stdout) > 0: - logger.error(exec_result.stdout) - raise RuntimeError() - return exec_result.returncode + + if exec_result.returncode != 0: + raise RuntimeError(exec_result.stdout, exec_result.stderr) + return exec_result @staticmethod def check_if_under_root_dir(strict: bool = True): - """ Check if under the root dir of PyPOTS project. + """Check if under the root dir of PyPOTS project. Parameters ---------- @@ -85,6 +83,10 @@ def check_if_under_root_dir(strict: bool = True): return check_result + @abstractmethod + def checkup(self): + raise NotImplementedError() + @abstractmethod def run(self): raise NotImplementedError() diff --git a/pypots/utils/commands/dev.py b/pypots/utils/commands/dev.py index de3c01cd..a5022be6 100644 --- a/pypots/utils/commands/dev.py +++ b/pypots/utils/commands/dev.py @@ -7,7 +7,7 @@ import os import shutil -from argparse import ArgumentParser, Namespace +from argparse import Namespace from pypots.utils.commands.base import BaseCommand from pypots.utils.logging import logger @@ -43,9 +43,10 @@ class DevCommand(BaseCommand): """ @staticmethod - def register_subcommand(parser: ArgumentParser): + def register_subcommand(parser): sub_parser = parser.add_parser( - "dev", help="CLI tools helping develop PyPOTS code" + "dev", + help="CLI tools helping develop PyPOTS code", ) sub_parser.add_argument( "--build", @@ -54,6 +55,7 @@ def register_subcommand(parser: ArgumentParser): help="Build PyPOTS into a wheel and package the source code into a .tar.gz file for distribution", ) sub_parser.add_argument( + "-c", "--cleanup", dest="cleanup", action="store_true", @@ -61,11 +63,13 @@ def register_subcommand(parser: ArgumentParser): ) sub_parser.add_argument( "--run_tests", + "--run-tests", dest="run_tests", action="store_true", help="Run all test cases", ) sub_parser.add_argument( + "--show-coverage", "--show_coverage", dest="show_coverage", action="store_true", @@ -86,6 +90,7 @@ def register_subcommand(parser: ArgumentParser): "matching is case-insensitive.", ) sub_parser.add_argument( + "--lint-code", "--lint_code", dest="lint_code", action="store_true", @@ -109,7 +114,7 @@ def __init__( self._show_coverage = show_coverage self._lint_code = lint_code - def check_arguments(self): + def checkup(self): """Run some checks on the arguments to avoid error usages""" self.check_if_under_root_dir() @@ -133,9 +138,8 @@ def check_arguments(self): def run(self): """Execute the given command.""" - - # check arguments first - self.check_arguments() + # run checks first + self.checkup() try: if self._cleanup: @@ -151,7 +155,6 @@ def run(self): if self._show_coverage else pytest_command ) - logger.info(f"Executing '{command_to_run_test}'...") self.execute_command(command_to_run_test) if self._show_coverage: self.execute_command("coverage report -m") diff --git a/pypots/utils/commands/doc.py b/pypots/utils/commands/doc.py index 462a0355..091d4422 100644 --- a/pypots/utils/commands/doc.py +++ b/pypots/utils/commands/doc.py @@ -7,11 +7,12 @@ import os import shutil -from argparse import ArgumentParser, Namespace +from argparse import Namespace + +from tsdb.data_processing import _download_and_extract from pypots.utils.commands.base import BaseCommand from pypots.utils.logging import logger -from tsdb.data_processing import _download_and_extract CLONED_LATEST_PYPOTS = "temp_pypots_latest" @@ -73,18 +74,22 @@ class DocCommand(BaseCommand): """ @staticmethod - def register_subcommand(parser: ArgumentParser): + def register_subcommand(parser): sub_parser = parser.add_parser( - "doc", help="CLI tools helping build PyPOTS documentation" + "doc", + help="CLI tools helping build PyPOTS documentation", + allow_abbrev=True, ) sub_parser.add_argument( + "--gene-rst", "--gene_rst", dest="gene_rst", action="store_true", help="Generate rst (reStructuredText) documentation according to the latest code on Github", ) sub_parser.add_argument( + "-b", "--branch", type=str, default="main", @@ -92,24 +97,28 @@ def register_subcommand(parser: ArgumentParser): help="Code on which branch will be used for documentation generating", ) sub_parser.add_argument( + "--gene-html", "--gene_html", dest="gene_html", action="store_true", help="Generate the sphinx documentation into static HTML files", ) sub_parser.add_argument( + "--view-doc", "--view_doc", dest="view_doc", action="store_true", help="Deploy the generated HTML documentation locally for view", ) sub_parser.add_argument( + "-p", "--port", type=int, default=9075, help="Use which port to deploy the web server for doc view", # 9075 looks like "POTS", so use it as default ) sub_parser.add_argument( + "-c", "--cleanup", dest="cleanup", action="store_true", @@ -134,7 +143,7 @@ def __init__( self._port = port self._cleanup = cleanup - def check_arguments(self): + def checkup(self): """Run some checks on the arguments to avoid error usages""" self.check_if_under_root_dir() @@ -146,9 +155,8 @@ def check_arguments(self): def run(self): """Execute the given command.""" - - # check arguments first - self.check_arguments() + # run checks first + self.checkup() try: if self._cleanup: diff --git a/pypots/utils/commands/env.py b/pypots/utils/commands/env.py new file mode 100644 index 00000000..fe69354c --- /dev/null +++ b/pypots/utils/commands/env.py @@ -0,0 +1,136 @@ +""" +CLI tools to help initialize environments for running and developing PyPOTS. +""" + +# Created by Wenjie Du +# License: GLP-v3 + +try: + # here try importing all dependencies in the scope `basic` defined in `setup.cfg` + import torch + + # import numpy + # import sklearn + # import pandas + # import tensorboard + # import scipy + # import h5py + # import tsdb + # import pycorruptor +except ImportError: + raise ImportError( + "Torch not installed. Using this tool supposes that you've already installed `pypots` " + "with at least the scope of `basic` dependencies." + ) + +from argparse import ArgumentParser, Namespace + +from setuptools.config import read_configuration + +from pypots.utils.commands.base import BaseCommand +from pypots.utils.logging import logger + + +def env_command_factory(args: Namespace): + return EnvCommand( + args.install, + args.tool, + ) + + +class EnvCommand(BaseCommand): + """CLI tools helping users and developer setup python environments for running and developing PyPOTS. + + Notes + ----- + Using this tool supposes that you've already installed `pypots` with at least the scope of `basic` dependencies. + Please refer to file setup.cfg in PyPOTS project's root dir for definitions of different dependency scopes. + + Examples + -------- + $ pypots-cli env --scope full --tool pip + $ pypots-cli env --scope full --tool pip + $ pypots-cli env --scope dev --tool conda -n + """ + + @staticmethod + def register_subcommand(parser: ArgumentParser): + sub_parser = parser.add_parser( + "env", + help="CLI tools helping users and developer setup python environments for running and developing PyPOTS", + allow_abbrev=True, + ) + + sub_parser.add_argument( + "--install", + dest="install", + type=str, + required=True, + choices=["dev", "full", "doc", "test", "optional"], + help="Install specified dependencies in the current python environment", + ) + sub_parser.add_argument( + "--tool", + dest="tool", + type=str, + required=True, + choices=["conda", "pip"], + help="Setup the environment with pip or conda, have to be specific", + ) + + sub_parser.set_defaults(func=env_command_factory) + + def __init__( + self, + install: bool, + tool: str, + ): + self._install = install + self._tool = tool + + def checkup(self): + """Run some checks on the arguments to avoid error usages""" + self.check_if_under_root_dir() + + def run(self): + """Execute the given command.""" + # run checks first + self.checkup() + + setup_cfg = read_configuration("setup.cfg") + + logger.info( + f"Installing the dependencies in scope `{self._install}` for you..." + ) + + if self._tool == "conda": + assert ( + self.execute_command("which conda").returncode == 0 + ), "Conda not installed, cannot set --tool=conda, please check your conda." + + self.execute_command( + "conda install pyg pytorch-scatter pytorch-sparse -c pyg" + ) + + dependencies = "" + for i in setup_cfg["options"]["extras_require"][self._install]: + dependencies += f"'{i}' " + + if "torch-geometric" in dependencies: + dependencies = dependencies.replace("'torch-geometric'", "") + dependencies = dependencies.replace("'torch-scatter'", "") + dependencies = dependencies.replace("'torch-sparse'", "") + + conda_comm = f"conda install {dependencies} -c conda-forge" + self.execute_command(conda_comm) + + else: # self._tool == "pip" + torch_version = torch.__version__ + + self.execute_command( + f"pip install -e '.[optional]' -f 'https://data.pyg.org/whl/torch-{torch_version}.html'" + ) + + if self._install != "optional": + self.execute_command(f"pip install -e '.[{self._install}]'") + logger.info("Installation finished. Enjoy your play with PyPOTS! Bye ;-)") diff --git a/pypots/utils/commands/pypots_cli.py b/pypots/utils/commands/pypots_cli.py index 4d9c9ea9..d1e89c6a 100644 --- a/pypots/utils/commands/pypots_cli.py +++ b/pypots/utils/commands/pypots_cli.py @@ -9,6 +9,7 @@ from pypots.utils.commands.dev import DevCommand from pypots.utils.commands.doc import DocCommand +from pypots.utils.commands.env import EnvCommand def main(): @@ -20,6 +21,7 @@ def main(): # Register commands here DevCommand.register_subcommand(commands_parser) DocCommand.register_subcommand(commands_parser) + EnvCommand.register_subcommand(commands_parser) # parse all arguments args = parser.parse_args()