From d552f6f5c69d127286c1b8740ee94368b17c443d Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 8 Nov 2023 09:57:01 -0800 Subject: [PATCH 01/44] Update readthedocs requirements.txt --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index b37591845f..b30f7a5b2a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -28,3 +28,4 @@ ply psutil ws4py Jinja2==3.1.2 +PyYaml From c3d2768e60db2cf3d0c069052e1a00eb39d7e52b Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:07:35 -0800 Subject: [PATCH 02/44] Update conf.py --- docs/source/conf.py | 50 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8a265dcba0..80eaeb23f6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,7 +23,49 @@ import sys import yaml -from volttron.platform.agent.utils import execute_command +# Copied from volttron.platform.agent.util because it is not required +# that volttron be installed to utilize this script. +def execute_command(cmds, + env=None, + cwd=None, + logger=None, + err_prefix=None, + use_shell=False) -> str: + """ Executes a command as a subprocess + + If the return code of the call is 0 then return stdout otherwise + raise a RuntimeError. If logger is specified then write the exception + to the logger otherwise this call will remain silent. + + :param cmds:list of commands to pass to subprocess.run + :param env: environment to run the command with + :param cwd: working directory for the command + :param logger: a logger to use if errors occure + :param err_prefix: an error prefix to allow better tracing through the error message + :return: stdout string if successful + + :raises RuntimeError: if the return code is not 0 from suprocess.run + """ + + results = subprocess.run(cmds, + env=env, + cwd=cwd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + shell=use_shell) + if results.returncode != 0: + err_prefix = err_prefix if err_prefix is not None else "Error executing command" + err_message = "\n{}: Below Command failed with non zero exit code.\n" \ + "Command:{} \nStderr:\n{}\n".format(err_prefix, + results.args, + results.stderr) + if logger: + logger.exception(err_message) + raise RuntimeError(err_message) + else: + raise RuntimeError(err_message) + + return results.stdout.decode('utf-8') class Mock(MagicMock): @@ -39,13 +81,13 @@ def __getattr__(cls, name): # -- Project information ----------------------------------------------------- project = 'VOLTTRON' -copyright = '2022, The VOLTTRON Community' +copyright = '2023, The VOLTTRON Community' author = 'The VOLTTRON Community' # The short X.Y version -version = '8.2' +version = '9.0' # The full version, including alpha/beta/rc tags -release = '8.2' +release = '9.0-rc' # -- General configuration --------------------------------------------------- From b19e8f5bd4ddb6db4b185834cb522febe6299d42 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:09:50 -0800 Subject: [PATCH 03/44] Remove pyproject.toml file --- pyproject.toml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 06556a1356..0000000000 --- a/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[tool.yapfignore] -ignore_patterns = [ - ".env/**", - ".pytest_cache/**", - "dist/**", - "docs/**", -] - -[tool.yapf] -based_on_style = "pep8" -spaces_before_comment = 4 -column_limit = 99 -split_before_logical_operator = true \ No newline at end of file From dc1ec9701c947eb369947f1903ef57662c0323a9 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:10:12 -0800 Subject: [PATCH 04/44] bump to 9.0.1rc0 --- volttron/platform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index b56f6f309a..e3ab2daa4c 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -35,7 +35,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '8.2' +__version__ = '9.0.1rc0' _log = logging.getLogger(__name__) From 1af66a55e854e9140102a239b0fcdd8a41f4ff95 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:11:59 -0800 Subject: [PATCH 05/44] use v6 of volttron-build-action --- .github/workflows/pytest-web.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest-web.yml b/.github/workflows/pytest-web.yml index 482fbe3160..43b0540aa4 100644 --- a/.github/workflows/pytest-web.yml +++ b/.github/workflows/pytest-web.yml @@ -57,7 +57,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v7 + uses: volttron/volttron-build-action@v6 with: python_version: ${{ matrix.python-version }} os: ${{ matrix.os }} From adbee058bda63d2fc75382932e4befce8bafcddc Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Mon, 13 Nov 2023 14:06:55 -0800 Subject: [PATCH 06/44] Update __init__.py Make rc 9.0rc0 the version for the rc. --- volttron/platform/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index e3ab2daa4c..d6ce14cdd2 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -35,7 +35,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '9.0.1rc0' +__version__ = '9.0rc0' _log = logging.getLogger(__name__) From c9e66f09e57bfdf2abe45cbb8f7c872d9120684d Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:55:40 +0000 Subject: [PATCH 07/44] Move 2030.5 and DNP3 Driver to proper spot. --- .../OldPlatformDrivers/Old2030_5Driver}/IEEE2030_5.py | 0 .../PlatformDriverAgent/platform_driver/interfaces/dnp3.py | 0 .../OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py | 0 .../OldDnp3/OldDnp3Driver/dnp3-driver.rst | 0 .../OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv | 0 .../OldDnp3Driverexamples/configurations/drivers/test_dnp3.config | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename {services/core/PlatformDriverAgent/platform_driver/interfaces => deprecated/OldPlatformDrivers/Old2030_5Driver}/IEEE2030_5.py (100%) rename deprecated/{ => OldPlatformDrivers}/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py (100%) rename deprecated/{ => OldPlatformDrivers}/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py (100%) rename deprecated/{ => OldPlatformDrivers}/OldDnp3/OldDnp3Driver/dnp3-driver.rst (100%) rename deprecated/{ => OldPlatformDrivers}/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv (100%) rename deprecated/{ => OldPlatformDrivers}/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config (100%) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py b/deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py similarity index 100% rename from services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py rename to deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py diff --git a/deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py diff --git a/deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py diff --git a/deprecated/OldDnp3/OldDnp3Driver/dnp3-driver.rst b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/dnp3-driver.rst similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driver/dnp3-driver.rst rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/dnp3-driver.rst diff --git a/deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv diff --git a/deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config similarity index 100% rename from deprecated/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config From e98a04307bc82484c2eb572f5940fda2709dec18 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Tue, 16 Jan 2024 13:32:08 -0800 Subject: [PATCH 08/44] Update requirements_demo.txt Add missing pandas requirement for demo --- services/core/IEEE_2030_5/requirements_demo.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/services/core/IEEE_2030_5/requirements_demo.txt b/services/core/IEEE_2030_5/requirements_demo.txt index cf84bba01d..0fdf400704 100644 --- a/services/core/IEEE_2030_5/requirements_demo.txt +++ b/services/core/IEEE_2030_5/requirements_demo.txt @@ -2,3 +2,4 @@ nicegui requests xsdata>=23.8 blinker +pandas From 7f088e02a502943fe029a5435a3c89dae16693d3 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Tue, 20 Feb 2024 16:09:04 -0800 Subject: [PATCH 09/44] work around for issue #3154 --- volttron/platform/instance_setup.py | 68 +++++++++++------------------ 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index 83066aa24c..86b919a20e 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -53,7 +53,6 @@ from gevent.subprocess import Popen from zmq import green as zmq -from requirements import extras_require from volttron.platform import is_rabbitmq_available from volttron.platform.auth import certs from volttron.platform import jsonapi @@ -350,12 +349,7 @@ def _get_dependencies(): return dependencies -def _check_dependencies_met(requirement): - try: - dependencies_needed = extras_require[requirement] - except KeyError: - print(f"ERROR: Requirement {requirement} was not found in requirements.py") - return False +def _check_dependencies_met(dependencies_needed): current_dependencies = _get_dependencies() for dependency in dependencies_needed: if "==" in dependency: @@ -372,15 +366,21 @@ def _check_dependencies_met(requirement): def set_dependencies(requirement): try: + # go up two level above env/bin + sys.path.append(os.path.dirname(os.path.dirname(sys.path[0]))) + from requirements import extras_require dependencies_needed = extras_require[requirement] except KeyError: - print("ERROR: Incorrect requirement chosen") + print(f"ERROR: Incorrect requirement chosen: {requirement}") + return + + if not _check_dependencies_met(dependencies_needed): + print(f"Installing {requirement} dependencies...") + cmds = [sys.executable, "-m", "pip", "install"] + for dependency in dependencies_needed: + cmds.append(dependency) + subprocess.check_call(cmds) return - cmds = [sys.executable, "-m", "pip", "install"] - for dependency in dependencies_needed: - cmds.append(dependency) - subprocess.check_call(cmds) - return def _create_web_certs(): @@ -907,10 +907,7 @@ def wizard(): prompt = 'Is this instance web enabled?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met('web'): - print("Web dependencies not installed. Installing now...") - set_dependencies('web') - print("Done!") + set_dependencies('web') if config_opts['message-bus'] == 'rmq': do_web_enabled_rmq(volttron_home) elif config_opts['message-bus'] == 'zmq': @@ -930,13 +927,9 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): - print("VCP dependencies not installed. Installing now...") - if not _check_dependencies_met("drivers"): - set_dependencies("drivers") - if not _check_dependencies_met("web"): - set_dependencies("web") - print("Done!") + print("Checking for VCP dependencies.....") + set_dependencies("drivers") + set_dependencies("web") do_vcp() prompt = 'Would you like to install a platform historian?' @@ -946,10 +939,8 @@ def wizard(): prompt = 'Would you like to install a platform driver?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met("drivers"): - print("Driver dependencies not installed. Installing now...") - set_dependencies("drivers") - print("Done!") + print("Checking Driver dependencies...") + set_dependencies("drivers") do_platform_driver() prompt = 'Would you like to install a listener agent?' @@ -968,13 +959,9 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): - print("VCP dependencies not installed. Installing now...") - if not _check_dependencies_met("drivers"): - set_dependencies("drivers") - if not _check_dependencies_met("web"): - set_dependencies("web") - print("Done!") + print("Checking VCP dependencies...") + set_dependencies("drivers") + set_dependencies("web") do_vcp() prompt = 'Would you like to install a platform historian?' @@ -984,10 +971,8 @@ def wizard(): prompt = 'Would you like to install a platform driver?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met("drivers"): - print("Driver dependencies not installed. Installing now...") - set_dependencies("drivers") - print("Done!") + print("Checking Driver dependencies...") + set_dependencies("drivers") do_platform_driver() prompt = 'Would you like to install a listener agent?' @@ -1128,10 +1113,9 @@ def process_rmq_inputs(args_dict, instance_name=None): vhome = get_home() - if args_dict['installation-type'] in ['federation', 'shovel'] and not _check_dependencies_met('web'): - print("Web dependencies not installed. Installing now...") + if args_dict['installation-type'] in ['federation', 'shovel']: + print("Checking Web dependencies...") set_dependencies('web') - print("Done!") if args_dict['config'] is not None: if not os.path.exists(vhome): From cbcdd28dc9a96b018219a25cef8b1d1bfc3320fe Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:09:49 -0700 Subject: [PATCH 10/44] fix when vctl is called with no arguments. --- volttron/platform/control/control_parser.py | 320 ++++++++++---------- 1 file changed, 157 insertions(+), 163 deletions(-) diff --git a/volttron/platform/control/control_parser.py b/volttron/platform/control/control_parser.py index 0c10fd44a9..5a256800d2 100644 --- a/volttron/platform/control/control_parser.py +++ b/volttron/platform/control/control_parser.py @@ -24,27 +24,26 @@ # Monkeypatch for gevent from volttron.utils import monkey_patch + monkey_patch() import argparse import collections import logging -import logging.handlers import logging.config +import logging.handlers import os import sys - -from datetime import timedelta, datetime +from datetime import datetime, timedelta import gevent import gevent.event -# noinspection PyUnresolvedReferences - from volttron.platform import aip as aipmod -from volttron.platform import config -from volttron.platform import get_home, get_address -from volttron.platform import jsonapi +from volttron.platform import config, get_address, get_home, is_rabbitmq_available, jsonapi +from volttron.platform.agent import utils +from volttron.platform.agent.known_identities import PLATFORM_HEALTH +from volttron.platform.agent.utils import is_secure_mode, wait_for_volttron_shutdown from volttron.platform.control.control_auth import add_auth_parser from volttron.platform.control.control_certs import add_certs_parser from volttron.platform.control.control_config import add_config_store_parser @@ -52,25 +51,18 @@ from volttron.platform.control.control_rmq import add_rabbitmq_parser from volttron.platform.control.control_rpc import add_rpc_agent_parser from volttron.platform.control.control_utils import ( - _list_agents, - _show_filtered_agents, - _show_filtered_agents_status, - filter_agent, - filter_agents, - get_filtered_agents - ) -from volttron.platform.agent import utils -from volttron.platform.agent.known_identities import PLATFORM_HEALTH + _list_agents, _show_filtered_agents, _show_filtered_agents_status, + filter_agent, filter_agents, get_filtered_agents) +from volttron.platform.control.install_agents import InstallRuntimeError, add_install_agent_parser from volttron.platform.jsonrpc import RemoteError from volttron.platform.keystore import KeyStore, KnownHostsStore +from volttron.platform.vip.agent.errors import Unreachable, VIPError + +# noinspection PyUnresolvedReferences -from volttron.platform.vip.agent.errors import VIPError, Unreachable -from volttron.platform.agent.utils import is_secure_mode, wait_for_volttron_shutdown -from volttron.platform.control.install_agents import add_install_agent_parser, InstallRuntimeError -from volttron.platform import is_rabbitmq_available if is_rabbitmq_available(): - from volttron.utils.rmq_setup import check_rabbit_status from volttron.utils.rmq_config_params import RMQConfig + from volttron.utils.rmq_setup import check_rabbit_status try: import volttron.restricted @@ -86,8 +78,7 @@ # will be volttron.platform.main or main.py instead of __main__ _log = logging.getLogger( - os.path.basename(sys.argv[0]) if __name__ == "__main__" else __name__ -) + os.path.basename(sys.argv[0]) if __name__ == "__main__" else __name__) # Allows server side logging. # _log.setLevel(logging.DEBUG) @@ -96,16 +87,16 @@ CHUNK_SIZE = 4096 -def log_to_file(file, level=logging.WARNING, + +def log_to_file(file, + level=logging.WARNING, handler_class=logging.StreamHandler): """Direct log output to a file (or something like one).""" handler = handler_class(file) handler.setLevel(level) handler.setFormatter( utils.AgentFormatter( - "%(asctime)s %(composite_name)s %(levelname)s: %(message)s" - ) - ) + "%(asctime)s %(composite_name)s %(levelname)s: %(message)s")) root = logging.getLogger() root.setLevel(level) root.addHandler(handler) @@ -123,17 +114,17 @@ def tag_agent(opts): msg = "multiple agents selected" else: msg = "agent not found" - _stderr.write( - "{}: error: {}: {}\n".format(opts.command, msg, opts.agent)) + _stderr.write("{}: error: {}: {}\n".format(opts.command, msg, + opts.agent)) return 10 - (agent,) = agents + (agent, ) = agents if opts.tag: _stdout.write("Tagging {} {}\n".format(agent.uuid, agent.name)) opts.aip.tag_agent(agent.uuid, opts.tag) elif opts.remove: if agent.tag is not None: - _stdout.write( - "Removing tag for {} {}\n".format(agent.uuid, agent.name)) + _stdout.write("Removing tag for {} {}\n".format( + agent.uuid, agent.name)) opts.aip.tag_agent(agent.uuid, None) else: if agent.tag is not None: @@ -144,27 +135,25 @@ def remove_agent(opts, remove_auth=True): agents = _list_agents(opts.aip) for pattern, match in filter_agents(agents, opts.pattern, opts): if not match: - _stderr.write( - "{}: error: agent not found: {}\n".format(opts.command, - pattern) - ) + _stderr.write("{}: error: agent not found: {}\n".format( + opts.command, pattern)) elif len(match) > 1 and not opts.force: _stderr.write( "{}: error: pattern returned multiple agents: {}\n".format( - opts.command, pattern - ) - ) + opts.command, pattern)) _stderr.write( "Use -f or --force to force removal of multiple agents.\n") return 10 for agent in match: _stdout.write("Removing {} {}\n".format(agent.uuid, agent.name)) - opts.connection.call("remove_agent", agent.uuid, + opts.connection.call("remove_agent", + agent.uuid, remove_auth=remove_auth) # TODO: Remove AIP def list_agents(opts): + def get_priority(agent): return opts.aip.agent_priority(agent.uuid) or "" @@ -193,11 +182,8 @@ def update_health_cache(opts): do_update = True # Make sure we update if we don't have any health dicts, or if the cache # has timed out. - if ( - health_cache_timeout_date is not None - and t_now < health_cache_timeout_date - and health_cache - ): + if (health_cache_timeout_date is not None + and t_now < health_cache_timeout_date and health_cache): do_update = False if do_update: @@ -205,12 +191,10 @@ def update_health_cache(opts): if opts.connection.server: health_cache.update( opts.connection.server.vip.rpc.call( - PLATFORM_HEALTH, "get_platform_health" - ).get(timeout=4) - ) + PLATFORM_HEALTH, "get_platform_health").get(timeout=4)) health_cache_timeout_date = datetime.now() + timedelta( - seconds=health_cache_timeout - ) + seconds=health_cache_timeout) + # TODO: Remove AIP def status_agents(opts): @@ -229,12 +213,14 @@ def status_agents(opts): agent = agents[uuid] agents[uuid] = agent._replace(agent_user=agent_user) except KeyError: - Agent = collections.namedtuple("Agent", - "name tag uuid vip_identity " - "agent_user") - agents[uuid] = agent = Agent( - name, None, uuid, vip_identity=identity, agent_user=agent_user - ) + Agent = collections.namedtuple( + "Agent", "name tag uuid vip_identity " + "agent_user") + agents[uuid] = agent = Agent(name, + None, + uuid, + vip_identity=identity, + agent_user=agent_user) status[uuid] = stat agents = list(agents.values()) @@ -268,6 +254,7 @@ def get_health(agent): _show_filtered_agents_status(opts, get_status, get_health, agents) + #TODO: Remove AIP def agent_health(opts): agents = {agent.uuid: agent for agent in _list_agents(opts.aip)}.values() @@ -295,21 +282,17 @@ def agent_health(opts): def clear_status(opts): opts.connection.call("clear_status", opts.clear_all) + # TODO: Remove AIP def enable_agent(opts): agents = _list_agents(opts.aip) for pattern, match in filter_agents(agents, opts.pattern, opts): if not match: - _stderr.write( - "{}: error: agent not found: {}\n".format(opts.command, - pattern) - ) + _stderr.write("{}: error: agent not found: {}\n".format( + opts.command, pattern)) for agent in match: - _stdout.write( - "Enabling {} {} with priority {}\n".format( - agent.uuid, agent.name, opts.priority - ) - ) + _stdout.write("Enabling {} {} with priority {}\n".format( + agent.uuid, agent.name, opts.priority)) opts.aip.prioritize_agent(agent.uuid, opts.priority) @@ -317,15 +300,13 @@ def disable_agent(opts): agents = _list_agents(opts.aip) for pattern, match in filter_agents(agents, opts.pattern, opts): if not match: - _stderr.write( - "{}: error: agent not found: {}\n".format(opts.command, - pattern) - ) + _stderr.write("{}: error: agent not found: {}\n".format( + opts.command, pattern)) for agent in match: priority = opts.aip.agent_priority(agent.uuid) if priority is not None: - _stdout.write( - "Disabling {} {}\n".format(agent.uuid, agent.name)) + _stdout.write("Disabling {} {}\n".format( + agent.uuid, agent.name)) opts.aip.prioritize_agent(agent.uuid, None) @@ -358,12 +339,16 @@ def act_on_agent(action, opts): agents, pattern_to_use = [a for a in agents if a.tag is not None], '*' # filter agents and update regex pattern - for pattern, filtered_agents in filter_agents(agents, pattern_to_use, opts): + for pattern, filtered_agents in filter_agents(agents, pattern_to_use, + opts): if not filtered_agents: - _stderr.write(f"Agents NOT found using 'vctl {opts.command}' on pattern: {pattern}\n") + _stderr.write( + f"Agents NOT found using 'vctl {opts.command}' on pattern: {pattern}\n" + ) for agent in filtered_agents: pid, status = call("agent_status", agent.uuid) - _call_action_on_agent(agent, pid, status, call, action) + _call_action_on_agent(agent, pid, status, call, action) + def _call_action_on_agent(agent, pid, status, call, action): if action == "start_agent": @@ -389,23 +374,17 @@ def shutdown_agents(opts): if "rmq" == utils.get_messagebus(): if not check_rabbit_status(): rmq_cfg = RMQConfig() - wait_period = ( - rmq_cfg.reconnect_delay() if rmq_cfg.reconnect_delay() < 60 - else 60 - ) + wait_period = (rmq_cfg.reconnect_delay() + if rmq_cfg.reconnect_delay() < 60 else 60) _stderr.write( "RabbitMQ server is not running.\n" "Waiting for {} seconds for possible reconnection and to " - "perform normal shutdown\n".format( - wait_period - ) - ) + "perform normal shutdown\n".format(wait_period)) gevent.sleep(wait_period) if not check_rabbit_status(): _stderr.write( "RabbitMQ server is still not running.\nShutting down " - "the platform forcefully\n" - ) + "the platform forcefully\n") opts.aip.brute_force_platform_shutdown() return opts.connection.call("shutdown") @@ -442,9 +421,8 @@ def send(): wheel.close() channel.close(linger=0) - result = connection.vip.rpc.call( - peer, "install_agent", os.path.basename(path), channel.name - ) + result = connection.vip.rpc.call(peer, "install_agent", + os.path.basename(path), channel.name) task = gevent.spawn(send) result.rawlink(lambda glt: task.kill(block=False)) _log.debug(f"Result is {result}") @@ -458,9 +436,6 @@ def send_agent(opts): return uuid - - - def do_stats(opts): call = opts.connection.call if opts.op == "status": @@ -478,7 +453,6 @@ def do_stats(opts): _stdout.write("%sabled\n" % ("en" if call("stats.enabled") else "dis")) - def priority(value): n = int(value) if not 0 <= n < 100: @@ -493,17 +467,18 @@ def get_keys(opts): key_store = KeyStore() publickey = key_store.public secretkey = key_store.secret - return {"publickey": publickey, "secretkey": secretkey, - "serverkey": serverkey} + return { + "publickey": publickey, + "secretkey": secretkey, + "serverkey": serverkey + } def main(): # Refuse to run as root if not getattr(os, "getuid", lambda: -1)(): - sys.stderr.write( - "%s: error: refusing to run as root to prevent " - "potential damage.\n" % os.path.basename(sys.argv[0]) - ) + sys.stderr.write("%s: error: refusing to run as root to prevent " + "potential damage.\n" % os.path.basename(sys.argv[0])) sys.exit(77) volttron_home = get_home() @@ -534,8 +509,7 @@ def main(): help="timeout in seconds for remote calls (default: %(default)g)", ) global_args.add_argument( - "--msgdebug", help="route all messages to an agent while debugging" - ) + "--msgdebug", help="route all messages to an agent while debugging") global_args.add_argument( "--vip-address", metavar="ZMQADDR", @@ -554,21 +528,24 @@ def main(): action="store_true", help="filter/search by agent name", ) - filterable.add_argument( - "--tag", dest="by_tag", action="store_true", - help="filter/search by tag name" - ) - filterable.add_argument( - "--all-tagged", dest="by_all_tagged", action="store_true", - help="filter/search by all tagged agents" - ) + filterable.add_argument("--tag", + dest="by_tag", + action="store_true", + help="filter/search by tag name") + filterable.add_argument("--all-tagged", + dest="by_all_tagged", + action="store_true", + help="filter/search by all tagged agents") filterable.add_argument( "--uuid", dest="by_uuid", action="store_true", help="filter/search by UUID (default)", ) - filterable.set_defaults(by_name=False, by_tag=False, by_all_tagged=False, by_uuid=False) + filterable.set_defaults(by_name=False, + by_tag=False, + by_all_tagged=False, + by_uuid=False) parser = config.ArgumentParser( prog=os.path.basename(sys.argv[0]), @@ -614,12 +591,13 @@ def main(): default=logging.WARNING, help="set logger verboseness", ) - parser.add_argument("--show-config", action="store_true", + parser.add_argument("--show-config", + action="store_true", help=argparse.SUPPRESS) - parser.add_argument( - "--json", action="store_true", default=False, - help="format output to json" - ) + parser.add_argument("--json", + action="store_true", + default=False, + help="format output to json") parser.add_help_argument() parser.set_defaults( @@ -627,9 +605,9 @@ def main(): volttron_home=volttron_home, ) - top_level_subparsers = parser.add_subparsers( - title="commands", metavar="", dest="command" - ) + top_level_subparsers = parser.add_subparsers(title="commands", + metavar="", + dest="command") def add_parser(*args, **kwargs) -> argparse.ArgumentParser: parents = kwargs.get("parents", []) @@ -643,28 +621,32 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: # ==================================================== add_install_agent_parser(add_parser, HAVE_RESTRICTED) - tag = add_parser("tag", parents=[filterable], + tag = add_parser("tag", + parents=[filterable], help="set, show, or remove agent tag") tag.add_argument("agent", help="UUID or name of agent") group = tag.add_mutually_exclusive_group() group.add_argument("tag", nargs="?", const=None, help="tag to give agent") - group.add_argument("-r", "--remove", action="store_true", + group.add_argument("-r", + "--remove", + action="store_true", help="remove tag") tag.set_defaults(func=tag_agent, tag=None, remove=False) remove = add_parser("remove", parents=[filterable], help="remove agent") remove.add_argument("pattern", nargs="+", help="UUID or name of agent") - remove.add_argument( - "-f", "--force", action="store_true", - help="force removal of multiple agents" - ) + remove.add_argument("-f", + "--force", + action="store_true", + help="force removal of multiple agents") remove.set_defaults(func=remove_agent, force=False) peers = add_parser("peerlist", help="list the peers connected to the platform") peers.set_defaults(func=list_peers) - list_ = add_parser("list", parents=[filterable], + list_ = add_parser("list", + parents=[filterable], help="list installed agent") list_.add_argument("pattern", nargs="*", help="UUID or name of agent") list_.add_argument( @@ -676,7 +658,8 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) list_.set_defaults(func=list_agents, min_uuid_len=1) - status = add_parser("status", parents=[filterable], + status = add_parser("status", + parents=[filterable], help="show status of agents") status.add_argument("pattern", nargs="*", help="UUID or name of agent") status.add_argument( @@ -688,9 +671,9 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) status.set_defaults(func=status_agents, min_uuid_len=1) - health = add_parser( - "health", parents=[filterable], help="show agent health as JSON" - ) + health = add_parser("health", + parents=[filterable], + help="show agent health as JSON") health.add_argument("pattern", nargs=1, help="UUID or name of agent") health.set_defaults(func=agent_health, min_uuid_len=1) @@ -704,27 +687,29 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) clear.set_defaults(func=clear_status, clear_all=False) - enable = add_parser( - "enable", parents=[filterable], - help="enable agent to start automatically" - ) + enable = add_parser("enable", + parents=[filterable], + help="enable agent to start automatically") enable.add_argument("pattern", nargs="+", help="UUID or name of agent") - enable.add_argument( - "-p", "--priority", type=priority, - help="2-digit priority from 00 to 99" - ) + enable.add_argument("-p", + "--priority", + type=priority, + help="2-digit priority from 00 to 99") enable.set_defaults(func=enable_agent, priority="50") - disable = add_parser( - "disable", parents=[filterable], - help="prevent agent from start automatically" - ) + disable = add_parser("disable", + parents=[filterable], + help="prevent agent from start automatically") disable.add_argument("pattern", nargs="+", help="UUID or name of agent") disable.set_defaults(func=disable_agent) - start = add_parser("start", parents=[filterable], + start = add_parser("start", + parents=[filterable], help="start installed agent") - start.add_argument("pattern", nargs="*", help="UUID or name of agent", default='') + start.add_argument("pattern", + nargs="*", + help="UUID or name of agent", + default='') if HAVE_RESTRICTED: start.add_argument( "--verify", @@ -741,11 +726,17 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: start.set_defaults(func=start_agent) stop = add_parser("stop", parents=[filterable], help="stop agent") - stop.add_argument("pattern", nargs="*", help="UUID or name of agent", default='') + stop.add_argument("pattern", + nargs="*", + help="UUID or name of agent", + default='') stop.set_defaults(func=stop_agent) restart = add_parser("restart", parents=[filterable], help="restart agent") - restart.add_argument("pattern", nargs="*", help="UUID or name of agent", default='') + restart.add_argument("pattern", + nargs="*", + help="UUID or name of agent", + default='') restart.set_defaults(func=restart_agent) run = add_parser("run", help="start any agent by path") @@ -765,7 +756,6 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: ) run.set_defaults(func=run_agent) - # ==================================================== # rpc commands # ==================================================== @@ -787,10 +777,9 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: add_config_store_parser(add_parser) shutdown = add_parser("shutdown", help="stop all agents") - shutdown.add_argument( - "--platform", action="store_true", - help="also stop the platform process" - ) + shutdown.add_argument("--platform", + action="store_true", + help="also stop the platform process") shutdown.set_defaults(func=shutdown_agents, platform=False) send = add_parser("send", help="send agent and start on a remote platform") @@ -800,9 +789,9 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: stats = add_parser("stats", help="manage router message statistics tracking") op = stats.add_argument( - "op", choices=["status", "enable", "disable", "dump", "pprint"], - nargs="?" - ) + "op", + choices=["status", "enable", "disable", "dump", "pprint"], + nargs="?") stats.set_defaults(func=do_stats, op="status") # ============================================================================== @@ -820,12 +809,14 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: "create-cgroups", help="setup VOLTTRON control group for restricted execution", ) - cgroup.add_argument( - "-u", "--user", metavar="USER", help="owning user name or ID" - ) - cgroup.add_argument( - "-g", "--group", metavar="GROUP", help="owning group name or ID" - ) + cgroup.add_argument("-u", + "--user", + metavar="USER", + help="owning user name or ID") + cgroup.add_argument("-g", + "--group", + metavar="GROUP", + help="owning group name or ID") cgroup.set_defaults(func=create_cgroups, user=None, group=None) # Parse and expand options @@ -840,10 +831,8 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: if args[0] not in ("list", "tag", "auth", "rabbitmq", "certs"): # check pid file if not utils.is_volttron_running(volttron_home): - _stderr.write( - "VOLTTRON is not running. This command " - "requires VOLTTRON platform to be running\n" - ) + _stderr.write("VOLTTRON is not running. This command " + "requires VOLTTRON platform to be running\n") return 10 conf = os.path.join(volttron_home, "config") @@ -868,13 +857,18 @@ def add_parser(*args, **kwargs) -> argparse.ArgumentParser: elif opts.log == "-": log_to_file(sys.stdout, level) elif opts.log: - log_to_file(opts.log, level, + log_to_file(opts.log, + level, handler_class=logging.handlers.WatchedFileHandler) else: log_to_file(None, 100, handler_class=lambda x: logging.NullHandler()) if opts.log_config: logging.config.fileConfig(opts.log_config) + if not hasattr(opts, "func"): + parser.print_help() + sys.exit(0) + opts.aip = aipmod.AIPplatform(opts) opts.aip.setup() From aa7f2f18f645ed0cc6f43e674f8084f4e15540a3 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Tue, 20 Feb 2024 16:09:04 -0800 Subject: [PATCH 11/44] work around for issue #3154 --- volttron/platform/instance_setup.py | 68 +++++++++++------------------ 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/volttron/platform/instance_setup.py b/volttron/platform/instance_setup.py index a022639ec2..56cbdb26a3 100644 --- a/volttron/platform/instance_setup.py +++ b/volttron/platform/instance_setup.py @@ -39,7 +39,6 @@ from gevent.subprocess import Popen from zmq import green as zmq -from requirements import extras_require from volttron.platform import is_rabbitmq_available from volttron.platform.auth import certs from volttron.platform import jsonapi @@ -336,12 +335,7 @@ def _get_dependencies(): return dependencies -def _check_dependencies_met(requirement): - try: - dependencies_needed = extras_require[requirement] - except KeyError: - print(f"ERROR: Requirement {requirement} was not found in requirements.py") - return False +def _check_dependencies_met(dependencies_needed): current_dependencies = _get_dependencies() for dependency in dependencies_needed: if "==" in dependency: @@ -358,15 +352,21 @@ def _check_dependencies_met(requirement): def set_dependencies(requirement): try: + # go up two level above env/bin + sys.path.append(os.path.dirname(os.path.dirname(sys.path[0]))) + from requirements import extras_require dependencies_needed = extras_require[requirement] except KeyError: - print("ERROR: Incorrect requirement chosen") + print(f"ERROR: Incorrect requirement chosen: {requirement}") + return + + if not _check_dependencies_met(dependencies_needed): + print(f"Installing {requirement} dependencies...") + cmds = [sys.executable, "-m", "pip", "install"] + for dependency in dependencies_needed: + cmds.append(dependency) + subprocess.check_call(cmds) return - cmds = [sys.executable, "-m", "pip", "install"] - for dependency in dependencies_needed: - cmds.append(dependency) - subprocess.check_call(cmds) - return def _create_web_certs(): @@ -893,10 +893,7 @@ def wizard(): prompt = 'Is this instance web enabled?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met('web'): - print("Web dependencies not installed. Installing now...") - set_dependencies('web') - print("Done!") + set_dependencies('web') if config_opts['message-bus'] == 'rmq': do_web_enabled_rmq(volttron_home) elif config_opts['message-bus'] == 'zmq': @@ -916,13 +913,9 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): - print("VCP dependencies not installed. Installing now...") - if not _check_dependencies_met("drivers"): - set_dependencies("drivers") - if not _check_dependencies_met("web"): - set_dependencies("web") - print("Done!") + print("Checking for VCP dependencies.....") + set_dependencies("drivers") + set_dependencies("web") do_vcp() prompt = 'Would you like to install a platform historian?' @@ -932,10 +925,8 @@ def wizard(): prompt = 'Would you like to install a platform driver?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met("drivers"): - print("Driver dependencies not installed. Installing now...") - set_dependencies("drivers") - print("Done!") + print("Checking Driver dependencies...") + set_dependencies("drivers") do_platform_driver() prompt = 'Would you like to install a listener agent?' @@ -954,13 +945,9 @@ def wizard(): prompt = 'Will this instance be controlled by volttron central?' response = prompt_response(prompt, valid_answers=y_or_n, default='Y') if response in y: - if not _check_dependencies_met("drivers") or not _check_dependencies_met("web"): - print("VCP dependencies not installed. Installing now...") - if not _check_dependencies_met("drivers"): - set_dependencies("drivers") - if not _check_dependencies_met("web"): - set_dependencies("web") - print("Done!") + print("Checking VCP dependencies...") + set_dependencies("drivers") + set_dependencies("web") do_vcp() prompt = 'Would you like to install a platform historian?' @@ -970,10 +957,8 @@ def wizard(): prompt = 'Would you like to install a platform driver?' response = prompt_response(prompt, valid_answers=y_or_n, default='N') if response in y: - if not _check_dependencies_met("drivers"): - print("Driver dependencies not installed. Installing now...") - set_dependencies("drivers") - print("Done!") + print("Checking Driver dependencies...") + set_dependencies("drivers") do_platform_driver() prompt = 'Would you like to install a listener agent?' @@ -1114,10 +1099,9 @@ def process_rmq_inputs(args_dict, instance_name=None): vhome = get_home() - if args_dict['installation-type'] in ['federation', 'shovel'] and not _check_dependencies_met('web'): - print("Web dependencies not installed. Installing now...") + if args_dict['installation-type'] in ['federation', 'shovel']: + print("Checking Web dependencies...") set_dependencies('web') - print("Done!") if args_dict['config'] is not None: if not os.path.exists(vhome): From 33672ff96a1c253efebf7211222f94142feb34cc Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:33:56 -0700 Subject: [PATCH 12/44] bump actions to not have warnings for node 16 --- .github/workflows/pytest-auth.yml | 4 ++-- .github/workflows/pytest-dbutils-backup_db.yml | 4 ++-- .github/workflows/pytest-dbutils-influxdbfuncts.yml | 4 ++-- .github/workflows/pytest-dbutils-mysqlfuncts.yml | 4 ++-- .github/workflows/pytest-dbutils-postgresqlfuncts.yml | 4 ++-- .github/workflows/pytest-dbutils-sqlitefuncts.yml | 4 ++-- .github/workflows/pytest-dbutils-timescaldbfuncts.yml | 4 ++-- .github/workflows/pytest-miscellaneous-tests.yml | 4 ++-- .github/workflows/pytest-testutils.yml | 4 ++-- .github/workflows/pytest-vctl.yml | 4 ++-- .github/workflows/pytest-web.yml | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pytest-auth.yml b/.github/workflows/pytest-auth.yml index 3b8c10c250..feaeca61bf 100644 --- a/.github/workflows/pytest-auth.yml +++ b/.github/workflows/pytest-auth.yml @@ -54,7 +54,7 @@ jobs: # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -69,7 +69,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-backup_db.yml b/.github/workflows/pytest-dbutils-backup_db.yml index beecf28097..bc677c6b8d 100644 --- a/.github/workflows/pytest-dbutils-backup_db.yml +++ b/.github/workflows/pytest-dbutils-backup_db.yml @@ -55,7 +55,7 @@ jobs: # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -75,7 +75,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-influxdbfuncts.yml b/.github/workflows/pytest-dbutils-influxdbfuncts.yml index ff3dcc5552..47b335e9e8 100644 --- a/.github/workflows/pytest-dbutils-influxdbfuncts.yml +++ b/.github/workflows/pytest-dbutils-influxdbfuncts.yml @@ -44,7 +44,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -64,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-mysqlfuncts.yml b/.github/workflows/pytest-dbutils-mysqlfuncts.yml index 37e3068ffc..6454fa4a86 100644 --- a/.github/workflows/pytest-dbutils-mysqlfuncts.yml +++ b/.github/workflows/pytest-dbutils-mysqlfuncts.yml @@ -44,7 +44,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -64,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-postgresqlfuncts.yml b/.github/workflows/pytest-dbutils-postgresqlfuncts.yml index 0398771cdb..87499a2384 100644 --- a/.github/workflows/pytest-dbutils-postgresqlfuncts.yml +++ b/.github/workflows/pytest-dbutils-postgresqlfuncts.yml @@ -44,7 +44,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -64,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-sqlitefuncts.yml b/.github/workflows/pytest-dbutils-sqlitefuncts.yml index 3848e6efa8..e5bb742d7d 100644 --- a/.github/workflows/pytest-dbutils-sqlitefuncts.yml +++ b/.github/workflows/pytest-dbutils-sqlitefuncts.yml @@ -43,7 +43,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -63,7 +63,7 @@ jobs: test_output_suffix: ${{ env.OUTPUT_SUFFIX }} - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-timescaldbfuncts.yml b/.github/workflows/pytest-dbutils-timescaldbfuncts.yml index 1aeb461487..7639d195c7 100644 --- a/.github/workflows/pytest-dbutils-timescaldbfuncts.yml +++ b/.github/workflows/pytest-dbutils-timescaldbfuncts.yml @@ -43,7 +43,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -63,7 +63,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-miscellaneous-tests.yml b/.github/workflows/pytest-miscellaneous-tests.yml index 9fc023b5a9..80dc6d8d27 100644 --- a/.github/workflows/pytest-miscellaneous-tests.yml +++ b/.github/workflows/pytest-miscellaneous-tests.yml @@ -51,7 +51,7 @@ jobs: # Setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -98,7 +98,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-testutils.yml b/.github/workflows/pytest-testutils.yml index 39cebe91aa..fa1a06887b 100644 --- a/.github/workflows/pytest-testutils.yml +++ b/.github/workflows/pytest-testutils.yml @@ -42,7 +42,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -57,7 +57,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-vctl.yml b/.github/workflows/pytest-vctl.yml index d0d5d868fb..b541753236 100644 --- a/.github/workflows/pytest-vctl.yml +++ b/.github/workflows/pytest-vctl.yml @@ -53,7 +53,7 @@ jobs: # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -68,7 +68,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-web.yml b/.github/workflows/pytest-web.yml index 43b0540aa4..fe0496f35a 100644 --- a/.github/workflows/pytest-web.yml +++ b/.github/workflows/pytest-web.yml @@ -51,7 +51,7 @@ jobs: # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -66,7 +66,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report From 32d54ffe1f2a210f378dbd56c76809b4026d2d44 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:49:19 -0700 Subject: [PATCH 13/44] Update version of rtd --- docs/source/conf.py | 78 +++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 80eaeb23f6..dc75f60eb2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,12 +16,14 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) -from glob import glob -from mock import Mock as MagicMock import os import subprocess import sys +from glob import glob + import yaml +from mock import Mock as MagicMock + # Copied from volttron.platform.agent.util because it is not required # that volttron be installed to utilize this script. @@ -69,14 +71,17 @@ def execute_command(cmds, class Mock(MagicMock): + @classmethod def __getattr__(cls, name): - return Mock() + return Mock() -autodoc_mock_imports = ['loadshape', 'numpy', 'sympy', 'xlrd', 'stomp', 'oadr2', 'pyodbc', 'lxml', 'pytest', - 'pint', 'pandas', 'suds', 'paho', 'pymongo', 'bson', 'subprocess32', 'heaters', 'meters', - 'hvac', 'blinds', 'vehicles'] +autodoc_mock_imports = [ + 'loadshape', 'numpy', 'sympy', 'xlrd', 'stomp', 'oadr2', 'pyodbc', 'lxml', + 'pytest', 'pint', 'pandas', 'suds', 'paho', 'pymongo', 'bson', + 'subprocess32', 'heaters', 'meters', 'hvac', 'blinds', 'vehicles' +] # -- Project information ----------------------------------------------------- @@ -87,8 +92,7 @@ def __getattr__(cls, name): # The short X.Y version version = '9.0' # The full version, including alpha/beta/rc tags -release = '9.0-rc' - +release = '9.0' # -- General configuration --------------------------------------------------- @@ -151,7 +155,6 @@ def __getattr__(cls, name): # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -180,13 +183,11 @@ def __getattr__(cls, name): # # html_sidebars = {} - # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'VOLTTRONdoc' - # -- Options for LaTeX output ------------------------------------------------ latex_elements = { @@ -215,16 +216,11 @@ def __getattr__(cls, name): 'The VOLTTRON Community', 'manual'), ] - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (main_doc, 'volttron', 'VOLTTRON Documentation', - [author], 1) -] - +man_pages = [(main_doc, 'volttron', 'VOLTTRON Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -232,20 +228,22 @@ def __getattr__(cls, name): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (main_doc, 'VOLTTRON', 'VOLTTRON Documentation', - author, 'VOLTTRON', 'One line description of project.', - 'Miscellaneous'), + (main_doc, 'VOLTTRON', 'VOLTTRON Documentation', author, 'VOLTTRON', + 'One line description of project.', 'Miscellaneous'), ] - # -- Extension configuration ------------------------------------------------- # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3.6': None, - 'volttron-ansible': ('https://volttron.readthedocs.io/projects/volttron-ansible/en/main/', - None)} +intersphinx_mapping = { + 'https://docs.python.org/3.6': + None, + 'volttron-ansible': + ('https://volttron.readthedocs.io/projects/volttron-ansible/en/main/', + None) +} # -- Options for todo extension ---------------------------------------------- @@ -290,11 +288,15 @@ def generate_apidoc(app): # generate api-docs for each api docs directory for docs_subdir in config.keys(): docs_subdir_path = os.path.join(apidocs_base_dir, docs_subdir) - agent_dirs = glob(os.path.join(volttron_root, config[docs_subdir]["path"], "*/")) + agent_dirs = glob( + os.path.join(volttron_root, config[docs_subdir]["path"], "*/")) file_excludes = [] if config[docs_subdir].get("file_excludes"): - for exclude_pattern in config[docs_subdir].get("file_excludes", []): - file_excludes.append(os.path.join(volttron_root, config[docs_subdir]["path"], exclude_pattern)) + for exclude_pattern in config[docs_subdir].get( + "file_excludes", []): + file_excludes.append( + os.path.join(volttron_root, config[docs_subdir]["path"], + exclude_pattern)) print("after file excludes. calling apidoc") agent_excludes = \ config[docs_subdir].get("agent_excludes") if config[docs_subdir].get("agent_excludes", []) else [] @@ -311,19 +313,24 @@ def run_apidoc(docs_dir, agent_dirs, agent_excludes, exclude_pattern): :param agent_excludes: agent directories to be skipped :param exclude_pattern: file name patterns to be excluded. This passed on to sphinx-apidoc command for exclude """ - print(f"In run apidoc params {docs_dir}, {agent_dirs}, {agent_excludes}, {exclude_pattern}") + print( + f"In run apidoc params {docs_dir}, {agent_dirs}, {agent_excludes}, {exclude_pattern}" + ) for agent_src_dir in agent_dirs: agent_src_dir = os.path.abspath(agent_src_dir) - agent_src_dir = agent_src_dir[:-1] if agent_src_dir.endswith("/") else agent_src_dir + agent_src_dir = agent_src_dir[:-1] if agent_src_dir.endswith( + "/") else agent_src_dir name = os.path.basename(agent_src_dir) agent_doc_dir = os.path.join(docs_dir, name) if name not in agent_excludes: sys.path.insert(0, agent_src_dir) - cmd = ["sphinx-apidoc", '-e', '-a', '-M', '-d 4', - '-t', os.path.join(script_dir, 'apidocs-templates'), - '--force', '-o', agent_doc_dir, agent_src_dir, - os.path.join(agent_src_dir, "setup.py"), os.path.join(agent_src_dir, "conftest.py") - ] + cmd = [ + "sphinx-apidoc", '-e', '-a', '-M', '-d 4', '-t', + os.path.join(script_dir, 'apidocs-templates'), '--force', '-o', + agent_doc_dir, agent_src_dir, + os.path.join(agent_src_dir, "setup.py"), + os.path.join(agent_src_dir, "conftest.py") + ] cmd.extend(exclude_pattern) subprocess.check_call(cmd) @@ -363,5 +370,6 @@ def clean_api_rst(app, exception): global apidocs_base_dir import shutil if os.path.exists(apidocs_base_dir): - print("Cleanup: Removing apidocs directory {}".format(apidocs_base_dir)) + print( + "Cleanup: Removing apidocs directory {}".format(apidocs_base_dir)) shutil.rmtree(apidocs_base_dir) From 169473deea22092979816de1fff7a1900691beed Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Mon, 25 Mar 2024 15:57:27 -0700 Subject: [PATCH 14/44] Remove slack from the community page --- docs/source/developing-volttron/community.rst | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/docs/source/developing-volttron/community.rst b/docs/source/developing-volttron/community.rst index 05f32dde04..24718c7d1c 100644 --- a/docs/source/developing-volttron/community.rst +++ b/docs/source/developing-volttron/community.rst @@ -12,18 +12,10 @@ Contributing back to the project, which is encouraged but not required, enhances To learn more, check out :ref:`Contributing ` and :ref:`Documentation `. -Slack Channel -============= - -volttron-community.slack.com is where the |VOLTTRON| community at large can ask questions and meet with others -using |VOLTTRON|. To be added to Slack please email the VOLTTRON team at -`volttron@pnnl.gov `__. - - Mailing List ============ -Join the mailing list by emailing `volttron@pnnl.gov `__. +Join the mailing list at Eclipse: https://projects.eclipse.org/projects/iot.volttron/contact Stack Overflow @@ -39,8 +31,8 @@ Office Hours PNNL hosts office hours every other week on Fridays at 11 AM (PST). These meetings are designed to be very informal where VOLTTRON developers can answer specific questions about the inner workings of VOLTTRON. These meetings are also available for topical discussions of different aspects of the VOLTTRON platform. Currently the office hours are -available through a Zoom meeting. To be invited to the link meeting, contact the volttron team via email: -``__ +available through a Zoom meeting. All members of the mailing list will receive invites to the meetings. Join the +mailing list https://projects.eclipse.org/projects/iot.volttron/contact. Meetings are recorded and can be reviewed `here `__. From 3177ff8a7f837bce99efeafa7e4bc66da509beb8 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:14:50 -0700 Subject: [PATCH 15/44] require 3.8 of python --- bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap.py b/bootstrap.py index 6abc0ce359..19548817a4 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -227,7 +227,7 @@ def main(argv=sys.argv): sys.exit(77) # Python3 for life! - if sys.version_info.major < 3 or sys.version_info.minor < 6: + if sys.version_info.major < 3 or sys.version_info.minor < 8: sys.stderr.write('error: Python >= 3.8 is required\n') sys.exit(1) From db467aac631f04e1e21bfe63963b43bac3efd230 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Tue, 26 Mar 2024 14:31:49 -0700 Subject: [PATCH 16/44] fix for issue#3159 (fix for deprecated pip --install-options) --- bootstrap.py | 2 +- requirements.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index 19548817a4..8121b03c4c 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -123,7 +123,7 @@ def update(operation, verbose=None, offline=False, optional_requirements=[], rab # option_requirements contains wheel as first entry # Build option_requirements separately to pass install options - build_option = '--build-option' if wheeling else '--install-option' + build_option = '--build-option' if wheeling else '--config-settings' for requirement, options in option_requirements: args = [] diff --git a/requirements.py b/requirements.py index 80fa44c40e..249d9b56ce 100644 --- a/requirements.py +++ b/requirements.py @@ -30,7 +30,7 @@ # wheel version 0.31 has removed metadata.json file # https://github.com/pypa/wheel/issues/195 # so sticking to 0.30 for now. Could upgrade to wheel 0.31 with code changes -option_requirements = [('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] +option_requirements = [('pip==24.0', []), ('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] install_requires = ['gevent==21.12.0', From cc71c445c749c04973a33a0bdcfdf94e26eeff57 Mon Sep 17 00:00:00 2001 From: Chandrika Sivaramakrishnan Date: Thu, 28 Mar 2024 11:00:42 -0700 Subject: [PATCH 17/44] handled minor gevent timeout --- volttrontesting/services/historian/test_historian.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/volttrontesting/services/historian/test_historian.py b/volttrontesting/services/historian/test_historian.py index afa49acf41..352cbfea6a 100644 --- a/volttrontesting/services/historian/test_historian.py +++ b/volttrontesting/services/historian/test_historian.py @@ -1443,6 +1443,8 @@ def test_invalid_query(request, historian, publish_agent, query_agent, start=now, count=20, order="LAST_TO_FIRST").get(timeout=10) + except gevent.timeout.Timeout: + assert True except Exception as error: print("exception: {}".format(error)) assert "No route to host:" in str(error) From 50290afbb6cc9176fda4d0f778e839bd2931626b Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Thu, 28 Mar 2024 10:33:16 -0700 Subject: [PATCH 18/44] Minor updates to bacnet and modbus_tk tests. --- .../interfaces/modbus_tk/tests/test_battery_meter.py | 2 +- services/core/PlatformDriverAgent/tests/test_bacnet.py | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py index 5c7a868b66..0ac107061f 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_battery_meter.py @@ -387,7 +387,7 @@ def scrape_all(self, agent, device_name): return agent.vip.rpc.call(PLATFORM_DRIVER, 'scrape_all', device_name)\ .get(timeout=10) - @pytest.mark.xfail(is_running_in_container(), reason='Fails to set points on this test setup, only in Docker.') + @pytest.mark.skip('This test has been unreliable.') def test_scrape_all(self, agent): for key in registers_dict.keys(): self.set_point(agent, 'modbus_tk', key, registers_dict[key]) diff --git a/services/core/PlatformDriverAgent/tests/test_bacnet.py b/services/core/PlatformDriverAgent/tests/test_bacnet.py index 6c608e42e9..476a4fc5f2 100644 --- a/services/core/PlatformDriverAgent/tests/test_bacnet.py +++ b/services/core/PlatformDriverAgent/tests/test_bacnet.py @@ -77,7 +77,15 @@ def test_get_point_should_succeed(bacnet_test_agent): @pytest.fixture(scope="module") def bacnet_proxy_agent(volttron_instance): - device_address = socket.gethostbyname(socket.gethostname() + ".local") + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.settimeout(0) + try: + s.connect(('8.8.8.8', 1)) + device_address = s.getsockname()[0] + except Exception: + device_address = '127.0.0.1' + finally: + s.close() print(f"Device address for proxy agent for testing: {device_address}") bacnet_proxy_agent_config = { "device_address": device_address, From 74503d988a147a0bad5a69bf4eb392e355ec9d15 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Thu, 28 Mar 2024 14:04:13 -0700 Subject: [PATCH 19/44] Fixed missing import in modbus_tk/tests/test_driver_demo_board.py --- .../interfaces/modbus_tk/tests/test_driver_demo_board.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py index ea58ded782..593c2a684f 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/tests/test_driver_demo_board.py @@ -2,6 +2,9 @@ import gevent import pytest + +from pathlib import Path + from volttron.platform.agent.known_identities import CONFIGURATION_STORE, PLATFORM_DRIVER from volttron.platform import jsonapi from volttrontesting.utils.platformwrapper import PlatformWrapper From 28e7f07836c0f2ce3a6a6105d4bfc6b4d3494ea5 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 29 Mar 2024 13:04:41 -0700 Subject: [PATCH 20/44] Update README.md --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 6999280281..bea8985a36 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ ![image](docs/source/files/VOLLTRON_Logo_Black_Horizontal_with_Tagline.png) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/fcf58045b4804edf8f4d3ecde3016f76)](https://app.codacy.com/gh/VOLTTRON/volttron?utm_source=github.com&utm_medium=referral&utm_content=VOLTTRON/volttron&utm_campaign=Badge_Grade_Settings) +# VOLTTRON + +This repository is for the current production VOLTTRON. We are working on VOLTTRON 10 (modular) which is available under +github at https://github.com/eclipse-volttron/. The modular version of VOLTTRON will help ease deployment and support +flexible deployment where in only required agents/applications can be installed, thereby simplifying setup and upgrade +steps for the end user. The VOLTTRON team are currently working on porting agents from monolithic VOLTTRON to the +modular version of VOLTTRON. To know more about modular VOLTTRON, please visit our new documentation site available +at https://eclipse-volttron.readthedocs.io/en/latest/. We would love for you to try it out and give us early +feedback. Also, until our work on modular VOLTTRON is completed, please continue cloning and using this +repository for your production systems. VOLTTRON™ is an open source platform for distributed sensing and control. The platform provides services for collecting and storing data from buildings and From 82863691365e75f617a0edf02c4826d4a842ca59 Mon Sep 17 00:00:00 2001 From: Chandrika Date: Fri, 19 Apr 2024 10:40:10 -0700 Subject: [PATCH 21/44] Fix for security issue #3168 (#3169) * Fix for security issue #3168 * handling clean up errors in test * testing group commands in different test module * moved group and role test to different module * moved group and role test to different module --- volttron/platform/main.py | 3 - volttron/platform/vip/agent/subsystems/rpc.py | 4 +- .../platform/auth_tests/test_auth_control.py | 14 -- .../auth_tests/test_auth_group_roles.py | 174 ++++++++++++++ .../auth_tests/test_auth_integration.py | 224 ++++++++++++++++++ volttrontesting/utils/platformwrapper.py | 25 +- 6 files changed, 414 insertions(+), 30 deletions(-) create mode 100644 volttrontesting/platform/auth_tests/test_auth_group_roles.py create mode 100644 volttrontesting/platform/auth_tests/test_auth_integration.py diff --git a/volttron/platform/main.py b/volttron/platform/main.py index a0fdd7ecf4..16db2285d1 100644 --- a/volttron/platform/main.py +++ b/volttron/platform/main.py @@ -435,9 +435,6 @@ def issue(self, topic, frames, extra=None): # return result def handle_subsystem(self, frames, user_id): - _log.debug( - f"Handling subsystem with frames: {frames} user_id: {user_id}") - subsystem = frames[5] if subsystem == 'quit': sender = frames[0] diff --git a/volttron/platform/vip/agent/subsystems/rpc.py b/volttron/platform/vip/agent/subsystems/rpc.py index 510ce9d099..d77108a481 100644 --- a/volttron/platform/vip/agent/subsystems/rpc.py +++ b/volttron/platform/vip/agent/subsystems/rpc.py @@ -278,8 +278,8 @@ def _iterate_exports(self): for method_name in self._exports: method = self._exports[method_name] caps = annotations(method, set, "rpc.allow_capabilities") - # if caps: - # self._exports[method_name] = self._add_auth_check(method, caps) + if caps: + self._exports[method_name] = self._add_auth_check(method, caps) def _add_auth_check(self, method, required_caps): """ diff --git a/volttrontesting/platform/auth_tests/test_auth_control.py b/volttrontesting/platform/auth_tests/test_auth_control.py index e76d63df8f..a17c6bf1b0 100644 --- a/volttrontesting/platform/auth_tests/test_auth_control.py +++ b/volttrontesting/platform/auth_tests/test_auth_control.py @@ -375,20 +375,6 @@ def test_auth_rpc_method_remove(auth_instance): assert entries[-1]['rpc_method_authorizations'] != {'test_method': ["test_auth"]} -@pytest.mark.control -def test_group_cmds(auth_instance): - """Test add-group, list-groups, update-group, and remove-group""" - _run_group_or_role_cmds(auth_instance, _add_group, _list_groups, - _update_group, _remove_group) - - -@pytest.mark.control -def test_role_cmds(auth_instance): - """Test add-role, list-roles, update-role, and remove-role""" - _run_group_or_role_cmds(auth_instance, _add_role, _list_roles, - _update_role, _remove_role) - - def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn): expected = [] key = '0' diff --git a/volttrontesting/platform/auth_tests/test_auth_group_roles.py b/volttrontesting/platform/auth_tests/test_auth_group_roles.py new file mode 100644 index 0000000000..bfcad699c5 --- /dev/null +++ b/volttrontesting/platform/auth_tests/test_auth_group_roles.py @@ -0,0 +1,174 @@ + +import os +import re +import subprocess + +import gevent +import pytest +from mock import MagicMock +from volttron.platform.auth.auth_protocols.auth_zmq import ZMQAuthorization, ZMQServerAuthentication + +from volttrontesting.platform.auth_tests.conftest import assert_auth_entries_same +from volttrontesting.utils.platformwrapper import with_os_environ +from volttrontesting.utils.utils import AgentMock +from volttron.platform.vip.agent import Agent +from volttron.platform.auth import AuthService +from volttron.platform.auth import AuthEntry +from volttron.platform import jsonapi + +@pytest.fixture(autouse=True) +def auth_instance(volttron_instance): + if not volttron_instance.auth_enabled: + pytest.skip("AUTH tests are not applicable if auth is disabled") + with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'r') as f: + auth_file = jsonapi.load(f) + print(auth_file) + try: + yield volttron_instance + finally: + with with_os_environ(volttron_instance.env): + with open(os.path.join(volttron_instance.volttron_home, "auth.json"), 'w') as f: + jsonapi.dump(auth_file, f) + + +def _run_group_or_role_cmds(platform, add_fn, list_fn, update_fn, remove_fn): + expected = [] + key = '0' + values = ['0', '1'] + expected.extend(values) + + add_fn(platform, key, values) + gevent.sleep(4) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update add single value + values = ['2'] + expected.extend(values) + update_fn(platform, key, values) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update add multiple values + values = ['3', '4'] + expected.extend(values) + update_fn(platform, key, values) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update remove single value + value = '0' + expected.remove(value) + update_fn(platform, key, [value], remove=True) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Update remove single value + values = ['1', '2'] + for value in values: + expected.remove(value) + update_fn(platform, key, values, remove=True) + gevent.sleep(2) + keys = list_fn(platform) + assert set(keys[key]) == set(expected) + + # Remove key + remove_fn(platform, key) + gevent.sleep(2) + keys = list_fn(platform) + assert key not in keys + + + +def _add_group_or_role(platform, cmd, name, list_): + with with_os_environ(platform.env): + args = ['volttron-ctl', 'auth', cmd, name] + args.extend(list_) + p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True) + p.communicate() + assert p.returncode == 0 + + +def _add_group(platform, group, roles): + _add_group_or_role(platform, 'add-group', group, roles) + + +def _add_role(platform, role, capabilities): + _add_group_or_role(platform, 'add-role', role, capabilities) + + +def _list_groups_or_roles(platform, cmd): + with with_os_environ(platform.env): + output = subprocess.check_output(['volttron-ctl', 'auth', cmd], + env=platform.env, universal_newlines=True) + # For these tests don't use names that contain space, [, comma, or ' + output = output.replace('[', '').replace("'", '').replace(']', '') + output = output.replace(',', '') + lines = output.split('\n') + + dict_ = {} + for line in lines[2:-1]: # skip two header lines and last (empty) line + list_ = ' '.join(line.split()).split() # combine multiple spaces + dict_[list_[0]] = list_[1:] + return dict_ + + +def _list_groups(platform): + return _list_groups_or_roles(platform, 'list-groups') + + +def _list_roles(platform): + return _list_groups_or_roles(platform, 'list-roles') + + +def _update_group_or_role(platform, cmd, key, values, remove): + with with_os_environ(platform.env): + args = ['volttron-ctl', 'auth', cmd, key] + args.extend(values) + if remove: + args.append('--remove') + p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True) + p.communicate() + assert p.returncode == 0 + + +def _update_group(platform, group, roles, remove=False): + _update_group_or_role(platform, 'update-group', group, roles, remove) + + +def _update_role(platform, role, caps, remove=False): + _update_group_or_role(platform, 'update-role', role, caps, remove) + + +def _remove_group_or_role(platform, cmd, key): + with with_os_environ(platform.env): + args = ['volttron-ctl', 'auth', cmd, key] + p = subprocess.Popen(args, env=platform.env, stdin=subprocess.PIPE, universal_newlines=True) + p.communicate() + assert p.returncode == 0 + + +def _remove_group(platform, group): + _remove_group_or_role(platform, 'remove-group', group) + + +def _remove_role(platform, role): + _remove_group_or_role(platform, 'remove-role', role) + + +@pytest.mark.control +def test_group_cmds(auth_instance): + """Test add-group, list-groups, update-group, and remove-group""" + _run_group_or_role_cmds(auth_instance, _add_group, _list_groups, + _update_group, _remove_group) + + +@pytest.mark.control +def test_role_cmds(auth_instance): + """Test add-role, list-roles, update-role, and remove-role""" + _run_group_or_role_cmds(auth_instance, _add_role, _list_roles, + _update_role, _remove_role) + diff --git a/volttrontesting/platform/auth_tests/test_auth_integration.py b/volttrontesting/platform/auth_tests/test_auth_integration.py new file mode 100644 index 0000000000..bc9dedc9b3 --- /dev/null +++ b/volttrontesting/platform/auth_tests/test_auth_integration.py @@ -0,0 +1,224 @@ +import os +import subprocess +import sys +import tempfile +import gevent +import pytest +from volttron.platform.agent.known_identities import AUTH +from volttron.platform import jsonrpc +from volttron.platform.messaging.health import STATUS_BAD + +called_agent_src = """ +import sys +from volttron.platform.agent import utils +from volttron.platform.vip.agent import Agent, Core +from volttron.platform.vip.agent.subsystems import RPC +import gevent +class CalledAgent(Agent): + def __init__(self, config_path, **kwargs): + super(CalledAgent, self).__init__(**kwargs) + @RPC.export + @RPC.allow("can_call_method") + def restricted_method(self, sender, **kwargs): + print("test") +def main(argv=sys.argv): + try: + utils.vip_main(CalledAgent, version='0.1') + except Exception as e: + print('unhandled exception: {}'.format(e)) +if __name__ == '__main__': + # Entry point for script + sys.exit(main()) +""" + +called_agent_setup = """ +from setuptools import setup +setup( + name='calledagent', + version='0.1', + install_requires=['volttron'], + packages=['calledagent'], + entry_points={ + 'setuptools.installation': [ + 'eggsecutable=calledagent.calledagent:main', + ] + } +) +""" + +caller_agent_src = """ +import sys +import gevent +import logging +from volttron.platform.agent import utils +from volttron.platform.vip.agent import Agent, Core +from volttron.platform.vip.agent.subsystems import RPC +from volttron.platform.scheduling import periodic +from volttron.platform.messaging.health import (STATUS_BAD, + STATUS_GOOD, Status) +from volttron.platform.agent.known_identities import AUTH +from volttron.platform import jsonrpc +from volttron.platform.messaging.health import STATUS_BAD + +_log = logging.getLogger(__name__) +class CallerAgent(Agent): + def __init__(self, config_path, **kwargs): + super(CallerAgent, self).__init__(**kwargs) + + # @Core.schedule(periodic(3)) + # def call_rpc_method(self): + @Core.receiver("onstart") + def onstart(self, sender, **kwargs): + try: + self.vip.rpc.call('called_agent', 'restricted_method').get(timeout=3) + except Exception as e: + self.vip.health.set_status(STATUS_BAD, f"{e}") +def main(argv=sys.argv): + try: + utils.vip_main(CallerAgent, version='0.1') + except Exception as e: + print('unhandled exception: {}'.format(e)) +if __name__ == '__main__': + # Entry point for script + sys.exit(main()) +""" + +caller_agent_setup = """ +from setuptools import setup +setup( + name='calleragent', + version='0.1', + install_requires=['volttron'], + packages=['calleragent'], + entry_points={ + 'setuptools.installation': [ + 'eggsecutable=calleragent.calleragent:main', + ] + } +) +""" + +@pytest.fixture +def install_two_agents(volttron_instance): + """Returns two agents for testing authorization + + The first agent is the "RPC callee." + The second agent is the unauthorized "RPC caller." + """ + """ + Test if control agent periodically monitors and restarts any crashed agents + :param volttron_instance: + :return: + """ + + tmpdir = volttron_instance.volttron_home+"/tmpdir" + os.mkdir(tmpdir) + tmpdir = volttron_instance.volttron_home+"/tmpdir" + "/called" + os.mkdir(tmpdir) + os.chdir(tmpdir) + + os.mkdir("calledagent") + with open(os.path.join("calledagent", "__init__.py"), "w") as file: + pass + with open(os.path.join("calledagent", "calledagent.py"), "w") as file: + file.write(called_agent_src) + with open(os.path.join("setup.py"), "w") as file: + file.write(called_agent_setup) + p = subprocess.Popen( + [sys.executable, "setup.py", "bdist_wheel"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + # print("out {}".format(stdout)) + # print("err {}".format(stderr)) + + wheel = os.path.join(tmpdir, "dist", "calledagent-0.1-py3-none-any.whl") + assert os.path.exists(wheel) + called_uuid = volttron_instance.install_agent(agent_wheel=wheel, + vip_identity="called_agent", + start=False) + assert called_uuid + gevent.sleep(1) + + + tmpdir = volttron_instance.volttron_home+"/tmpdir" + "/caller" + os.mkdir(tmpdir) + os.chdir(tmpdir) + os.mkdir("calleragent") + with open(os.path.join("calleragent", "__init__.py"), "w") as file: + pass + with open(os.path.join("calleragent", "calleragent.py"), "w") as file: + file.write(caller_agent_src) + with open(os.path.join("setup.py"), "w") as file: + file.write(caller_agent_setup) + p = subprocess.Popen( + [sys.executable, "setup.py", "bdist_wheel"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + # print("out {}".format(stdout)) + # print("err {}".format(stderr)) + + wheel = os.path.join(tmpdir, "dist", "calleragent-0.1-py3-none-any.whl") + assert os.path.exists(wheel) + caller_uuid = volttron_instance.install_agent(agent_wheel=wheel, + vip_identity="caller_agent", + start=False) + assert caller_uuid + gevent.sleep(1) + + try: + yield caller_uuid, called_uuid + finally: + #volttron_instance.remove_agent(caller_uuid) + #volttron_instance.remove_agent(called_uuid) + # TODO if we have to wait for auth propagation anyways why do we create new agents for each test case + # we should just update capabilities, at least we will save on agent creation and tear down time + gevent.sleep(1) + + +@pytest.fixture(autouse=True) +def build_volttron_instance(volttron_instance): + if not volttron_instance.auth_enabled: + pytest.skip("AUTH tests are not applicable if auth is disabled") + + +@pytest.mark.auth +def test_unauthorized_rpc_call(volttron_instance, install_two_agents): + """Tests an agent with no capabilities calling a method that + requires one capability ("can_call_foo") + """ + (caller_agent_uuid, called_agent_uuid) = install_two_agents + + # check auth error for newly installed agents + check_auth_error(volttron_instance, caller_agent_uuid, called_agent_uuid) + + volttron_instance.restart_platform() + gevent.sleep(3) + + # check auth error for already installed agent + check_auth_error(volttron_instance, caller_agent_uuid, called_agent_uuid) + +def check_auth_error(volttron_instance, caller_agent_uuid, called_agent_uuid): + + expected_auth_err = ('volttron.platform.jsonrpc.Error(' + '-32001, "method \'restricted_method\' ' + 'requires capabilities {\'can_call_method\'}, ' + 'but capability {\'edit_config_store\': {\'identity\': \'caller_agent\'}}' + ' was provided for user caller_agent")') + volttron_instance.start_agent(called_agent_uuid) + gevent.sleep(1) + volttron_instance.start_agent(caller_agent_uuid) + + # If the agent is not authorized health status is updated + health = volttron_instance.dynamic_agent.vip.rpc.call( + "caller_agent", "health.get_status").get(timeout=2) + + assert health.get('status') == STATUS_BAD + assert health.get('context') == expected_auth_err + + + + diff --git a/volttrontesting/utils/platformwrapper.py b/volttrontesting/utils/platformwrapper.py index 5fc82bf4b0..79cd4e4267 100644 --- a/volttrontesting/utils/platformwrapper.py +++ b/volttrontesting/utils/platformwrapper.py @@ -1564,17 +1564,20 @@ def shutdown_platform(self): return running_pids = [] - if self.dynamic_agent: # because we are not creating dynamic agent in setupmode - for agnt in self.list_agents(): - pid = self.agent_pid(agnt['uuid']) - if pid is not None and int(pid) > 0: - running_pids.append(int(pid)) - if not self.skip_cleanup: - self.remove_all_agents() - # don't wait indefinetly as shutdown will not throw an error if RMQ is down/has cert errors - self.dynamic_agent.vip.rpc(CONTROL, 'shutdown').get(timeout=10) - self.dynamic_agent.core.stop() - self.dynamic_agent = None + if self.dynamic_agent: + try:# because we are not creating dynamic agent in setupmode + for agnt in self.list_agents(): + pid = self.agent_pid(agnt['uuid']) + if pid is not None and int(pid) > 0: + running_pids.append(int(pid)) + if not self.skip_cleanup: + self.remove_all_agents() + # don't wait indefinetly as shutdown will not throw an error if RMQ is down/has cert errors + self.dynamic_agent.vip.rpc(CONTROL, 'shutdown').get(timeout=10) + self.dynamic_agent.core.stop() + self.dynamic_agent = None + except BaseException as e: + self.logit(f"Exception while shutting down. {e}") if self.p_process is not None: try: From d131597dc07347152e60d3254ee0304e2641f716 Mon Sep 17 00:00:00 2001 From: Andrew Rodgers Date: Tue, 23 Apr 2024 09:24:00 -0400 Subject: [PATCH 22/44] Added a cache for agent names since platform start --- volttron/platform/aip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/volttron/platform/aip.py b/volttron/platform/aip.py index dc7859ec89..07e8219126 100644 --- a/volttron/platform/aip.py +++ b/volttron/platform/aip.py @@ -247,6 +247,7 @@ def __init__(self, env, **kwargs): if self.message_bus == 'rmq': self.rmq_mgmt = RabbitMQMgmt() self.instance_name = get_platform_instance_name() + self.agent_uuid_name_map = {} def add_agent_user_group(self): user = pwd.getpwuid(os.getuid()) @@ -682,11 +683,14 @@ def remove_agent(self, agent_uuid, remove_auth=True): self.remove_agent_user(volttron_agent_user) def agent_name(self, agent_uuid): + if cached_name := self.agent_uuid_name_map.get(agent_uuid): + return cached_name agent_path = os.path.join(self.install_dir, agent_uuid) for agent_name in os.listdir(agent_path): dist_info = os.path.join( agent_path, agent_name, agent_name + '.dist-info') if os.path.exists(dist_info): + self.agent_uuid_name_map[agent_uuid] = agent_name return agent_name raise KeyError(agent_uuid) From 5e77b59a119d4cd248bac06327d7fb3ea169fd8b Mon Sep 17 00:00:00 2001 From: Andrew Rodgers Date: Mon, 22 Apr 2024 16:04:58 -0400 Subject: [PATCH 23/44] Fixes process overload from file events --- volttron/utils/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/volttron/utils/__init__.py b/volttron/utils/__init__.py index daa1e1aaa7..e236d5185f 100644 --- a/volttron/utils/__init__.py +++ b/volttron/utils/__init__.py @@ -108,7 +108,7 @@ def __init__(self, filetowatch, callback): _log.debug("patterns is {}".format([get_home() + '/' + filetowatch])) self._callback = callback - def on_any_event(self, event): + def on_closed(self, event): _log.debug("Calling callback on event {}. Calling {}".format(event, self._callback)) try: self._callback() @@ -133,7 +133,7 @@ def __init__(self, filetowatch, callback): def watchfile(self): return self._filetowatch - def on_any_event(self, event): + def on_closed(self, event): _log.debug("Calling callback on event {}. Calling {}".format(event, self._callback)) try: self._callback(self._filetowatch) From 90d6f25ce85f71f35a1625a258cb4ef52daf2f96 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 10 May 2024 10:14:10 -0700 Subject: [PATCH 24/44] Releases/9.0.1 (#3171) (#3181) * Update readthedocs requirements.txt * Update conf.py * Update requirements_demo.txt Add missing pandas requirement for demo * work around for issue #3154 * Fix for security issue #3168 (#3169) * Fix for security issue #3168 * handling clean up errors in test * testing group commands in different test module * moved group and role test to different module * moved group and role test to different module * Added a cache for agent names since platform start * Fixes process overload from file events * fixed issue with variable definition. * Remove PersistentDict from web-user.json file. * Update admin_endpoints.py Handle behavior of removing PersistentDict * Update version to 9.0.1 --------- Co-authored-by: Chandrika Sivaramakrishnan Co-authored-by: Chandrika Co-authored-by: Andrew Rodgers --- docs/source/conf.py | 4 ++-- volttron/platform/__init__.py | 2 +- volttron/platform/vip/agent/subsystems/auth.py | 6 ++---- volttron/platform/web/admin_endpoints.py | 15 +++++++++++---- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index dc75f60eb2..10c31ff659 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -90,9 +90,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '9.0' +version = '9.0.1' # The full version, including alpha/beta/rc tags -release = '9.0' +release = '9.0.1' # -- General configuration --------------------------------------------------- diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index d6ce14cdd2..5a7525b81e 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -35,7 +35,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '9.0rc0' +__version__ = '9.0.1' _log = logging.getLogger(__name__) diff --git a/volttron/platform/vip/agent/subsystems/auth.py b/volttron/platform/vip/agent/subsystems/auth.py index 8ba4e2dcba..9337d0e513 100644 --- a/volttron/platform/vip/agent/subsystems/auth.py +++ b/volttron/platform/vip/agent/subsystems/auth.py @@ -265,6 +265,7 @@ def update_rpc_method_capabilities(self): """ rpc_method_authorizations = {} rpc_methods = self.get_rpc_exports() + updated_rpc_authorizations = None for method in rpc_methods: if len(method.split(".")) > 1: pass @@ -295,9 +296,7 @@ def update_rpc_method_capabilities(self): _log.info( f"Skipping updating rpc auth capabilities for agent " f"{self._core().identity} connecting to remote address: {self._core().address} ") - updated_rpc_authorizations = None except gevent.timeout.Timeout: - updated_rpc_authorizations = None _log.warning(f"update_id_rpc_authorization rpc call timed out for {self._core().identity} {rpc_method_authorizations}") except MethodNotFound: _log.warning("update_id_rpc_authorization method is missing from " @@ -306,7 +305,6 @@ def update_rpc_method_capabilities(self): "dynamic RPC authorizations.") return except Exception as e: - updated_rpc_authorizations = None _log.exception(f"Exception when calling rpc method update_id_rpc_authorizations for identity: " f"{self._core().identity} Exception:{e}") if updated_rpc_authorizations is None: @@ -318,7 +316,7 @@ def update_rpc_method_capabilities(self): f"the identity of the agent" ) return - if rpc_method_authorizations != updated_rpc_authorizations: + if rpc_method_authorizations != updated_rpc_authorizations and updated_rpc_authorizations is not None: for method in updated_rpc_authorizations: self.set_rpc_authorizations( method, updated_rpc_authorizations[method] diff --git a/volttron/platform/web/admin_endpoints.py b/volttron/platform/web/admin_endpoints.py index 9de8d05a5c..d307b2f353 100644 --- a/volttron/platform/web/admin_endpoints.py +++ b/volttron/platform/web/admin_endpoints.py @@ -46,7 +46,6 @@ from volttron.platform import get_home from volttron.platform import jsonapi from volttron.utils import VolttronHomeFileReloader -from volttron.utils.persistance import PersistentDict _log = logging.getLogger(__name__) @@ -84,7 +83,7 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None) else: self._ssl_public_key = None - self._userdict = None + self._userdict = {} self.reload_userdict() self._observer = Observer() @@ -96,7 +95,14 @@ def __init__(self, rmq_mgmt=None, ssl_public_key: bytes = None, rpc_caller=None) def reload_userdict(self): webuserpath = os.path.join(get_home(), 'web-users.json') - self._userdict = PersistentDict(webuserpath, format="json") + if os.path.exists(webuserpath): + with open(webuserpath) as fp: + try: + self._userdict = jsonapi.loads(fp.read()) + except json.decoder.JSONDecodeError: + self._userdict = {} + # Keep same behavior as with PersistentDict + raise ValueError("File not in a supported format") def get_routes(self): """ @@ -339,4 +345,5 @@ def add_user(self, username, unencrypted_pw, groups=None, overwrite=False): groups=groups ) - self._userdict.sync() + with open(os.path.join(get_home(), 'web-users.json'), 'w') as fp: + fp.write(jsonapi.dumps(self._userdict, indent=2)) From 02533d6910afc425aa6012d30400ef68294414f0 Mon Sep 17 00:00:00 2001 From: "C. Allwardt" <3979063+craig8@users.noreply.github.com> Date: Wed, 1 May 2024 07:48:52 -0700 Subject: [PATCH 25/44] Modbustk error handler doesn't include .message --- .../platform_driver/interfaces/modbus_tk/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py index beea69b023..6581e3cff2 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/modbus_tk/client.py @@ -669,9 +669,10 @@ def read_request(self, request): ) self._data.update(request.parse_values(results)) except (AttributeError, ModbusError) as err: - if "Exception code" in err.message: - raise Exception("{0}: {1}".format(err.message, - helpers.TABLE_EXCEPTION_CODE.get(err.message[-1], "UNDEFINED"))) + if err is ModbusError: + code = err.get_exception_code() + raise Exception(f'{err.args[0]}, {helpers.TABLE_EXCEPTION_CODE.get(code, "UNDEFINED")}') + logger.warning("modbus read_all() failure on request: %s\tError: %s", request, err) def read_all(self): From 10fcabf80025d8e571352e0a2399182c1451ec95 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 10 Apr 2024 14:21:58 -0700 Subject: [PATCH 26/44] Fixed saving of state. --- services/core/ActuatorAgent/actuator/scheduler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/core/ActuatorAgent/actuator/scheduler.py b/services/core/ActuatorAgent/actuator/scheduler.py index c1190e5491..8bb8b4bf2a 100644 --- a/services/core/ActuatorAgent/actuator/scheduler.py +++ b/services/core/ActuatorAgent/actuator/scheduler.py @@ -21,14 +21,14 @@ # # ===----------------------------------------------------------------------=== # }}} - - import bisect import logging -from pickle import dumps, loads + +from base64 import b64encode from collections import defaultdict, namedtuple from copy import deepcopy from datetime import timedelta +from pickle import dumps, loads from volttron.platform.agent import utils @@ -340,7 +340,7 @@ def save_state(self, now): try: self._cleanup(now) - self.save_state_callback(dumps(self.tasks)) + self.save_state_callback(b64encode(dumps(self.tasks)).decode("utf-8")) except Exception: _log.error('Failed to save scheduler state!') From 8605a4ae8a560faf6921bd3b00837f75d917641f Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 10 Apr 2024 14:24:52 -0700 Subject: [PATCH 27/44] Fixed missing preempted task information when scheduling. --- services/core/ActuatorAgent/actuator/scheduler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/core/ActuatorAgent/actuator/scheduler.py b/services/core/ActuatorAgent/actuator/scheduler.py index 8bb8b4bf2a..b2fcc4ae52 100644 --- a/services/core/ActuatorAgent/actuator/scheduler.py +++ b/services/core/ActuatorAgent/actuator/scheduler.py @@ -411,7 +411,10 @@ def request_slots(self, agent_id, id_, requests, priority, now=None): self.save_state(now) - return RequestResult(True, preempted_tasks, '') + if preempted_tasks: + return RequestResult(True, list(preempted_tasks), 'TASK_WERE_PREEMPTED') + else: + return RequestResult(True, {}, '') def cancel_task(self, agent_id, task_id, now): if task_id not in self.tasks: From c24f01152b0bbc4b23cd608845bc0818ec2fc752 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Tue, 16 Apr 2024 18:08:12 -0700 Subject: [PATCH 28/44] Allow preempted device information to reach user. --- services/core/ActuatorAgent/actuator/agent.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/core/ActuatorAgent/actuator/agent.py b/services/core/ActuatorAgent/actuator/agent.py index 369489ed75..82f666c9ca 100644 --- a/services/core/ActuatorAgent/actuator/agent.py +++ b/services/core/ActuatorAgent/actuator/agent.py @@ -1382,11 +1382,8 @@ def _request_new_schedule(self, sender, task_id, priority, requests, publish_res 'data': {'agentID': sender, 'taskID': task_id}}) - # If we are successful we do something else with the real result data - data = result.data if not result.success else {} - results = {'result': success, - 'data': data, + 'data': result.data, 'info': result.info_string} if publish_result: From 99f8eda86a99efd671b6115a65ad9d68d66ac928 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Tue, 16 Apr 2024 18:08:56 -0700 Subject: [PATCH 29/44] Fixes to driver exception handling related to bacnet pings and heartbeats. --- services/core/PlatformDriverAgent/platform_driver/agent.py | 5 ++++- .../platform_driver/interfaces/bacnet.py | 7 +++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/agent.py b/services/core/PlatformDriverAgent/platform_driver/agent.py index 967e2c81f1..5c3d097433 100644 --- a/services/core/PlatformDriverAgent/platform_driver/agent.py +++ b/services/core/PlatformDriverAgent/platform_driver/agent.py @@ -487,7 +487,10 @@ def heart_beat(self): """ _log.debug("sending heartbeat") for device in self.instances.values(): - device.heart_beat() + try: + device.heart_beat() + except (Exception, gevent.Timeout) as e: + _log.warning(f'Failed to set heart_beat point on device: {device.device_name} -- {e}.') @RPC.export def revert_point(self, path, point_name, **kwargs): diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index b2fc973e10..d7f7c606e1 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -22,7 +22,7 @@ # ===----------------------------------------------------------------------=== # }}} - +import gevent import logging from datetime import datetime, timedelta @@ -98,9 +98,8 @@ def ping_target(self): pinged = True except errors.Unreachable: _log.warning("Unable to reach BACnet proxy.") - - except errors.VIPError: - _log.warning("Error trying to ping device.") + except (Exception, gevent.Timeout) as e: + _log.warning(f"Error trying to ping device: {e}") self.scheduled_ping = None From 8fbfc051e6ffbc70ac4ed7d533e1094532df6523 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 10 May 2024 12:59:00 -0700 Subject: [PATCH 30/44] Added device information to error messages. --- .../PlatformDriverAgent/platform_driver/interfaces/bacnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index d7f7c606e1..d591bc8b45 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -99,7 +99,7 @@ def ping_target(self): except errors.Unreachable: _log.warning("Unable to reach BACnet proxy.") except (Exception, gevent.Timeout) as e: - _log.warning(f"Error trying to ping device: {e}") + _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}: {e}") self.scheduled_ping = None From 84ca12b2ff98bd40dfa9521f888138d6098bf2c6 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 28 Jun 2024 13:24:47 -0700 Subject: [PATCH 31/44] Added on_property method to get_point and set_point. Replaced and extended broken functionality to get_priortity_array on get_point. --- .../PlatformDriverAgent/platform_driver/interfaces/bacnet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index d591bc8b45..6ae4a4c8fa 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -97,9 +97,10 @@ def ping_target(self): self.vip.rpc.call(self.proxy_address, 'ping_device', self.target_address, self.device_id).get(timeout=self.timeout) pinged = True except errors.Unreachable: - _log.warning("Unable to reach BACnet proxy.") + _log.warning(f"Unable to reach BACnet proxy at: {self.proxy_address}.") except (Exception, gevent.Timeout) as e: - _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}: {e}") + _log.warning(f"Error trying to ping device with device_id '{self.device_id}' at {self.target_address}" + f"through proxy {self.proxy_address}: {e}") self.scheduled_ping = None From 94c908017c5ca729294c6e4abf614fd15382dcc6 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 28 Jun 2024 11:21:15 -0700 Subject: [PATCH 32/44] Added on_property method to get_point and set_point. Replaced and extended broken functionality to get_priortity_array on get_point. --- .../platform_driver/interfaces/bacnet.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py index 6ae4a4c8fa..370a3d2d9d 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/bacnet.py @@ -108,16 +108,25 @@ def ping_target(self): if not pinged: self.schedule_ping() - def get_point(self, point_name, get_priority_array=False): + def get_point(self, point_name, on_property=None): register = self.get_register_by_name(point_name) - property_name = "priorityArray" if get_priority_array else register.property - register_index = None if get_priority_array else register.index - result = self.vip.rpc.call(self.proxy_address, 'read_property', - self.target_address, register.object_type, - register.instance_number, property_name, register_index).get(timeout=self.timeout) + if on_property is None: + result = self.vip.rpc.call(self.proxy_address, 'read_property', + self.target_address, register.object_type, + register.instance_number, register.property, register.index).get(timeout=self.timeout) + else: + point_map = {} + point_map[register.point_name] = [register.object_type, + register.instance_number, + on_property, + register.index] + result = self.vip.rpc.call(self.proxy_address, 'read_properties', + self.target_address, point_map, + self.max_per_request, True).get(timeout=self.timeout) + result = list(result.values())[0] return result - def set_point(self, point_name, value, priority=None): + def set_point(self, point_name, value, priority=None, on_property=None): # TODO: support writing from an array. register = self.get_register_by_name(point_name) if register.read_only: @@ -130,7 +139,7 @@ def set_point(self, point_name, value, priority=None): args = [self.target_address, value, register.object_type, register.instance_number, - register.property, + on_property if on_property is not None else register.property, priority if priority is not None else register.priority, register.index] result = self.vip.rpc.call(self.proxy_address, 'write_property', *args).get(timeout=self.timeout) From adf79e10b10558e9fb202eb67bd60b7cb4596b9c Mon Sep 17 00:00:00 2001 From: jiangyilin123 Date: Mon, 26 Feb 2024 13:06:37 -0800 Subject: [PATCH 33/44] modified chargepoint interface for platform driver --- .../interfaces/chargepoint/__init__.py | 31 ++++++++++--------- .../interfaces/chargepoint/async_service.py | 10 +++--- .../chargepoint/credential_check.py | 6 ++-- .../interfaces/chargepoint/service.py | 25 ++++++++------- update_scripts/update.driver | 2 ++ 5 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 update_scripts/update.driver diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index 0f8d26e5bb..0e915905c8 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -27,12 +27,11 @@ import logging import abc import sys - from . import service as cps -from . import async_service as async - +from . import async_service as async_service from .. import BaseInterface, BaseRegister, BasicRevert, DriverInterfaceError -from suds.sudsobject import asdict +#from suds.sudsobject import asdict +from zeep.helpers import serialize_object _log = logging.getLogger(__name__) @@ -54,7 +53,7 @@ point_name_mapping = {"Status.TimeStamp": "TimeStamp"} service = {} -gevent.spawn(async.web_service) +gevent.spawn(async_service.web_service) def recursive_asdict(d): @@ -64,7 +63,7 @@ def recursive_asdict(d): http://stackoverflow.com/questions/2412486/serializing-a-suds-object-in-python """ out = {} - for k, v in asdict(d).items(): + for k, v in serialize_object(d, dict).items(): if hasattr(v, '__keylist__'): out[k] = recursive_asdict(v) elif isinstance(v, list): @@ -196,7 +195,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStations - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() return self.get_register(result.value, method) @@ -237,7 +236,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getLoad - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() return self.get_register(result.value, method) @@ -263,7 +262,7 @@ def value(self, x): kwargs = {'stationID': self.station_id} if self.attribute_name == 'shedState' and not value: method = service[self.username].clearShedState - result = async.CPRequest.request(method, 0, stationID=self.station_id) + result = async_service.CPRequest.request(method, 0, stationID=self.station_id) elif self.attribute_name == 'shedState': _log.error('shedState may only be written with value 0. If you want to shedLoad, write to ' 'allowedLoad or percentShed') @@ -273,7 +272,7 @@ def value(self, x): kwargs[self.attribute_name] = value if self.port: kwargs['portNumber'] = self.port - result = async.CPRequest.request(method, 0, **kwargs) + result = async_service.CPRequest.request(method, 0, **kwargs) result.wait() if result.value.responseCode != "100": @@ -322,7 +321,7 @@ def value(self): if self.port: kwargs['portNumber'] = self.port - result = async.CPRequest.request(method, self.timeout, **kwargs) + result = async_service.CPRequest.request(method, self.timeout, **kwargs) result.wait() return self.get_register(result.value, method, False) @@ -348,7 +347,7 @@ def value(self, x): if self.attribute_name == 'clearAlarms' and value: kwargs = {'stationID': self.station_id} method = service[self.username].clearAlarms - result = async.CPRequest.request(method, 0, **kwargs) + result = async_service.CPRequest.request(method, 0, **kwargs) result.wait() if result.value.responseCode not in ['100', '153']: @@ -383,7 +382,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getChargingSessionData - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() # Of Note, due to API limitations, port number is ignored for these calls @@ -418,7 +417,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStationStatus - result = async.CPRequest.request(method, self.timeout, self.station_id) + result = async_service.CPRequest.request(method, self.timeout, self.station_id) result.wait() return self.get_register(result.value, method) @@ -455,7 +454,7 @@ def __init__(self, read_only, point_name, attribute_name, units, data_type, stat def value(self): global service method = service[self.username].getStationRights - result = async.CPRequest.request(method, self.timeout, stationID=self.station_id) + result = async_service.CPRequest.request(method, self.timeout, stationID=self.station_id) result.wait() # Note: this does not go through get_register, as it is of a unique type, 'dictionary.' @@ -523,6 +522,8 @@ def parse_config(self, config_dict, registry_config_str): return for regDef in registry_config_str: + print(regDef) + _log.debug(f'RegDef is {regDef}') # Skip lines that have no address yet. if not regDef['Attribute Name']: continue diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py index ca98c8a753..4a273cbc80 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py @@ -47,15 +47,16 @@ import gevent.event import gevent.queue import logging -import suds +#import suds +import zeep from gevent import monkey from .service import CPAPIException from datetime import datetime, timedelta monkey.patch_all() _log = logging.getLogger(__name__) -SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" - +#SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" +SERVICE_WSDL_URL = "http://localhost:8080/cp_api_5.1.wsdl" # Queue for Web API requests and responses. It is managed by the long running # web_service() greenlet. web_service_queue = gevent.queue.Queue() @@ -253,7 +254,8 @@ def web_service(): web_cache[item_key] = cache_item if not client_set: - client_set.add(suds.client.Client(SERVICE_WSDL_URL)) + #client_set.add(suds.client.Client(SERVICE_WSDL_URL)) + client_set.add(zeep.Client(SERVICE_WSDL_URL)) client = client_set.pop() gevent.spawn(web_call, item, client) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py index 9687d08d63..9f8a8f1586 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py @@ -1,5 +1,6 @@ from . import service as cps -import suds +#import suds +import zeep import io station_csv = { @@ -176,5 +177,6 @@ else: print("Some other error happened") - except suds.WebFault as a: + #except suds.WebFault as a: + except zeep.exception.Fault as e: print("Sorry, your API credentials are invalid. Please contact Chargepoint for assistance.") diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index da845c676e..0ef7d381a1 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -22,8 +22,10 @@ # ===----------------------------------------------------------------------=== # }}} -import suds.client -import suds.wsse +#import suds.client +#import suds.wsse +import zeep +from zeep.wsse.username import UsernameToken import logging logger = logging.getLogger('chargepoint') @@ -769,18 +771,18 @@ def __init__(self, username=None, password=None): """ self._username = username self._password = password - self._suds_client = None + self._zeep_client = None @property def _client(self): """Initialize the SUDS client if necessary.""" - if self._suds_client is None: - self._suds_client = suds.client.Client(SERVICE_WSDL_URL) + if self._zeep_client is None: + self._zeep_client = zeep.Client(SERVICE_WSDL_URL) # Add SOAP Security tokens self.set_security_token() - return self._suds_client + return self._zeep_client @property def _soap_service(self): @@ -788,13 +790,14 @@ def _soap_service(self): def set_security_token(self): # Add SOAP Security tokens - security = suds.wsse.Security() - token = suds.wsse.UsernameToken(self._username, self._password) - security.tokens.append(token) - self._suds_client.set_options(wsse=security) + #security = suds.wsse.Security() + #token = suds.wsse.UsernameToken(self._username, self._password) + #security.tokens.append(token) + #self._zeep_client.set_options(wsse=security) + self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password)) def set_client(self, client): - self._suds_client = client + self._zeep_client = client self.set_security_token() def clearAlarms(self, **kwargs): diff --git a/update_scripts/update.driver b/update_scripts/update.driver new file mode 100644 index 0000000000..9f9c3346f8 --- /dev/null +++ b/update_scripts/update.driver @@ -0,0 +1,2 @@ +#!/bin/bash +python ~/volttron/scripts/install-agent.py -s /home/vboxuser/chargepoint/volttron_chargepoint/services/core/PlatformDriverAgent -c /home/vboxuser/chargepoint/volttron_chargepoint/config/driver.config -i platform.driver -t driver --force --start --priority 60 From 6b77994787ae7b360ed7e2d3418ddf46cca32317 Mon Sep 17 00:00:00 2001 From: jiangyilin123 Date: Tue, 5 Mar 2024 16:27:11 -0800 Subject: [PATCH 34/44] replace suds to zeep for chargepoint interface in platform.driver --- .../interfaces/chargepoint/README.rst | 2 +- .../interfaces/chargepoint/__init__.py | 4 +- .../interfaces/chargepoint/async_service.py | 7 +-- .../chargepoint/credential_check.py | 2 - .../interfaces/chargepoint/requirements.txt | 2 +- .../interfaces/chargepoint/service.py | 52 +++++++++---------- update_scripts/update.driver | 2 - 7 files changed, 31 insertions(+), 40 deletions(-) delete mode 100644 update_scripts/update.driver diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst index 218b1dba8d..4da6ce77a3 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/README.rst @@ -12,7 +12,7 @@ activated environment: :: - pip install suds-jurko + pip install zeep Alternatively requirements can be installed from requirements.txt using: diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index 0e915905c8..de3c7786ac 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -522,8 +522,6 @@ def parse_config(self, config_dict, registry_config_str): return for regDef in registry_config_str: - print(regDef) - _log.debug(f'RegDef is {regDef}') # Skip lines that have no address yet. if not regDef['Attribute Name']: continue @@ -559,7 +557,7 @@ def parse_config(self, config_dict, registry_config_str): description=description, port_number=port_num, username=config_dict['username'], - timeout=config_dict['cacheExpiration'] + timeout=config_dict.get('cacheExpiration',0) ) self.insert_register(register) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py index 4a273cbc80..e142a30d86 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/async_service.py @@ -47,16 +47,14 @@ import gevent.event import gevent.queue import logging -#import suds import zeep from gevent import monkey from .service import CPAPIException from datetime import datetime, timedelta -monkey.patch_all() _log = logging.getLogger(__name__) -#SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" -SERVICE_WSDL_URL = "http://localhost:8080/cp_api_5.1.wsdl" + +SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.1.wsdl" # Queue for Web API requests and responses. It is managed by the long running # web_service() greenlet. web_service_queue = gevent.queue.Queue() @@ -254,7 +252,6 @@ def web_service(): web_cache[item_key] = cache_item if not client_set: - #client_set.add(suds.client.Client(SERVICE_WSDL_URL)) client_set.add(zeep.Client(SERVICE_WSDL_URL)) client = client_set.pop() gevent.spawn(web_call, item, client) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py index 9f8a8f1586..e06c51b25d 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/credential_check.py @@ -1,5 +1,4 @@ from . import service as cps -#import suds import zeep import io @@ -177,6 +176,5 @@ else: print("Some other error happened") - #except suds.WebFault as a: except zeep.exception.Fault as e: print("Sorry, your API credentials are invalid. Please contact Chargepoint for assistance.") diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt index 274ad9e069..f61b24a644 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/requirements.txt @@ -1 +1 @@ -suds-jurko==0.6 \ No newline at end of file +zeep==4.2.1 \ No newline at end of file diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index 0ef7d381a1..559f78e5ca 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -22,16 +22,14 @@ # ===----------------------------------------------------------------------=== # }}} -#import suds.client -#import suds.wsse import zeep from zeep.wsse.username import UsernameToken +from zeep import Settings import logging logger = logging.getLogger('chargepoint') -SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.0.wsdl" - +SERVICE_WSDL_URL = "https://webservices.chargepoint.com/cp_api_5.1.wsdl" CPAPI_SUCCESS = '100' XMPP_EVENTS = [ @@ -158,13 +156,13 @@ class CPStation: """Wrapper around the getStations() return by Chargepoint API. Data surrounding a Chargepoint Station can generally be categorized as static or dynamic. Chargepoint API has two - basic calls, getLoad and getStation, that each return station data. getLoad returns the stationLoadData SUDS - object, and getStation returns the stationDataExtended SUDS object. These are each kept as separate meta-data + basic calls, getLoad and getStation, that each return station data. getLoad returns the stationLoadData object, + and getStation returns the stationDataExtended object. These are each kept as separate meta-data parameters. :param cps: Chargepoint Service object. - :param sld: stationLoadData SUDS object. - :param sde: stationDataExtended SUDS object. + :param sld: stationLoadData object. + :param sde: stationDataExtended object. (stationDataExtended){ stationID = "1:00001" @@ -760,8 +758,8 @@ class CPService: """ Python wrapper around the Chargepoint WebServices API. - Current Version: 5.0 - Docs: ChargePoint_Web_Services_API_Guide_Ver4.1_Rev5.pdf + Current Version: 5.1 + Docs: ChargePoint_Web_Services_API_Guide_Ver5.1_Rev1.13.pdf """ def __init__(self, username=None, password=None): @@ -775,7 +773,7 @@ def __init__(self, username=None, password=None): @property def _client(self): - """Initialize the SUDS client if necessary.""" + """Initialize the ZEEP client if necessary.""" if self._zeep_client is None: self._zeep_client = zeep.Client(SERVICE_WSDL_URL) @@ -790,11 +788,10 @@ def _soap_service(self): def set_security_token(self): # Add SOAP Security tokens - #security = suds.wsse.Security() - #token = suds.wsse.UsernameToken(self._username, self._password) - #security.tokens.append(token) - #self._zeep_client.set_options(wsse=security) - self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password)) + #TODO:might need to put this in config + #NOTE: wihtout this setting, zeep will not get result + settins = Settings(strict=False, xml_huge_tree=True, xsd_ignore_sequence_order=True) + self._zeep_client = zeep.Client(SERVICE_WSDL_URL, wsse=UsernameToken(self._username, self._password),settings=settins) def set_client(self, client): self._zeep_client = client @@ -823,7 +820,7 @@ def clearAlarms(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('clearAlarmsSearchQuery') + searchQuery = self._client.get_type('ns0:clearAlarmsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.clearAlarms(searchQuery) @@ -839,7 +836,7 @@ def clearShedState(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('shedQueryInputData') + searchQuery = self._client.get_type('ns0:shedQueryInputData')() if 'stationID' in kwargs.keys(): setattr(searchQuery, 'shedStation', {'stationID': kwargs['stationID']}) elif 'sgID' in kwargs.keys(): @@ -893,7 +890,7 @@ def getAlarms(self, **kwargs): } """ - searchQuery = self._client.factory.create('getAlarmsSearchQuery') + searchQuery = self._client.get_type('ns0:getAlarmsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getAlarms(searchQuery) @@ -968,7 +965,7 @@ def getChargingSessionData(self, **kwargs): } """ - searchQuery = self._client.factory.create('sessionSearchdata') + searchQuery = self._client.get_type('ns0:sessionSearchdata')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getChargingSessionData(searchQuery) @@ -1021,7 +1018,8 @@ def getLoad(self, **kwargs): """ # @ToDo: Figure out what type of request searchQuery should be here. - searchQuery = self._client.factory.create('stationSearchRequestExtended') + # @Note: Looks like it should be {sgID: xsd:int, stationID: xsd:string, sessionID: xsd:long} + searchQuery = {} for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getLoad(searchQuery) @@ -1062,7 +1060,7 @@ def getOrgsAndStationGroups(self, **kwargs): } """ - searchQuery = self._client.factory.create('getOrgsAndStationGroupsSearchQuery') + searchQuery = self._client.get_type('ns0:getOrgsAndStationGroupsSearchQuery')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getOrgsAndStationGroups(searchQuery) @@ -1217,7 +1215,7 @@ def getStationRights(self, **kwargs): } """ - searchQuery = self._client.factory.create('stationRightsSearchRequest') + searchQuery = self._client.get_type('ns0:stationRightsSearchRequest')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getStationRights(searchQuery) @@ -1362,7 +1360,8 @@ def getStations(self, **kwargs): moreFlag = 0 } """ - searchQuery = self._client.factory.create('stationSearchRequestExtended') + + searchQuery = self._client.get_type('ns0:stationSearchRequestExtended')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getStations(searchQuery) @@ -1449,7 +1448,7 @@ def getUsers(self, **kwargs): } """ - searchQuery = self._client.factory.create('getUsersSearchRequest') + searchQuery = self._client.get_type('ns0:getUsersSearchRequest')() for k, v in kwargs.items(): setattr(searchQuery, k, v) response = self._soap_service.getUsers(searchQuery) @@ -1487,7 +1486,8 @@ def shedLoad(self, **kwargs): :returns SOAP reply object. If successful, there will be a responseCode of '100'. """ - searchQuery = self._client.factory.create('shedLoadQueryInputData') + + searchQuery = self._client.get_type('ns0:shedLoadQueryInputData')() port = kwargs.pop('portNumber', None) query_params = {'stationID': kwargs['stationID']} if port: diff --git a/update_scripts/update.driver b/update_scripts/update.driver deleted file mode 100644 index 9f9c3346f8..0000000000 --- a/update_scripts/update.driver +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -python ~/volttron/scripts/install-agent.py -s /home/vboxuser/chargepoint/volttron_chargepoint/services/core/PlatformDriverAgent -c /home/vboxuser/chargepoint/volttron_chargepoint/config/driver.config -i platform.driver -t driver --force --start --priority 60 From 87650e1fbd11c5ce7ec87a872e0262e237ddfc24 Mon Sep 17 00:00:00 2001 From: jiangyilin123 Date: Fri, 15 Mar 2024 16:07:59 -0700 Subject: [PATCH 35/44] correct ChargingSessionData published format --- .../interfaces/chargepoint/__init__.py | 18 +++++++++++++++--- .../interfaces/chargepoint/service.py | 16 ++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py index de3c7786ac..a106540650 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/__init__.py @@ -138,6 +138,16 @@ def read_only_check(self): raise IOError("Trying to write to a point configured read only: {0}".format(self.attribute_name)) return True + def get_last_non_none_value(self,lst): + """ + Depends on port number, the result could be a list with None value + get last non-None value as result + """ + for item in reversed(lst): + if item is not None: + return item + return None + def get_register(self, result, method, port_flag=True): """Gets correct register from API response. @@ -150,9 +160,10 @@ def get_register(self, result, method, port_flag=True): :return: Correct register value cast to appropriate python type. Returns None if there is an error. """ try: - value = getattr(result, self.attribute_name)(self.port)[0] \ + _log.debug(f'In get_register, to get {self.attribute_name}, the port_flag is {port_flag}') + value = self.get_last_non_none_value(getattr(result, self.attribute_name)(self.port)) \ if port_flag \ - else getattr(result, self.attribute_name)(None)[0] + else self.get_last_non_none_value(getattr(result, self.attribute_name)(None)) return self.sanitize_output(self.data_type, value) except cps.CPAPIException as exception: if exception._responseCode not in ['153']: @@ -386,7 +397,8 @@ def value(self): result.wait() # Of Note, due to API limitations, port number is ignored for these calls - return self.get_register(result.value, method, False) + # NOTE: Change this port number for Chargingsession data. + return self.get_register(result.value, method) @value.setter def value(self, x): diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py index 559f78e5ca..f9b404a723 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/chargepoint/service.py @@ -416,8 +416,17 @@ def get_port_value(port_number, data, attribute): if flag: logger.warning("Station does not have a definition for port {0}".format(port_number)) else: - logger.warning("Response does not have Ports defined") - return None + if (attribute in ['sessionID', 'startTime', 'endTime', 'Energy', 'rfidSerialNumber', 'driverAccountNumber', + 'driverName']) and int(data['portNumber']) == port_number: + try: + data_attribute = data[attribute] + return data_attribute + except: + logger.warning(f'Response does not have {attribute} field') + return None + else: + logger.warning("Response does not have Ports defined") + return None @staticmethod def check_output(attribute, parent_dict): @@ -441,6 +450,7 @@ def get_attr_from_response(name_string, response, portNum=None): else CPAPIResponse.is_not_found(name_string)) else: list.append(CPAPIResponse.get_port_value(portNum, item, name_string)) + logger.debug(f'{name_string} list for {portNum} is {list}') return list @@ -632,12 +642,14 @@ def Type(self, port=None): def startTime(self, port=None): if port: + logger.debug(f'startTime port is {port}') return CPAPIResponse.get_attr_from_response('startTime', self.stations, port) else: return [self.pricing_helper('startTime', station) for station in self.stations] def endTime(self, port=None): if port: + logger.debug(f'endTime port is {port}') return CPAPIResponse.get_attr_from_response('endTime', self.stations, port) else: return [self.pricing_helper('endTime', station) for station in self.stations] From e0cbf326b58e2222246f43d8584e8fb4d5bc4a37 Mon Sep 17 00:00:00 2001 From: riley206 Date: Tue, 20 Aug 2024 13:09:56 -0700 Subject: [PATCH 36/44] added more logging --- services/ops/EmailerAgent/emailer/agent.py | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/services/ops/EmailerAgent/emailer/agent.py b/services/ops/EmailerAgent/emailer/agent.py index 00cdef0b14..cf0dd5de77 100644 --- a/services/ops/EmailerAgent/emailer/agent.py +++ b/services/ops/EmailerAgent/emailer/agent.py @@ -25,6 +25,7 @@ # Import the email modules we'll need from email.mime.text import MIMEText +from smtplib import SMTPException import logging import socket @@ -251,17 +252,27 @@ def _send_email(self, from_address, to_addresses, mime_message): self.vip.health.set_status(STATUS_GOOD, "Successfully sent email.") send_successful = True + except SMTPException as smtp_err: + _log.error(f"SMTP error occurred: {smtp_err}") + _log.error(f"Unable to send email message: {mime_message.as_string()}") + self.vip.health.set_status( + STATUS_BAD, + "SMTP configuration or authentication issue. Please check your SMTP settings and credentials.") + + except OSError as os_err: + _log.error(f"Network-related error occurred: {os_err}") + _log.error(f"Unable to send email message: {mime_message.as_string()}") + self.vip.health.set_status( + STATUS_BAD, "Network issue. Please check your internet connection and SMTP server accessibility.") + except Exception as e: - _log.error( - 'Unable to send email message: %s' % mime_message.as_string()) - _log.error(e.args) - self.vip.health.set_status(STATUS_BAD, - "Unable to send email to recipients") + _log.error(f"An unexpected error occurred: {e}") + _log.error(f"Unable to send email message: {mime_message.as_string()}") + self.vip.health.set_status(STATUS_BAD, f"Unable to send email to recipients: {e}") finally: if sent_email_record is not None: sent_email_record['successful'] = send_successful - self.vip.pubsub.publish("pubsub", "record/sent_email", - message=sent_email_record) + self.vip.pubsub.publish("pubsub", "record/sent_email", message=sent_email_record) def send_email(self, from_address, to_addresses, subject, message): """ From 9ce2b2dd2cde1d435d7e614d8ded9a90a9f07ace Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Mon, 11 Nov 2024 15:27:22 -0800 Subject: [PATCH 37/44] Pinned setuptools in requirements.py. Newer versions that 70 are incompatible with wheel 0.30.0. --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index 249d9b56ce..36a0e7c682 100644 --- a/requirements.py +++ b/requirements.py @@ -42,7 +42,7 @@ 'python-dateutil==2.8.2', 'pytz==2022.1', 'PyYAML==6.0', - 'setuptools>=40.0.0', + 'setuptools>=40.0.0,<=70.0.0', # tzlocal 3.0 breaks without the backports.tzinfo package on python < 3.9 https://pypi.org/project/tzlocal/3.0/ 'tzlocal==2.1', #'pyOpenSSL==19.0.0', From bb1ef84597393074a4a788aabdd08ea77d9dc6a1 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 8 Nov 2024 19:27:09 -0800 Subject: [PATCH 38/44] Update gevent to 23.9.1 https://github.com/gevent/gevent/issues/1989 --- requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.py b/requirements.py index 36a0e7c682..2ded1c44bb 100644 --- a/requirements.py +++ b/requirements.py @@ -33,7 +33,7 @@ option_requirements = [('pip==24.0', []), ('wheel==0.30', []), ('pyzmq==22.2.1', ['--zmq=bundled'])] -install_requires = ['gevent==21.12.0', +install_requires = ['gevent==23.9.1', 'grequests==0.6.0', 'requests==2.23.0', 'idna<3,>=2.5', From 1eb39735aa3e1c8266c9dd7f6270739e002b724e Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Mon, 2 Dec 2024 12:17:34 -0800 Subject: [PATCH 39/44] Fixed missing Interface attribute required by BACnet COV. --- .../platform_driver/interfaces/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py b/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py index 128bc617d1..e2713a65f8 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py +++ b/services/core/PlatformDriverAgent/platform_driver/interfaces/__init__.py @@ -237,12 +237,12 @@ class BaseInterface(object, metaclass=abc.ABCMeta): """ - def __init__(self, vip=None, core=None, **kwargs): + def __init__(self, vip=None, core=None, device_path=None, **kwargs): # Object does not take any arguments to the init. super(BaseInterface, self).__init__() self.vip = vip self.core = core - + self.device_path = device_path self.point_map = {} self.build_register_map() From a87f3a3dfff511c6e52a86c656f50c24fb45b3f0 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Fri, 24 Jan 2025 13:26:04 -0800 Subject: [PATCH 40/44] Pinned watchdog to less than version 5, since version 5.0.0 explictly removed support for python 3.8. --- requirements.py | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.py b/requirements.py index 2ded1c44bb..727f98bd31 100644 --- a/requirements.py +++ b/requirements.py @@ -47,6 +47,7 @@ 'tzlocal==2.1', #'pyOpenSSL==19.0.0', 'cryptography==37.0.4', + 'watchdog<5.0', 'watchdog-gevent==0.1.1', 'deprecated==1.2.14'] From 7750a8d4379c34b36d7b3efaa23d65a406af59c4 Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Mon, 20 Jan 2025 16:29:04 -0800 Subject: [PATCH 41/44] Added try blocks to avoid possible race condition where driver interface is not ready before RPC calls. --- .../platform_driver/driver.py | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/services/core/PlatformDriverAgent/platform_driver/driver.py b/services/core/PlatformDriverAgent/platform_driver/driver.py index b86688f465..8fe1e37e30 100644 --- a/services/core/PlatformDriverAgent/platform_driver/driver.py +++ b/services/core/PlatformDriverAgent/platform_driver/driver.py @@ -326,29 +326,47 @@ def get_paths_for_point(self, point): return depth_first, breadth_first def get_point(self, point_name, **kwargs): - return self.interface.get_point(point_name, **kwargs) + try: + return self.interface.get_point(point_name, **kwargs) + except AttributeError as e: + _log.warning(e) + def set_point(self, point_name, value, **kwargs): - return self.interface.set_point(point_name, value, **kwargs) + try: + return self.interface.set_point(point_name, value, **kwargs) + except AttributeError as e: + _log.warning(e) def scrape_all(self): - return self.interface.scrape_all() + try: + return self.interface.scrape_all() + except AttributeError as e: + _log.warning(e) def get_multiple_points(self, point_names, **kwargs): - return self.interface.get_multiple_points(self.device_name, - point_names, - **kwargs) + try: + return self.interface.get_multiple_points(self.device_name, point_names, **kwargs) + except AttributeError as e: + _log.warning(e) def set_multiple_points(self, point_names_values, **kwargs): - return self.interface.set_multiple_points(self.device_name, - point_names_values, - **kwargs) + try: + return self.interface.set_multiple_points(self.device_name, point_names_values, **kwargs) + except AttributeError as e: + _log.warning(e) def revert_point(self, point_name, **kwargs): - self.interface.revert_point(point_name, **kwargs) + try: + self.interface.revert_point(point_name, **kwargs) + except AttributeError as e: + _log.warning(e) def revert_all(self, **kwargs): - self.interface.revert_all(**kwargs) + try: + self.interface.revert_all(**kwargs) + except AttributeError as e: + _log.warning(e) def publish_cov_value(self, point_name, point_values): """ From 9920b8c654de38c47eaead0b7e25412cbc67db01 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:01:15 -0800 Subject: [PATCH 42/44] Bump to 9.0.3 --- docs/source/conf.py | 4 ++-- volttron/platform/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 10c31ff659..91ebf436cf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -90,9 +90,9 @@ def __getattr__(cls, name): author = 'The VOLTTRON Community' # The short X.Y version -version = '9.0.1' +version = '9.0.3' # The full version, including alpha/beta/rc tags -release = '9.0.1' +release = '9.0.3' # -- General configuration --------------------------------------------------- diff --git a/volttron/platform/__init__.py b/volttron/platform/__init__.py index 5a7525b81e..b7b950618c 100644 --- a/volttron/platform/__init__.py +++ b/volttron/platform/__init__.py @@ -35,7 +35,7 @@ from urllib.parse import urlparse from ..utils.frozendict import FrozenDict -__version__ = '9.0.1' +__version__ = '9.0.3' _log = logging.getLogger(__name__) From 3a31977571660ece5094f9339c5cc67c0217a87d Mon Sep 17 00:00:00 2001 From: "David M. Raker" Date: Wed, 29 Jan 2025 11:06:29 -0800 Subject: [PATCH 43/44] Documentation update to the RESTful Web API. Clarified agents which may be required to utilize the devices endpoints. --- docs/source/platform-features/web-api/introduction.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/source/platform-features/web-api/introduction.rst b/docs/source/platform-features/web-api/introduction.rst index 0713d0e895..de138c96d9 100644 --- a/docs/source/platform-features/web-api/introduction.rst +++ b/docs/source/platform-features/web-api/introduction.rst @@ -105,8 +105,14 @@ set to ``https://localhost:8443`` the following HTTP request (with a proper GET https://localhost:8443/vui/ -Access to the API may be disabled by removing "vui" from the list of groups in ``$VOLTTRON_HOME/web-users.json`` for any user which should not have access -to the API. +Access to the API may be disabled by removing "vui" from the list of groups in ``$VOLTTRON_HOME/web-users.json`` for +any user which should not have access to the API. + +.. Note:: + Certain features of the API require that appropriate agents be installed to handle requests. In particular, + devices endpoints require at least the Platform Driver Agent and the Actuator Agent to both be installed. + For use with BACnet devices, the BACnet Proxy Agent may also be required. Missing agents are likely to result + in timeout errors from the API as the request has been sent to an agent which is not there to respond. Path Structure --------------- From 7d5721e85156cf6e78050688ee13318ab705edb6 Mon Sep 17 00:00:00 2001 From: Craig <3979063+craig8@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:52:14 -0800 Subject: [PATCH 44/44] Update utils.py Fix error to have the correct ENV Variable. --- volttron/platform/agent/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/volttron/platform/agent/utils.py b/volttron/platform/agent/utils.py index eb9f2ec238..dfa9f7c560 100644 --- a/volttron/platform/agent/utils.py +++ b/volttron/platform/agent/utils.py @@ -466,7 +466,7 @@ def vip_main(agent_class, identity=None, version='0.1', **kwargs): # TODO: Make required for all agents. Handle it through vctl and aip. if not os.environ.get("_LAUNCHED_BY_PLATFORM"): if not publickey or not secretkey: - raise ValueError("AGENT_PUBLIC and AGENT_SECRET environmental variables must " + raise ValueError("AGENT_PUBLICKEY and AGENT_SECRETKEY environmental variables must " "be set to run without the platform.") address = get_address()