From b689413612309aad42908ee882a16bc43bf81939 Mon Sep 17 00:00:00 2001 From: gavin-aguiar <80794152+gavin-aguiar@users.noreply.github.com> Date: Fri, 25 Aug 2023 15:58:21 -0500 Subject: [PATCH 1/5] SettingsApi ModuleNotFoundError fix when extensions enabled (#1305) * ModuleNotFound error fix when extensions is enabled * Fixed flake8 errors * Fixed flake8 validation * Fixed unit tests * Addressed comments * Fixed log formatiting * Fixed flake8 validation * Removed debug logging check from unittest * Added py311 tests for TPTC * Skipped TestThreadPoolSettingsPython38 for py37 * Fixed flake8 validation * Updated threadpool tests * Test unit tests failures * Fixing unit tests failures * Added exception message to test case * Fixed logging error * Addressed comments * Address comments * Fixed unit tests --------- Co-authored-by: Gavin Aguiar --- azure_functions_worker/constants.py | 10 ++- azure_functions_worker/dispatcher.py | 55 ++++++++----- azure_functions_worker/extension.py | 5 +- azure_functions_worker/loader.py | 14 ++-- azure_functions_worker/logging.py | 5 -- azure_functions_worker/utils/common.py | 40 +++++---- azure_functions_worker/utils/dependency.py | 10 +-- .../test_linux_consumption.py | 25 +++++- tests/unittests/test_dispatcher.py | 81 +++++++------------ tests/unittests/test_extension.py | 36 +++++++-- tests/unittests/test_rpc_messages.py | 7 +- tests/unittests/test_utilities.py | 15 +++- 12 files changed, 182 insertions(+), 121 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 44e7461f..d094fb14 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +import os +import pathlib import sys # Capabilities @@ -40,14 +41,19 @@ PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True +PYTHON_EXTENSIONS_RELOAD_FUNCTIONS = "PYTHON_EXTENSIONS_RELOAD_FUNCTIONS" # External Site URLs MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound" # new programming model script file name SCRIPT_FILE_NAME = "function_app.py" - PYTHON_LANGUAGE_RUNTIME = "python" # Settings for V2 programming model RETRY_POLICY = "retry_policy" + +# Paths +CUSTOMER_PACKAGES_PATH = os.path.join(pathlib.Path.home(), + pathlib.Path( + "site/wwwroot/.python_packages")) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 988f2686..66f89e21 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -26,10 +26,9 @@ PYTHON_THREADPOOL_THREAD_COUNT_MAX_37, PYTHON_THREADPOOL_THREAD_COUNT_MIN, PYTHON_ENABLE_DEBUG_LOGGING, SCRIPT_FILE_NAME, - PYTHON_LANGUAGE_RUNTIME) + PYTHON_LANGUAGE_RUNTIME, CUSTOMER_PACKAGES_PATH) from .extension import ExtensionManager from .logging import disable_console_logging, enable_console_logging -from .logging import enable_debug_logging_recommendation from .logging import (logger, error_logger, is_system_log_category, CONSOLE_LOG_PREFIX, format_exception) from .utils.common import get_app_setting, is_envvar_true @@ -263,9 +262,14 @@ async def _dispatch_grpc_request(self, request): async def _handle__worker_init_request(self, request): logger.info('Received WorkerInitRequest, ' - 'python version %s, worker version %s, request ID %s', - sys.version, VERSION, self.request_id) - enable_debug_logging_recommendation() + 'python version %s, ' + 'worker version %s, ' + 'request ID %s.' + ' To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging', + sys.version, + VERSION, + self.request_id) worker_init_request = request.worker_init_request host_capabilities = worker_init_request.capabilities @@ -293,6 +297,10 @@ async def _handle__worker_init_request(self, request): "Importing azure functions in WorkerInitRequest") import azure.functions # NoQA + if CUSTOMER_PACKAGES_PATH not in sys.path: + logger.warning("Customer packages not in sys path. " + "This should never happen! ") + # loading bindings registry and saving results to a static # dictionary which will be later used in the invocation request bindings.load_binding_registry() @@ -363,6 +371,7 @@ async def _handle__function_load_request(self, request): 'Received WorkerLoadRequest, request ID %s, function_id: %s,' 'function_name: %s,', self.request_id, function_id, function_name) + programming_model = "V1" try: if not self._functions.get_function(function_id): if function_metadata.properties.get("worker_indexed", False) \ @@ -371,8 +380,7 @@ async def _handle__function_load_request(self, request): # indexing is enabled and load request is called without # calling the metadata request. In this case we index the # function and update the workers registry - logger.info(f"Indexing function {function_name} in the " - f"load request") + programming_model = "V2" _ = self.index_functions(function_path) else: # legacy function @@ -385,17 +393,24 @@ async def _handle__function_load_request(self, request): self._functions.add_function( function_id, func, func_request.metadata) - ExtensionManager.function_load_extension( + try: + ExtensionManager.function_load_extension( + function_name, + func_request.metadata.directory + ) + except Exception as ex: + logging.error("Failed to load extensions: ", ex) + raise + + logger.info('Successfully processed FunctionLoadRequest, ' + 'request ID: %s, ' + 'function ID: %s,' + 'function Name: %s,' + 'programming model: %s', + self.request_id, + function_id, function_name, - func_request.metadata.directory - ) - - logger.info('Successfully processed FunctionLoadRequest, ' - 'request ID: %s, ' - 'function ID: %s,' - 'function Name: %s', self.request_id, - function_id, - function_name) + programming_model) return protos.StreamingMessage( request_id=self.request_id, @@ -532,8 +547,10 @@ async def _handle__function_environment_reload_request(self, request): """ try: logger.info('Received FunctionEnvironmentReloadRequest, ' - 'request ID: %s', self.request_id) - enable_debug_logging_recommendation() + 'request ID: %s,' + ' To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging', + self.request_id) func_env_reload_request = \ request.function_environment_reload_request diff --git a/azure_functions_worker/extension.py b/azure_functions_worker/extension.py index ceb619e5..568b94f9 100644 --- a/azure_functions_worker/extension.py +++ b/azure_functions_worker/extension.py @@ -236,9 +236,8 @@ def _try_get_sdk_with_extension_enabled(cls) -> Optional[ModuleType]: @classmethod def _info_extension_is_enabled(cls, sdk): logger.info( - 'Python Worker Extension is enabled in azure.functions (%s).', - get_sdk_version(sdk) - ) + 'Python Worker Extension is enabled in azure.functions (%s). ' + 'Sdk path: %s', get_sdk_version(sdk), sdk.__file__) @classmethod def _info_discover_extension_list(cls, function_name, sdk): diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index 49782c98..618a8c64 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -18,18 +18,12 @@ from . import protos, functions from .bindings.retrycontext import RetryPolicy from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ - PYTHON_LANGUAGE_RUNTIME, RETRY_POLICY + PYTHON_LANGUAGE_RUNTIME, RETRY_POLICY, CUSTOMER_PACKAGES_PATH from .utils.wrappers import attach_message_to_exception _AZURE_NAMESPACE = '__app__' _DEFAULT_SCRIPT_FILENAME = '__init__.py' _DEFAULT_ENTRY_POINT = 'main' - -PKGS_PATH = pathlib.Path("site/wwwroot/.python_packages") -home = pathlib.Path.home() -pkgs_path = os.path.join(home, PKGS_PATH) - - _submodule_dirs = [] @@ -140,7 +134,8 @@ def process_indexed_function(functions_registry: functions.Registry, f' guide: {MODULE_NOT_FOUND_TS_URL} ', debug_logs='Error in load_function. ' f'Sys Path: {sys.path}, Sys Module: {sys.modules},' - f'python-packages Path exists: {os.path.exists(pkgs_path)}') + 'python-packages Path exists: ' + f'{os.path.exists(CUSTOMER_PACKAGES_PATH)}') def load_function(name: str, directory: str, script_file: str, entry_point: Optional[str]): dir_path = pathlib.Path(directory) @@ -191,7 +186,8 @@ def load_function(name: str, directory: str, script_file: str, message=f'Troubleshooting Guide: {MODULE_NOT_FOUND_TS_URL}', debug_logs='Error in index_function_app. ' f'Sys Path: {sys.path}, Sys Module: {sys.modules},' - f'python-packages Path exists: {os.path.exists(pkgs_path)}') + 'python-packages Path exists: ' + f'{os.path.exists(CUSTOMER_PACKAGES_PATH)}') def index_function_app(function_path: str): module_name = pathlib.Path(function_path).stem imported_module = importlib.import_module(module_name) diff --git a/azure_functions_worker/logging.py b/azure_functions_worker/logging.py index 0e6bc79c..91a64c3b 100644 --- a/azure_functions_worker/logging.py +++ b/azure_functions_worker/logging.py @@ -91,11 +91,6 @@ def enable_console_logging() -> None: logger.addHandler(handler) -def enable_debug_logging_recommendation(): - logging.info("To enable debug level logging, please refer to " - "https://aka.ms/python-enable-debug-logging") - - def is_system_log_category(ctg: str) -> bool: """Check if the logging namespace belongs to system logs. Category starts with the following name will be treated as system logs. diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index f508a78f..0f12a5f5 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -1,10 +1,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import Optional, Callable -from types import ModuleType +import importlib import os import sys -import importlib +from types import ModuleType +from typing import Optional, Callable + +from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH, \ + PYTHON_EXTENSIONS_RELOAD_FUNCTIONS def is_true_like(setting: str) -> bool: @@ -110,19 +113,26 @@ def get_sdk_from_sys_path() -> ModuleType: ModuleType The azure.functions that is loaded from the first sys.path entry """ - backup_azure_functions = None - backup_azure = None - if 'azure.functions' in sys.modules: - backup_azure_functions = sys.modules.pop('azure.functions') - if 'azure' in sys.modules: - backup_azure = sys.modules.pop('azure') + if is_envvar_true(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS): + backup_azure_functions = None + backup_azure = None + + if 'azure.functions' in sys.modules: + backup_azure_functions = sys.modules.pop('azure.functions') + if 'azure' in sys.modules: + backup_azure = sys.modules.pop('azure') + + module = importlib.import_module('azure.functions') + + if backup_azure: + sys.modules['azure'] = backup_azure + if backup_azure_functions: + sys.modules['azure.functions'] = backup_azure_functions - module = importlib.import_module('azure.functions') + return module - if backup_azure: - sys.modules['azure'] = backup_azure - if backup_azure_functions: - sys.modules['azure.functions'] = backup_azure_functions + if CUSTOMER_PACKAGES_PATH not in sys.path: + sys.path.insert(0, CUSTOMER_PACKAGES_PATH) - return module + return importlib.import_module('azure.functions') diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 9a8d75bd..a213b5e8 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -1,15 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from azure_functions_worker.utils.common import is_true_like -from typing import List, Optional -from types import ModuleType import importlib import inspect import os import re import sys +from types import ModuleType +from typing import List, Optional -from ..logging import logger +from azure_functions_worker.utils.common import is_true_like from ..constants import ( AZURE_WEBJOBS_SCRIPT_ROOT, CONTAINER_NAME, @@ -17,6 +16,7 @@ PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT, PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 ) +from ..logging import logger from ..utils.common import is_python_version from ..utils.wrappers import enable_feature_by @@ -226,7 +226,7 @@ def reload_azure_google_namespace_from_worker_deps(cls): logger.info('Reloaded azure.functions module now at %s', inspect.getfile(sys.modules['azure.functions'])) except Exception as ex: - logger.info( + logger.warning( 'Unable to reload azure.functions. Using default. ' 'Exception:\n%s', ex) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 681ae866..92589c7b 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -217,7 +217,7 @@ def test_pinning_functions_to_older_version(self): "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url( "PinningFunctions"), - "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1", }) req = Request('GET', f'{ctrl.url}/api/HttpTrigger1') resp = ctrl.send_request(req) @@ -225,6 +225,29 @@ def test_pinning_functions_to_older_version(self): self.assertEqual(resp.status_code, 200) self.assertIn("Func Version: 1.11.1", resp.text) + @skipIf(sys.version_info.minor != 10, + "This is testing only for python310") + def test_opencensus_with_extensions_enabled(self): + """A function app with extensions enabled containing the + following libraries: + + azure-functions, azure-eventhub, azure-storage-blob, numpy, + cryptography, pyodbc, requests + + should return 200 after importing all libraries. + """ + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: + ctrl.assign_container(env={ + "AzureWebJobsStorage": self._storage, + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("Opencensus"), + "PYTHON_ENABLE_WORKER_EXTENSIONS": "1", + "AzureWebJobsFeatureFlags": "EnableWorkerIndexing" + }) + req = Request('GET', f'{ctrl.url}/api/opencensus') + resp = ctrl.send_request(req) + self.assertEqual(resp.status_code, 200) + def _get_blob_url(self, scenario_name: str) -> str: return ( f'https://pythonworker{self._py_shortform}sa.blob.core.windows.net/' diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index dfdcb56d..c7b127da 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -39,7 +39,7 @@ class TestThreadPoolSettingsPython37(testutils.AsyncTestCase): NEW_TYPING = sys.version_info[:3] >= (3, 7, 0) # PEP 560 """ - def setUp(self): + def setUp(self, version=SysVersionInfo(3, 7, 0, 'final', 0)): self._ctrl = testutils.start_mockhost( script_root=DISPATCHER_FUNCTIONS_DIR) self._default_workers: Optional[ @@ -49,7 +49,7 @@ def setUp(self): self._pre_env = dict(os.environ) self.mock_version_info = patch( 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 7, 0, 'final', 0)) + version) self.mock_version_info.start() def tearDown(self): @@ -98,13 +98,6 @@ async def test_dispatcher_initialize_worker_logging(self): 1 ) - self.assertEqual( - len([log for log in r.logs if log.message.startswith( - 'To enable debug level logging' - )]), - 1 - ) - async def test_dispatcher_environment_reload_logging(self): """Test if the sync threadpool will pick up app setting in placeholder mode (Linux Consumption) @@ -121,13 +114,6 @@ async def test_dispatcher_environment_reload_logging(self): 1 ) - self.assertEqual( - len([log for log in r.logs if log.message.startswith( - 'To enable debug level logging' - )]), - 1 - ) - async def test_dispatcher_send_worker_request(self): """Test if the worker status response will be sent correctly when a worker status request is received @@ -409,8 +395,9 @@ async def _check_if_function_is_ok(self, host) -> Tuple[str, str, str]: function_name = "show_context" func_id, load_r = await host.load_function(function_name) self.assertEqual(load_r.response.function_id, func_id) + ex = load_r.response.result.exception self.assertEqual(load_r.response.result.status, - protos.StatusResult.Success) + protos.StatusResult.Success, msg=ex) # Ensure the function can be properly invoked invoke_id, call_r = await host.invoke_function( @@ -457,20 +444,18 @@ async def _check_if_async_function_is_ok(self, host) -> Tuple[str, str]: return func_id, invoke_id, function_name +@unittest.skipIf(sys.version_info.minor != 8, + "Run the tests only for Python 3.8. In other platforms, " + "as the default passed is None, the cpu_count determines the " + "number of max_workers and we cannot mock the os.cpu_count() " + "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython38(TestThreadPoolSettingsPython37): - def setUp(self): - super(TestThreadPoolSettingsPython38, self).setUp() - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 8, 0, 'final', 0)) - self._over_max_workers: int = 10000 + def setUp(self, version=SysVersionInfo(3, 8, 0, 'final', 0)): + super(TestThreadPoolSettingsPython38, self).setUp(version) self._allowed_max_workers: int = self._over_max_workers - self.mock_version_info.start() def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) - self.mock_version_info.stop() + super(TestThreadPoolSettingsPython38, self).tearDown() async def test_dispatcher_sync_threadpool_in_placeholder_above_max(self): """Test if the sync threadpool will use any value and there isn't any @@ -497,25 +482,18 @@ async def test_dispatcher_sync_threadpool_in_placeholder_above_max(self): "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython39(TestThreadPoolSettingsPython38): - def setUp(self): - super(TestThreadPoolSettingsPython39, self).setUp() + def setUp(self, version=SysVersionInfo(3, 9, 0, 'final', 0)): + super(TestThreadPoolSettingsPython39, self).setUp(version) self.mock_os_cpu = patch( 'os.cpu_count', return_value=2) # 6 - based on 2 cores - min(32, (os.cpu_count() or 1) + 4) - 2 + 4 self._default_workers: Optional[int] = 6 - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 9, 0, 'final', 0)) - self.mock_os_cpu.start() - self.mock_version_info.start() def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) self.mock_os_cpu.stop() - self.mock_version_info.stop() + super(TestThreadPoolSettingsPython39, self).tearDown() @unittest.skipIf(sys.version_info.minor != 10, @@ -524,25 +502,24 @@ def tearDown(self): "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython310(TestThreadPoolSettingsPython39): - def setUp(self): - super(TestThreadPoolSettingsPython310, self).setUp() + def setUp(self, version=SysVersionInfo(3, 10, 0, 'final', 0)): + super(TestThreadPoolSettingsPython310, self).setUp(version) - self.mock_os_cpu = patch( - 'os.cpu_count', return_value=2) - # 6 - based on 2 cores - min(32, (os.cpu_count() or 1) + 4) - 2 + 4 - self._default_workers: Optional[int] = 6 - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 10, 0, 'final', 0)) + def tearDown(self): + super(TestThreadPoolSettingsPython310, self).tearDown() - self.mock_os_cpu.start() - self.mock_version_info.start() + +@unittest.skipIf(sys.version_info.minor != 11, + "Run the tests only for Python 3.11. In other platforms, " + "as the default passed is None, the cpu_count determines the " + "number of max_workers and we cannot mock the os.cpu_count() " + "in the concurrent.futures.ThreadPoolExecutor") +class TestThreadPoolSettingsPython311(TestThreadPoolSettingsPython310): + def setUp(self, version=SysVersionInfo(3, 11, 0, 'final', 0)): + super(TestThreadPoolSettingsPython311, self).setUp(version) def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) - self.mock_os_cpu.stop() - self.mock_version_info.stop() + super(TestThreadPoolSettingsPython310, self).tearDown() class TestDispatcherStein(testutils.AsyncTestCase): diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index 4727f4f8..13140c0b 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -1,13 +1,17 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +import importlib import logging import os +import pathlib import sys import unittest -from unittest.mock import patch, Mock, call from importlib import import_module +from unittest.mock import patch, Mock, call + from azure_functions_worker._thirdparty import aio_compat +from azure_functions_worker.constants import PYTHON_ENABLE_WORKER_EXTENSIONS, \ + CUSTOMER_PACKAGES_PATH from azure_functions_worker.extension import ( ExtensionManager, APP_EXT_POST_FUNCTION_LOAD, FUNC_EXT_POST_FUNCTION_LOAD, @@ -15,7 +19,6 @@ APP_EXT_POST_INVOCATION, FUNC_EXT_POST_INVOCATION ) from azure_functions_worker.utils.common import get_sdk_from_sys_path -from azure_functions_worker.constants import PYTHON_ENABLE_WORKER_EXTENSIONS class MockContext: @@ -54,6 +57,7 @@ def setUp(self): 'resources', 'mock_azure_functions' ) + self._dummy_sdk = Mock(__file__="test") # Initialize mock context self._mock_arguments = {'req': 'request'} @@ -91,7 +95,26 @@ def test_extension_is_not_supported_by_mock_sdk(self): sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) self.assertFalse(sdk_enabled) - @patch('azure_functions_worker.extension.get_sdk_from_sys_path') + def test_extension_in_worker(self): + """Test if worker contains support for extensions + """ + sys.path.insert(0, pathlib.Path.home()) + module = importlib.import_module('azure.functions') + sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) + self.assertTrue(sdk_enabled) + + def test_extension_if_sdk_not_in_path(self): + """Test if the detection works when an azure.functions SDK does not + support extension management. + """ + + module = get_sdk_from_sys_path() + self.assertIn(CUSTOMER_PACKAGES_PATH, sys.path) + sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) + self.assertTrue(sdk_enabled) + + @patch('azure_functions_worker.extension.get_sdk_from_sys_path', + return_value=importlib.import_module('azure.functions')) def test_function_load_extension_enable_when_feature_flag_is_on( self, get_sdk_from_sys_path_mock: Mock @@ -163,7 +186,8 @@ def test_function_load_extension_should_invoke_extension_call( any_order=True ) - @patch('azure_functions_worker.extension.get_sdk_from_sys_path') + @patch('azure_functions_worker.extension.get_sdk_from_sys_path', + return_value=importlib.import_module('azure.functions')) def test_invocation_extension_enable_when_feature_flag_is_on( self, get_sdk_from_sys_path_mock: Mock @@ -697,7 +721,7 @@ def test_info_extension_is_enabled(self, info_mock: Mock): self._instance._info_extension_is_enabled(sdk) info_mock.assert_called_once_with( 'Python Worker Extension is enabled in azure.functions ' - '(%s).', sdk.__version__ + '(%s). Sdk path: %s', sdk.__version__, sdk.__file__ ) @patch('azure_functions_worker.extension.logger.info') diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index f9919984..a1129a31 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -38,12 +38,15 @@ async def _verify_environment_reloaded( try: r = await disp._handle__function_environment_reload_request( request_msg) + status = r.function_environment_reload_response.result.status + exp = r.function_environment_reload_response.result.exception + self.assertEqual(status, protos.StatusResult.Success, + f"Exception in Reload request: {exp}") environ_dict = os.environ.copy() self.assertDictEqual(environ_dict, test_env) self.assertEqual(os.getcwd(), test_cwd) - status = r.function_environment_reload_response.result.status - self.assertEqual(status, protos.StatusResult.Success) + finally: self._reset_environ() diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 70feb742..c71bcce3 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -1,11 +1,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os +import pathlib import sys import typing import unittest from unittest.mock import patch +from azure_functions_worker.constants import PYTHON_EXTENSIONS_RELOAD_FUNCTIONS from azure_functions_worker.utils import common, wrappers TEST_APP_SETTING_NAME = "TEST_APP_SETTING_NAME" @@ -342,9 +344,9 @@ def test_get_sdk_from_sys_path_after_updating_sys_path(self): """ sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() - self.assertEqual( + self.assertNotEqual( os.path.dirname(module.__file__), - os.path.join(self._dummy_sdk_sys_path, 'azure', 'functions') + os.path.join(pathlib.Path.home(), 'azure', 'functions') ) def test_get_sdk_version(self): @@ -361,6 +363,15 @@ def test_get_sdk_dummy_version(self): sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) + self.assertNotEqual(sdk_version, 'dummy') + + def test_get_sdk_dummy_version_with_flag_enabled(self): + """Test if sdk version can get dummy sdk version + """ + os.environ[PYTHON_EXTENSIONS_RELOAD_FUNCTIONS] = '1' + sys.path.insert(0, self._dummy_sdk_sys_path) + module = common.get_sdk_from_sys_path() + sdk_version = common.get_sdk_version(module) self.assertEqual(sdk_version, 'dummy') def _unset_feature_flag(self): From d0135b091e8df03f8ac131894e0ec66f780de049 Mon Sep 17 00:00:00 2001 From: AzureFunctionsPython Date: Fri, 25 Aug 2023 21:01:34 +0000 Subject: [PATCH 2/5] Update Python Worker Version to 4.17.0 --- azure_functions_worker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_functions_worker/version.py b/azure_functions_worker/version.py index 602f1604..5c1f8485 100644 --- a/azure_functions_worker/version.py +++ b/azure_functions_worker/version.py @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -VERSION = '4.16.0' +VERSION = '4.17.0' From b8291fb75fbc6828008660836878a49f3deeedbd Mon Sep 17 00:00:00 2001 From: pdthummar <101662222+pdthummar@users.noreply.github.com> Date: Wed, 20 Sep 2023 14:53:49 -0500 Subject: [PATCH 3/5] Enable support for handling new command line arguments (#1318) * Enable support for handling new command line arguments with functions- prefix --- azure_functions_worker/main.py | 13 +++++- tests/unittests/test_main.py | 79 ++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/unittests/test_main.py diff --git a/azure_functions_worker/main.py b/azure_functions_worker/main.py index 3edf2840..4e1f70a7 100644 --- a/azure_functions_worker/main.py +++ b/azure_functions_worker/main.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. """Main entrypoint.""" - import argparse @@ -26,6 +25,18 @@ def parse_args(): 'syslog, or a file path') parser.add_argument('--grpcMaxMessageLength', type=int, dest='grpc_max_msg_len') + parser.add_argument('--functions-uri', dest='functions_uri', type=str, + help='URI with IP Address and Port used to' + ' connect to the Host via gRPC.') + parser.add_argument('--functions-request-id', dest='functions_request_id', + type=str, help='Request ID used for gRPC communication ' + 'with the Host.') + parser.add_argument('--functions-worker-id', + dest='functions_worker_id', type=str, + help='Worker ID assigned to this language worker.') + parser.add_argument('--functions-grpc-max-message-length', type=int, + dest='functions_grpc_max_msg_len', + help='Max grpc message length for Functions') return parser.parse_args() diff --git a/tests/unittests/test_main.py b/tests/unittests/test_main.py new file mode 100644 index 00000000..2fe29c8c --- /dev/null +++ b/tests/unittests/test_main.py @@ -0,0 +1,79 @@ +import unittest +import sys +from unittest.mock import patch +from azure_functions_worker.main import parse_args + + +class TestMain(unittest.TestCase): + + @patch.object(sys, 'argv', + ['xxx', '--host', '127.0.0.1', + '--port', '50821', + '--workerId', 'e9efd817-47a1-45dc-9e20-e6f975d7a025', + '--requestId', 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51', + '--grpcMaxMessageLength', '2147483647', + '--functions-uri', 'http://127.0.0.1:50821', + '--functions-worker-id', + 'e9efd817-47a1-45dc-9e20-e6f975d7a025', + '--functions-request-id', + 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51', + '--functions-grpc-max-message-length', '2147483647']) + def test_all_args(self): + args = parse_args() + self.assertEqual(args.host, '127.0.0.1') + self.assertEqual(args.port, 50821) + self.assertEqual(args.worker_id, + 'e9efd817-47a1-45dc-9e20-e6f975d7a025') + self.assertEqual(args.request_id, + 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51') + self.assertEqual(args.grpc_max_msg_len, 2147483647) + self.assertEqual(args.functions_uri, 'http://127.0.0.1:50821') + self.assertEqual(args.functions_worker_id, + 'e9efd817-47a1-45dc-9e20-e6f975d7a025') + self.assertEqual(args.functions_request_id, + 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51') + self.assertEqual(args.functions_grpc_max_msg_len, 2147483647) + + @patch.object(sys, 'argv', + ['xxx', '--host', '127.0.0.1', + '--port', '50821', + '--workerId', 'e9efd817-47a1-45dc-9e20-e6f975d7a025', + '--requestId', 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51', + '--grpcMaxMessageLength', '2147483647']) + def test_old_args(self): + args = parse_args() + self.assertEqual(args.host, '127.0.0.1') + self.assertEqual(args.port, 50821) + self.assertEqual(args.worker_id, + 'e9efd817-47a1-45dc-9e20-e6f975d7a025') + self.assertEqual(args.request_id, + 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51') + self.assertEqual(args.grpc_max_msg_len, 2147483647) + self.assertIsNone(args.functions_uri) + self.assertIsNone(args.functions_worker_id) + self.assertIsNone(args.functions_request_id) + self.assertIsNone(args.functions_grpc_max_msg_len) + + @patch.object(sys, 'argv', + ['xxx', '--functions-uri', 'http://127.0.0.1:50821', + '--functions-worker-id', + 'e9efd817-47a1-45dc-9e20-e6f975d7a025', + '--functions-request-id', + 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51', + '--functions-grpc-max-message-length', '2147483647']) + def test_new_args(self): + args = parse_args() + self.assertEqual(args.functions_uri, 'http://127.0.0.1:50821') + self.assertEqual(args.functions_worker_id, + 'e9efd817-47a1-45dc-9e20-e6f975d7a025') + self.assertEqual(args.functions_request_id, + 'cbef5957-cdb3-4462-9ee7-ac9f91be0a51') + self.assertEqual(args.functions_grpc_max_msg_len, 2147483647) + + @patch.object(sys, 'argv', ['xxx', '--host', 'dummy_host', + '--port', '12345', + '--invalid-arg', 'invalid_value']) + def test_invalid_args(self): + with self.assertRaises(SystemExit) as context: + parse_args() + self.assertEqual(context.exception.code, 2) From 9d0ee4cada08c512752bd1b0e140401061b37206 Mon Sep 17 00:00:00 2001 From: AzureFunctionsPython Date: Wed, 20 Sep 2023 21:39:31 +0000 Subject: [PATCH 4/5] Update Python Worker Version to 4.18.0 --- azure_functions_worker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_functions_worker/version.py b/azure_functions_worker/version.py index 5c1f8485..4e97eb60 100644 --- a/azure_functions_worker/version.py +++ b/azure_functions_worker/version.py @@ -1,4 +1,4 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -VERSION = '4.17.0' +VERSION = '4.18.0' From 201013ea7065305539910b4dbdabc7104800a7ac Mon Sep 17 00:00:00 2001 From: gavin-aguiar <80794152+gavin-aguiar@users.noreply.github.com> Date: Mon, 2 Oct 2023 17:45:15 -0500 Subject: [PATCH 5/5] Update Python SDK Version to 1.18.0b3 (#1320) * Update Python SDK Version to 1.18.0b3 * Update setup.py --------- Co-authored-by: AzureFunctionsPython --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d6fea428..8aa0671c 100644 --- a/setup.py +++ b/setup.py @@ -106,7 +106,7 @@ ] INSTALL_REQUIRES = [ - "azure-functions==1.16.0", + "azure-functions==1.18.0b3", "python-dateutil~=2.8.2" ]