From ef69ad1f03c5bca2d42d71b88ca143d649d139df Mon Sep 17 00:00:00 2001 From: Antoni Ivanov Date: Tue, 16 Jan 2024 17:28:29 +0200 Subject: [PATCH] vdk-oracle: fixes 1. Fix the tests to be timezone independant. Currently they were failing because fromtimestamp is timezone dependant so it will return different reseult depending on the timezone 2. Simplify passing secrets as configuration by removing redudant configuration options and note it's just a prototype until such change is in core. It is unnecsary to pass 3 configuration options to start passing secret. Just read everytime secrets if there's one use if not no. Kept option "ORACLE_USE_SECRETS" as a kill switch just in case. --- projects/vdk-plugins/vdk-oracle/README.md | 2 -- .../vdk/plugin/oracle/oracle_configuration.py | 26 ++++--------------- .../src/vdk/plugin/oracle/oracle_plugin.py | 5 ++-- .../10_ingest.py | 2 +- .../20_ingest.py | 2 +- .../oracle-ingest-job-no-table/10_ingest.py | 6 ++--- .../tests/jobs/oracle-ingest-job/20_ingest.py | 2 +- 7 files changed, 14 insertions(+), 31 deletions(-) diff --git a/projects/vdk-plugins/vdk-oracle/README.md b/projects/vdk-plugins/vdk-oracle/README.md index e3092fb4f3..acdab95758 100644 --- a/projects/vdk-plugins/vdk-oracle/README.md +++ b/projects/vdk-plugins/vdk-oracle/README.md @@ -19,8 +19,6 @@ pip install vdk-oracle |--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------| | oracle_user | Username used when connecting to Oracle database | my_user | | oracle_password | Password used when connecting to Oracle database | super_secret_shhhh | -| oracle_user_secret | The user name secret key if using secrets to connect to Oracle | user_secret_key | -| oracle_password_secret | The password secret key if using secrets to connect to Oracle | password_secret_key | | oracle_use_secrets | Set to True to use secrets to connect to Oracle | True | | oracle_connection_string | The Oracle connection string | localhost:1521/free | | oracle_thick_mode | Python-oracledb is said to be in Thick mode when Oracle Client libraries are used. True by default. Set to False to disable Oracle Thick mode. More info: https://python-oracledb.readthedocs.io/en/latest/user_guide/appendix_b.html | True | diff --git a/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_configuration.py b/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_configuration.py index 9f79041877..a15b582645 100644 --- a/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_configuration.py +++ b/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_configuration.py @@ -13,8 +13,6 @@ ORACLE_PASSWORD = "ORACLE_PASSWORD" ORACLE_USE_SECRETS = "ORACLE_USE_SECRETS" ORACLE_THICK_MODE = "ORACLE_THICK_MODE" -ORACLE_USER_SECRET = "ORACLE_USER_SECRET" -ORACLE_PASSWORD_SECRET = "ORACLE_PASSWORD_SECRET" ORACLE_CONNECTION_STRING = "ORACLE_CONNECTION_STRING" @@ -28,12 +26,6 @@ def get_oracle_user(self) -> str: def get_oracle_password(self) -> str: return self.__config.get_value(ORACLE_PASSWORD) - def get_oracle_user_secret(self) -> str: - return self.__config.get_value(ORACLE_USER_SECRET) - - def get_oracle_password_secret(self) -> str: - return self.__config.get_value(ORACLE_PASSWORD_SECRET) - def get_oracle_connection_string(self) -> str: return self.__config.get_value(ORACLE_CONNECTION_STRING) @@ -65,21 +57,13 @@ def add_definitions(config_builder: ConfigurationBuilder): ) config_builder.add( key=ORACLE_USE_SECRETS, - default_value=False, - description="Set this flag to use secrets to connect to Oracle", - ) - config_builder.add( - key=ORACLE_USER_SECRET, - default_value=None, - description="The user secret key if using secrets to connect to Oracle", - ) - config_builder.add( - key=ORACLE_PASSWORD_SECRET, - default_value=None, - description="The password secret key if using secrets to connect to Oracle", + default_value=True, + description="Set this flag to use secrets to connect to Oracle. This is protype option. " + "Leave default value unless there are issues.", ) config_builder.add( key=ORACLE_THICK_MODE, default_value=True, - description="Use oracle thick mode, default is True. Set to False to disable oracle thick mode. More info: https://python-oracledb.readthedocs.io/en/latest/user_guide/appendix_b.html", + description="Use oracle thick mode. Set to False to disable oracle thick mode. " + "More info: https://python-oracledb.readthedocs.io/en/latest/user_guide/appendix_b.html", ) diff --git a/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_plugin.py b/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_plugin.py index 07ee1dde9e..570a6de70a 100644 --- a/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_plugin.py +++ b/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_plugin.py @@ -36,9 +36,10 @@ def initialize_job(self, context: JobContext): conf = OracleConfiguration(context.core_context.configuration) oracle_user, oracle_pass = conf.get_oracle_user(), conf.get_oracle_password() if conf.oracle_use_secrets(): + # TODO: this will be removed once support for reading configuration from secrets is added in core job_secrets = context.job_input.get_all_secrets() - oracle_user = job_secrets[conf.get_oracle_user_secret()] - oracle_pass = job_secrets[conf.get_oracle_password_secret()] + oracle_user = job_secrets.get(ORACLE_USER.lower(), oracle_user) + oracle_pass = job_secrets.get(ORACLE_PASSWORD.lower(), oracle_pass) context.connections.add_open_connection_factory_method( "ORACLE", lambda: OracleConnection( diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads-no-table/10_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads-no-table/10_ingest.py index 064028228e..fbc4e32b46 100644 --- a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads-no-table/10_ingest.py +++ b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads-no-table/10_ingest.py @@ -36,7 +36,7 @@ def run(job_input): "int_data": 12, "float_data": 1.2, "bool_data": True, - "timestamp_data": datetime.datetime.fromtimestamp(1700554373), + "timestamp_data": datetime.datetime.utcfromtimestamp(1700554373), }, { "id": 6, diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads/20_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads/20_ingest.py index 064028228e..fbc4e32b46 100644 --- a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads/20_ingest.py +++ b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads/20_ingest.py @@ -36,7 +36,7 @@ def run(job_input): "int_data": 12, "float_data": 1.2, "bool_data": True, - "timestamp_data": datetime.datetime.fromtimestamp(1700554373), + "timestamp_data": datetime.datetime.utcfromtimestamp(1700554373), }, { "id": 6, diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-no-table/10_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-no-table/10_ingest.py index 298aba10db..48758b6bee 100644 --- a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-no-table/10_ingest.py +++ b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-no-table/10_ingest.py @@ -21,7 +21,7 @@ def run(job_input): 12, 1.2, True, - datetime.datetime.fromtimestamp(1700554373), + datetime.datetime.utcfromtimestamp(1700554373), Decimal(1.1), ], [ @@ -30,7 +30,7 @@ def run(job_input): 12, 1.2, True, - datetime.datetime.fromtimestamp(1700554373), + datetime.datetime.utcfromtimestamp(1700554373), Decimal(1.1), ], [ @@ -39,7 +39,7 @@ def run(job_input): 12, 1.2, True, - datetime.datetime.fromtimestamp(1700554373), + datetime.datetime.utcfromtimestamp(1700554373), Decimal(1.1), ], ] diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job/20_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job/20_ingest.py index 4388cb85a5..329ca656bd 100644 --- a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job/20_ingest.py +++ b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job/20_ingest.py @@ -11,7 +11,7 @@ def run(job_input): "int_data": 12, "float_data": 1.2, "bool_data": True, - "timestamp_data": datetime.datetime.fromtimestamp(1700554373), + "timestamp_data": datetime.datetime.utcfromtimestamp(1700554373), "decimal_data": Decimal(0.1), }