vdk-oracle: fixes in tests and passing secrets (#3031)

1. Fix the tests to be timezone independant. Currently they were failing because fromtimestamp is timezone dependant so it will return different reseult depending on the timezone 2. Simplify passing secrets as configuration by removing redudant configuration options and note it's just a prototype until such change is in core. It is unnecsary to pass 3 configuration options to start passing secret. Just read everytime secrets if there's one use if not no. Kept option "ORACLE_USE_SECRETS" as a kill switch just in case.
vmware · Jan 17, 2024 · 7309289 · 7309289
1 parent 3124222
commit 7309289
Show file tree

Hide file tree

Showing 7 changed files with 14 additions and 31 deletions.
diff --git a/projects/vdk-plugins/vdk-oracle/README.md b/projects/vdk-plugins/vdk-oracle/README.md
@@ -19,8 +19,6 @@ pip install vdk-oracle
 |--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
 | oracle_user              | Username used when connecting to Oracle database                                                                                                                                                                                      | my_user             |
 | oracle_password          | Password used when connecting to Oracle database                                                                                                                                                                                      | super_secret_shhhh  |
-| oracle_user_secret       | The user name secret key if using secrets to connect to Oracle                                                                                                                                                                        | user_secret_key     |
-| oracle_password_secret   | The password secret key if using secrets to connect to Oracle                                                                                                                                                                         | password_secret_key |
 | oracle_use_secrets       | Set to True to use secrets to connect to Oracle                                                                                                                                                                                       | True                |
 | oracle_connection_string | The Oracle connection string                                                                                                                                                                                                          | localhost:1521/free |
 | oracle_thick_mode        | Python-oracledb is said to be in Thick mode when Oracle Client libraries are used. True by default. Set to False to disable Oracle Thick mode. More info: https://python-oracledb.readthedocs.io/en/latest/user_guide/appendix_b.html | True                |

diff --git a/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_configuration.py b/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_configuration.py
@@ -13,8 +13,6 @@
 ORACLE_PASSWORD = "ORACLE_PASSWORD"
 ORACLE_USE_SECRETS = "ORACLE_USE_SECRETS"
 ORACLE_THICK_MODE = "ORACLE_THICK_MODE"
-ORACLE_USER_SECRET = "ORACLE_USER_SECRET"
-ORACLE_PASSWORD_SECRET = "ORACLE_PASSWORD_SECRET"
 ORACLE_CONNECTION_STRING = "ORACLE_CONNECTION_STRING"
 
 
@@ -28,12 +26,6 @@ def get_oracle_user(self) -> str:
     def get_oracle_password(self) -> str:
         return self.__config.get_value(ORACLE_PASSWORD)
 
-    def get_oracle_user_secret(self) -> str:
-        return self.__config.get_value(ORACLE_USER_SECRET)
-
-    def get_oracle_password_secret(self) -> str:
-        return self.__config.get_value(ORACLE_PASSWORD_SECRET)
-
     def get_oracle_connection_string(self) -> str:
         return self.__config.get_value(ORACLE_CONNECTION_STRING)
 
@@ -65,21 +57,13 @@ def add_definitions(config_builder: ConfigurationBuilder):
         )
         config_builder.add(
             key=ORACLE_USE_SECRETS,
-            default_value=False,
-            description="Set this flag to use secrets to connect to Oracle",
-        )
-        config_builder.add(
-            key=ORACLE_USER_SECRET,
-            default_value=None,
-            description="The user secret key if using secrets to connect to Oracle",
-        )
-        config_builder.add(
-            key=ORACLE_PASSWORD_SECRET,
-            default_value=None,
-            description="The password secret key if using secrets to connect to Oracle",
+            default_value=True,
+            description="Set this flag to use secrets to connect to Oracle. This is protype option. "
+            "Leave default value unless there are issues.",
         )
         config_builder.add(
             key=ORACLE_THICK_MODE,
             default_value=True,
-            description="Use oracle thick mode, default is True. Set to False to disable oracle thick mode. More info: https://python-oracledb.readthedocs.io/en/latest/user_guide/appendix_b.html",
+            description="Use oracle thick mode. Set to False to disable oracle thick mode. "
+            "More info: https://python-oracledb.readthedocs.io/en/latest/user_guide/appendix_b.html",
         )
diff --git a/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_plugin.py b/projects/vdk-plugins/vdk-oracle/src/vdk/plugin/oracle/oracle_plugin.py
@@ -36,9 +36,10 @@ def initialize_job(self, context: JobContext):
         conf = OracleConfiguration(context.core_context.configuration)
         oracle_user, oracle_pass = conf.get_oracle_user(), conf.get_oracle_password()
         if conf.oracle_use_secrets():
+            # TODO: this will be removed once support for reading configuration from secrets is added in core
             job_secrets = context.job_input.get_all_secrets()
-            oracle_user = job_secrets[conf.get_oracle_user_secret()]
-            oracle_pass = job_secrets[conf.get_oracle_password_secret()]
+            oracle_user = job_secrets.get(ORACLE_USER.lower(), oracle_user)
+            oracle_pass = job_secrets.get(ORACLE_PASSWORD.lower(), oracle_pass)
         context.connections.add_open_connection_factory_method(
             "ORACLE",
             lambda: OracleConnection(

diff --git a/...-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads-no-table/10_ingest.py b/...-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads-no-table/10_ingest.py
@@ -36,7 +36,7 @@ def run(job_input):
             "int_data": 12,
             "float_data": 1.2,
             "bool_data": True,
-            "timestamp_data": datetime.datetime.fromtimestamp(1700554373),
+            "timestamp_data": datetime.datetime.utcfromtimestamp(1700554373),
         },
         {
             "id": 6,

diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads/20_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-different-payloads/20_ingest.py
@@ -36,7 +36,7 @@ def run(job_input):
             "int_data": 12,
             "float_data": 1.2,
             "bool_data": True,
-            "timestamp_data": datetime.datetime.fromtimestamp(1700554373),
+            "timestamp_data": datetime.datetime.utcfromtimestamp(1700554373),
         },
         {
             "id": 6,

diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-no-table/10_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job-no-table/10_ingest.py
@@ -21,7 +21,7 @@ def run(job_input):
             12,
             1.2,
             True,
-            datetime.datetime.fromtimestamp(1700554373),
+            datetime.datetime.utcfromtimestamp(1700554373),
             Decimal(1.1),
         ],
         [
@@ -30,7 +30,7 @@ def run(job_input):
             12,
             1.2,
             True,
-            datetime.datetime.fromtimestamp(1700554373),
+            datetime.datetime.utcfromtimestamp(1700554373),
             Decimal(1.1),
         ],
         [
@@ -39,7 +39,7 @@ def run(job_input):
             12,
             1.2,
             True,
-            datetime.datetime.fromtimestamp(1700554373),
+            datetime.datetime.utcfromtimestamp(1700554373),
             Decimal(1.1),
         ],
     ]

diff --git a/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job/20_ingest.py b/projects/vdk-plugins/vdk-oracle/tests/jobs/oracle-ingest-job/20_ingest.py
@@ -11,7 +11,7 @@ def run(job_input):
         "int_data": 12,
         "float_data": 1.2,
         "bool_data": True,
-        "timestamp_data": datetime.datetime.fromtimestamp(1700554373),
+        "timestamp_data": datetime.datetime.utcfromtimestamp(1700554373),
         "decimal_data": Decimal(0.1),
     }