vdk-heartbeat: cover requirements.txt automatic installs

Upon `vdk deploy` the Control Service is responsible for installing all
external dependecies defined in requirements.txt of the data job
so they can be used during a cloud run.

Recent outage in one of our customers which caused data jobs have not
had their requirements installed. It was trace to be due to a release of
[data job base
image](https://hub.docker.com/layers/versatiledatakit/data-job-base-python-3.11)

This was not caught by CICD of the open source VDK.

This change aim to address this gap and to cover this improtnat
functionality for correct installation of data job's python
requirements.

projects/vdk-heartbeat/.gitlab-ci.yml

@@ -28,6 +28,28 @@ vdk-heartbeat-test:
       junit: tests.xml
 
 
+vdk-heartbeat-release-acceptance-test:
+  stage: pre_release
+  before_script:
+    - cd projects/vdk-heartbeat
+  script:
+    - set -x
+    - python setup.py sdist --formats=gztar
+    - pip install vdk-heartbeat --extra-index-url $PIP_EXTRA_INDEX_URL
+    - export VDK_HEARTBEAT_API_TOKEN=$VDK_API_TOKEN
+    - export VDK_HEARTBEAT_OP_ID="vdk-heartbeat-$CI_PIPELINE_ID"
+    - export JOB_NAME=vdk-heartbeat-release-test-job-$(date +%s)
+    - vdk-heartbeat -f cicd/cicd_heartbeat_config.ini
+  artifacts:
+    when: always
+    reports:
+      junit: tests.xml
+  only:
+    refs: # TODO switch to "main" so it is used as a pre-release test not pre-merge test
+      - external_pull_requests
+    changes: *vdk_heartbeat_changes_locations
+
+
 vdk-heartbeat-release:
   before_script:
     - cd projects/vdk-heartbeat

projects/vdk-heartbeat/cicd/cicd_heartbeat_config.ini

@@ -0,0 +1,10 @@
+[DEFAULT]
+# Genereate with https://console-stg.cloud.vmware.com/csp/gateway/portal/#/user/tokens
+# VDKCLI_OAUTH2_REFRESH_TOKEN= passed as environment variable
+VDKCLI_OAUTH2_URI=https://console-stg.cloud.vmware.com/csp/gateway/am/api/auth/api-tokens/authorize
+CONTROL_API_URL=http://cicd-control-service-svc:8092
+# https://iaclqhm5xk.execute-api.us-west-1.amazonaws.com/data-jobs
+
+JOB_RUN_TEST_MODULE_NAME=vdk.internal.heartbeat.simple_run_test
+JOB_RUN_TEST_CLASS_NAME=SimpleRunTest
+DATAJOB_DIRECTORY_NAME=simple

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/empty/10_do_nothing.py

@@ -2,10 +2,15 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vdk.api.job_input import IJobInput
 
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     log.info(f"Empty job that just logs and exits")

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/empty/requirements.txt

@@ -0,0 +1 @@
+Faker

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/impala/05_set_properties.py

@@ -2,12 +2,17 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vacloud.api.job_input import IJobInput
 
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     # Test creates dynamically file 06_override_properties.py
     # which will override properties
 

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/simple/10_simple.py

@@ -2,15 +2,20 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vdk.api.job_input import IJobInput
 
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     log.info(f"Get current properties ")
     props = job_input.get_all_properties()
     props["succeeded"] = "true"
-    log.info(f"Save new properties ")
+    log.info(f"Save new properties {props}")
     job_input.set_all_properties(props)
     log.info(f"Updated property now to {props['succeeded']}")

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/simple/requirements.txt

@@ -0,0 +1,2 @@
+
+Faker

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/trino/05_set_properties.py

@@ -2,12 +2,18 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vdk.api.job_input import IJobInput
 
+
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     # Test creates dynamically file 06_override_properties.py
     # which will override properties
 

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/trino/requirements.txt

@@ -1 +1,2 @@
+Faker
 vdk-trino