vdk-heartbeat: cover requirements.txt automatic installs (#2393)

Upon `vdk deploy` the Control Service is responsible for installing all
external dependecies defined in requirements.txt of the data job so they
can be used during a cloud run.

A recent outage in one of our customers caused data jobs to ignore their
requirements.txt and not install any external libraries. It was trace to
be due to a release of [data job base

image](https://hub.docker.com/layers/versatiledatakit/data-job-base-python-3.11)

CICD of the open source VDK did not catch this. 

I am extending VDK heartbeat to test for the issue so we can catch it in
the future.
 
See also #2391

projects/vdk-heartbeat/.gitlab-ci.yml

@@ -28,6 +28,28 @@ vdk-heartbeat-test:
       junit: tests.xml
 
 
+vdk-heartbeat-release-acceptance-test:
+  stage: pre_release
+  before_script:
+    - cd projects/vdk-heartbeat
+  script:
+    - set -x
+    - python setup.py sdist --formats=gztar
+    - pip install vdk-heartbeat --extra-index-url $PIP_EXTRA_INDEX_URL
+    - export VDK_HEARTBEAT_API_TOKEN=$VDK_API_TOKEN
+    - export VDK_HEARTBEAT_OP_ID="vdk-heartbeat-$CI_PIPELINE_ID"
+    - export JOB_NAME=vdk-heartbeat-release-test-job-$(date +%s)
+    - vdk-heartbeat -f cicd/cicd_heartbeat_config.ini
+  artifacts:
+    when: always
+    reports:
+      junit: tests.xml
+  only:
+    refs: # TODO switch to "main" so it is used as a pre-release test not pre-merge test
+      - external_pull_requests
+    changes: *vdk_heartbeat_changes_locations
+
+
 vdk-heartbeat-release:
   before_script:
     - cd projects/vdk-heartbeat

projects/vdk-heartbeat/cicd/cicd_heartbeat_config.ini

@@ -0,0 +1,10 @@
+[DEFAULT]
+# Genereate with https://console-stg.cloud.vmware.com/csp/gateway/portal/#/user/tokens
+# VDKCLI_OAUTH2_REFRESH_TOKEN= passed as environment variable
+VDKCLI_OAUTH2_URI=https://console-stg.cloud.vmware.com/csp/gateway/am/api/auth/api-tokens/authorize
+CONTROL_API_URL=http://cicd-control-service-svc:8092
+# https://iaclqhm5xk.execute-api.us-west-1.amazonaws.com/data-jobs
+
+JOB_RUN_TEST_MODULE_NAME=vdk.internal.heartbeat.simple_run_test
+JOB_RUN_TEST_CLASS_NAME=SimpleRunTest
+DATAJOB_DIRECTORY_NAME=simple

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/empty/10_do_nothing.py

@@ -2,10 +2,15 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vdk.api.job_input import IJobInput
 
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     log.info(f"Empty job that just logs and exits")

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/empty/requirements.txt

@@ -0,0 +1 @@
+Faker

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/impala/05_set_properties.py

@@ -2,12 +2,17 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vacloud.api.job_input import IJobInput
 
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     # Test creates dynamically file 06_override_properties.py
     # which will override properties
 

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/simple/10_simple.py

@@ -2,15 +2,20 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vdk.api.job_input import IJobInput
 
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     log.info(f"Get current properties ")
     props = job_input.get_all_properties()
     props["succeeded"] = "true"
-    log.info(f"Save new properties ")
+    log.info(f"Save new properties {props}")
     job_input.set_all_properties(props)
     log.info(f"Updated property now to {props['succeeded']}")

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/simple/requirements.txt

@@ -0,0 +1,2 @@
+
+Faker

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/trino/05_set_properties.py

@@ -2,12 +2,18 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
 
+from faker import Faker
 from vdk.api.job_input import IJobInput
 
+
 log = logging.getLogger(__name__)
 
 
 def run(job_input: IJobInput):
+    # this is just added to test that external libraries are installed and imported correctly from requirements.txt
+    fake = Faker()
+    log.info(f"Fake name: {fake.name()}")
+
     # Test creates dynamically file 06_override_properties.py
     # which will override properties
 

projects/vdk-heartbeat/src/vdk/internal/heartbeat/vdk_heartbeat_data_job/trino/requirements.txt

@@ -1 +1,2 @@
+Faker
 vdk-trino