From a9ae5ce22fdeb2cb4d2c357351fcfbe33c7d47c4 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Mon, 9 Oct 2023 19:38:13 +0530 Subject: [PATCH 1/9] LR-627 | Ansible and config changes for Cloud agnostic --- .../tasks/spark-cluster.yaml | 45 +++++++ .../lern-data-products-deploy/tasks/main.yml | 124 +++++------------- .../templates/common.conf.j2 | 99 +++----------- .../tasks/spark-cluster.yaml | 53 ++++++++ 4 files changed, 151 insertions(+), 170 deletions(-) create mode 100644 ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml create mode 100644 ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml new file mode 100644 index 000000000..d2da9f1fb --- /dev/null +++ b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml @@ -0,0 +1,45 @@ +- name: Ensure azure blob storage container exists + command: az storage container create --name {{ bucket }} + when: dp_object_store_type == "azure" + tags: + - always + +- name: Copy Core Data Products to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} + async: 3600 + poll: 10 + when: dp_object_store_type == "azure" + tags: + - dataproducts-spark-cluster + +- name: Copy Ed Data Products to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar + async: 3600 + poll: 10 + when: dp_object_store_type == "azure" + tags: + - lern-dataproducts-spark-cluster + +- name: Copy Framework Library to azure blob + command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} + async: 3600 + poll: 10 + when: dp_object_store_type == "azure" + tags: + - framework-spark-cluster + +- name: Copy Scruid Library to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} + async: 3600 + poll: 10 + when: dp_object_store_type == "azure" + tags: + - framework-spark-cluster + +- name: Copy configuration file to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf + async: 3600 + poll: 10 + when: dp_object_store_type == "azure" + tags: + - framework-spark-cluster \ No newline at end of file diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 646f25edc..4cd977061 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -1,20 +1,11 @@ ## Data products deployment ## -- name: Ensure azure blob storage container exists - command: az storage container create --name {{ bucket }} - when: dp_object_store_type == "azure" - tags: - - always - -- name: Ensure oci oss bucket exists - command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}" - register: check_bucket - when: dp_object_store_type == "oci" - tags: - - always - -- name: Create oci oss bucket - command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" - when: dp_object_store_type == "oci" and check_bucket.rc !=0 +- name: Ensure the storage container exists + include_role: + name: "{{ cloud_service_provider }}-cloud-storage" + tasks_from: spark-cluster.yml + apply: + tags: + - always tags: - always @@ -23,22 +14,15 @@ tags: - dataproducts -- name: Copy Core Data Products to azure blob - command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} - async: 3600 - poll: 10 - when: dp_object_store_type == "azure" - tags: - - dataproducts-spark-cluster - -- name: Copy Core Data Products to oci oss - command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force - async: 3600 - poll: 10 - when: dp_object_store_type == "oci" +- name: Copy Core Data Products + include_role: + name: "{{ cloud_service_provider }}-cloud-storage" + tasks_from: spark-cluster.yml + apply: + tags: + - dataproducts-spark-cluster tags: - - dataproducts-spark-cluster - + - dataproducts-spark-cluster - name: Unarchive Ed Data Products become: yes @@ -46,63 +30,35 @@ tags: - lern-dataproducts -- name: Copy Ed Data Products to azure blob - command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar - async: 3600 - poll: 10 - when: dp_object_store_type == "azure" +- name: Copy LERN Data Products + include_role: + name: "{{ cloud_service_provider }}-cloud-storage" + tasks_from: spark-cluster.yml + apply: + tags: + - lern-dataproducts-spark-cluster tags: - - lern-dataproducts-spark-cluster - -- name: Copy Ed Data Products to oci oss - command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force - async: 3600 - poll: 10 - when: dp_object_store_type == "oci" - tags: - - lern-dataproducts-spark-cluster + - lern-dataproducts-spark-cluster - name: Copy Framework Library copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - framework -- name: Copy Framework Library to azure blob - command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} - async: 3600 - poll: 10 - when: dp_object_store_type == "azure" - tags: - - framework-spark-cluster - -- name: Copy Framework Library to oci oss - command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force - async: 3600 - poll: 10 - when: dp_object_store_type == "oci" +- name: Copy Analytics Framework related libraries + include_role: + name: "{{ cloud_service_provider }}-cloud-storage" + tasks_from: spark-cluster.yml + apply: + tags: + - framework-spark-cluster tags: - framework-spark-cluster - name: Copy Scruid Library copy: src={{ scruid_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - - framework - -- name: Copy Scruid Library to azure blob - command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} - async: 3600 - poll: 10 - when: dp_object_store_type == "azure" - tags: - - framework-spark-cluster - -- name: Copy Scruid Library to oci oss - command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force - async: 3600 - poll: 10 - when: dp_object_store_type == "oci" - tags: - - framework-spark-cluster + - framework - name: Copy Job Manager copy: src={{ analytics_job_manager_artifact }} dest={{ analytics.home }}/models-{{ model_version }} @@ -136,23 +92,7 @@ regexp: '^log.appender.kafka.enable="false"' line: 'log.appender.kafka.enable="true"' tags: - - framework-spark-cluster - -- name: Copy configuration file to azure blob - command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf - async: 3600 - poll: 10 - when: dp_object_store_type == "azure" - tags: - - framework-spark-cluster - -- name: Copy configuration file to oci oss - command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force - async: 3600 - poll: 10 - when: dp_object_store_type == "oci" - tags: - - framework-spark-cluster + - framework-spark-cluster - name: Copy log4j2 xml file template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }} diff --git a/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 b/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 index fb018399e..225d74582 100644 --- a/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 @@ -14,26 +14,11 @@ storage.key.config="{{ dp_storage_key_config }}" storage.secret.config="{{ dp_storage_secret_config }}" reports.storage.key.config="{{ dp_reports_storage_key_config }}" reports.storage.secret.config="{{ dp_reports_storage_secret_config }}" -{% if dp_object_store_type == "azure" %} -cloud_storage_type="azure" -{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3" or dp_object_store_type == "oci") %} +cloud_storage_type="{{dp_object_store_type}}" cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\1') }}" cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}" -{% if cloud_service_provider == "oci" %} -cloud_storage_type="oci" -storage.endpoint.config="{{ dp_storage_endpoint_config }}" -oci_storage_key="{{ s3_storage_key }}" -oci_storage_secret="{{ s3_storage_secret }}" -{% else %} -cloud_storage_type="s3" -aws_storage_key="{{ s3_storage_key }}" -aws_storage_secret="{{ s3_storage_secret }}" -{% endif %} - -{% endif %} - lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M" # Joblog Kafka appender config for cluster execution @@ -56,13 +41,13 @@ spark.driver_memory="{{ spark.driver_memory }}" #Monitor Jobs monitor { - notification { - webhook_url = "{{ data_exhaust_webhook_url }}" - channel = "{{ data_exhaust_Channel }}" - token = "{{ data_exhaust_token }}" - slack = true + notification { + webhook_url = "{{ data_exhaust_webhook_url }}" + channel = "{{ data_exhaust_Channel }}" + token = "{{ data_exhaust_token }}" + slack = true name = "{{ data_exhaust_name }}" - } + } } #App ID & Channel ID @@ -71,15 +56,6 @@ default.channel.id="in.ekstep" default.creation.app.id="no_value" -# Media Service Type -media_service_type = "azure" - -azure_tenant="{{ media_service_azure_tenant }}" -azure_subscription_id="{{ media_service_azure_subscription_id }}" -azure_account_name="{{ media_service_azure_account_name }}" -azure_resource_group_name="{{ media_service_azure_resource_group_name }}" -azure_token_client_key="{{ media_service_azure_token_client_key }}" -azure_token_client_secret="{{ media_service_azure_token_client_secret }}" elasticsearch.service.endpoint="http://{{groups['composite-search-cluster'][0]}}:9200" elasticsearch.index.compositesearch.name="{{ es_search_index }}" @@ -88,42 +64,10 @@ org.search.api.key="{{ searchServiceAuthorizationToken }}" hierarchy.search.api.url="{{ hierarchySearchServiceUrl }}" hierarchy.search.api.path="{{ hierarchySearchServicEndpoint }}" -# Azure Media Service Config -azure { - location = "centralindia" - tenant = "tenant name" - subscription_id = "subscription id" - - api { - endpoint="Media Service API End Point" - version = "2018-07-01" - } - - account_name = "account name" - resource_group_name = "Resource Group Name" - - transform { - default = "media_transform_default" - hls = "media_transform_hls" - } - - stream { - base_url = "{{ stream_base_url }}" - endpoint_name = "default" - protocol = "Hls" - policy_name = "Predefined_ClearStreamingOnly" - } - - token { - client_key = "client key" - client_secret = "client secret" - } -} - ## Reports - Global config cloud.container.reports="{{reports_container}}" -# course metrics container in azure +# course metrics container course.metrics.cassandra.sunbirdKeyspace="sunbird" course.metrics.cassandra.sunbirdCoursesKeyspace="sunbird_courses" course.metrics.cassandra.sunbirdHierarchyStore="{{ cassandra_hierarchy_store_keyspace }}" @@ -134,7 +78,6 @@ es.port="9200" es.composite.host="{{groups['composite-search-cluster'][0]}}" # State admin user reports -# Uses azure only - course.metrics.cloud.provider admin.metrics.cloud.objectKey="" admin.metrics.temp.dir="/mount/data/analytics/admin-user-reports" @@ -169,23 +112,23 @@ druid.summaryDatasource.count.query="{\"query\": \"SELECT COUNT(*) AS \\\"total\ #Pipeline Audit Jobs pipeline_audit { - notification { - webhook_url = "{{ data_exhaust_webhook_url }}" - channel = "{{ data_exhaust_Channel }}" - token = "{{ data_exhaust_token }}" - slack = true - name = "Pipeline Audit" - } + notification { + webhook_url = "{{ data_exhaust_webhook_url }}" + channel = "{{ data_exhaust_Channel }}" + token = "{{ data_exhaust_token }}" + slack = true + name = "Pipeline Audit" + } } #Druid Query Processor druid = { - hosts = "{{druid_broker_host}}:8082" - secure = false - url = "/druid/v2/" - datasource = "telemetry-events" - response-parsing-timeout = 300000 + hosts = "{{druid_broker_host}}:8082" + secure = false + url = "/druid/v2/" + datasource = "telemetry-events" + response-parsing-timeout = 300000 client-backend = "com.ing.wbaa.druid.client.DruidAdvancedHttpClient" client-config = { druid-advanced-http-client ={ @@ -325,4 +268,4 @@ sunbird.course.redis.port=6379 sunbird.course.redis.relationCache.id=5 org.search.private.api.url="{{ org_search_service_private_endpoint }}" -tenant.pref.read.private.api.url="{{ tenant_preferance_read_private_service_endpoint }}" +tenant.pref.read.private.api.url="{{ tenant_preferance_read_private_service_endpoint }}" \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml new file mode 100644 index 000000000..24ccb8e7e --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml @@ -0,0 +1,53 @@ +- name: Ensure oci oss bucket exists + command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}" + register: check_bucket + when: dp_object_store_type == "oci" + tags: + - always + +- name: Create oci oss bucket + command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" + when: dp_object_store_type == "oci" and check_bucket.rc !=0 + tags: + - always + +- name: Copy Core Data Products to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - dataproducts-spark-cluster + +- name: Copy Ed Data Products to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - lern-dataproducts-spark-cluster + + +- name: Copy Framework Library to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - framework-spark-cluster + +- name: Copy Scruid Library to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - framework-spark-cluster + +- name: Copy configuration file to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - framework-spark-cluster \ No newline at end of file From 3dc05f56164131ac7ffa66b0798e722fda8991b1 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Mon, 9 Oct 2023 19:41:28 +0530 Subject: [PATCH 2/9] LR-627 | Ansible changes for Cloud agnostic --- .../roles/azure-cloud-storage/tasks/spark-cluster.yaml | 6 ------ ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml | 8 +------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml index d2da9f1fb..be479a732 100644 --- a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml +++ b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml @@ -1,6 +1,5 @@ - name: Ensure azure blob storage container exists command: az storage container create --name {{ bucket }} - when: dp_object_store_type == "azure" tags: - always @@ -8,7 +7,6 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" tags: - dataproducts-spark-cluster @@ -16,7 +14,6 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar async: 3600 poll: 10 - when: dp_object_store_type == "azure" tags: - lern-dataproducts-spark-cluster @@ -24,7 +21,6 @@ command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" tags: - framework-spark-cluster @@ -32,7 +28,6 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" tags: - framework-spark-cluster @@ -40,6 +35,5 @@ command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf async: 3600 poll: 10 - when: dp_object_store_type == "azure" tags: - framework-spark-cluster \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml index 24ccb8e7e..de8aa287e 100644 --- a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml +++ b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml @@ -1,13 +1,12 @@ - name: Ensure oci oss bucket exists command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}" register: check_bucket - when: dp_object_store_type == "oci" tags: - always - name: Create oci oss bucket command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" - when: dp_object_store_type == "oci" and check_bucket.rc !=0 + when: check_bucket.rc !=0 tags: - always @@ -15,7 +14,6 @@ command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force async: 3600 poll: 10 - when: dp_object_store_type == "oci" tags: - dataproducts-spark-cluster @@ -23,7 +21,6 @@ command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force async: 3600 poll: 10 - when: dp_object_store_type == "oci" tags: - lern-dataproducts-spark-cluster @@ -32,7 +29,6 @@ command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force async: 3600 poll: 10 - when: dp_object_store_type == "oci" tags: - framework-spark-cluster @@ -40,7 +36,6 @@ command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force async: 3600 poll: 10 - when: dp_object_store_type == "oci" tags: - framework-spark-cluster @@ -48,6 +43,5 @@ command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force async: 3600 poll: 10 - when: dp_object_store_type == "oci" tags: - framework-spark-cluster \ No newline at end of file From ba6da20ea19796114bfc45e3fcd39f30513ca439 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 11:12:33 +0530 Subject: [PATCH 3/9] LR-627 | Config changes for Cloud agnostic --- .../lern-data-products-deploy/templates/common.conf.j2 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 b/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 index 225d74582..b3c068f29 100644 --- a/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 @@ -14,12 +14,10 @@ storage.key.config="{{ dp_storage_key_config }}" storage.secret.config="{{ dp_storage_secret_config }}" reports.storage.key.config="{{ dp_reports_storage_key_config }}" reports.storage.secret.config="{{ dp_reports_storage_secret_config }}" -cloud_storage_type="{{dp_object_store_type}}" +cloud_storage_type="{{cloud_service_provider}}" -cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\1') }}" -cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}" - -lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M" +cloud_storage_endpoint="{{ cloud_private_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\1') }}" +cloud_storage_endpoint_with_protocol="{{ cloud_private_storage_endpoint }}" # Joblog Kafka appender config for cluster execution log.appender.kafka.enable="false" From 1c1dbf623309c2722dd2b39a63b663d3af3273a4 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 11:19:34 +0530 Subject: [PATCH 4/9] LR-627 | File name changes --- .../tasks/{spark-cluster.yaml => spark-cluster.yml} | 0 .../tasks/{spark-cluster.yaml => spark-cluster.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ansible/roles/azure-cloud-storage/tasks/{spark-cluster.yaml => spark-cluster.yml} (100%) rename ansible/roles/oci-cloud-storage/tasks/{spark-cluster.yaml => spark-cluster.yml} (100%) diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml similarity index 100% rename from ansible/roles/azure-cloud-storage/tasks/spark-cluster.yaml rename to ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml diff --git a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml similarity index 100% rename from ansible/roles/oci-cloud-storage/tasks/spark-cluster.yaml rename to ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml From 841abf9550fded10d1a8d450a4e3780d983ada99 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 11:50:45 +0530 Subject: [PATCH 5/9] LR-627 | Config changes for Cloud agnostic --- ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml | 2 +- ansible/roles/lern-data-products-deploy/tasks/main.yml | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml index be479a732..1374769bd 100644 --- a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml +++ b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml @@ -1,7 +1,7 @@ - name: Ensure azure blob storage container exists command: az storage container create --name {{ bucket }} tags: - - always + - ensure-container - name: Copy Core Data Products to azure blob command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 4cd977061..5b3bde9a3 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -5,9 +5,11 @@ tasks_from: spark-cluster.yml apply: tags: - - always + - ensure-container tags: - - always + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster - name: Copy Core Data Products copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }} From 11e2d6ac10a254e13f77800db963db72f73f84c6 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 12:02:26 +0530 Subject: [PATCH 6/9] LR-627 | Ansible changes for Cloud agnostic --- .../azure-cloud-storage/tasks/spark-cluster.yml | 2 +- .../lern-data-products-deploy/tasks/main.yml | 16 ++++------------ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml index 1374769bd..67691f480 100644 --- a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml +++ b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml @@ -10,7 +10,7 @@ tags: - dataproducts-spark-cluster -- name: Copy Ed Data Products to azure blob +- name: Copy LERN Data Products to azure blob command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar async: 3600 poll: 10 diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 5b3bde9a3..b58503f3a 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -3,9 +3,7 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - apply: - tags: - - ensure-container + tags: ensure-container tags: - dataproducts-spark-cluster - lern-dataproducts-spark-cluster @@ -20,9 +18,7 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - apply: - tags: - - dataproducts-spark-cluster + tags: dataproducts-spark-cluster tags: - dataproducts-spark-cluster @@ -36,9 +32,7 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - apply: - tags: - - lern-dataproducts-spark-cluster + tags: lern-dataproducts-spark-cluster tags: - lern-dataproducts-spark-cluster @@ -51,9 +45,7 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - apply: - tags: - - framework-spark-cluster + tags: framework-spark-cluster tags: - framework-spark-cluster From 81449a3bd09f03305f949647a4d2e3d59fcb120c Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 12:09:35 +0530 Subject: [PATCH 7/9] LR-627 | Ansible changes for Cloud agnostic --- .../roles/lern-data-products-deploy/tasks/main.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index b58503f3a..6c91274e4 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -3,7 +3,8 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - tags: ensure-container + apply: + tags: ensure-container tags: - dataproducts-spark-cluster - lern-dataproducts-spark-cluster @@ -18,7 +19,8 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - tags: dataproducts-spark-cluster + apply: + tags: dataproducts-spark-cluster tags: - dataproducts-spark-cluster @@ -32,7 +34,8 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - tags: lern-dataproducts-spark-cluster + apply: + tags: lern-dataproducts-spark-cluster tags: - lern-dataproducts-spark-cluster @@ -45,7 +48,8 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - tags: framework-spark-cluster + apply: + tags: framework-spark-cluster tags: - framework-spark-cluster From 68abe912ee33b9daf1bb9930d320d0a8a3d031ec Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 12:16:30 +0530 Subject: [PATCH 8/9] LR-627 | Ansible changes for Cloud agnostic --- .../tasks/spark-cluster.yml | 4 ++- .../lern-data-products-deploy/tasks/main.yml | 31 +------------------ 2 files changed, 4 insertions(+), 31 deletions(-) diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml index 67691f480..e318d2072 100644 --- a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml +++ b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml @@ -1,7 +1,9 @@ - name: Ensure azure blob storage container exists command: az storage container create --name {{ bucket }} tags: - - ensure-container + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster - name: Copy Core Data Products to azure blob command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 6c91274e4..683462b16 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -3,8 +3,6 @@ include_role: name: "{{ cloud_service_provider }}-cloud-storage" tasks_from: spark-cluster.yml - apply: - tags: ensure-container tags: - dataproducts-spark-cluster - lern-dataproducts-spark-cluster @@ -15,44 +13,17 @@ tags: - dataproducts -- name: Copy Core Data Products - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: dataproducts-spark-cluster - tags: - - dataproducts-spark-cluster - -- name: Unarchive Ed Data Products +- name: Unarchive LERN Data Products become: yes unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }} tags: - lern-dataproducts -- name: Copy LERN Data Products - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: lern-dataproducts-spark-cluster - tags: - - lern-dataproducts-spark-cluster - - name: Copy Framework Library copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - framework -- name: Copy Analytics Framework related libraries - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: framework-spark-cluster - tags: - - framework-spark-cluster - - name: Copy Scruid Library copy: src={{ scruid_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: From c8ba7486ec15071d7d8037064c5b2a1ef327610e Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 10 Oct 2023 12:22:22 +0530 Subject: [PATCH 9/9] LR-627 | Ansible changes for Cloud agnostic --- .../lern-data-products-deploy/tasks/main.yml | 20 +++++++++---------- .../oci-cloud-storage/tasks/spark-cluster.yml | 11 ++++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 683462b16..608637aac 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -1,13 +1,4 @@ ## Data products deployment ## -- name: Ensure the storage container exists - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - tags: - - dataproducts-spark-cluster - - lern-dataproducts-spark-cluster - - framework-spark-cluster - - name: Copy Core Data Products copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: @@ -62,7 +53,16 @@ line: 'log.appender.kafka.enable="true"' tags: - framework-spark-cluster - + +- name: Ensure the storage container exists + include_role: + name: "{{ cloud_service_provider }}-cloud-storage" + tasks_from: spark-cluster.yml + tags: + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster + - name: Copy log4j2 xml file template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }} tags: [ dataproducts, framework, lern-dataproducts ] diff --git a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml index de8aa287e..5d71e6a73 100644 --- a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml +++ b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml @@ -2,13 +2,17 @@ command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}" register: check_bucket tags: - - always + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster - name: Create oci oss bucket command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" when: check_bucket.rc !=0 tags: - - always + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster - name: Copy Core Data Products to oci oss command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force @@ -17,14 +21,13 @@ tags: - dataproducts-spark-cluster -- name: Copy Ed Data Products to oci oss +- name: Copy LERN Data Products to oci oss command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force async: 3600 poll: 10 tags: - lern-dataproducts-spark-cluster - - name: Copy Framework Library to oci oss command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force async: 3600