ED-2087: Add OCI support for LernDataproducts (#52) (#61)

Co-authored-by: Deepak Devadathan <[email protected]>

Author: AmiableAnil

.gitignore

@@ -21,3 +21,4 @@ lern-data-products/src/test/resources.csv
 lern-data-products/src/test/resources/tpd_metrics/
 **.csv.crc
 **/bin/*
+.vscode

ansible/artifacts-download.yml

@@ -41,3 +41,15 @@
         aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
       when: cloud_service_provider == "aws"  
 
+    - name: download artifact from oci oss
+      include_role:
+        name: oci-cloud-storage
+        apply:
+          environment:
+           OCI_CLI_AUTH: "instance_principal"
+        tasks_from: download.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        oss_path: "{{ artifact }}"
+      when: cloud_service_provider == "oci" 

ansible/artifacts-upload.yml

@@ -41,3 +41,16 @@
         aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
         aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
       when: cloud_service_provider == "aws"
+
+    - name: upload artifact to oci oss
+      include_role:
+        name: oci-cloud-storage
+        apply:
+          environment:
+           OCI_CLI_AUTH: "instance_principal"
+        tasks_from: upload.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        oss_path: "{{ artifact }}"
+      when: cloud_service_provider == "oci"

ansible/lpa_lern-data-products_deploy.yml

@@ -7,5 +7,6 @@
   environment:
     AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
     AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+    OCI_CLI_AUTH: "instance_principal"
   roles:
    - lern-data-products-deploy

ansible/roles/lern-data-products-deploy/tasks/main.yml

@@ -5,6 +5,19 @@
   tags:
     - always
 
+- name: Ensure oci oss bucket exists
+  command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}"
+  register: check_bucket
+  when: dp_object_store_type == "oci"
+  tags:
+    - always
+
+- name: Create oci oss bucket 
+  command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}"
+  when: dp_object_store_type == "oci" and check_bucket.rc !=0
+  tags:
+    - always
+
 - name: Copy Core Data Products
   copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
@@ -14,9 +27,19 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
+  tags:
+    - dataproducts-spark-cluster 
+
+- name: Copy Core Data Products to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
   tags:
     - dataproducts-spark-cluster 
 
+
 - name: Unarchive Ed Data Products
   become: yes
   unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }}
@@ -27,9 +50,18 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
   tags:
     - lern-dataproducts-spark-cluster
 
+- name: Copy Ed Data Products to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
+  tags:
+    - lern-dataproducts-spark-cluster 
+
 - name: Copy Framework Library
   copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
@@ -39,6 +71,15 @@
   command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }}
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
+  tags:
+    - framework-spark-cluster
+
+- name: Copy Framework Library to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
   tags:
     - framework-spark-cluster
 
@@ -51,9 +92,18 @@
   command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }}
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
   tags:
     - framework-spark-cluster  
 
+- name: Copy Scruid Library to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
+  tags:
+    - framework-spark-cluster 
+
 - name: Copy Job Manager
   copy: src={{ analytics_job_manager_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
@@ -92,9 +142,18 @@
   command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf
   async: 3600
   poll: 10
+  when: dp_object_store_type == "azure"
   tags:
     - framework-spark-cluster    
 
+- name: Copy configuration file to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force
+  async: 3600
+  poll: 10
+  when: dp_object_store_type == "oci"
+  tags:
+    - framework-spark-cluster  
+    
 - name: Copy log4j2 xml file
   template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   tags: [ dataproducts, framework, lern-dataproducts ]

ansible/roles/lern-data-products-deploy/templates/common.conf.j2

@@ -16,14 +16,24 @@ reports.storage.key.config="{{ dp_reports_storage_key_config }}"
 reports.storage.secret.config="{{ dp_reports_storage_secret_config }}"
 {% if dp_object_store_type == "azure" %}
 cloud_storage_type="azure"
-{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %}
-cloud_storage_type="s3"
-cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}"
+{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3" or dp_object_store_type == "oci") %}
+
+cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\1') }}"
 cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}"
+
+{% if cloud_service_provider == "oci" %}
+cloud_storage_type="oci"
+storage.endpoint.config="{{ dp_storage_endpoint_config }}"
+oci_storage_key="{{ s3_storage_key }}"
+oci_storage_secret="{{ s3_storage_secret }}"
+{% else %}
+cloud_storage_type="s3"
 aws_storage_key="{{ s3_storage_key }}"
 aws_storage_secret="{{ s3_storage_secret }}"
 {% endif %}
 
+{% endif %}
+
 lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M"
 
 # Joblog Kafka appender config for cluster execution
@@ -112,7 +122,7 @@ azure {
 }
 
 ## Reports - Global config
-cloud.container.reports="reports"
+cloud.container.reports="{{reports_container}}"
 
 # course metrics container in azure
 course.metrics.cassandra.sunbirdKeyspace="sunbird"
@@ -252,9 +262,9 @@ etbtextbook.filename="ETB_textbook_data.csv"
 etb.dialcode.druid.length={{ etb_dialcode_list_druid_length }}
 
 
-druid.report.default.storage="azure"
+druid.report.default.storage="{{druid_storage_type}}"
 druid.report.date.format="yyyy-MM-dd"
-druid.report.default.container="report-verification"
+druid.report.default.container="{{cloud_storage_report_verfication_bucketname}}"
 
 ## Collection Exhaust Jobs Configuration -- Start ##