Issue #LR-627 | Ansible changes for Cloud agnostic

ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml

@@ -0,0 +1,41 @@
+- name: Ensure azure blob storage container exists
+  command: az storage container create --name {{ bucket }}
+  tags:
+    - dataproducts-spark-cluster
+    - lern-dataproducts-spark-cluster
+    - framework-spark-cluster
+
+- name: Copy Core Data Products to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}
+  async: 3600
+  poll: 10
+  tags:
+    - dataproducts-spark-cluster
+
+- name: Copy LERN Data Products to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar
+  async: 3600
+  poll: 10
+  tags:
+    - lern-dataproducts-spark-cluster
+
+- name: Copy Framework Library to azure blob
+  command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }}
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster
+
+- name: Copy Scruid Library to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }}
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster
+
+- name: Copy configuration file to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster

ansible/roles/lern-data-products-deploy/tasks/main.yml

@@ -1,60 +1,20 @@
 ## Data products deployment ##
-- name: Ensure the storage container exists
-  include_role:
-    name: "{{ cloud_service_provider }}-cloud-storage"
-    tasks_from: spark-cluster.yml
-    apply:
-      tags:
-        - always
-  tags:
-    - always
-
 - name: Copy Core Data Products
   copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
     - dataproducts
 
-- name: Copy Core Data Products
-  include_role:
-    name: "{{ cloud_service_provider }}-cloud-storage"
-    tasks_from: spark-cluster.yml
-    apply:
-      tags:
-        - dataproducts-spark-cluster
-  tags:
-    - dataproducts-spark-cluster
-
-- name: Unarchive Ed Data Products
+- name: Unarchive LERN Data Products
   become: yes
   unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }}
   tags:
     - lern-dataproducts
 
-- name: Copy LERN Data Products
-  include_role:
-    name: "{{ cloud_service_provider }}-cloud-storage"
-    tasks_from: spark-cluster.yml
-    apply:
-      tags:
-        - lern-dataproducts-spark-cluster
-  tags:
-    - lern-dataproducts-spark-cluster    
-
 - name: Copy Framework Library
   copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
     - framework
 
-- name: Copy Analytics Framework related libraries
-  include_role:
-    name: "{{ cloud_service_provider }}-cloud-storage"
-    tasks_from: spark-cluster.yml
-    apply:
-      tags:
-        - framework-spark-cluster
-  tags:
-    - framework-spark-cluster
-
 - name: Copy Scruid Library
   copy: src={{ scruid_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
@@ -93,7 +53,16 @@
     line: 'log.appender.kafka.enable="true"'
   tags:
     - framework-spark-cluster
-
+
+- name: Run spark cluster tasks
+  include_role:
+    name: "{{ cloud_service_provider }}-cloud-storage"
+    tasks_from: spark-cluster.yml
+  tags:
+    - dataproducts-spark-cluster
+    - lern-dataproducts-spark-cluster
+    - framework-spark-cluster    
+
 - name: Copy log4j2 xml file
   template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   tags: [ dataproducts, framework, lern-dataproducts ]

ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml

@@ -0,0 +1,50 @@
+- name: Ensure oci oss bucket exists
+  command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}"
+  register: check_bucket
+  tags:
+    - dataproducts-spark-cluster
+    - lern-dataproducts-spark-cluster
+    - framework-spark-cluster
+
+- name: Create oci oss bucket 
+  command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}"
+  when: check_bucket.rc !=0
+  tags:
+    - dataproducts-spark-cluster
+    - lern-dataproducts-spark-cluster
+    - framework-spark-cluster
+
+- name: Copy Core Data Products to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force
+  async: 3600
+  poll: 10
+  tags:
+    - dataproducts-spark-cluster
+
+- name: Copy LERN Data Products to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force
+  async: 3600
+  poll: 10
+  tags:
+    - lern-dataproducts-spark-cluster
+
+- name: Copy Framework Library to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster
+
+- name: Copy Scruid Library to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster
+
+- name: Copy configuration file to oci oss
+  command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster