diff --git a/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml new file mode 100644 index 000000000..e318d2072 --- /dev/null +++ b/ansible/roles/azure-cloud-storage/tasks/spark-cluster.yml @@ -0,0 +1,41 @@ +- name: Ensure azure blob storage container exists + command: az storage container create --name {{ bucket }} + tags: + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster + +- name: Copy Core Data Products to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} + async: 3600 + poll: 10 + tags: + - dataproducts-spark-cluster + +- name: Copy LERN Data Products to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar + async: 3600 + poll: 10 + tags: + - lern-dataproducts-spark-cluster + +- name: Copy Framework Library to azure blob + command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} + async: 3600 + poll: 10 + tags: + - framework-spark-cluster + +- name: Copy Scruid Library to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} + async: 3600 + poll: 10 + tags: + - framework-spark-cluster + +- name: Copy configuration file to azure blob + command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf + async: 3600 + poll: 10 + tags: + - framework-spark-cluster \ No newline at end of file diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 4cd977061..32ef032bb 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -1,60 +1,20 @@ ## Data products deployment ## -- name: Ensure the storage container exists - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: - - always - tags: - - always - - name: Copy Core Data Products copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - dataproducts -- name: Copy Core Data Products - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: - - dataproducts-spark-cluster - tags: - - dataproducts-spark-cluster - -- name: Unarchive Ed Data Products +- name: Unarchive LERN Data Products become: yes unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }} tags: - lern-dataproducts -- name: Copy LERN Data Products - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: - - lern-dataproducts-spark-cluster - tags: - - lern-dataproducts-spark-cluster - - name: Copy Framework Library copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: - framework -- name: Copy Analytics Framework related libraries - include_role: - name: "{{ cloud_service_provider }}-cloud-storage" - tasks_from: spark-cluster.yml - apply: - tags: - - framework-spark-cluster - tags: - - framework-spark-cluster - - name: Copy Scruid Library copy: src={{ scruid_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: @@ -93,7 +53,16 @@ line: 'log.appender.kafka.enable="true"' tags: - framework-spark-cluster - + +- name: Run spark cluster tasks + include_role: + name: "{{ cloud_service_provider }}-cloud-storage" + tasks_from: spark-cluster.yml + tags: + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster + - name: Copy log4j2 xml file template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }} tags: [ dataproducts, framework, lern-dataproducts ] diff --git a/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml new file mode 100644 index 000000000..5d71e6a73 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/spark-cluster.yml @@ -0,0 +1,50 @@ +- name: Ensure oci oss bucket exists + command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}" + register: check_bucket + tags: + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster + +- name: Create oci oss bucket + command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" + when: check_bucket.rc !=0 + tags: + - dataproducts-spark-cluster + - lern-dataproducts-spark-cluster + - framework-spark-cluster + +- name: Copy Core Data Products to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force + async: 3600 + poll: 10 + tags: + - dataproducts-spark-cluster + +- name: Copy LERN Data Products to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force + async: 3600 + poll: 10 + tags: + - lern-dataproducts-spark-cluster + +- name: Copy Framework Library to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force + async: 3600 + poll: 10 + tags: + - framework-spark-cluster + +- name: Copy Scruid Library to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force + async: 3600 + poll: 10 + tags: + - framework-spark-cluster + +- name: Copy configuration file to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force + async: 3600 + poll: 10 + tags: + - framework-spark-cluster \ No newline at end of file