diff --git a/.gitignore b/.gitignore index 1a80f3601..5b6d77223 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ lern-data-products/src/test/resources.csv lern-data-products/src/test/resources/tpd_metrics/ **.csv.crc **/bin/* +.vscode \ No newline at end of file diff --git a/ansible/artifacts-download.yml b/ansible/artifacts-download.yml index c06130864..f5c1e6d41 100644 --- a/ansible/artifacts-download.yml +++ b/ansible/artifacts-download.yml @@ -41,3 +41,15 @@ aws_secret_access_key: "{{ cloud_artifact_storage_secret }}" when: cloud_service_provider == "aws" + - name: download artifact from oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: download.yml + vars: + local_file_or_folder_path: "{{ artifact_path }}" + oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}" + oss_path: "{{ artifact }}" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/artifacts-upload.yml b/ansible/artifacts-upload.yml index 305492afc..3e444ef16 100644 --- a/ansible/artifacts-upload.yml +++ b/ansible/artifacts-upload.yml @@ -41,3 +41,16 @@ aws_access_key_id: "{{ cloud_artifact_storage_accountname }}" aws_secret_access_key: "{{ cloud_artifact_storage_secret }}" when: cloud_service_provider == "aws" + + - name: upload artifact to oci oss + include_role: + name: oci-cloud-storage + apply: + environment: + OCI_CLI_AUTH: "instance_principal" + tasks_from: upload.yml + vars: + local_file_or_folder_path: "{{ artifact_path }}" + oss_bucket_name: "{{ cloud_storage_artifacts_bucketname }}" + oss_path: "{{ artifact }}" + when: cloud_service_provider == "oci" \ No newline at end of file diff --git a/ansible/lpa_lern-data-products_deploy.yml b/ansible/lpa_lern-data-products_deploy.yml index 07285fbc6..e979c97fb 100644 --- a/ansible/lpa_lern-data-products_deploy.yml +++ b/ansible/lpa_lern-data-products_deploy.yml @@ -7,5 +7,6 @@ environment: AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}" AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}" + OCI_CLI_AUTH: "instance_principal" roles: - lern-data-products-deploy diff --git a/ansible/roles/lern-data-products-deploy/tasks/main.yml b/ansible/roles/lern-data-products-deploy/tasks/main.yml index 761974341..646f25edc 100644 --- a/ansible/roles/lern-data-products-deploy/tasks/main.yml +++ b/ansible/roles/lern-data-products-deploy/tasks/main.yml @@ -5,6 +5,19 @@ tags: - always +- name: Ensure oci oss bucket exists + command: "/home/{{analytics_user}}/bin/oci os bucket get --name {{ bucket }}" + register: check_bucket + when: dp_object_store_type == "oci" + tags: + - always + +- name: Create oci oss bucket + command: "/home/{{analytics_user}}/bin/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" + when: dp_object_store_type == "oci" and check_bucket.rc !=0 + tags: + - always + - name: Copy Core Data Products copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: @@ -14,9 +27,19 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} async: 3600 poll: 10 + when: dp_object_store_type == "azure" + tags: + - dataproducts-spark-cluster + +- name: Copy Core Data Products to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" tags: - dataproducts-spark-cluster + - name: Unarchive Ed Data Products become: yes unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }} @@ -27,9 +50,18 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/lern-data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/lern-data-products-1.0/lern-data-products-1.0.jar async: 3600 poll: 10 + when: dp_object_store_type == "azure" tags: - lern-dataproducts-spark-cluster +- name: Copy Ed Data Products to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - lern-dataproducts-spark-cluster + - name: Copy Framework Library copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: @@ -39,6 +71,15 @@ command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} async: 3600 poll: 10 + when: dp_object_store_type == "azure" + tags: + - framework-spark-cluster + +- name: Copy Framework Library to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" tags: - framework-spark-cluster @@ -51,9 +92,18 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} async: 3600 poll: 10 + when: dp_object_store_type == "azure" tags: - framework-spark-cluster +- name: Copy Scruid Library to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - framework-spark-cluster + - name: Copy Job Manager copy: src={{ analytics_job_manager_artifact }} dest={{ analytics.home }}/models-{{ model_version }} tags: @@ -92,9 +142,18 @@ command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/lernapplication.conf --name models-{{ model_version }}/lernapplication.conf async: 3600 poll: 10 + when: dp_object_store_type == "azure" tags: - framework-spark-cluster +- name: Copy configuration file to oci oss + command: /home/{{analytics_user}}/bin/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force + async: 3600 + poll: 10 + when: dp_object_store_type == "oci" + tags: + - framework-spark-cluster + - name: Copy log4j2 xml file template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }} tags: [ dataproducts, framework, lern-dataproducts ] diff --git a/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 b/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 index 201444a19..6bbb77bca 100644 --- a/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/lern-data-products-deploy/templates/common.conf.j2 @@ -16,14 +16,24 @@ reports.storage.key.config="{{ dp_reports_storage_key_config }}" reports.storage.secret.config="{{ dp_reports_storage_secret_config }}" {% if dp_object_store_type == "azure" %} cloud_storage_type="azure" -{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %} -cloud_storage_type="s3" -cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}" +{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3" or dp_object_store_type == "oci") %} + +cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\1') }}" cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}" + +{% if cloud_service_provider == "oci" %} +cloud_storage_type="oci" +storage.endpoint.config="{{ dp_storage_endpoint_config }}" +oci_storage_key="{{ s3_storage_key }}" +oci_storage_secret="{{ s3_storage_secret }}" +{% else %} +cloud_storage_type="s3" aws_storage_key="{{ s3_storage_key }}" aws_storage_secret="{{ s3_storage_secret }}" {% endif %} +{% endif %} + lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M" # Joblog Kafka appender config for cluster execution @@ -112,7 +122,7 @@ azure { } ## Reports - Global config -cloud.container.reports="reports" +cloud.container.reports="{{reports_container}}" # course metrics container in azure course.metrics.cassandra.sunbirdKeyspace="sunbird" @@ -252,9 +262,9 @@ etbtextbook.filename="ETB_textbook_data.csv" etb.dialcode.druid.length={{ etb_dialcode_list_druid_length }} -druid.report.default.storage="azure" +druid.report.default.storage="{{druid_storage_type}}" druid.report.date.format="yyyy-MM-dd" -druid.report.default.container="report-verification" +druid.report.default.container="{{cloud_storage_report_verfication_bucketname}}" ## Collection Exhaust Jobs Configuration -- Start ## diff --git a/ansible/roles/lern-data-products-deploy/templates/lern-model-config.j2 b/ansible/roles/lern-data-products-deploy/templates/lern-model-config.j2 index faaaa4bad..9e218876e 100644 --- a/ansible/roles/lern-data-products-deploy/templates/lern-model-config.j2 +++ b/ansible/roles/lern-data-products-deploy/templates/lern-model-config.j2 @@ -30,16 +30,16 @@ config() { if [ ! -z "$2" ]; then keyword=$2; fi case "$1" in "assessment-correction") - echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}' + echo '{"search":{"type":"{{dp_object_store_type}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}' ;; "assessment-archival") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"azure","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}' ;; "assessment-archived-removal") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"azure","blobExt":"csv.gz","reportPath":"archived-data/","container":"reports"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"{{dp_object_store_type}}","blobExt":"csv.gz","reportPath":"archived-data/","container":"{{reports_container}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}' ;; "collection-summary-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"], "contentType": "Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"azure","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' ;; "score-metric-migration-job") echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.ScoreMetricMigrationJob","modelParams":{"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Score Metric Migration Job"}' @@ -48,31 +48,31 @@ config() { echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.AssessmentScoreCorrectionJob","modelParams":{"assessment.score.correction.batches":"","cassandraReadConsistency":"QUORUM","cassandraWriteConsistency":"QUORUM","csvPath":"/mount/data/analytics/score_correction","isDryRunMode":true,"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":30,"appName":"Assessment Score Correction Job"}' ;; "course-batch-status-updater") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"azure","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{learningservice_ip}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{learningservice_ip}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' ;; "collection-summary-report-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"druid_storage_account_key","storageSecretConfig":"druid_storage_account_secret","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"azure","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.audit.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{dp_object_store_type}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' ;; "userinfo-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' ;; "response-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' ;; "response-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' ;; "progress-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' ;; "progress-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' ;; "cassandra-migration") echo '{"search":{"type":"none"},"model":"org.sunbird.lms.updater.CassandraMigratorJob","modelParams":{"cassandraDataHost":"{{ core_cassandra_host }}","cassandraMigrateHost":"{{ report_cassandra_host }}","keyspace":"sunbird_courses","cassandraDataTable":"user_enrolments","cassandraMigrateTable":"{{ report_user_enrolment_table }}","repartitionColumns":"batchid"},"parallelization":10,"appName":"Cassandra Migrator","deviceMapping":false}' ;; "job-manager") - echo '{"jobsCount":'$jobManagerJobsCount',"topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}' + echo '{"jobsCount":"'$jobManagerJobsCount'","topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}' ;; "admin-user-reports") echo '{"search":{"type":"none"},"model":"org.sunbird.userorg.job.report.StateAdminReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' @@ -81,10 +81,10 @@ config() { echo '{"search":{"type":"none"},"model":"org.sunbird.userorg.job.report.StateAdminGeoReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' ;; "course-enrollment-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.lms.job.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.job.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' ;; "course-consumption-report") - echo '{"search": {"type": "none"},"model": "org.sunbird.lms.job.report.CourseConsumptionJob","modelParams": {"esConfig": {"request": {"filters": {"objectType": ["Content"],"contentType": ["Course"],"identifier": [],"status": ["Live"]}}},"reportConfig": {"id": "tpd_metrics","labels": {"date": "Date","status": "Batch Status","timespent": "Timespent in mins","courseName": "Course Name","batchName": "Batch Name"},"dateRange": {"staticInterval": "LastDay","granularity": "all"},"metrics": [{"metric": "totalCoursePlays","label": "Total Course Plays (in mins)","druidQuery": {"queryType": "groupBy","dataSource": "summary-events","intervals":"LastDay","aggregations": [{"name": "sum__edata_time_spent","type": "doubleSum","fieldName": "edata_time_spent"}],"dimensions": [{"fieldName": "object_rollup_l1","aliasName": "courseId"}, {"fieldName": "uid","aliasName": "userId"}, {"fieldName": "context_cdata_id","aliasName": "batchId"}],"filters": [{"type": "equals","dimension": "eid","value": "ME_WORKFLOW_SUMMARY"}, {"type": "in","dimension": "dimensions_pdata_id","values": ["'$producerEnv'.app", "'$producerEnv'.portal"]}, {"type": "equals","dimension": "dimensions_type","value": "content"}, {"type": "equals","dimension": "dimensions_mode","value": "play"}, {"type": "equals","dimension": "context_cdata_type","value": "batch"}],"postAggregation": [{"type": "arithmetic","name": "timespent","fields": {"leftField": "sum__edata_time_spent","rightField": 60,"rightFieldType": "constant"},"fn": "/"}],"descending": "false"}}],"output": [{"type": "csv","metrics": ["timespent"],"dims": []}],"queryType": "groupBy"},"store": "azure","format":"csv","key": "druid-reports/","filePath": "druid-reports/","container":"'$bucket'","folderPrefix": ["slug", "reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "TPD Course Consumption Metrics Model","deviceMapping": false}' + echo '{"search":{"type":"none"},"model":"org.sunbird.lms.job.report.CourseConsumptionJob","modelParams":{"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]}}},"reportConfig":{"id":"tpd_metrics","labels":{"date":"Date","status":"Batch Status","timespent":"Timespent in mins","courseName":"Course Name","batchName":"Batch Name"},"dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalCoursePlays","label":"Total Course Plays (in mins)","druidQuery":{"queryType":"groupBy","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"object_rollup_l1","aliasName":"courseId"},{"fieldName":"uid","aliasName":"userId"},{"fieldName":"context_cdata_id","aliasName":"batchId"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"context_cdata_type","value":"batch"}],"postAggregation":[{"type":"arithmetic","name":"timespent","fields":{"leftField":"sum__edata_time_spent","rightField":60,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}}],"output":[{"type":"csv","metrics":["timespent"],"dims":[]}],"queryType":"groupBy"},"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Consumption Metrics Model","deviceMapping":false}' ;; "*") echo "Unknown model code" diff --git a/ansible/roles/lern-data-products-deploy/templates/lern-model-config.json.j2 b/ansible/roles/lern-data-products-deploy/templates/lern-model-config.json.j2 index 067ea9095..130ff0c18 100644 --- a/ansible/roles/lern-data-products-deploy/templates/lern-model-config.json.j2 +++ b/ansible/roles/lern-data-products-deploy/templates/lern-model-config.json.j2 @@ -73,7 +73,7 @@ "limit": 10000 } }, - "store": "azure", + "store": "{{dp_object_store_type}}", "format":"csv", "key": "druid-reports/", "filePath": "druid-reports/", @@ -184,7 +184,7 @@ }], "queryType": "groupBy" }, - "store": "azure", + "store": "{{dp_object_store_type}}", "format":"csv", "key": "druid-reports/", "filePath": "druid-reports/", @@ -213,7 +213,7 @@ { "name": "denorm", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "bucket": "{{ bucket }}", @@ -259,7 +259,7 @@ { "name": "failed", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "bucket": "{{ bucket }}", @@ -273,7 +273,7 @@ { "name": "unique", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "bucket": "{{ bucket }}", @@ -287,7 +287,7 @@ { "name": "raw", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "bucket": "{{ bucket }}", @@ -301,7 +301,7 @@ { "name": "channel-raw", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "folder": true, @@ -316,7 +316,7 @@ { "name": "channel-summary", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "folder": true, @@ -331,7 +331,7 @@ { "name": "derived", "search": { - "type": "azure", + "type": "{{dp_object_store_type}}", "queries": [ { "bucket": "{{ bucket }}", diff --git a/ansible/roles/oci-cloud-storage/defaults/main.yml b/ansible/roles/oci-cloud-storage/defaults/main.yml new file mode 100644 index 000000000..72727de16 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/defaults/main.yml @@ -0,0 +1,3 @@ +oss_bucket_name: "" +oss_path: "" +local_file_or_folder_path: "" diff --git a/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml b/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml new file mode 100644 index 000000000..6ed4e6b8b --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/delete-folder.yml @@ -0,0 +1,5 @@ +--- +- name: delete files and folders recursively + shell: "oci os object bulk-delete -ns {{oss_namespace}} -bn {{oss_bucket_name}} --prefix {{oss_path}} --force" + async: 3600 + poll: 10 diff --git a/ansible/roles/oci-cloud-storage/tasks/delete.yml b/ansible/roles/oci-cloud-storage/tasks/delete.yml new file mode 100644 index 000000000..65d18843c --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/delete.yml @@ -0,0 +1,7 @@ +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: Upload to oci oss bucket + command: oci os object delete -bn {{ oss_bucket_name }} --name {{ oss_path }} --force + async: 3600 + poll: 10 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/download.yml b/ansible/roles/oci-cloud-storage/tasks/download.yml new file mode 100644 index 000000000..63e776c34 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/download.yml @@ -0,0 +1,7 @@ +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: download files from oci oss bucket + command: oci os object bulk-download -bn {{ oss_bucket_name }} --prefix {{ oss_path }} --dest-dir {{ local_file_or_folder_path }} + async: 3600 + poll: 10 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/main.yml b/ansible/roles/oci-cloud-storage/tasks/main.yml new file mode 100644 index 000000000..6f9dca6b6 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: delete files from oci oss bucket + include: delete.yml + +- name: delete folders from oci oss bucket recursively + include: delete-folder.yml + + +- name: download file from oss + include: download.yml + +- name: upload files from a local to oci oss + include: upload.yml + +- name: upload files and folder from local directory to oci oss + include: upload-folder.yml + + diff --git a/ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml b/ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml new file mode 100644 index 000000000..8ad3f257e --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/oss-delete-batch-no-poll.yml @@ -0,0 +1,5 @@ +--- +- name: delete files and folders - deleting {{ oss_bucket_name }}/{{ object_prefix }} + shell: oci os object bulk-delete -bn {{oss_bucket_name}} --prefix {{object_prefix}} --force + async: 1800 + poll: 0 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml b/ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml new file mode 100644 index 000000000..6d01756df --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/oss-upload-batch-no-poll.yml @@ -0,0 +1,5 @@ +--- +- name: upload files and folders - uploading {{ oss_bucket_name }}/{{ object_prefix }} + shell: oci os object bulk-upload -bn {{oss_bucket_name}} --prefix {{object_prefix}} --src-dir {{local_file_or_folder_path}} --overwrite + async: 1800 + poll: 0 \ No newline at end of file diff --git a/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml b/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml new file mode 100644 index 000000000..2e0d45bcb --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/upload-folder.yml @@ -0,0 +1,15 @@ +--- +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: Upload folder to oci oss bucket + command: oci os object bulk-upload -bn {{ oss_bucket_name }} --src-dir {{ local_file_or_folder_path }} --content-type auto --overwrite + async: 3600 + poll: 10 + when: oss_path|length == 0 + +- name: Upload folder to oci oss bucket + command: oci os object bulk-upload -bn {{ oss_bucket_name }} --prefix {{oss_path}} --src-dir {{ local_file_or_folder_path }} --content-type auto --overwrite + async: 3600 + poll: 10 + when: oss_path|length > 0 diff --git a/ansible/roles/oci-cloud-storage/tasks/upload.yml b/ansible/roles/oci-cloud-storage/tasks/upload.yml new file mode 100644 index 000000000..9e1ceb428 --- /dev/null +++ b/ansible/roles/oci-cloud-storage/tasks/upload.yml @@ -0,0 +1,8 @@ +--- +- name: Ensure oci oss bucket exists + command: oci os bucket get --name {{ oss_bucket_name }} + +- name: Upload to oci oss bucket + command: oci os object put -bn {{ oss_bucket_name }} --name {{ oss_path }} --file {{ local_file_or_folder_path }} --content-type auto --force + async: 3600 + poll: 10 diff --git a/lern-data-products/pom.xml b/lern-data-products/pom.xml index ec653711b..a6bb02abd 100644 --- a/lern-data-products/pom.xml +++ b/lern-data-products/pom.xml @@ -12,8 +12,8 @@ 1.4.11 2.12 2.12.10 - 3.0 - 3.1.0 + 3.2 + 3.2.1 @@ -104,6 +104,14 @@ org.apache.hadoop hadoop-client + + org.apache.hadoop + hadoop-client-api + + + org.apache.hadoop + hadoop-client-runtime + org.apache.xbean xbean-asm6-shaded @@ -281,7 +289,7 @@ org.sunbird cloud-store-sdk_2.12 - 1.4.4 + 1.4.6 com.microsoft.azure diff --git a/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala b/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala index 8705fab6f..3b6edf2b9 100644 --- a/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala +++ b/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala @@ -51,7 +51,7 @@ trait BaseReportsJob { val storageKey = modelParams.getOrElse("storageKeyConfig", "reports_storage_key").asInstanceOf[String]; val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; store.toLowerCase() match { - case "s3" => + case "s3" | "oci" => spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(storageKey)); spark.sparkContext.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(storageSecret)); case "azure" => diff --git a/lern-data-products/src/main/scala/org/sunbird/core/exhaust/OnDemandExhaustJob.scala b/lern-data-products/src/main/scala/org/sunbird/core/exhaust/OnDemandExhaustJob.scala index a37777a88..e88be13eb 100644 --- a/lern-data-products/src/main/scala/org/sunbird/core/exhaust/OnDemandExhaustJob.scala +++ b/lern-data-products/src/main/scala/org/sunbird/core/exhaust/OnDemandExhaustJob.scala @@ -171,7 +171,7 @@ trait OnDemandExhaustJob { fc.getHadoopFileUtil().delete(conf, tempDir); val filePrefix = storageConfig.store.toLowerCase() match { // $COVERAGE-OFF$ Disabling scoverage - case "s3" => + case "s3" | "oci" => CommonUtil.getS3File(storageConfig.container, "") case "azure" => CommonUtil.getAzureFile(storageConfig.container, "", storageConfig.accountKey.getOrElse("azure_storage_key")) diff --git a/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala b/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala index eb13fc8d8..854c704bd 100644 --- a/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala +++ b/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala @@ -75,7 +75,7 @@ trait BaseReportsJob { val reportsStorageAccountSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; if (reportsStorageAccountKey != null && reportsStorageAccountSecret.nonEmpty) { store.toLowerCase() match { - case "s3" => + case "s3" | "oci" => sc.hadoopConfiguration.set("fs.s3n.awsAccessKeyId", AppConf.getConfig(reportsStorageAccountKey)); sc.hadoopConfiguration.set("fs.s3n.awsSecretAccessKey", AppConf.getConfig(reportsStorageAccountSecret)); case "azure" =>