From d00373d2b3e4a0d368b72d3e5d657529f055b512 Mon Sep 17 00:00:00 2001 From: kumarks1122 Date: Tue, 19 Sep 2023 14:31:46 +0530 Subject: [PATCH] ED-2856 | Report upload fixes added --- .../sunbird/core/exhaust/BaseReportsJob.scala | 22 +++++++++++++++---- .../lms/job/report/BaseReportsJob.scala | 22 +++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala b/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala index 806925744..bf00a332f 100644 --- a/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala +++ b/lern-data-products/src/main/scala/org/sunbird/core/exhaust/BaseReportsJob.scala @@ -48,8 +48,15 @@ trait BaseReportsJob { val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); val store = modelParams.getOrElse("store", "local").asInstanceOf[String]; - val storageKey = modelParams.getOrElse("storageKeyConfig", "reports_storage_key").asInstanceOf[String]; - val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; + val storageKeyConfig = modelParams.getOrElse("storageKeyConfig", "").asInstanceOf[String]; + val storageSecretConfig = modelParams.getOrElse("storageSecretConfig", "").asInstanceOf[String]; + + val storageKey = if (storageKeyConfig.nonEmpty) { + AppConf.getConfig(storageKeyConfig) + } else "reports_storage_key" + val storageSecret = if (storageSecretConfig.nonEmpty) { + AppConf.getConfig(storageSecretConfig) + } else "reports_storage_secret" CloudStorageProviders.setSparkCSPConfigurations(spark.sparkContext, AppConf.getConfig("cloud_storage_type"), Option(storageKey), Option(storageSecret)) } @@ -58,8 +65,15 @@ trait BaseReportsJob { val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()); val container = modelParams.getOrElse("storageContainer", "reports").asInstanceOf[String] - val storageKey = modelParams.getOrElse("storageKeyConfig", "reports_storage_key").asInstanceOf[String]; - val storageSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; + val storageKeyConfig = modelParams.getOrElse("storageKeyConfig", "").asInstanceOf[String]; + val storageSecretConfig = modelParams.getOrElse("storageSecretConfig", "").asInstanceOf[String]; + + val storageKey = if (storageKeyConfig.nonEmpty) { + AppConf.getConfig(storageKeyConfig) + } else "reports_storage_key" + val storageSecret = if (storageSecretConfig.nonEmpty) { + AppConf.getConfig(storageSecretConfig) + } else "reports_storage_secret" val store = modelParams.getOrElse("store", "local").asInstanceOf[String] StorageConfig(store, container, key, Option(storageKey), Option(storageSecret)); } diff --git a/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala b/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala index 372d4ea28..dad094eec 100644 --- a/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala +++ b/lern-data-products/src/main/scala/org/sunbird/lms/job/report/BaseReportsJob.scala @@ -71,20 +71,28 @@ trait BaseReportsJob { def setReportsStorageConfiguration(sc: SparkContext, config: JobConfig) { val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()) val store = modelParams.getOrElse("store", "local").asInstanceOf[String]; - val storageAccKey = modelParams.getOrElse("storageKeyConfig", "reports_storage_key").asInstanceOf[String]; - val storageAccSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; - CloudStorageProviders.setSparkCSPConfigurations(sc, AppConf.getConfig("cloud_storage_type"), Option(storageAccKey), Option(storageAccSecret)) + val storageKeyConfig = modelParams.getOrElse("storageKeyConfig", "").asInstanceOf[String]; + val storageSecretConfig = modelParams.getOrElse("storageSecretConfig", "").asInstanceOf[String]; + + val storageKey = if (storageKeyConfig.nonEmpty) { + AppConf.getConfig(storageKeyConfig) + } else "reports_storage_key" + val storageSecret = if (storageSecretConfig.nonEmpty) { + AppConf.getConfig(storageSecretConfig) + } else "reports_storage_secret" + CloudStorageProviders.setSparkCSPConfigurations(sc, AppConf.getConfig("cloud_storage_type"), Option(storageKey), Option(storageSecret)) } // $COVERAGE-ON$ Enabling scoverage for all other functions def getStorageConfig(container: String, key: String, config: JobConfig = JSONUtils.deserialize[JobConfig]("""{}""")): org.ekstep.analytics.framework.StorageConfig = { val modelParams = config.modelParams.getOrElse(Map[String, Option[AnyRef]]()) - val reportsStorageAccountKey = modelParams.getOrElse("storageKeyConfig", "reports_storage_key").asInstanceOf[String]; - val reportsStorageAccountSecret = modelParams.getOrElse("storageSecretConfig", "reports_storage_secret").asInstanceOf[String]; + val storageKeyConfig = modelParams.getOrElse("storageKeyConfig", "").asInstanceOf[String]; + val storageSecretConfig = modelParams.getOrElse("storageSecretConfig", "").asInstanceOf[String]; + val provider = AppConf.getConfig("cloud_storage_type") - if (reportsStorageAccountKey != null && reportsStorageAccountSecret.nonEmpty) { - org.ekstep.analytics.framework.StorageConfig(provider, container, key, Option(reportsStorageAccountKey), Option(reportsStorageAccountSecret)) + if (storageKeyConfig != null && storageSecretConfig.nonEmpty) { + org.ekstep.analytics.framework.StorageConfig(provider, container, key, Option(AppConf.getConfig(storageKeyConfig)), Option(AppConf.getConfig(storageSecretConfig))) } else { org.ekstep.analytics.framework.StorageConfig(provider, container, key, Option(provider), Option(provider)); }