common.conf.j2

application.env="{{ env }}"
telemetry.version="2.1"
default.parallelization="10"
spark_output_temp_dir="/mount/data/analytics/tmp/"
lp.url="{{lp_url}}"
service.search.url="{{ service.search.url }}"
service.search.path="{{ service.search.path }}"
spark.cassandra.connection.host="{{groups['dp-cassandra'][0]}}"
cassandra.keyspace_prefix="{{ cassandra_keyspace_prefix }}"
cassandra.hierarchy_store_prefix="{{ cassandra_hierarchy_store_prefix }}"


storage.key.config="{{ dp_storage_key_config }}"
storage.secret.config="{{ dp_storage_secret_config }}"
reports.storage.key.config="{{ dp_reports_storage_key_config }}"
reports.storage.secret.config="{{ dp_reports_storage_secret_config }}"
{% if dp_object_store_type == "azure" %}
cloud_storage_type="azure"
{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3" or dp_object_store_type == "oci") %}

cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\1') }}"
cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}"

{% if cloud_service_provider == "oci" %}
cloud_storage_type="oci"
storage.endpoint.config="{{ dp_storage_endpoint_config }}"
oci_storage_key="{{ s3_storage_key }}"
oci_storage_secret="{{ s3_storage_secret }}"
{% else %}
cloud_storage_type="s3"
aws_storage_key="{{ s3_storage_key }}"
aws_storage_secret="{{ s3_storage_secret }}"
{% endif %}

{% endif %}

lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M"

# Joblog Kafka appender config for cluster execution
log.appender.kafka.enable="false"
log.appender.kafka.broker_host="{{groups['processing-cluster-kafka'][0]}}:9092"
log.appender.kafka.topic="{{ env }}.druid.events.log"

# Kafka connection configuration
kafka.consumer.brokerlist="{{groups['processing-cluster-kafka'][0]}}:9092"
kafka.consumer.topic="{{ env }}.analytics.job_queue"
no_of_jobs=42

# Spark Driver
spark.driver.memory=6g

spark.memory_fraction={{ spark.memory_fraction }}
spark.storage_fraction={{ spark.storage_fraction }}
spark.driver_memory="{{ spark.driver_memory }}" 

#Monitor Jobs

monitor {
	notification {
		webhook_url = "{{ data_exhaust_webhook_url }}"
		channel = "{{ data_exhaust_Channel }}"
		token = "{{ data_exhaust_token }}"
		slack = true
        name = "{{ data_exhaust_name }}"
	}
}

#App ID & Channel ID
default.consumption.app.id="no_value"
default.channel.id="in.ekstep"
default.creation.app.id="no_value"


# Media Service Type
media_service_type = "azure"

azure_tenant="{{ media_service_azure_tenant }}"
azure_subscription_id="{{ media_service_azure_subscription_id }}"
azure_account_name="{{ media_service_azure_account_name }}"
azure_resource_group_name="{{ media_service_azure_resource_group_name }}"
azure_token_client_key="{{ media_service_azure_token_client_key }}" 
azure_token_client_secret="{{ media_service_azure_token_client_secret }}"
elasticsearch.service.endpoint="http://{{groups['composite-search-cluster'][0]}}:9200"
elasticsearch.index.compositesearch.name="{{ es_search_index }}"

org.search.api.url="{{ channelSearchServiceEndpoint }}"
org.search.api.key="{{ searchServiceAuthorizationToken }}"

hierarchy.search.api.url="{{ hierarchySearchServiceUrl }}"
hierarchy.search.api.path="{{ hierarchySearchServicEndpoint }}"

# Azure Media Service Config
azure {
  location = "centralindia"
  tenant = "tenant name"
  subscription_id = "subscription id"

  api {
    endpoint="Media Service API End Point"
    version = "2018-07-01"
  }

  account_name = "account name"
  resource_group_name = "Resource Group Name"

  transform {
    default = "media_transform_default"
    hls = "media_transform_hls"
  }

  stream {
    base_url = "{{ stream_base_url }}"
    endpoint_name = "default"
    protocol = "Hls"
    policy_name = "Predefined_ClearStreamingOnly"
  }

  token {
    client_key = "client key"
    client_secret = "client secret"
  }
}

## Reports - Global config
cloud.container.reports="{{reports_container}}"

# course metrics container in azure
course.metrics.cassandra.sunbirdKeyspace="sunbird"
course.metrics.cassandra.sunbirdCoursesKeyspace="sunbird_courses"
course.metrics.cassandra.sunbirdHierarchyStore="{{ cassandra_hierarchy_store_keyspace }}"
course.metrics.cloud.objectKey=""
course.metrics.cassandra.input.consistency="QUORUM"
es.host="http://{{groups['core-es'][0]}}"
es.port="9200"
es.composite.host="{{groups['composite-search-cluster'][0]}}"

# State admin user reports
# Uses azure only - course.metrics.cloud.provider
admin.metrics.cloud.objectKey=""
admin.metrics.temp.dir="/mount/data/analytics/admin-user-reports"

#Assessment report config
es.scroll.size = 1000

#BestScore or Latst Updated Score
assessment.metrics.bestscore.report=true
assessment.metrics.supported.contenttype="SelfAssess"
assessment.metrics.supported.primaryCategories="{{ assessment_metric_primary_category }}"
spark.sql.caseSensitive=true

# content rating configurations

druid.sql.host="http://{{druid_broker_host}}:8082/druid/v2/sql/"
druid.unique.content.query="{\"query\":\"SELECT DISTINCT \\\"object_id\\\" AS \\\"Id\\\"\\nFROM \\\"druid\\\".\\\"summary-events\\\" WHERE \\\"__time\\\"  BETWEEN TIMESTAMP '%s' AND TIMESTAMP '%s'\"}"
druid.content.rating.query="{\"query\":\"SELECT \\\"object_id\\\" AS contentId, COUNT(*) AS \\\"totalRatingsCount\\\", SUM(edata_rating) AS \\\"Total Ratings\\\", SUM(edata_rating)/COUNT(*) AS \\\"averageRating\\\" FROM \\\"druid\\\".\\\"telemetry-feedback-events\\\" WHERE \\\"eid\\\" = 'FEEDBACK' AND \\\"edata_rating\\\">0 GROUP BY \\\"object_id\\\"\"}"
druid.content.consumption.query="{\"query\":\"SELECT COUNT(*) as \\\"play_sessions_count\\\", object_id as \\\"contentId\\\", SUM(total_time_spent) as \\\"total_time_spent\\\", dimensions_pdata_id, object_id\\nFROM \\\"summary-events\\\"\\nWHERE \\\"dimensions_mode\\\" = 'play' AND \\\"dimensions_type\\\" ='content' AND \\\"dimensions_pdata_pid\\\" != 'creation-portal' \\nGROUP BY object_id, dimensions_pdata_id\"}"
lp.system.update.base.url="{{lp_url}}/system/v3/content/update"


#Experiment Configuration

user.search.api.url="{{sunbird_learner_service_url}}/private/user/v1/search"
user.search.limit="10000"

# pipeline auditing
druid.pipeline_metrics.audit.query="{\"query\":\"SELECT \\\"job-name\\\", SUM(\\\"success-message-count\\\") AS \\\"success-message-count\\\", SUM(\\\"failed-message-count\\\") AS \\\"failed-message-count\\\", SUM(\\\"duplicate-event-count\\\") AS \\\"duplicate-event-count\\\", SUM(\\\"batch-success-count\\\") AS \\\"batch-success-count\\\", SUM(\\\"batch-error-count\\\") AS \\\"batch-error-count\\\", SUM(\\\"primary-route-success-count\\\") AS \\\"primary-route-success-count\\\", SUM(\\\"secondary-route-success-count\\\") AS \\\"secondary-route-success-count\\\" FROM \\\"druid\\\".\\\"pipeline-metrics\\\" WHERE \\\"job-name\\\" IN (%s) AND \\\"__time\\\" BETWEEN TIMESTAMP '%s' AND TIMESTAMP '%s' GROUP BY \\\"job-name\\\" \"}"
druid.telemetryDatasource.count.query="{ \"query\": \"SELECT COUNT(*) AS \\\"total\\\" FROM \\\"druid\\\".\\\"telemetry-events\\\" WHERE TIME_FORMAT(MILLIS_TO_TIMESTAMP(\\\"syncts\\\"), 'yyyy-MM-dd HH:mm:ss.SSS', 'Asia/Kolkata') BETWEEN TIMESTAMP '%s' AND '%s' AND  \\\"__time\\\" BETWEEN TIMESTAMP '%s' AND TIMESTAMP '%s'\" }"
druid.summaryDatasource.count.query="{\"query\": \"SELECT COUNT(*) AS \\\"total\\\" FROM \\\"druid\\\".\\\"summary-events\\\" WHERE \\\"__time\\\" BETWEEN TIMESTAMP '%s' AND TIMESTAMP '%s'\" }"

#Pipeline Audit Jobs

pipeline_audit {
	notification {
		webhook_url = "{{ data_exhaust_webhook_url }}"
		channel = "{{ data_exhaust_Channel }}"
		token = "{{ data_exhaust_token }}"
		slack = true
		name = "Pipeline Audit"
	}
}

#Druid Query Processor

druid = {
	hosts = "{{druid_broker_host}}:8082"
	secure = false
	url = "/druid/v2/"
	datasource = "telemetry-events"
	response-parsing-timeout = 300000
        client-backend = "com.ing.wbaa.druid.client.DruidAdvancedHttpClient"
        client-config = {
           druid-advanced-http-client ={
           queue-size = 32768
           queue-overflow-strategy = "Backpressure"
           query-retries = 5
           query-retry-delay = 10 ms
           host-connection-pool = {
             max-connections = 32
             min-connections = 0
             max-open-requests = 128
             max-connection-lifetime = 20 min
             idle-timeout = 15 min
          client = {
            # The time after which an idle connection will be automatically closed.
            # Set to `infinite` to completely disable idle timeouts.
            idle-timeout = 10 min
           parsing.max-chunk-size = 10m
        }
        }
    }

  }
}
druid.rollup.host="{{druid_rollup_broker_host}}"
druid.rollup.port=8082
druid.query.wait.time.mins=10
druid.report.upload.wait.time.mins=10
druid.scan.batch.size=100
druid.scan.batch.bytes=2000000
druid.query.batch.buffer=500000


// Metric event config
metric.producer.id="pipeline.monitoring"
metric.producer.pid="dataproduct.metrics"
push.metrics.kafka=true
metric.kafka.broker="{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
metric.kafka.topic="{{ env }}.prom.monitoring.metrics"

//Postgres Config
postgres.db="{{postgres.db_name}}"
postgres.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/"
postgres.user="{{postgres.db_username}}"
postgres.pass="{{postgres.db_password}}"
postgres.program.table="program"
postgres.nomination.table="nomination"
postgres.usertable="\"V_User\""
postgres.org.table="\"V_User_Org\""

druid.ingestion.path="/druid/indexer/v1/task"
druid.segment.path="/druid/coordinator/v1/metadata/datasources/"
druid.deletesegment.path="/druid/coordinator/v1/datasources/"

postgres.druid.db="{{ druid_report_postgres_db_name }}"
postgres.druid.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/"
postgres.druid.user="{{ druid_report_postgres_db_username }}"
postgres.druid.pass="{{ dp_vault_druid_postgress_pass }}"


location.search.url="https://{{location_search_url}}/v1/location/search"
location.search.token="{{ location_search_token }}"
location.search.request="{\"request\": {\"filters\": {\"type\" :[\"state\",\"district\"]},\"limit\" : 10000}}"

druid.state.lookup.url = "http://{{groups['raw-coordinator'][0]}}:8081/druid/coordinator/v1/lookups/config/__default/stateSlugLookup"

sunbird_encryption_key="{{ core_vault_sunbird_encryption_key }}"

dcedialcode.filename="DCE_dialcode_data.csv"
etbdialcode.filename="ETB_dialcode_data.csv"
dcetextbook.filename="DCE_textbook_data.csv"
etbtextbook.filename="ETB_textbook_data.csv"
etb.dialcode.druid.length={{ etb_dialcode_list_druid_length }}


druid.report.default.storage="{{druid_storage_type}}"
druid.report.date.format="yyyy-MM-dd"
druid.report.default.container="{{cloud_storage_report_verfication_bucketname}}"

## Collection Exhaust Jobs Configuration -- Start ##

sunbird.user.keyspace="{{ user_table_keyspace }}"
sunbird.courses.keyspace="{{ course_keyspace }}"
sunbird.content.hierarchy.keyspace="{{ cassandra_hierarchy_store_keyspace }}"
sunbird.user.cluster.host="{{ core_cassandra_host }}"
sunbird.courses.cluster.host="{{ core_cassandra_host }}"
sunbird.content.cluster.host="{{ core_cassandra_host }}"
sunbird.report.cluster.host="{{ report_cassandra_cluster_host }}"
sunbird.user.report.keyspace="{{ report_user_table_keyspace }}"
collection.exhaust.store.prefix=""
postgres.table.job_request="{{ job_request_table }}"
postgres.table.dataset_metadata="{{ dataset_metadata_table }}"

## Collection Exhaust Jobs Configuration -- End ##

## Exhaust throttling variables
exhaust.batches.limit.per.channel={{ exhaust_batches_limit_per_channel }}
exhaust.file.size.limit.per.channel={{ exhaust_file_size_limit_bytes_per_channel }}

exhaust.parallel.batch.load.limit={{ exhaust_parallel_batch_load_limit }}
exhaust.user.parallelism={{ exhaust_user_parallelism }}

data_exhaust.batch.limit.per.request={{ data_exhaust_batch_limit_per_request }}



//START of UCI Postgres Config

uci.conversation.postgres.db="{{ uci_postgres.conversation_db_name }}"
uci.conversation.postgres.url="jdbc:postgresql://{{uci_postgres.conversation_db_host}}:{{uci_postgres.conversation_db_port}}/"

uci.fushionauth.postgres.db="{{ uci_postgres.fushionauth_db_name }}"
uci.fushionauth.postgres.url="jdbc:postgresql://{{uci_postgres.fushionauth_db_host}}:{{uci_postgres.fushionauth_db_port}}/"

uci.postgres.table.conversation="{{ uci_postgres.conversation_table_name }}"
uci.postgres.table.user="{{ uci_postgres.user_table_name }}"
uci.postgres.table.user_registration="{{ uci_postgres.user_registration_table_name }}"
uci.postgres.table.identities="{{ uci_postgres.user_identities_table_name }}"

uci.conversation.postgres.user="{{ uci_postgres.conversation_db_user }}"
uci.conversation.postgres.pass="{{ uci_postgres.conversation_db_psss }}"

uci.fushionauth.postgres.user="{{ uci_postgres.fushionauth_db_user }}"
uci.fushionauth.postgres.pass="{{ uci_postgres.fushionauth_db_psss }}"

uci.exhaust.store.prefix=""
uci.encryption.secret="{{ uci_encryption_secret_key }}"

// END OF UCI Related Job Configs

sunbird.course.optionalnodes="optionalnodes"
sunbird.course.redis.host={{ groups['redisall'][0] }}
sunbird.course.redis.port=6379
sunbird.course.redis.relationCache.id=5