forked from Sunbird-Lern/data-pipeline
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmain.yml
225 lines (202 loc) · 7.31 KB
/
main.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
imagepullsecrets: sunbird-registry-secret
### Job manager related vars
jobmanager_rpc_port: 6123
jobmanager_blob_port: 6124
jobmanager_query_port: 6125
jobmanager_ui_port: 8081
jobmanager_prom_port: 9250
jobmanager_heap_memory: 1024
jobmanager_ui_rest_port: 80
jobmanager_ui_tcp_port: 8081
jobmanager_ui_service:
type: LoadBalancer
annotations:
service.beta.kubernetes.io/azure-load-balancer-internal: "true"
### taskmanager related vars
healthcheck: true
taskmanager_liveness:
livenessProbe:
tcpSocket:
port: 6122
initialDelaySeconds: 30
periodSeconds: 60
taskmanager_prom_port: 9251
taskmanager_rpc_port: 6122
taskmanager_heap_memory: 1024
### base-config related vars
postgres_max_connections: 2
flink_container_name: "{{ cloud_storage_flink_bucketname }}"
checkpoint_interval: 60000
checkpoint_pause_between_seconds: 5000
checkpoint_compression_enabled: true
restart_attempts: 3
restart_delay: 30000 # in milli-seconds
producer_max_request_size: 1572864
producer_batch_size: 98304
producer_linger_ms: 10
redis_timeout: 30000
### Activity Aggregate job related vars
activity_agg_consumer_parallelism: 1
activity_agg_dedup_parallelism: 1
activity_agg_parallelism: 1
enrolment_complete_parallelism: 1
middleware_consumption_table: "user_content_consumption"
middleware_user_activity_agg_table: "user_activity_agg"
activity_agg_checkpointing_interval: 300000
activity_agg_checkpointing_pause_interval: 90000
activity_agg_batch_interval: 60
activity_agg_batch_read_size: 1
activity_agg_batch_write_size: 10
activity_agg_window_shards: 1000
activity_agg_dedup_index: 13
activity_agg_dedup_expiry: 604800
activity_module_aggs_enabled: true
activity_input_dedup_enabled: true
activity_agg_enrolment_filter_processe_enabled: true
activity_agg_collection_status_cache_expiry_time: 3600
### Relation Cache Updater Job related Vars
extractor_consumer_parallelism: 1
relation_cache_updater_consumer_parallelism: 1
relation_cache_updater_parallelism: 1
middleware_content_hierarchy_table: "content_hierarchy"
### Certificate Job related Vars
certificate_generator_consumer_parallelism: 1
certificate_generator_parallelism: 1
### Enrolment Reconciliation
enrolment_reconciliation_consumer_parallelism: 1
enrolment_reconciliation_parallelism: 1
enrolment_reconciliation_batch_write_size: 10
enrolment_reconciliation_filter_processe_enabled: true
enrolment_reconciliation_collection_status_cache_expiry_time: 3600
### Collection Cert Pre Processor
collection_cert_pre_processor_consumer_parallelism: 1
generate_certificate_parallelism: 1
middleware_course_batch_table: "course_batch"
middleware_user_enrolments_table: "user_enrolments"
middleware_assessment_aggregator_table: "assessment_aggregator"
### Collection Generator Job related Vars
collection_certificate_generator_consumer_parallelism: 1
collection_certificate_generator_parallelism: 1
collection_certificate_generator_enable_suppress_exception: "{{ enable_suppress_exception | lower }}"
collection_certificate_generator_enable_rc_certificate: "{{ enable_rc_certificate | lower }}"
collection_certificate_pre_processor_enable_suppress_exception: "{{ enable_suppress_exception | lower }}"
collection_certificate_generator_rc_badcharlist: "{{ rc_bad_char_list | default('\"\\\\x00,\\\\\\\\aaa,\\\\aaa,Ø,Ý,\\\\\"') }}"
registry_sunbird_keyspace: "sunbird"
cert_registry_table: "cert_registry"
### Merge User Courses Job related Vars
merge_user_courses_consumer_parallelism: 1
merge_user_courses_parallelism: 1
merge_user_courses_course_batch_parallelism: 1
merge_user_courses_course_date_format: "yyyy-MM-dd HH:mm:ss:SSSZ"
### Notification Job related Vars
notification_job_consumer_parallelism: 1
notification_job_parallelism: 1
### assessment-aggregator related vars
assessaggregator_parallelism: 1
assessaggregator_consumer_parallelism: 1
assessaggregator_downstream_parallelism: 1
assessaggregator_scoreaggregator_parallelism: 1
middleware_cassandra_courses_keyspace: sunbird_courses
middleware_cassandra_assessment_aggregator_table: assessment_aggregator
middleware_cassandra_assessment_question_type : question
middleware_cassandra_user_enrolments_table: user_enrolments
middleware_cassandra_user_activity_agg_table: user_activity_agg
### User-cache-update Job related Vars
user_cache_updater_job_consumer_parallelism: 1
user_cache_updater_job_parallelism: 1
### Program-User-Info Job vars
programuserinfo_consumer_parallelism: 1
programuserinfo_downstream_parallelism: 1
programuserinfo_programuser_parallelism: 1
middleware_cassandra_programuserinfo_keyspace: sunbird_programs
middleware_cassandra_programuserinfo_table: program_enrollment
### to be removed
job_classname: ""
### class name vars
flink_job_names:
activity-aggregate-updater:
job_class_name: 'org.sunbird.job.aggregate.task.ActivityAggregateUpdaterStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
relation-cache-updater:
job_class_name: 'org.sunbird.job.relationcache.task.RelationCacheUpdaterStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
enrolment-reconciliation:
job_class_name: 'org.sunbird.job.recounciliation.task.EnrolmentReconciliationStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
collection-cert-pre-processor:
job_class_name: 'org.sunbird.job.collectioncert.task.CollectionCertPreProcessorTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
collection-certificate-generator:
job_class_name: 'org.sunbird.job.certgen.task.CertificateGeneratorStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
merge-user-courses:
job_class_name: 'org.sunbird.job.merge.user.courses.task.MergeUserCoursesStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
assessment-aggregator:
job_class_name: 'org.sunbird.dp.assessment.task.AssessmentAggregatorStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskmanager_process_memory: 1700m
jobmanager_process_memory: 1600m
taskslots: 1
cpu_requests: 0.3
scale_enabled: false
notification-job:
job_class_name: 'org.sunbird.job.notification.task.NotificationStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskslots: 1
cpu_requests: 0.3
user-cache-updater-v2:
job_class_name: 'org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTaskV2'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskmanager_process_memory: 1700m
jobmanager_process_memory: 1600m
taskslots: 1
cpu_requests: 0.3
scale_enabled: false
program-user-info:
job_class_name: 'org.sunbird.dp.userinfo.task.ProgramUserInfoStreamTask'
replica: 1
jobmanager_memory: 1024m
taskmanager_memory: 1024m
taskmanager_process_memory: 1700m
jobmanager_process_memory: 1600m
taskslots: 1
cpu_requests: 0.3
scale_enabled: false
### Global vars
middleware_course_keyspace: "sunbird_courses"
cloud_store: azure
service_monitor_enabled: true
### controlling the flink jobs log level
flink_jobs_console_log_level: INFO
flink_libraries_log_level: ERROR