Skip to content

Commit 722ea5d

Browse files
reshmi-nairreshmi-nair
authored
Merge pull request #103 from shikshalokam/release-5.2.0
Add ml-jobs for ProgramUser Info
2 parents 2468d2b + 2acedb1 commit 722ea5d

File tree

17 files changed

+876
-1
lines changed

17 files changed

+876
-1
lines changed

‎dp-core/src/main/resources/base-config.conf

+7-1
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,10 @@ lms-cassandra {
5050
host = "localhost"
5151
port = "9042"
5252
isMultiDCEnabled = false
53-
}
53+
}
54+
55+
ml-cassandra {
56+
host = "localhost"
57+
port = "9042"
58+
isMultiDCEnabled = false
59+
}

‎dp-core/src/test/resources/base-test.conf

+6
Original file line numberDiff line numberDiff line change
@@ -56,3 +56,9 @@ lms-cassandra {
5656
port = 9142
5757
isMultiDCEnabled = false
5858
}
59+
60+
ml-cassandra {
61+
host = "localhost"
62+
port = 9142
63+
isMultiDCEnabled = false
64+
}

‎kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml

+18
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,13 @@ middleware_cassandra_user_activity_agg_table: user_activity_agg
120120
user_cache_updater_job_consumer_parallelism: 1
121121
user_cache_updater_job_parallelism: 1
122122

123+
### Program-User-Info Job vars
124+
programuserinfo_consumer_parallelism: 1
125+
programuserinfo_downstream_parallelism: 1
126+
programuserinfo_programuser_parallelism: 1
127+
middleware_cassandra_programuserinfo_keyspace: sunbird_programs
128+
middleware_cassandra_programuserinfo_table: program_enrollment
129+
123130
### to be removed
124131
job_classname: ""
125132

@@ -194,6 +201,16 @@ flink_job_names:
194201
taskslots: 1
195202
cpu_requests: 0.3
196203
scale_enabled: false
204+
program-user-info:
205+
job_class_name: 'org.sunbird.dp.userinfo.task.ProgramUserInfoStreamTask'
206+
replica: 1
207+
jobmanager_memory: 1024m
208+
taskmanager_memory: 1024m
209+
taskmanager_process_memory: 1700m
210+
jobmanager_process_memory: 1600m
211+
taskslots: 1
212+
cpu_requests: 0.3
213+
scale_enabled: false
197214

198215

199216
### Global vars
@@ -205,3 +222,4 @@ service_monitor_enabled: true
205222
### controlling the flink jobs log level
206223
flink_jobs_console_log_level: INFO
207224
flink_libraries_log_level: ERROR
225+

‎kubernetes/helm_charts/datapipeline_jobs/values.j2

+36
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@ scale_properties:
2020
scale_target_value: {{ flink_job_names['assessment-aggregator'].scale_target_value | default(0) }}
2121
min_replica: {{ flink_job_names['assessment-aggregator'].min_replica | default(1) }}
2222
max_replica: {{ flink_job_names['assessment-aggregator'].max_replica | default(2) }}
23+
program-user-info:
24+
enabled: {{ flink_job_names['program-user-info'].scale_enabled | lower}}
25+
scale_target_value: {{ flink_job_names['program-user-info'].scale_target_value | default(0) }}
26+
min_replica: {{ flink_job_names['program-user-info'].min_replica | default(1) }}
27+
max_replica: {{ flink_job_names['program-user-info'].max_replica | default(2) }}
2328

2429
jobmanager:
2530
rpc_port: {{ jobmanager_rpc_port }}
@@ -534,3 +539,34 @@ user-cache-updater-v2:
534539
jobmanager.execution.failover-strategy: region
535540
taskmanager.memory.network.fraction: 0.1
536541

542+
program-user-info:
543+
program-user-info: |+
544+
include file("/data/flink/conf/base-config.conf")
545+
kafka {
546+
producer.broker-servers = "{{ kafka_brokers }}"
547+
consumer.broker-servers = "{{ kafka_brokers }}"
548+
zookeeper = "{{ zookeepers }}"
549+
input.topic = {{ kafka_topic_programuser_info }}
550+
groupId = {{ kafka_group_programuser_info }}
551+
}
552+
task {
553+
consumer.parallelism = {{ programuserinfo_consumer_parallelism }}
554+
downstream.parallelism = {{ programuserinfo_downstream_parallelism }}
555+
programuser {
556+
parallelism = {{ programuserinfo_programuser_parallelism }}
557+
}
558+
559+
}
560+
ml-cassandra {
561+
keyspace = "{{ middleware_cassandra_programuserinfo_keyspace }}"
562+
table = "{{ middleware_cassandra_programuserinfo_table }}"
563+
}
564+
565+
flink-conf: |+
566+
jobmanager.memory.flink.size: {{ flink_job_names['program-user-info'].jobmanager_memory }}
567+
taskmanager.memory.flink.size: {{ flink_job_names['program-user-info'].taskmanager_memory }}
568+
taskmanager.numberOfTaskSlots: {{ flink_job_names['program-user-info'].taskslots }}
569+
parallelism.default: 1
570+
jobmanager.execution.failover-strategy: region
571+
taskmanager.memory.network.fraction: 0.1
572+

‎ml-jobs/pom.xml

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
6+
<parent>
7+
<groupId>org.sunbird</groupId>
8+
<artifactId>data-pipeline</artifactId>
9+
<version>1.0</version>
10+
<relativePath>../pom.xml</relativePath>
11+
</parent>
12+
13+
<artifactId>ml-jobs</artifactId>
14+
<packaging>pom</packaging>
15+
<name>ml-jobs</name>
16+
17+
18+
<modules>
19+
<module>program-user-info</module>
20+
</modules>
21+
22+
<build>
23+
<pluginManagement>
24+
<plugins>
25+
<plugin>
26+
<groupId>org.apache.maven.plugins</groupId>
27+
<artifactId>maven-compiler-plugin</artifactId>
28+
<version>3.8.1</version>
29+
<configuration>
30+
<source>11</source>
31+
<target>11</target>
32+
</configuration>
33+
</plugin>
34+
<plugin>
35+
<groupId>org.scoverage</groupId>
36+
<artifactId>scoverage-maven-plugin</artifactId>
37+
<version>${scoverage.plugin.version}</version>
38+
<configuration>
39+
<scalaVersion>${scala.version}</scalaVersion>
40+
<aggregate>true</aggregate>
41+
<highlighting>true</highlighting>
42+
<excludedPackages>org.sunbird.incredible</excludedPackages>
43+
</configuration>
44+
</plugin>
45+
</plugins>
46+
</pluginManagement>
47+
</build>
48+
49+
50+
</project>
51+

‎ml-jobs/program-user-info/pom.xml

+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
3+
<project xmlns="http://maven.apache.org/POM/4.0.0"
4+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
5+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
6+
<modelVersion>4.0.0</modelVersion>
7+
<parent>
8+
<groupId>org.sunbird</groupId>
9+
<artifactId>data-pipeline</artifactId>
10+
<version>1.0</version>
11+
<relativePath>../../pom.xml</relativePath>
12+
</parent>
13+
<artifactId>program-user-info</artifactId>
14+
<version>1.0.0</version>
15+
<packaging>jar</packaging>
16+
<name>ProgramUserInfo</name>
17+
<description>
18+
Job to insert user pii events details to cassandra
19+
</description>
20+
21+
<properties>
22+
<encoding>UTF-8</encoding>
23+
<scoverage.plugin.version>1.4.0</scoverage.plugin.version>
24+
</properties>
25+
26+
<dependencies>
27+
<dependency>
28+
<groupId>org.apache.flink</groupId>
29+
<artifactId>flink-clients_${scala.version}</artifactId>
30+
<version>${flink.version}</version>
31+
</dependency>
32+
<dependency>
33+
<groupId>org.apache.flink</groupId>
34+
<artifactId>flink-streaming-scala_${scala.version}</artifactId>
35+
<version>${flink.version}</version>
36+
<scope>provided</scope>
37+
</dependency>
38+
<dependency>
39+
<groupId>org.sunbird</groupId>
40+
<artifactId>dp-core</artifactId>
41+
<version>1.0.0</version>
42+
</dependency>
43+
<dependency>
44+
<groupId>org.sunbird</groupId>
45+
<artifactId>dp-core</artifactId>
46+
<version>1.0.0</version>
47+
<type>test-jar</type>
48+
<scope>test</scope>
49+
</dependency>
50+
<dependency>
51+
<groupId>org.apache.flink</groupId>
52+
<artifactId>flink-test-utils_${scala.version}</artifactId>
53+
<version>${flink.version}</version>
54+
<scope>test</scope>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.apache.flink</groupId>
58+
<artifactId>flink-streaming-java_${scala.version}</artifactId>
59+
<version>${flink.version}</version>
60+
<scope>test</scope>
61+
<classifier>tests</classifier>
62+
</dependency>
63+
<dependency>
64+
<groupId>org.scalatest</groupId>
65+
<artifactId>scalatest_${scala.version}</artifactId>
66+
<version>3.0.6</version>
67+
<scope>test</scope>
68+
</dependency>
69+
<dependency>
70+
<groupId>org.mockito</groupId>
71+
<artifactId>mockito-core</artifactId>
72+
<version>3.3.3</version>
73+
<scope>test</scope>
74+
</dependency>
75+
<dependency>
76+
<groupId>com.opentable.components</groupId>
77+
<artifactId>otj-pg-embedded</artifactId>
78+
<version>0.13.3</version>
79+
<scope>test</scope>
80+
</dependency>
81+
<dependency>
82+
<groupId>com.fiftyonred</groupId>
83+
<artifactId>mock-jedis</artifactId>
84+
<version>0.4.0</version>
85+
<scope>test</scope>
86+
</dependency>
87+
<dependency>
88+
<groupId>org.cassandraunit</groupId>
89+
<artifactId>cassandra-unit</artifactId>
90+
<version>3.11.2.0</version>
91+
<scope>test</scope>
92+
</dependency>
93+
</dependencies>
94+
95+
<build>
96+
<sourceDirectory>src/main/scala</sourceDirectory>
97+
<testSourceDirectory>src/test/scala</testSourceDirectory>
98+
<plugins>
99+
<plugin>
100+
<groupId>org.apache.maven.plugins</groupId>
101+
<artifactId>maven-compiler-plugin</artifactId>
102+
<version>3.8.1</version>
103+
<configuration>
104+
<release>11</release>
105+
</configuration>
106+
</plugin>
107+
<plugin>
108+
<groupId>org.apache.maven.plugins</groupId>
109+
<artifactId>maven-shade-plugin</artifactId>
110+
<version>3.2.1</version>
111+
<executions>
112+
<!-- Run shade goal on package phase -->
113+
<execution>
114+
<phase>package</phase>
115+
<goals>
116+
<goal>shade</goal>
117+
</goals>
118+
<configuration>
119+
<artifactSet>
120+
<excludes>
121+
<exclude>com.google.code.findbugs:jsr305</exclude>
122+
</excludes>
123+
</artifactSet>
124+
<filters>
125+
<filter>
126+
<!-- Do not copy the signatures in the META-INF folder.
127+
Otherwise, this might cause SecurityExceptions when using the JAR. -->
128+
<artifact>*:*</artifact>
129+
<excludes>
130+
<exclude>META-INF/*.SF</exclude>
131+
<exclude>META-INF/*.DSA</exclude>
132+
<exclude>META-INF/*.RSA</exclude>
133+
</excludes>
134+
</filter>
135+
</filters>
136+
<transformers>
137+
<transformer
138+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
139+
<mainClass>org.sunbird.dp.userinfo.task.ProgramUserInfoStreamTask</mainClass>
140+
</transformer>
141+
<!-- append default configs -->
142+
<transformer
143+
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
144+
<resource>reference.conf</resource>
145+
</transformer>
146+
</transformers>
147+
</configuration>
148+
</execution>
149+
</executions>
150+
</plugin>
151+
152+
<plugin>
153+
<groupId>net.alchim31.maven</groupId>
154+
<artifactId>scala-maven-plugin</artifactId>
155+
<version>4.4.0</version>
156+
<configuration>
157+
<source>${java.target.runtime}</source>
158+
<target>${java.target.runtime}</target>
159+
<scalaVersion>${scala.maj.version}</scalaVersion>
160+
<checkMultipleScalaVersions>false</checkMultipleScalaVersions>
161+
</configuration>
162+
<executions>
163+
<execution>
164+
<id>scala-compile-first</id>
165+
<phase>process-resources</phase>
166+
<goals>
167+
<goal>add-source</goal>
168+
<goal>compile</goal>
169+
</goals>
170+
</execution>
171+
<execution>
172+
<id>scala-test-compile</id>
173+
<phase>process-test-resources</phase>
174+
<goals>
175+
<goal>testCompile</goal>
176+
</goals>
177+
</execution>
178+
</executions>
179+
</plugin>
180+
181+
<plugin>
182+
<artifactId>maven-surefire-plugin</artifactId>
183+
<version>2.22.2</version>
184+
<configuration>
185+
<skipTests>true</skipTests>
186+
</configuration>
187+
</plugin>
188+
189+
<plugin>
190+
<groupId>org.scalatest</groupId>
191+
<artifactId>scalatest-maven-plugin</artifactId>
192+
<version>1.0</version>
193+
<configuration>
194+
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
195+
<junitxml>.</junitxml>
196+
<filereports>program-user-info-testsuite.txt</filereports>
197+
</configuration>
198+
<executions>
199+
<execution>
200+
<id>test</id>
201+
<goals>
202+
<goal>test</goal>
203+
</goals>
204+
</execution>
205+
</executions>
206+
</plugin>
207+
<plugin>
208+
<groupId>org.scoverage</groupId>
209+
<artifactId>scoverage-maven-plugin</artifactId>
210+
<version>${scoverage.plugin.version}</version>
211+
<configuration>
212+
<scalaVersion>${scala.version}</scalaVersion>
213+
<aggregate>true</aggregate>
214+
<highlighting>true</highlighting>
215+
</configuration>
216+
</plugin>
217+
</plugins>
218+
</build>
219+
220+
</project>
221+

0 commit comments

Comments
 (0)