Skip to content
This repository was archived by the owner on Aug 31, 2022. It is now read-only.

Commit 3481461

Browse files
honnixhonnix
authored
Merge pull request #212 from aleksandr-spotify/update_scio_version_0_8
Update to scio 0.8 and other dependencies that required version update
2 parents f2cd96c + dbfb1d9 commit 3481461

File tree

7 files changed

+105
-49
lines changed

7 files changed

+105
-49
lines changed

‎contrib/flo-scio_2.11/pom.xml

+2-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
<properties>
1818
<scala.baseVersion>2.11</scala.baseVersion>
1919
<scala.version>2.11.12</scala.version>
20-
<scio.version>0.6.1</scio.version>
20+
<scio.version>0.8.0</scio.version>
2121
</properties>
2222

2323
<dependencies>
@@ -50,7 +50,6 @@
5050
<artifactId>scala-library</artifactId>
5151
<version>${scala.version}</version>
5252
</dependency>
53-
5453
<!--test deps-->
5554
<dependency>
5655
<groupId>com.spotify</groupId>
@@ -60,7 +59,7 @@
6059
<dependency>
6160
<groupId>org.scalatest</groupId>
6261
<artifactId>scalatest_${scala.baseVersion}</artifactId>
63-
<version>3.0.5</version>
62+
<version>3.1.0</version>
6463
<scope>test</scope>
6564
</dependency>
6665
</dependencies>

‎contrib/flo-scio_2.12/pom.xml

+3-3
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616

1717
<properties>
1818
<scala.baseVersion>2.12</scala.baseVersion>
19-
<scala.version>2.12.6</scala.version>
20-
<scio.version>0.6.1</scio.version>
19+
<scala.version>2.12.10</scala.version>
20+
<scio.version>0.8.0</scio.version>
2121
</properties>
2222

2323
<dependencies>
@@ -61,7 +61,7 @@
6161
<dependency>
6262
<groupId>org.scalatest</groupId>
6363
<artifactId>scalatest_${scala.baseVersion}</artifactId>
64-
<version>3.0.5</version>
64+
<version>3.1.0</version>
6565
<scope>test</scope>
6666
</dependency>
6767
</dependencies>

‎contrib/flo-scio_2.12/src/main/scala/com/spotify/flo/contrib/scio/ScioOperator.scala

+9-9
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,15 @@ class ScioOperator[T] extends TaskOperator[ScioJobSpec.Provider[T], ScioJobSpec[
6767
spec.pipeline(sc)
6868

6969
// Start job
70-
val scioResult = Try(sc.close())
71-
scioResult match {
70+
val sec = Try(sc.run())
71+
sec match {
7272
case Failure(t) => return spec.failure(t)
7373
case _ =>
7474
}
7575

7676
// Wait for job to complete
77-
val done = Try(scioResult.get.waitUntilDone())
78-
done match {
77+
val scioResult = sec.map(_.waitUntilDone())
78+
scioResult match {
7979
case Failure(t) => return spec.failure(t)
8080
case _ =>
8181
}
@@ -117,21 +117,21 @@ class ScioOperator[T] extends TaskOperator[ScioJobSpec.Provider[T], ScioJobSpec[
117117
spec.pipeline(sc)
118118

119119
// Start job
120-
val scioResult = Try(sc.close())
121-
scioResult match {
120+
val sec = Try(sc.run())
121+
sec match {
122122
case Failure(t) => return spec.failure(t)
123123
case _ =>
124124
}
125125

126126
// Report job id
127-
scioResult.get.internal match {
127+
sec.get.pipelineResult match {
128128
case job: DataflowPipelineJob => reportDataflowJob(spec.taskId, job, listener)
129129
case _ =>
130130
}
131131

132132
// Wait for job to complete
133-
val done = Try(scioResult.get.waitUntilDone())
134-
done match {
133+
val scioResult = sec.map(_.waitUntilDone())
134+
scioResult match {
135135
case Failure(t) => return spec.failure(t)
136136
case _ =>
137137
}

‎contrib/flo-scio_2.12/src/test/scala/com/spotify/flo/contrib/scio/ScioOperatorTest.scala

+3-2
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,13 @@ import com.spotify.flo.context.FloRunner
2929
import com.spotify.flo.contrib.scio.ScioOperatorTest.{JobError, lineCountingTask}
3030
import com.spotify.flo.status.NotRetriable
3131
import com.spotify.scio.ScioMetrics
32-
import com.spotify.scio.testing.{PipelineSpec, TextIO}
32+
import com.spotify.scio.testing.PipelineSpec
3333
import org.apache.beam.sdk.metrics.Counter
3434
import org.scalatest._
35-
3635
import scala.collection.JavaConverters._
3736

37+
import com.spotify.scio.io.TextIO
38+
3839
class ScioOperatorTest extends PipelineSpec with Matchers {
3940

4041
it should "be able to run a scio job with mocked result" in {

‎flo-scala_2.11/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
<dependency>
4343
<groupId>org.scalatest</groupId>
4444
<artifactId>scalatest_${scala.baseVersion}</artifactId>
45-
<version>3.0.5</version>
45+
<version>3.1.0</version>
4646
<scope>test</scope>
4747
</dependency>
4848
</dependencies>

‎flo-scala_2.12/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
<properties>
1717
<scala.baseVersion>2.12</scala.baseVersion>
18-
<scala.version>2.12.6</scala.version>
18+
<scala.version>2.12.10</scala.version>
1919
</properties>
2020

2121
<dependencies>

‎pom.xml

+86-30
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@
5353
</modules>
5454

5555
<properties>
56-
<grpc.version>1.13.1</grpc.version>
57-
<jackson.version>2.9.10</jackson.version>
58-
<jackson.databind.version>${jackson.version}.1</jackson.databind.version>
59-
<google.client.version>1.27.0</google.client.version>
60-
<google.auth.version>0.9.1</google.auth.version>
56+
<grpc.version>1.22.1</grpc.version>
57+
<jackson.version>2.10.2</jackson.version>
58+
<jackson.databind.version>${jackson.version}</jackson.databind.version>
59+
<google.client.version>1.29.0</google.client.version>
60+
<google.auth.version>0.19.0</google.auth.version>
6161
</properties>
6262

6363
<dependencyManagement>
@@ -85,7 +85,7 @@
8585
<dependency>
8686
<groupId>org.slf4j</groupId>
8787
<artifactId>slf4j-api</artifactId>
88-
<version>1.7.25</version>
88+
<version>1.7.30</version>
8989
</dependency>
9090
<dependency>
9191
<groupId>com.typesafe</groupId>
@@ -95,7 +95,7 @@
9595
<dependency>
9696
<groupId>org.apache.commons</groupId>
9797
<artifactId>commons-lang3</artifactId>
98-
<version>3.7</version>
98+
<version>3.9</version>
9999
</dependency>
100100
<dependency>
101101
<groupId>org.trimou</groupId>
@@ -111,7 +111,7 @@
111111
<dependency>
112112
<groupId>com.google.auto.service</groupId>
113113
<artifactId>auto-service</artifactId>
114-
<version>1.0-rc3</version>
114+
<version>1.0-rc6</version>
115115
<scope>provided</scope>
116116
</dependency>
117117
<dependency>
@@ -145,7 +145,7 @@
145145
<dependency>
146146
<groupId>com.google.truth</groupId>
147147
<artifactId>truth</artifactId>
148-
<version>0.44</version>
148+
<version>0.46</version>
149149
<scope>test</scope>
150150
</dependency>
151151
<dependency>
@@ -163,7 +163,13 @@
163163
<dependency>
164164
<groupId>org.hamcrest</groupId>
165165
<artifactId>hamcrest-library</artifactId>
166-
<version>1.3</version>
166+
<version>2.2</version>
167+
<scope>test</scope>
168+
</dependency>
169+
<dependency>
170+
<groupId>org.hamcrest</groupId>
171+
<artifactId>hamcrest-core</artifactId>
172+
<version>2.2</version>
167173
<scope>test</scope>
168174
</dependency>
169175
<dependency>
@@ -206,19 +212,34 @@
206212
<dependency>
207213
<groupId>org.apache.beam</groupId>
208214
<artifactId>beam-runners-direct-java</artifactId>
209-
<version>2.6.0</version>
215+
<version>2.17.0</version>
210216
</dependency>
211217
<dependency>
212218
<groupId>org.apache.beam</groupId>
213219
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
214-
<version>2.6.0</version>
220+
<version>2.16.0</version>
221+
</dependency>
222+
<dependency>
223+
<groupId>org.apache.beam</groupId>
224+
<artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
225+
<version>2.17.0</version>
226+
</dependency>
227+
<dependency>
228+
<groupId>com.google.apis</groupId>
229+
<artifactId>google-api-services-storage</artifactId>
230+
<version>v1-rev20181109-1.27.0</version>
231+
</dependency>
232+
<dependency>
233+
<groupId>com.google.auto.value</groupId>
234+
<artifactId>auto-value-annotations</artifactId>
235+
<version>1.7</version>
215236
</dependency>
216237

217238
<!-- version resolution -->
218239
<dependency>
219240
<groupId>com.google.guava</groupId>
220241
<artifactId>guava</artifactId>
221-
<version>27.1-jre</version>
242+
<version>28.1-jre</version>
222243
</dependency>
223244
<dependency>
224245
<groupId>com.google.code.findbugs</groupId>
@@ -233,7 +254,7 @@
233254
<dependency>
234255
<groupId>joda-time</groupId>
235256
<artifactId>joda-time</artifactId>
236-
<version>2.9.9</version>
257+
<version>2.10.5</version>
237258
</dependency>
238259
<dependency>
239260
<groupId>com.thoughtworks.paranamer</groupId>
@@ -243,17 +264,17 @@
243264
<dependency>
244265
<groupId>com.google.errorprone</groupId>
245266
<artifactId>error_prone_annotations</artifactId>
246-
<version>2.3.1</version>
267+
<version>2.3.4</version>
247268
</dependency>
248269
<dependency>
249270
<groupId>com.google.protobuf</groupId>
250271
<artifactId>protobuf-java</artifactId>
251-
<version>3.6.0</version>
272+
<version>3.11.1</version>
252273
</dependency>
253274
<dependency>
254275
<groupId>com.google.protobuf</groupId>
255276
<artifactId>protobuf-java-util</artifactId>
256-
<version>3.6.0</version>
277+
<version>3.10.0</version>
257278
</dependency>
258279
<dependency>
259280
<groupId>com.google.api-client</groupId>
@@ -263,12 +284,12 @@
263284
<dependency>
264285
<groupId>com.google.http-client</groupId>
265286
<artifactId>google-http-client</artifactId>
266-
<version>${google.client.version}</version>
287+
<version>1.34.0</version>
267288
</dependency>
268289
<dependency>
269290
<groupId>com.google.http-client</groupId>
270291
<artifactId>google-http-client-jackson2</artifactId>
271-
<version>${google.client.version}</version>
292+
<version>1.33.0</version>
272293
</dependency>
273294
<dependency>
274295
<groupId>com.google.http-client</groupId>
@@ -293,7 +314,12 @@
293314
<dependency>
294315
<groupId>com.google.api</groupId>
295316
<artifactId>api-common</artifactId>
296-
<version>1.6.0</version>
317+
<version>1.8.1</version>
318+
</dependency>
319+
<dependency>
320+
<groupId>com.google.code.gson</groupId>
321+
<artifactId>gson</artifactId>
322+
<version>2.8.6</version>
297323
</dependency>
298324
<dependency>
299325
<groupId>io.grpc</groupId>
@@ -323,12 +349,22 @@
323349
<dependency>
324350
<groupId>io.opencensus</groupId>
325351
<artifactId>opencensus-api</artifactId>
326-
<version>0.12.3</version>
352+
<version>0.24.0</version>
353+
</dependency>
354+
<dependency>
355+
<groupId>io.opencensus</groupId>
356+
<artifactId>opencensus-contrib-http-util</artifactId>
357+
<version>0.24.0</version>
358+
</dependency>
359+
<dependency>
360+
<groupId>io.opencensus</groupId>
361+
<artifactId>opencensus-contrib-grpc-util</artifactId>
362+
<version>0.24.0</version>
327363
</dependency>
328364
<dependency>
329365
<groupId>com.google.api.grpc</groupId>
330366
<artifactId>proto-google-common-protos</artifactId>
331-
<version>1.12.0</version>
367+
<version>1.17.0</version>
332368
</dependency>
333369
<dependency>
334370
<groupId>com.google.cloud.bigtable</groupId>
@@ -343,27 +379,32 @@
343379
<dependency>
344380
<groupId>io.netty</groupId>
345381
<artifactId>netty-tcnative-boringssl-static</artifactId>
346-
<version>2.0.6.Final</version>
382+
<version>2.0.28.Final</version>
347383
</dependency>
348384
<dependency>
349385
<groupId>com.google.api.grpc</groupId>
350386
<artifactId>proto-google-cloud-spanner-admin-database-v1</artifactId>
351-
<version>0.1.11</version>
387+
<version>1.6.0</version>
352388
</dependency>
353389
<dependency>
354390
<groupId>com.google.api</groupId>
355391
<artifactId>gax</artifactId>
356-
<version>1.29.0</version>
392+
<version>1.52.0</version>
357393
</dependency>
358394
<dependency>
359-
<groupId>com.google.cloud</groupId>
360-
<artifactId>google-cloud-core</artifactId>
361-
<version>1.36.0</version>
395+
<groupId>javax.annotation</groupId>
396+
<artifactId>javax.annotation-api</artifactId>
397+
<version>1.3.2</version>
362398
</dependency>
363399
<dependency>
364400
<groupId>com.google.api.grpc</groupId>
365401
<artifactId>proto-google-iam-v1</artifactId>
366-
<version>0.12.0</version>
402+
<version>0.13.0</version>
403+
</dependency>
404+
<dependency>
405+
<groupId>commons-codec</groupId>
406+
<artifactId>commons-codec</artifactId>
407+
<version>1.13</version>
367408
</dependency>
368409
<dependency>
369410
<groupId>com.fasterxml.jackson.core</groupId>
@@ -383,10 +424,25 @@
383424
<dependency>
384425
<groupId>io.netty</groupId>
385426
<artifactId>netty-bom</artifactId>
386-
<version>4.1.25.Final</version>
427+
<version>4.1.35.Final</version>
387428
<type>pom</type>
388429
<scope>import</scope>
389430
</dependency>
431+
<dependency>
432+
<groupId>org.xerial.snappy</groupId>
433+
<artifactId>snappy-java</artifactId>
434+
<version>1.1.4</version>
435+
</dependency>
436+
<dependency>
437+
<groupId>org.tukaani</groupId>
438+
<artifactId>xz</artifactId>
439+
<version>1.8</version>
440+
</dependency>
441+
<dependency>
442+
<groupId>org.apache.xbean</groupId>
443+
<artifactId>xbean-asm7-shaded</artifactId>
444+
<version>4.15</version>
445+
</dependency>
390446
</dependencies>
391447
</dependencyManagement>
392448

0 commit comments

Comments
 (0)