Skip to content
This repository was archived by the owner on Aug 31, 2022. It is now read-only.

Update to scio 0.8 and other dependencies that required version update #212

Merged
merged 3 commits into from
Feb 10, 2020
Merged

Update to scio 0.8 and other dependencies that required version update #212

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d3b8fe4
update to scio 0.8 and other dependencies that required version update
aleksandr-spotify Feb 3, 2020
53f5420
clean up variable names
honnix Feb 10, 2020
dbfb1d9
do not rely on implicit
honnix Feb 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions contrib/flo-scio_2.11/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
<properties>
<scala.baseVersion>2.11</scala.baseVersion>
<scala.version>2.11.12</scala.version>
<scio.version>0.6.1</scio.version>
<scio.version>0.8.0</scio.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -50,7 +50,6 @@
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
</dependency>

<!--test deps-->
<dependency>
<groupId>com.spotify</groupId>
Expand All @@ -60,7 +59,7 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.baseVersion}</artifactId>
<version>3.0.5</version>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down
6 changes: 3 additions & 3 deletions contrib/flo-scio_2.12/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

<properties>
<scala.baseVersion>2.12</scala.baseVersion>
<scala.version>2.12.6</scala.version>
<scio.version>0.6.1</scio.version>
<scala.version>2.12.10</scala.version>
<scio.version>0.8.0</scio.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -61,7 +61,7 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.baseVersion}</artifactId>
<version>3.0.5</version>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down
18 changes: 9 additions & 9 deletions contrib/flo-scio_2.12/src/main/scala/com/spotify/flo/contrib/scio/ScioOperator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,15 +67,15 @@ class ScioOperator[T] extends TaskOperator[ScioJobSpec.Provider[T], ScioJobSpec[
spec.pipeline(sc)

// Start job
val scioResult = Try(sc.close())
scioResult match {
val sec = Try(sc.run())
sec match {
case Failure(t) => return spec.failure(t)
case _ =>
}

// Wait for job to complete
val done = Try(scioResult.get.waitUntilDone())
done match {
val scioResult = sec.map(_.waitUntilDone())
scioResult match {
case Failure(t) => return spec.failure(t)
case _ =>
}
Expand Down Expand Up @@ -117,21 +117,21 @@ class ScioOperator[T] extends TaskOperator[ScioJobSpec.Provider[T], ScioJobSpec[
spec.pipeline(sc)

// Start job
val scioResult = Try(sc.close())
scioResult match {
val sec = Try(sc.run())
sec match {
case Failure(t) => return spec.failure(t)
case _ =>
}

// Report job id
scioResult.get.internal match {
sec.get.pipelineResult match {
case job: DataflowPipelineJob => reportDataflowJob(spec.taskId, job, listener)
case _ =>
}

// Wait for job to complete
val done = Try(scioResult.get.waitUntilDone())
done match {
val scioResult = sec.map(_.waitUntilDone())
scioResult match {
case Failure(t) => return spec.failure(t)
case _ =>
}
Expand Down
5 changes: 3 additions & 2 deletions contrib/flo-scio_2.12/src/test/scala/com/spotify/flo/contrib/scio/ScioOperatorTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,13 @@ import com.spotify.flo.context.FloRunner
import com.spotify.flo.contrib.scio.ScioOperatorTest.{JobError, lineCountingTask}
import com.spotify.flo.status.NotRetriable
import com.spotify.scio.ScioMetrics
import com.spotify.scio.testing.{PipelineSpec, TextIO}
import com.spotify.scio.testing.PipelineSpec
import org.apache.beam.sdk.metrics.Counter
import org.scalatest._

import scala.collection.JavaConverters._

import com.spotify.scio.io.TextIO

class ScioOperatorTest extends PipelineSpec with Matchers {

it should "be able to run a scio job with mocked result" in {
Expand Down
2 changes: 1 addition & 1 deletion flo-scala_2.11/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.baseVersion}</artifactId>
<version>3.0.5</version>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
</dependencies>
Expand Down
2 changes: 1 addition & 1 deletion flo-scala_2.12/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

<properties>
<scala.baseVersion>2.12</scala.baseVersion>
<scala.version>2.12.6</scala.version>
<scala.version>2.12.10</scala.version>
</properties>

<dependencies>
Expand Down
116 changes: 86 additions & 30 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@
</modules>

<properties>
<grpc.version>1.13.1</grpc.version>
<jackson.version>2.9.10</jackson.version>
<jackson.databind.version>${jackson.version}.1</jackson.databind.version>
<google.client.version>1.27.0</google.client.version>
<google.auth.version>0.9.1</google.auth.version>
<grpc.version>1.22.1</grpc.version>
<jackson.version>2.10.2</jackson.version>
<jackson.databind.version>${jackson.version}</jackson.databind.version>
<google.client.version>1.29.0</google.client.version>
<google.auth.version>0.19.0</google.auth.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -85,7 +85,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
<version>1.7.30</version>
</dependency>
<dependency>
<groupId>com.typesafe</groupId>
Expand All @@ -95,7 +95,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
<version>3.9</version>
</dependency>
<dependency>
<groupId>org.trimou</groupId>
Expand All @@ -111,7 +111,7 @@
<dependency>
<groupId>com.google.auto.service</groupId>
<artifactId>auto-service</artifactId>
<version>1.0-rc3</version>
<version>1.0-rc6</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -145,7 +145,7 @@
<dependency>
<groupId>com.google.truth</groupId>
<artifactId>truth</artifactId>
<version>0.44</version>
<version>0.46</version>
<scope>test</scope>
</dependency>
<dependency>
Expand All @@ -163,7 +163,13 @@
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<version>1.3</version>
<version>2.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<version>2.2</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down Expand Up @@ -206,19 +212,34 @@
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-direct-java</artifactId>
<version>2.6.0</version>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
<version>2.6.0</version>
<version>2.16.0</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-extensions-google-cloud-platform-core</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-storage</artifactId>
<version>v1-rev20181109-1.27.0</version>
</dependency>
<dependency>
<groupId>com.google.auto.value</groupId>
<artifactId>auto-value-annotations</artifactId>
<version>1.7</version>
</dependency>

<!-- version resolution -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>27.1-jre</version>
<version>28.1-jre</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
Expand All @@ -233,7 +254,7 @@
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
<version>2.9.9</version>
<version>2.10.5</version>
</dependency>
<dependency>
<groupId>com.thoughtworks.paranamer</groupId>
Expand All @@ -243,17 +264,17 @@
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<version>2.3.1</version>
<version>2.3.4</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.6.0</version>
<version>3.11.1</version>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java-util</artifactId>
<version>3.6.0</version>
<version>3.10.0</version>
</dependency>
<dependency>
<groupId>com.google.api-client</groupId>
Expand All @@ -263,12 +284,12 @@
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
<version>${google.client.version}</version>
<version>1.34.0</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client-jackson2</artifactId>
<version>${google.client.version}</version>
<version>1.33.0</version>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
Expand All @@ -293,7 +314,12 @@
<dependency>
<groupId>com.google.api</groupId>
<artifactId>api-common</artifactId>
<version>1.6.0</version>
<version>1.8.1</version>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
<version>2.8.6</version>
</dependency>
<dependency>
<groupId>io.grpc</groupId>
Expand Down Expand Up @@ -323,12 +349,22 @@
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-api</artifactId>
<version>0.12.3</version>
<version>0.24.0</version>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-contrib-http-util</artifactId>
<version>0.24.0</version>
</dependency>
<dependency>
<groupId>io.opencensus</groupId>
<artifactId>opencensus-contrib-grpc-util</artifactId>
<version>0.24.0</version>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-common-protos</artifactId>
<version>1.12.0</version>
<version>1.17.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud.bigtable</groupId>
Expand All @@ -343,27 +379,32 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-tcnative-boringssl-static</artifactId>
<version>2.0.6.Final</version>
<version>2.0.28.Final</version>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-cloud-spanner-admin-database-v1</artifactId>
<version>0.1.11</version>
<version>1.6.0</version>
</dependency>
<dependency>
<groupId>com.google.api</groupId>
<artifactId>gax</artifactId>
<version>1.29.0</version>
<version>1.52.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-core</artifactId>
<version>1.36.0</version>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>com.google.api.grpc</groupId>
<artifactId>proto-google-iam-v1</artifactId>
<version>0.12.0</version>
<version>0.13.0</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.13</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
Expand All @@ -383,10 +424,25 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-bom</artifactId>
<version>4.1.25.Final</version>
<version>4.1.35.Final</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.4</version>
</dependency>
<dependency>
<groupId>org.tukaani</groupId>
<artifactId>xz</artifactId>
<version>1.8</version>
</dependency>
<dependency>
<groupId>org.apache.xbean</groupId>
<artifactId>xbean-asm7-shaded</artifactId>
<version>4.15</version>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down