From 5d9b3e25ab12dce11b61ab8565315d3f4de75c76 Mon Sep 17 00:00:00 2001 From: Rob Hudson Date: Mon, 27 Nov 2017 15:34:42 -0800 Subject: [PATCH] Update project settings and dependencies This adds a Docker container to run locally for development and for testing, as well as updates various dependencies to latest versions, and moves to CircleCI for testing and publishing. This commit differs from the prior commits by using sbt 1.0.4 which fixes a problem with `sbt publish`. See: https://github.com/sbt/librarymanagement/pull/183 --- .gitignore | 2 ++ .travis.yml | 3 ++- Dockerfile | 2 +- README.md | 25 ++++++++++++++---- build.sbt | 16 +++--------- circle.yml | 26 +++++++------------ project/build.properties | 1 + project/plugins.sbt | 10 +++---- .../com/mozilla/telemetry/heka/package.scala | 20 +++++++------- 9 files changed, 53 insertions(+), 52 deletions(-) create mode 100644 project/build.properties diff --git a/.gitignore b/.gitignore index cfd0eca..4be43e1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ target/ .idea/ .idea_modules/ .DS_Store +.sbt/ +.ivy2/ diff --git a/.travis.yml b/.travis.yml index 16a4b5a..dfa431d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,9 +3,10 @@ sudo: false cache: directories: - $HOME/.ivy2 + - $HOME/.sbt matrix: include: - - jdk: openjdk7 + - jdk: openjdk8 scala: 2.11.8 python: 3.5 env: TEST_SPARK_VERSION="2.1.0" AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=foo diff --git a/Dockerfile b/Dockerfile index 16fc3ad..ee18492 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM openjdk:8 ENV SCALA_VERSION=2.12.4 \ - SBT_VERSION=1.0.3 + SBT_VERSION=1.0.4 # Install Scala. RUN \ diff --git a/README.md b/README.md index c7d6c25..a94e729 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,11 @@ [![CircleCi](https://circleci.com/gh/mozilla/moztelemetry.svg?style=shield&circle-token=3fff2168f7d8da61b47bd1481c4fcb984ec88ef5)](https://circleci.com/gh/mozilla/moztelemetry) # moztelemetry + Mozilla's Telemetry API for Scala ## Using moztelemetry + In SBT: ``` resolvers += "S3 local maven snapshots" at "s3://net-mozaws-data-us-west-2-ops-mavenrepo/snapshots" @@ -13,14 +15,27 @@ libraryDependencies += "com.mozilla.telemetry" %% "moztelemetry" % "1.0-SNAPSHOT ``` ## Testing -To run the tests you have to start a mock S3 service first with moto: + +To run the tests, build the docker container and run the tests. + +Build the container. You only have to do this once or when you update the Dockerfile: + +``` +docker build -t moztelemetry . +``` + +Run the tests in the docker container: ``` -pip install moto -moto_server s3 -p 8001 & -AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=foo sbt test +./bin/test +``` + +Other test tasks can by run by passing the task through the test script, e.g.: + +``` +./bin/test "testOnly com.mozilla.telemetry.stats.StatsTest" ``` ## Publishing snapshots -Snapshots will now be published to our local maven repo in s3 on every commit merged into master via a circleci build +Snapshots will now be published to our local maven repo in s3 on every commit merged into master via a circleci build diff --git a/build.sbt b/build.sbt index 888c25a..92b7d2a 100644 --- a/build.sbt +++ b/build.sbt @@ -6,19 +6,19 @@ organization := "com.mozilla.telemetry" scalaVersion := "2.11.8" -sparkVersion := "2.1.0" - -sparkComponents ++= Seq("core") +val sparkVersion = "2.2.0" resolvers += Resolver.bintrayRepo("findify", "maven") libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-core" % sparkVersion % "provided", "org.scalatest" %% "scalatest" % "2.2.6" % "test", - "org.apache.commons" % "commons-io" % "1.3.2" % "test", + "commons-io" % "commons-io" % "1.3.2" % "test", "com.github.seratch" %% "awscala" % "0.5.+", "com.amazonaws" % "aws-java-sdk" % "1.11.83", "com.google.protobuf" % "protobuf-java" % "2.5.0" ) + /* The HBase client requires protobuf-java 2.5.0 but scalapb uses protobuf-java 3.x so we have to force the dependency here. This should be fine as we are using only @@ -26,12 +26,6 @@ libraryDependencies ++= Seq( */ dependencyOverrides += "com.google.protobuf" % "protobuf-java" % "2.5.0" -// Shade PB files -assemblyShadeRules in assembly := Seq( - ShadeRule.rename("com.google.protobuf.**" -> "shadeproto.@1").inAll, - ShadeRule.rename("com.trueaccord.scalapb.**" -> "shadescalapb.@1").inAll -) - // Compile proto files PB.targets in Compile := Seq( scalapb.gen() -> (sourceManaged in Compile).value @@ -40,8 +34,6 @@ PB.targets in Compile := Seq( // Exclude generated classes from the coverage coverageExcludedPackages := "com\\.mozilla\\.telemetry\\.heka\\.(Field|Message|Header)" -credentials += Credentials(Path.userHome / ".ivy2" / ".sbtcredentials") - publishMavenStyle := true publishTo := { diff --git a/circle.yml b/circle.yml index 74bf9ff..a3ddd92 100644 --- a/circle.yml +++ b/circle.yml @@ -1,31 +1,23 @@ machine: pre: - # Install sbt 0.13.16 + # Install sbt 1.0.4 - sudo apt-get install openjdk-8-jdk - - wget -q https://dl.bintray.com/sbt/debian/sbt-0.13.16.deb - - sudo dpkg -i sbt-0.13.16.deb + - wget -q https://dl.bintray.com/sbt/debian/sbt-1.0.4.deb + - sudo dpkg -i sbt-1.0.4.deb cache_directories: - "~/.ivy2" - "~/.sbt" + services: + - docker dependencies: - pre: - - sudo apt-get update - - sudo apt-get install libffi-dev libssl-dev python-dev - - sudo apt-get install python - - python --version - - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - - sudo python get-pip.py - - sudo pip install --upgrade pip - - sudo pip install --upgrade setuptools - - sudo pip install pyopenssl ndg-httpsclient pyasn1 - - sudo pip install moto --ignore-installed six - - sudo pip install flask - - moto_server s3 -p 8001 & + override: + - docker info + - docker build --rm=false -t circleci/moztelemetry:$CIRCLE_SHA1 . test: override: - - 'true' + - docker run -t -i -p 8001:8001 circleci/moztelemetry:$CIRCLE_SHA1 ./bin/run_tests deployment: latest: diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 0000000..394cb75 --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.0.4 diff --git a/project/plugins.sbt b/project/plugins.sbt index b4a9da8..1f81c42 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,13 +1,11 @@ resolvers += "bintray-spark-packages" at "https://dl.bintray.com/spark-packages/maven/" -addSbtPlugin("org.spark-packages" % "sbt-spark-package" % "0.2.5") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") +addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") -addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") - -addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.3") +addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.12") addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.12.0") -libraryDependencies += "com.trueaccord.scalapb" %% "compilerplugin" % "0.5.47" +libraryDependencies += "com.trueaccord.scalapb" %% "compilerplugin" % "0.6.6" diff --git a/src/main/scala/com/mozilla/telemetry/heka/package.scala b/src/main/scala/com/mozilla/telemetry/heka/package.scala index aa41b62..aefbc8b 100644 --- a/src/main/scala/com/mozilla/telemetry/heka/package.scala +++ b/src/main/scala/com/mozilla/telemetry/heka/package.scala @@ -19,7 +19,7 @@ package object heka { private def field(f: Field): Any = { // I am assuming there is only one value f.getValueType match { - case Field.ValueType.BYTES => { + case Field.ValueTypeEnum.BYTES => { val bytes = f.valueBytes(0) // Our JSON bytes fields sometimes contain non-UTF8 strings that can // still be parsed as JSON. For now, we attempt to coerce all bytes @@ -28,10 +28,10 @@ package object heka { // for details. bytes.toStringUtf8 } - case Field.ValueType.STRING => f.valueString(0) - case Field.ValueType.BOOL => f.valueBool(0) - case Field.ValueType.DOUBLE => f.valueDouble(0) - case Field.ValueType.INTEGER => f.valueInteger(0) + case Field.ValueTypeEnum.STRING => f.valueString(0) + case Field.ValueTypeEnum.BOOL => f.valueBool(0) + case Field.ValueTypeEnum.DOUBLE => f.valueDouble(0) + case Field.ValueTypeEnum.INTEGER => f.valueInteger(0) case _ => assert(false) } } @@ -42,19 +42,19 @@ package object heka { def apply (uuid: String, fieldsMap: Map[String, Any], payload: Option[String], timestamp: Long=0): Message = { val fields = fieldsMap.toList.map{ case (k: String, v: ByteString) => { - Field(k, Some(Field.ValueType.BYTES), valueBytes=Seq(v)) + Field(k, Some(Field.ValueTypeEnum.BYTES), valueBytes=Seq(v)) } case (k: String, v: String) => { - Field(k, Some(Field.ValueType.STRING), valueString=Seq(v)) + Field(k, Some(Field.ValueTypeEnum.STRING), valueString=Seq(v)) } case (k: String, v: Boolean) => { - Field(k, Some(Field.ValueType.BOOL), valueBool=Seq(v)) + Field(k, Some(Field.ValueTypeEnum.BOOL), valueBool=Seq(v)) } case (k: String, v: Double) => { - Field(k, Some(Field.ValueType.DOUBLE), valueDouble=Seq(v)) + Field(k, Some(Field.ValueTypeEnum.DOUBLE), valueDouble=Seq(v)) } case (k: String, v: Long) => { - Field(k, Some(Field.ValueType.INTEGER), valueInteger=Seq(v)) + Field(k, Some(Field.ValueTypeEnum.INTEGER), valueInteger=Seq(v)) } }.toSeq Message(ByteString.copyFromUtf8(uuid), timestamp, payload=payload, fields=fields)