Merge branch 'master' into fixes/d2iq-archive#4948/lazy_event_parsing

Jenkinsfile.scale

@@ -0,0 +1,20 @@
+#!/usr/bin/env groovy
+
+node {
+  properties([
+     parameters([
+       stringParam(
+         description: 'Sets Marathn endpoint.',
+         name: 'marathonUrl'
+       )
+     ])
+  ])
+
+  stage("Connect to CCM Cluster") {
+   println params.marathonUrl
+  }
+
+  stage("Run Scale Tests"){}
+
+  stage("Archive Results"){}
+}

src/main/scala/mesosphere/marathon/api/v2/Validation.scala

@@ -1,4 +1,5 @@
-package mesosphere.marathon.api.v2
+package mesosphere.marathon
+package api.v2
 
 import java.net._
 

src/main/scala/mesosphere/marathon/api/v2/validation/PodsValidation.scala

@@ -237,7 +237,7 @@ trait PodsValidation {
             } else {
               Success
             }
-          case GroupBy | MaxPer =>
+          case GroupBy =>
             if (c.value.fold(true)(i => Try(i.toInt).isSuccess)) {
               Success
             } else {
@@ -246,6 +246,15 @@ trait PodsValidation {
                 "Value was specified but is not a number",
                 Some("GROUP_BY may either have no value or an integer value"))))
             }
+          case MaxPer =>
+            if (c.value.fold(false)(i => Try(i.toInt).isSuccess)) {
+              Success
+            } else {
+              Failure(Set(RuleViolation(
+                c,
+                "Value was not specified or is not a number",
+                Some("MAX_PER must have an integer value"))))
+            }
           case Like | Unlike =>
             c.value.fold[Result] {
               Failure(Set(RuleViolation(c, "A regular expression value must be provided", None)))

src/main/scala/mesosphere/marathon/state/AppDefinition.scala

@@ -595,8 +595,8 @@ object AppDefinition extends GeneralPurposeCombinators {
             } else {
               Failure(Set(RuleViolation(
                 c,
-                "Value was specified but is not a number",
-                Some("MAX_PER may have an integer value"))))
+                "Value was not specified or is not a number",
+                Some("MAX_PER must have an integer value"))))
             }
           case _ =>
             Failure(Set(

src/test/scala/mesosphere/marathon/api/v2/ValidationTest.scala

@@ -0,0 +1,19 @@
+package mesosphere.marathon
+package api.v2
+
+import mesosphere.UnitTest
+import com.wix.accord.{Failure, RuleViolation}
+import play.api.libs.json._
+
+class ValidationTest extends UnitTest {
+  import Validation._
+
+  "The failure format" should {
+    "write validations errors" in {
+      val violation = RuleViolation(value = Some("foo"), constraint = "is a number", description = Some("id"))
+      val failure = Failure(Set(violation))
+      val json = Json.toJson(failure)
+      json.toString should be("""{"message":"Object is not valid","details":[{"path":"/id","errors":["is a number"]}]}""")
+    }
+  }
+}

src/test/scala/mesosphere/marathon/api/v2/validation/PodsValidationTest.scala

@@ -4,7 +4,7 @@ package v2.validation
 import mesosphere.UnitTest
 import com.wix.accord.scalatest.ResultMatchers
 import mesosphere.marathon.api.v2.validation.PodsValidation
-import mesosphere.marathon.raml.{ Endpoint, Network, NetworkMode, Pod, PodContainer, Resources, Volume, VolumeMount }
+import mesosphere.marathon.raml.{ Constraint, ConstraintOperator, Endpoint, Network, NetworkMode, Pod, PodContainer, Resources, Volume, VolumeMount }
 import mesosphere.marathon.util.SemanticVersion
 
 class PodsValidationTest extends UnitTest with ResultMatchers with PodsValidation {
@@ -53,6 +53,17 @@ class PodsValidationTest extends UnitTest with ResultMatchers with PodsValidatio
     }
   }
 
+  "A constraint definition" should {
+
+    "MaxPer is accepted with an integer value" in {
+      PodsValidation.complyWithConstraintRules(Constraint("foo", ConstraintOperator.MaxPer, Some("3"))).isSuccess shouldBe true
+    }
+
+    "MaxPer is rejected with no value" in {
+      PodsValidation.complyWithConstraintRules(Constraint("foo", ConstraintOperator.MaxPer)).isSuccess shouldBe false
+    }
+  }
+
   class Fixture {
     val validContainer = PodContainer(
       name = "ct1",

src/test/scala/mesosphere/marathon/integration/MesosAppIntegrationTest.scala

@@ -1,12 +1,14 @@
 package mesosphere.marathon
 package integration
 
+import java.util.concurrent.atomic.AtomicInteger
+
 import mesosphere.marathon.core.health.{ MesosHttpHealthCheck, PortReference }
 import mesosphere.marathon.core.pod.{ HostNetwork, HostVolume, MesosContainer, PodDefinition }
 import mesosphere.marathon.integration.facades.MarathonFacade._
 import mesosphere.marathon.integration.setup.{ EmbeddedMarathonTest, MesosConfig, WaitTestSupport }
 import mesosphere.marathon.raml.PodInstanceState
-import mesosphere.marathon.state.{ AppDefinition, Container, PathId }
+import mesosphere.marathon.state.{ AppDefinition, Container }
 import mesosphere.{ AkkaIntegrationTest, WhenEnvSet }
 
 import scala.collection.immutable.Seq
@@ -18,15 +20,19 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
   // Integration tests using docker image provisioning with the Mesos containerizer need to be
   // run as root in a Linux environment. They have to be explicitly enabled through an env variable.
   val envVar = "RUN_MESOS_INTEGRATION_TESTS"
+
+  val currentAppId = new AtomicInteger()
+
   // Configure Mesos to provide the Mesos containerizer with Docker image support.
   override lazy val mesosConfig = MesosConfig(
     launcher = "linux",
     containerizers = "mesos",
     isolation = Some("filesystem/linux,docker/runtime"),
     imageProviders = Some("docker"))
 
-  private[this] def simplePod(podId: PathId): PodDefinition = PodDefinition(
-    id = podId,
+
+  private[this] def simplePod(podId: String): PodDefinition = PodDefinition(
+    id = testBasePath / s"$podId-${currentAppId.incrementAndGet()}",
     containers = Seq(
       MesosContainer(
         name = "task1",
@@ -45,7 +51,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
     "deploy a simple Docker app using the Mesos containerizer" taggedAs WhenEnvSet(envVar) in {
       Given("a new Docker app")
       val app = AppDefinition(
-        id = testBasePath / "mesosdockerapp",
+        id = testBasePath / s"mesos-docker-app-${currentAppId.incrementAndGet()}",
         cmd = Some("sleep 600"),
         container = Some(Container.MesosDocker(image = "busybox")),
         resources = raml.Resources(cpus = 0.2, mem = 16.0),
@@ -56,7 +62,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
       val result = marathon.createAppV2(app)
 
       Then("The app is created")
-      result.code should be(201) // Created
+      result.code should be(201) withClue s"Response: ${result.entityString}" // Created
       extractDeploymentIds(result) should have size 1
       waitForDeployment(result)
       waitForTasks(app.id, 1) // The app has really started
@@ -65,7 +71,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
     "deploy a simple Docker app that uses Entrypoint/Cmd using the Mesos containerizer" taggedAs WhenEnvSet(envVar) in {
       Given("a new Docker app the uses 'Cmd' in its Dockerfile")
       val app = AppDefinition(
-        id = testBasePath / "mesosdockerapp",
+        id = testBasePath / s"mesos-docker-app-${currentAppId.incrementAndGet()}",
         container = Some(Container.MesosDocker(image = "hello-world")),
         resources = raml.Resources(cpus = 0.1, mem = 32.0),
         instances = 1
@@ -75,21 +81,21 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
       val result = marathon.createAppV2(app)
 
       Then("The app is created")
-      result.code should be(201) // Created
+      result.code should be(201) withClue s"Response: ${result.entityString}" // Created
       extractDeploymentIds(result) should have size 1
       waitForDeployment(result)
       waitForTasks(app.id, 1) // The app has really started
     }
 
     "deploy a simple pod" taggedAs WhenEnvSet(envVar) in {
       Given("a pod with a single task")
-      val pod = simplePod(testBasePath / "simplepod")
+      val pod = simplePod("simplepod")
 
       When("The pod is deployed")
       val createResult = marathon.createPodV2(pod)
 
       Then("The pod is created")
-      createResult.code should be(201) // Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" // Created
       waitForDeployment(createResult)
       waitForPod(pod.id)
 
@@ -114,7 +120,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
       val homeDir = sys.props.getOrElse("user.home", "~")
 
       Given("a pod with two tasks that are health checked")
-      val podId = testBasePath / "healthypod"
+      val podId = testBasePath / s"healthypod-${currentAppId.incrementAndGet()}"
       val containerDir = "/opt/marathon"
 
       def appMockCommand(port: String) = appProxyCommand(podId, "v1", containerDir, port)
@@ -164,7 +170,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
       val createResult = marathon.createPodV2(pod)
 
       Then("The pod is created")
-      createResult.code should be(201) //Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" //Created
       // The timeout is 5 minutes because downloading and provisioning the Python image can take some time.
       waitForDeployment(createResult, 300.seconds)
       waitForPod(podId)
@@ -194,7 +200,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
 
     "deploy a pod with Entrypoint/Cmd" taggedAs WhenEnvSet(envVar) in {
       Given("A pod using the 'hello' image that sets Cmd in its Dockerfile")
-      val pod = simplePod(testBasePath / "simplepod").copy(
+      val pod = simplePod("simplepod").copy(
         containers = Seq(MesosContainer(
           name = "hello",
           resources = raml.Resources(cpus = 0.1, mem = 32.0),
@@ -206,16 +212,16 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
       val createResult = marathon.createPodV2(pod)
 
       Then("The pod is created")
-      createResult.code should be(201) // Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" // Created
       waitForDeployment(createResult)
       waitForPod(pod.id)
     }
 
     "deleting a group deletes pods deployed in the group" taggedAs WhenEnvSet(envVar) in {
       Given("a deployed pod")
-      val pod = simplePod(testBasePath / "simplepod")
+      val pod = simplePod("simplepod")
       val createResult = marathon.createPodV2(pod)
-      createResult.code should be(201) //Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" //Created
       waitForDeployment(createResult)
       waitForPod(pod.id)
       marathon.listPodsInBaseGroup.value should have size 1
@@ -235,9 +241,9 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
 
     "list pod versions" taggedAs WhenEnvSet(envVar) in {
       Given("a new pod")
-      val pod = simplePod(testBasePath / "simplepod")
+      val pod = simplePod("simplepod")
       val createResult = marathon.createPodV2(pod)
-      createResult.code should be(201) //Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" //Created
       waitForDeployment(createResult)
       waitForPod(pod.id)
 
@@ -252,7 +258,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
 
     "correctly version pods" taggedAs WhenEnvSet(envVar) in {
       Given("a new pod")
-      val pod = simplePod(testBasePath / "simplepod")
+      val pod = simplePod("simplepod")
       val createResult = marathon.createPodV2(pod)
       createResult.code should be(201)
       //Created
@@ -286,13 +292,12 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
     "stop (forcefully delete) a pod deployment" taggedAs WhenEnvSet(envVar) in {
       Given("a pod with constraints that cannot be fulfilled")
       val constraint = Protos.Constraint.newBuilder().setField("nonExistent").setOperator(Protos.Constraint.Operator.CLUSTER).setValue("na").build()
-      val pod = simplePod(testBasePath / "simplepod").copy(
+      val pod = simplePod("simplepod").copy(
         constraints = Set(constraint)
       )
 
       val createResult = marathon.createPodV2(pod)
-      createResult.code should be(201)
-      //Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" //Created
       val deploymentId = createResult.originalResponse.headers.find(_.name == "Marathon-Deployment-Id").map(_.value)
       deploymentId shouldBe defined
 
@@ -316,13 +321,12 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
     "rollback a pod deployment" taggedAs WhenEnvSet(envVar) in {
       Given("a pod with constraints that cannot be fulfilled")
       val constraint = Protos.Constraint.newBuilder().setField("nonExistent").setOperator(Protos.Constraint.Operator.CLUSTER).setValue("na").build()
-      val pod = simplePod(testBasePath / "simplepod").copy(
+      val pod = simplePod("simplepod").copy(
         constraints = Set(constraint)
       )
 
       val createResult = marathon.createPodV2(pod)
-      createResult.code should be(201)
-      //Created
+      createResult.code should be(201) withClue s"Response: ${createResult.entityString}" //Created
       val deploymentId = createResult.originalResponse.headers.find(_.name == "Marathon-Deployment-Id").map(_.value)
       deploymentId shouldBe defined
 
@@ -346,7 +350,7 @@ class MesosAppIntegrationTest extends AkkaIntegrationTest with EmbeddedMarathonT
 
     "delete pod instances" taggedAs WhenEnvSet(envVar) in {
       Given("a new pod with 2 instances")
-      val pod = simplePod(testBasePath / "simplepod").copy(
+      val pod = simplePod("simplepod").copy(
         instances = 3
       )
 

src/test/scala/mesosphere/marathon/state/PathIdTest.scala

@@ -3,10 +3,11 @@ package state
 
 import mesosphere.UnitTest
 import mesosphere.marathon.state.PathId._
+import com.wix.accord.scalatest.ResultMatchers
 
 import scala.collection.SortedSet
 
-class PathIdTest extends UnitTest {
+class PathIdTest extends UnitTest with ResultMatchers {
   "A PathId" can {
     "be parsed from string" in {
       Given("A base id")
@@ -164,4 +165,24 @@ class PathIdTest extends UnitTest {
       SortedSet(ac, ab, aa).toSeq should equal(Seq(aa, ab, ac))
     }
   }
+
+  "The PathId validation" when {
+
+    "passed legal characters" should {
+      "be valid" in {
+        val path = PathId("/foobar-0")
+        val validation = PathId.pathIdValidator(path)
+        validation shouldBe aSuccess
+      }
+    }
+
+    "passed illegal characters" should {
+      "be invalid" in {
+        val path = PathId("/@§\'foobar-0")
+        val validation = PathId.pathIdValidator(path)
+        val expectedViolation = RuleViolationMatcher(value = "@§'foobar-0", constraint = "must fully match regular expression '^(([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])\\.)*([a-z0-9]|[a-z0-9][a-z0-9\\-]*[a-z0-9])|(\\.|\\.\\.)$'")
+        validation should failWith(expectedViolation)
+      }
+    }
+  }
 }