Use the port file and dynamic port generation in client/server tests. (…

…#10604)

* Use the port file and dynamic port generation in client/server tests.

This creates a runner named `runner_with_port_file` which knows how to
interpolate two variables, `%PORT_FILE%` and `%PORT%`. This allows us to
use the `port-file` argument to the kvutils runner rather than
hard-coding a port for conformance tests.

For now, we only use this for generating the kvutils reference ledger
export.

CHANGELOG_BEGIN
CHANGELOG_END

* Simplify the runner_with_port_file considerably.

It doesn't need to check if the port is open; we trust that the process
will do it.

This also makes sure the port file will be cleaned up, and reduces the
number of dependencies by making use of more functions in `extra`.

* Simplify port file generation in the new client-server runner.

Co-authored-by: Moritz Kiefer <[email protected]>

* Simplify the runner_with_port_file further.

This doesn't need to work if the server doesn't take a port file.

Co-authored-by: Moritz Kiefer <[email protected]>

bazel_tools/client_server/runner_with_port_file/BUILD.bazel

@@ -0,0 +1,19 @@
+# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+load("//bazel_tools:haskell.bzl", "da_haskell_binary")
+
+da_haskell_binary(
+    name = "runner_with_port_file",
+    srcs = ["Main.hs"],
+    hackage_deps = [
+        "base",
+        "directory",
+        "extra",
+        "filepath",
+        "process",
+        "temporary",
+    ],
+    visibility = ["//visibility:public"],
+    deps = ["//libs-haskell/da-hs-base"],
+)

bazel_tools/client_server/runner_with_port_file/Main.hs

@@ -0,0 +1,34 @@
+-- Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+-- SPDX-License-Identifier: Apache-2.0
+
+module Main (main) where
+
+import Control.Monad (unless)
+import Data.List.Extra (replace, splitOn, stripInfix)
+import Data.Maybe (isJust)
+import System.Environment (getArgs)
+import System.FilePath ((</>))
+import System.Process (callProcess, proc, withCreateProcess)
+import System.Exit (exitFailure)
+import System.IO (hPutStrLn, stderr)
+import System.IO.Temp (withSystemTempDirectory)
+
+import DA.PortFile
+
+main :: IO ()
+main = do
+  [clientExe, clientArgs, serverExe, serverArgs, _] <- getArgs
+  let splitArgs = filter (/= "") . splitOn " "
+  let splitServerArgs = splitArgs serverArgs
+  let splitClientArgs = splitArgs clientArgs
+  unless (any (isJust . stripInfix "%PORT_FILE%") splitServerArgs) $ do
+    hPutStrLn stderr "No server parameters accept a port file."
+    exitFailure
+  withSystemTempDirectory "runner" $ \tempDir -> do
+    let portFile = tempDir </> "portfile"
+    let interpolatedServerArgs = map (replace "%PORT_FILE%" portFile) splitServerArgs
+    let serverProc = proc serverExe interpolatedServerArgs
+    withCreateProcess serverProc $ \_stdin _stdout _stderr _ph -> do
+      port <- readPortFile maxRetries portFile
+      let interpolatedClientArgs = map (replace "%PORT%" (show port)) splitClientArgs
+      callProcess clientExe interpolatedClientArgs

ledger/participant-state/kvutils/BUILD.bazel

@@ -238,8 +238,6 @@ proto_jars(
 
 REFERENCE_LEDGER_EXPORT_NAME = "reference-ledger-export"
 
-REFERENCE_LEDGER_EXPORT_PORT = 65102
-
 # Generates a ledger export by running the test tool against a kvutils-based ledger.
 client_server_build(
     name = REFERENCE_LEDGER_EXPORT_NAME,
@@ -249,15 +247,15 @@ client_server_build(
     client_args = [
         "--concurrent-test-runs=4",
         "--timeout-scale-factor=20",
-        "localhost:%d" % REFERENCE_LEDGER_EXPORT_PORT,
+        "localhost:%PORT%",
     ],
     output_env = "KVUTILS_LEDGER_EXPORT",
-    runner = "@//bazel_tools/client_server/runner_with_port_check:runner",
-    runner_args = [str(REFERENCE_LEDGER_EXPORT_PORT)],
+    runner = "@//bazel_tools/client_server/runner_with_port_file",
+    runner_args = [],
     server = "//ledger/ledger-on-memory:app",
     server_args = [
         "--contract-id-seeding=testing-weak",
-        "--participant=participant-id=%s,port=%d" % (REFERENCE_LEDGER_EXPORT_NAME, REFERENCE_LEDGER_EXPORT_PORT),
+        "--participant=participant-id=export,port=0,port-file=%PORT_FILE%",
     ],
     visibility = [":__subpackages__"],
 ) if not is_windows else None