workbench: restore the CI test; bug fixes all over the place

Makefile

@@ -3,7 +3,6 @@ help: ## Print documentation
 
 include lib.mk
 include nix.mk
-include legacy.mk
 
 PROJECT_NAME = cardano-node
 NUM_PROC     = $(nproc --all)
@@ -35,7 +34,7 @@ trace-documentation:
 ###
 ### Workbench
 ###
-CI_TARGETS := hlint ci-test-plutus-autonix
+CI_TARGETS := hlint workbench-ci-test
 ci:  ci-report ci-targets
 ci-report:
 	@echo -e "\033[34mGoals under test\033[0m:  \033[33m$(CI_TARGETS)\033[0m"
@@ -45,7 +44,7 @@ ci-targets:  $(CI_TARGETS)
 ## Base targets:
 ##
 shell:                                           ## Nix shell, (workbench from /nix/store), vars: PROFILE, CMD, RUN
-	nix-shell -A 'cluster-shell' --max-jobs 8 --cores 0 --show-trace --argstr profileName ${PROFILE} ${ARGS} ${if ${CMD},--command "${CMD}"} ${if ${RUN},--run "${RUN}"}
+	nix-shell -A 'workbench-shell' --max-jobs 8 --cores 0 --show-trace --argstr profileName ${PROFILE} ${ARGS} ${if ${CMD},--command "${CMD}"} ${if ${RUN},--run "${RUN}"}
 shell-dev shell-prof shell-nix: shell
 shell-nix: ARGS += --arg 'workbenchDevMode' false ## Nix shell, (workbench from Nix store), vars: PROFILE, CMD, RUN
 shell-prof: ARGS += --arg 'profiled' true        ## Nix shell, everything Haskell built profiled
@@ -102,4 +101,4 @@ full-clean: clean
 cls:
 	echo -en "\ec"
 
-.PHONY: cabal-hashes clean cli cls cluster-profiles cluster-shell help node run-test shell shell-dev stylish-haskell $(SHELL_PROFILES) workbench-ci-test workbench-ci-bench
+.PHONY: cabal-hashes clean cli cls cluster-profiles help node run-test shell shell-dev stylish-haskell $(SHELL_PROFILES) workbench-ci-test

bench/locli/src/Cardano/Analysis/API.hs

@@ -138,6 +138,7 @@ data BPErrorKind
   | BPEUnexpectedForObserver !Phase
   | BPEUnexpectedForForger   !Phase
   | BPEUnexpectedAsFirst     !Phase
+  | BPENoBlocks
   | BPEDuplicateForge
   | BPEMissingPhase          !Phase
   | BPENegativePhase         !Phase !NominalDiffTime

bench/locli/src/Cardano/Analysis/BlockProp.hs

@@ -248,14 +248,19 @@ type MachBlockMap a
 
 data MachView
   = MachView
-  { mvBlocks  :: !(MachBlockMap UTCTime)
+  { mvHost    :: !Host
+  , mvBlocks  :: !(MachBlockMap UTCTime)
   , mvChecked :: Maybe UTCTime
   , mvLeading :: Maybe UTCTime
   }
   deriving (FromJSON, Generic, NFData, ToJSON)
 
-blockMapMaxBlock :: MachBlockMap a -> MachBlockEvents a
-blockMapMaxBlock = maximumBy ordBlockEv . Map.elems
+machViewMaxBlock :: MachView -> MachBlockEvents UTCTime
+machViewMaxBlock MachView{..} =
+  Map.elems mvBlocks
+  & \case
+       [] -> MBE $ BPError { eHost=mvHost, eBlock=Hash "Genesis", eLO=Nothing, eDesc=BPENoBlocks }
+       xs -> maximumBy ordBlockEv xs
 
 beForgedAt :: BlockEvents -> UTCTime
 beForgedAt BlockEvents{beForge=BlockForge{..}} =
@@ -273,7 +278,7 @@ rebuildChain run@Run{genesis} xs@(fmap snd -> machViews) = do
  where
    eventMaps      = mvBlocks <$> machViews
 
-   finalBlockEv   = maximumBy ordBlockEv $ blockMapMaxBlock <$> eventMaps
+   finalBlockEv   = maximumBy ordBlockEv $ machViewMaxBlock <$> machViews
 
    tipHash        = rewindChain eventMaps 1 (mbeBlock finalBlockEv)
    tipBlock       = getBlockForge eventMaps tipHash
@@ -498,16 +503,15 @@ witherToDistrib distrify proj xs =
 
 -- | Given a single machine's log object stream, recover its block map.
 blockEventMapsFromLogObjects :: Run -> (JsonLogfile, [LogObject]) -> MachView
-blockEventMapsFromLogObjects run (f@(unJsonLogfile -> fp), xs) =
-  if Map.size (mvBlocks view) == 0
-  then error $ mconcat
-       ["No block events in ",fp," : ","LogObject count: ",show (length xs)]
-  else view
+blockEventMapsFromLogObjects run (f@(unJsonLogfile -> fp), []) =
+  error $ mconcat ["0 LogObjects in ", fp]
+blockEventMapsFromLogObjects run (f@(unJsonLogfile -> fp), xs@(x:_)) =
+  foldl' (blockPropMachEventsStep run f) initial xs
  where
-   view = foldl' (blockPropMachEventsStep run f) initial xs
    initial =
      MachView
-     { mvBlocks  = mempty
+     { mvHost    = loHost x
+     , mvBlocks  = mempty
      , mvChecked = Nothing
      , mvLeading = Nothing
      }

flake.nix

@@ -270,22 +270,15 @@
               ## TODO: drop external membench, once we bump 'node-snapshot'
               # snapshot = membench.outputs.packages.x86_64-linux.snapshot;
               # membenches = pkgs.membench-node-this-5.batch-report;
-              # workbench-smoke-test =
-              #   (pkgs.supervisord-workbench-for-profile
-              #     {
-              #       inherit supervisord-workbench;
-              #       profileName = "smoke-alzo";
-              #     }
-              #   ).profile-run { trace = true; };
-              # workbench-ci-test =
-              #   (pkgs.supervisord-workbench-for-profile
-              #     {
-              #       inherit supervisord-workbench;
-              #       profileName = "k6-600slots-1000kU-1000kD-64kbs-10tps-fixed-loaded-alzo";
-              #     }
-              #   ).profile-run { };
-              # workbench-smoke-analysis = workbench-smoke-test.analysis;
-              # workbench-ci-analysis = workbench-ci-test.analysis;
+
+              ## This is a very light profile, no caching&pinning needed.
+              workbench-ci-test =
+                (pkgs.supervisord-workbench-for-profile
+                  {
+                    # inherit supervisord-workbench; ## Not required, as long as it's fast.
+                    profileName = "ci-test-bage";
+                  }).profile-run {};
+
               all-profiles-json = pkgs.all-profiles-json;
             }
             # Add checks to be able to build them individually
@@ -419,8 +412,6 @@
         let
           nonPrJobs = map lib.hasPrefix [
             "linux.native.membenches"
-            "linux.native.workbench-ci-analysis"
-            "linux.native.workbench-ci-test"
           ];
         in
         (lib.mapAttrsRecursiveCond (v: !(lib.isDerivation v))

nix.mk

@@ -13,13 +13,5 @@ membench-5:    ## Membench:  5 iterations, current commit
 membench-5-at: ## Membench:  5 iterations, set commit by:  make membench-5-at REV=[master]
 	nix build .#membench-node-this-5.batch-report      --out-link result-batch-5-report --override-input node-measured github:input-output-hk/cardano-node/${REV}
 
-test-smoke: smoke ## Build the 'workbench-smoke-test' flake attr, same as the Hydra job
-smoke:
-	nix build -f 'default.nix' 'workbench-smoke-test'     --out-link result-smoke-run      --cores 0
-
-test-analysis: smoke-analysis ## Build the 'workbench-smoke-analysis' flake attr, same as the Hydra job
-smoke-analysis:
-	nix build -f 'default.nix' 'workbench-smoke-analysis' --out-link result-smoke-analysis --cores 0 --show-trace
-
-ci-analysis:
-	nix build -f 'default.nix' 'workbench-ci-analysis'    --out-link result-ci-analysis    --cores 0 --show-trace
+workbench-ci-test smoke: ## Workbench:  test a-la Hydra, the ci-test profile, full Nix engaged
+	nix build '.#hydraJobsPr.linux.native.workbench-ci-test' --cores 0

nix/workbench/analyse.nix

@@ -29,9 +29,9 @@ pkgs.runCommand "workbench-run-analysis-${profileNix.name}"
   ''${cmd[@]} 2>&1 |
       tee $out/wb-analyse.log
 
-  cat > $out/nix-support/hydra-build-products <<EOF
-  report block-propagation      $out block-propagation.txt
-  report node-1-timeline        $out logs-node-1.timeline.txt
-  report block-propagation.json $out block-propagation.json
+  cd $out
+  for x in $(ls *.json *.org *.txt | grep -v 'flt\.json$')
+  do echo "report $x $out $x" >> $out/nix-support/hydra-build-products
+  done
   EOF
   ''

nix/workbench/default.nix

@@ -118,7 +118,7 @@ let
 
       topology = profile-topology { inherit profileNix profile; };
 
-      genesis = profile-topology-genesis { inherit profileNix profile topology; };
+      genesis = profile-topology-genesis { inherit profile; };
     in {
       inherit
         profileNix profile

nix/workbench/genesis.nix

@@ -1,32 +1,36 @@
 { pkgs }:
 
-{ profileNix, profile, topology }:
-pkgs.runCommand "workbench-profile-genesis-cache-${profileNix.name}"
+{ profile }:
+pkgs.runCommand "workbench-profile-genesis-cache-${profile.name}"
   { requiredSystemFeatures = [ "benchmark" ];
     nativeBuildInputs = with pkgs.haskellPackages; with pkgs;
       [ bash cardano-cli coreutils gnused jq moreutils workbench.workbench ];
   }
   ''
   mkdir $out
 
-  cache_key_input=$(wb genesis profile-cache-key-input ${profileNix.JSON})
-  cache_key=$(wb genesis profile-cache-key ${profileNix.JSON})
+  cache_key_input=$(wb genesis profile-cache-key-input ${profile}/profile.json)
+  cache_key=$(      wb genesis profile-cache-key       ${profile}/profile.json)
 
-  keepalive() {
-      while test ! -e $out/profile; do echo 'keepalive for Hydra'; sleep 60s; done
-      echo 'keepalive done'
+  genesis_keepalive() {
+      while test ! -e $out/profile; do echo 'genesis_keepalive for Hydra'; sleep 10s; done
   }
-  keepalive &
+  genesis_keepalive &
+  __genesis_keepalive_pid=$!
+  __genesis_keepalive_termination() {
+      kill $__genesis_keepalive_pid 2>/dev/null || true
+  }
+  trap __genesis_keepalive_termination EXIT
 
   args=(
      genesis actually-genesis
-     ${profileNix.JSON}
-     ${topology}
+     ${profile}/profile.json
+     ${profile}/node-specs.json
      $out
      "$cache_key_input"
      "$cache_key"
   )
-  time wb ''${args[@]}
+  time wb "''${args[@]}"
 
   touch done
 

nix/workbench/genesis.sh

@@ -148,6 +148,8 @@ case "$op" in
         local cache_key_input=$4
         local cache_key=$5
 
+        progress "genesis" "new one:  $(yellow profile) $(blue $profile_json) $(yellow node_specs) $(blue $node_specs) $(yellow dir) $(blue $dir) $(yellow cache_key) $(blue $cache_key) $(yellow cache_key_input) $(blue $cache_key_input)"
+
         rm -rf   "$dir"/{*-keys,byron,pools,nodes,*.json,*.params,*.version}
         mkdir -p "$dir"
 
@@ -158,7 +160,6 @@ case "$op" in
            "$global_basedir"/profiles/presets/mainnet/genesis/genesis.alonzo.json \
            >   "$dir"/genesis.alonzo.spec.json
 
-        msg "genesis:  creating initial genesis"
         cardano-cli genesis create --genesis-dir "$dir"/ \
             $(jq '.cli_args.createSpec | join(" ")' "$profile_json" --raw-output)
 
@@ -287,7 +288,9 @@ Massage_the_key_file_layout_to_match_AWS() {
 
     set -euo pipefail
 
-    local pool_density_map=$(topology density-map "$profile_json" "$node_specs")
+    local pool_density_map=$(topology density-map "$node_specs")
+    if test -z "$pool_density_map"
+    then fatal "failed: topology density-map '$node_specs'"; fi
     msg "genesis: pool density map:  $pool_density_map"
 
     __KEY_ROOT=$dir

nix/workbench/lib-cabal.sh

@@ -1,3 +1,4 @@
+echo 'lib-cabal args: '"$*"
 progress "workbench"  "cabal-inside-nix-shell mode enabled, calling cardano-* via '$(white cabal run)' (instead of using Nix store); $(red lib-cabal.sh) flags: $(yellow $*)"
 
 if test ! -v WB_PROFILED; then export WB_PROFILED= ; fi

nix/workbench/lib.sh

@@ -64,6 +64,7 @@ colorise_colors=(
     red green blue yellow white cyan
 )
 colorise() {
+    local i
     for ((i=0; $#!=0; i++))
     do echo -n "$(with_color ${colorise_colors[$((i % 6))]} $1) "
        shift

nix/workbench/manifest.sh

@@ -96,14 +96,20 @@ case "${op}" in
 
 manifest_git_head_commit() {
     local dir=$1
-    git -C "$dir" rev-parse HEAD
+    if test -d "$dir"/.git
+    then git -C "$dir" rev-parse HEAD
+    else echo      -n "0000000000000000000000000000000000000000"
+    fi
 }
 
 manifest_git_checkout_state_desc() {
     local dir=$1
-    if git -C "$dir" diff --quiet --exit-code
-    then echo -n "clean"
-    else echo -n "modified"
+    if test -d "$dir"/.git
+    then if git -C "$dir" diff --quiet --exit-code
+         then echo -n "clean"
+         else echo -n "modified"
+         fi
+    else echo      -n "no-git"
     fi
 }
 
@@ -115,7 +121,10 @@ manifest_cabal_project_dep_pin_hash() {
 }
 
 manifest_local_repo_branch() {
-        local dir=$1 rev=${2:-HEAD}
-        git -C "$dir" describe --all "$rev" |
-                sed 's_^\(.*/\|\)\([^/]*\)$_\2_'
+    local dir=$1 rev=${2:-HEAD}
+    if test -d "$dir"/.git
+    then git -C "$dir" describe --all "$rev" |
+            sed 's_^\(.*/\|\)\([^/]*\)$_\2_'
+    else echo      -n "no-git"
+    fi
 }

nix/workbench/profile.nix

@@ -47,3 +47,6 @@ pkgs.runCommand "workbench-profile-output-${profileNix.name}"
 
   wb profile node-specs $out/profile.json > $out/node-specs.json
   ''
+// { inherit (profileNix) name;
+     inherit workbench;
+   }

nix/workbench/profiles/variants.jq

@@ -291,7 +291,7 @@ def all_profile_variants:
     { desc: "Stop as soon as we've seen a single block"
     }) as $startstop_base
   |
-   ($scenario_fixed_loaded * $doublet * $dataset_miniature * $for_3blk *
+   ($scenario_fixed_loaded * $doublet * $dataset_empty * $for_3blk *
     { desc: "Miniature dataset, CI-friendly duration, test scale"
     }) as $citest_base
   |