Merge #4208

4208: workbench: template-based benchmark reporting & fixes r=deepfire a=deepfire 1. Fix `aws-get` & speed up `fetch-analysis` 2. Templated benchmark reporting: `wb analyse compare RUN1 RUN...` Co-authored-by: Kosyrev Serge <[email protected]>
IntersectMBO · Aug 4, 2022 · 866c421 · 866c421
2 parents d6d2dc7 + eb4f35d
commit 866c421
Show file tree

Hide file tree

Showing 18 changed files with 516 additions and 164 deletions.
diff --git a/Makefile b/Makefile
@@ -68,7 +68,7 @@ ps:                                              ## Plain-text list of profiles
 ##
 PROFILES_BASE         := default plutus oldtracing
 PROFILES_STARTSTOP    := startstop startstop-p2p startstop-plutus startstop-notracer startstop-oldtracing
-PROFILES_CI_TEST      := ci-test ci-test-p2p ci-test-plutus ci-test-notracer
+PROFILES_CI_TEST      := ci-test ci-test-p2p ci-test-plutus ci-test-notracer ci-test-dense10
 PROFILES_CI_BENCH     := ci-bench ci-bench-p2p ci-bench-plutus ci-bench-notracer
 PROFILES_10           := 10 10-p2p 10-plutus 10-notracer
 PROFILES_FORGE_STRESS := forge-stress forge-stress-p2p forge-stress-plutus forge-stress-plutus-singleton forge-stress-notracer

diff --git a/bench/locli/locli.cabal b/bench/locli/locli.cabal
@@ -47,6 +47,9 @@ common base
 
   build-depends:       base
 
+  if os(windows)
+    buildable: False
+
 library
   import:              base
   hs-source-dirs:      src
@@ -56,6 +59,7 @@ library
                        Data.DataDomain
 
                        Cardano.Command
+                       Cardano.Report
                        Cardano.TopHandler
                        Cardano.Util
 
@@ -91,6 +95,7 @@ library
                      , containers
                      , deepseq
                      , directory
+                     , ede
                      , extra
                      , filepath
                      , file-embed
@@ -114,6 +119,7 @@ library
                      , trace-resources
                      , transformers
                      , transformers-except
+                     , unix
                      , unordered-containers
                      , utf8-string
                      , vector

diff --git a/bench/locli/src/Cardano/Analysis/API.hs b/bench/locli/src/Cardano/Analysis/API.hs
@@ -37,7 +37,7 @@ import Cardano.Util
 -- | Results of block propagation analysis.
 data BlockProp f
   = BlockProp
-    { bpVersion             :: !Version
+    { bpVersion             :: !Cardano.Analysis.Version.Version
     , bpDomainSlots         :: !(DataDomain SlotNo)
     , bpDomainBlocks        :: !(DataDomain BlockNo)
     , bpForgerChecks        :: !(CDF f NominalDiffTime)
@@ -148,7 +148,7 @@ data BPErrorKind
 -- | The top-level representation of the machine timeline analysis results.
 data MachPerf f
   = MachPerf
-    { sVersion              :: !Version
+    { sVersion              :: !Cardano.Analysis.Version.Version
     , sDomainSlots          :: !(DataDomain SlotNo)
     -- distributions
     , sMissCDF              :: !(CDF f Double)

diff --git a/bench/locli/src/Cardano/Analysis/Context.hs b/bench/locli/src/Cardano/Analysis/Context.hs
@@ -1,10 +1,12 @@
 {-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE GeneralisedNewtypeDeriving #-}
 {-# LANGUAGE StrictData #-}
 module Cardano.Analysis.Context (module Cardano.Analysis.Context) where
 
 import Cardano.Prelude
 
-import Data.Aeson (FromJSON, ToJSON)
+import Data.Aeson (FromJSON (..), ToJSON (..), withObject, object, (.:), (.:?), (.=))
+import Data.Text qualified as T
 import Data.Time.Clock (UTCTime, NominalDiffTime)
 
 
@@ -52,10 +54,74 @@ data GeneratorProfile
   }
   deriving (Generic, Show, FromJSON, ToJSON)
 
+newtype Commit   = Commit  { unCommit  :: Text } deriving newtype (Show, FromJSON, ToJSON)
+newtype Branch   = Branch  { unBranch  :: Text } deriving newtype (Show, FromJSON, ToJSON)
+newtype Version  = Version { unVersion :: Text } deriving newtype (Show, FromJSON, ToJSON)
+
+unsafeShortenCommit :: Int -> Commit -> Commit
+unsafeShortenCommit n (Commit c) = Commit (T.take n c)
+
+data Manifest
+  = Manifest
+    { mNode          :: !Commit
+    , mNodeApproxVer :: !Version
+    , mNodeBranch    :: !Branch
+    , mNodeStatus    :: !Text
+    , mNetwork       :: !Commit
+    , mLedger        :: !Commit
+    , mPlutus        :: !Commit
+    , mCrypto        :: !Commit
+    , mBase          :: !Commit
+    , mPrelude       :: !Commit
+    }
+  deriving (Generic, Show)
+
+unsafeShortenManifest :: Int -> Manifest -> Manifest
+unsafeShortenManifest n m@Manifest{..} =
+  m { mNode    = unsafeShortenCommit n mNode
+    , mNetwork = unsafeShortenCommit n mNetwork
+    , mLedger  = unsafeShortenCommit n mLedger
+    , mPlutus  = unsafeShortenCommit n mPlutus
+    , mCrypto  = unsafeShortenCommit n mCrypto
+    , mBase    = unsafeShortenCommit n mBase
+    , mPrelude = unsafeShortenCommit n mPrelude
+    }
+
+instance FromJSON Manifest where
+  parseJSON = withObject "Manifest" $ \v -> do
+    mNode          <- v .: "cardano-node"
+    mNodeBranch    <- v .:? "cardano-node-branch"  <&> fromMaybe (Branch "unknown")
+    mNodeApproxVer <- v .:? "cardano-node-version" <&> fromMaybe (Version "unknown")
+    mNodeStatus    <- v .: "cardano-node-status"
+    mNetwork       <- v .: "ouroboros-network"
+    mLedger        <- v .: "cardano-ledger"
+    mPlutus        <- v .: "plutus"
+    mCrypto        <- v .: "cardano-crypto"
+    mBase          <- v .: "cardano-base"
+    mPrelude       <- v .: "cardano-prelude"
+    pure Manifest{..}
+
+instance ToJSON Manifest where
+  toJSON Manifest{..} =
+    object
+      [ "cardano-node"         .= mNode
+      , "cardano-node-branch"  .= mNodeBranch
+      , "cardano-node-version" .= mNodeApproxVer
+      , "cardano-node-status"  .= mNodeStatus
+      , "ouroboros-network"    .= mNetwork
+      , "cardano-ledger"       .= mLedger
+      , "plutus"               .= mPlutus
+      , "cardano-crypto"       .= mCrypto
+      , "cardano-base"         .= mBase
+      , "cardano-prelude"      .= mPrelude
+      ]
+
 data Metadata
   = Metadata
   { tag       :: Text
+  , batch     :: Text
   , profile   :: Text
   , era       :: Text
+  , manifest  :: Manifest
   }
   deriving (Generic, Show, FromJSON, ToJSON)
diff --git a/bench/locli/src/Cardano/Analysis/Ground.hs b/bench/locli/src/Cardano/Analysis/Ground.hs
@@ -82,28 +82,21 @@ data HostDeduction
 ---
 --- Files
 ---
-newtype JsonGenesisFile
-  = JsonGenesisFile { unJsonGenesisFile :: FilePath }
-  deriving (Show, Eq)
-
-newtype JsonRunMetafile
-  = JsonRunMetafile { unJsonRunMetafile :: FilePath }
-  deriving (Show, Eq)
-
-newtype JsonDomainFile
-  = JsonDomainFile { unJsonDomainFile :: FilePath }
+newtype InputDir
+  = InputDir { unInputDir :: FilePath }
   deriving (Show, Eq)
+  deriving newtype (NFData)
 
 newtype JsonLogfile
   = JsonLogfile { unJsonLogfile :: FilePath }
   deriving (Show, Eq)
   deriving newtype (NFData)
 
-newtype JsonInputFile
+newtype JsonInputFile (a :: Type)
   = JsonInputFile { unJsonInputFile :: FilePath }
   deriving (Show, Eq)
 
-newtype JsonOutputFile
+newtype JsonOutputFile (a :: Type)
   = JsonOutputFile { unJsonOutputFile :: FilePath }
   deriving (Show, Eq)
 
@@ -115,6 +108,10 @@ newtype OrgOutputFile
   = OrgOutputFile { unOrgOutputFile :: FilePath }
   deriving (Show, Eq)
 
+newtype TextInputFile
+  = TextInputFile { unTextInputFile :: FilePath }
+  deriving (Show, Eq)
+
 newtype TextOutputFile
   = TextOutputFile { unTextOutputFile :: FilePath }
   deriving (Show, Eq)
@@ -127,59 +124,15 @@ newtype OutputFile
   = OutputFile { unOutputFile :: FilePath }
   deriving (Show, Eq)
 
-data MachineTimelineOutputFiles
-  = MachineTimelineOutputFiles
-  { mtofSlotStats          :: Maybe JsonOutputFile
-  , mtofAnalysis           :: Maybe JsonOutputFile
-  , mtofFullStatsPretty    :: Maybe TextOutputFile
-  , mtofReportStatsPretty  :: Maybe TextOutputFile
-  , mtofTimelinePretty     :: Maybe TextOutputFile
-  , mtofTimelineCsv        :: Maybe  CsvOutputFile
-  , mtofFullStatsCsv       :: Maybe  CsvOutputFile
-  , mtofDerivedVectors0Csv :: Maybe  CsvOutputFile
-  , mtofDerivedVectors1Csv :: Maybe  CsvOutputFile
-  }
-  deriving (Show)
-
-data BlockPropOutputFiles
-  = BlockPropOutputFiles
-  { bpofForgerPretty       :: Maybe TextOutputFile
-  , bpofPeersPretty        :: Maybe TextOutputFile
-  , bpofPropagationPretty  :: Maybe TextOutputFile
-  , bpofFullStatsPretty    :: Maybe TextOutputFile
-  , bpofMachViews          :: Maybe JsonOutputFile
-  , bpofChainPretty        :: Maybe TextOutputFile
-  , bpofChain              :: Maybe JsonOutputFile
-  , bpofChainRaw           :: Maybe JsonOutputFile
-  , bpofAnalysis           :: Maybe JsonOutputFile
-  }
-  deriving (Show)
-
 ---
 --- Parsers
 ---
-optJsonGenesisFile :: String -> String -> Parser JsonGenesisFile
-optJsonGenesisFile optname desc =
-  fmap JsonGenesisFile $
+optInputDir :: String -> String -> Parser InputDir
+optInputDir optname desc =
+  fmap InputDir $
     Opt.option Opt.str
       $ long optname
-      <> metavar "GENESIS-FILE"
-      <> help desc
-
-optJsonRunMetafile :: String -> String -> Parser JsonRunMetafile
-optJsonRunMetafile optname desc =
-  fmap JsonRunMetafile $
-    Opt.option Opt.str
-      $ long optname
-      <> metavar "RUN-METAFILE"
-      <> help desc
-
-optJsonDomainFile :: String -> String -> Parser JsonDomainFile
-optJsonDomainFile optname desc =
-  fmap JsonDomainFile $
-    Opt.option Opt.str
-      $ long optname
-      <> metavar "DOMAINFILE"
+      <> metavar "DIR"
       <> help desc
 
 optJsonLogfile :: String -> String -> Parser JsonLogfile
@@ -195,15 +148,15 @@ argJsonLogfile =
   JsonLogfile <$>
     Opt.argument Opt.str (Opt.metavar "LOGFILE")
 
-optJsonInputFile :: String -> String -> Parser JsonInputFile
+optJsonInputFile :: String -> String -> Parser (JsonInputFile a)
 optJsonInputFile optname desc =
   fmap JsonInputFile $
     Opt.option Opt.str
       $ long optname
       <> metavar "JSON-FILE"
       <> help desc
 
-optJsonOutputFile :: String -> String -> Parser JsonOutputFile
+optJsonOutputFile :: String -> String -> Parser (JsonOutputFile a)
 optJsonOutputFile optname desc =
   fmap JsonOutputFile $
     Opt.option Opt.str
@@ -219,6 +172,14 @@ optGnuplotOutputFile optname desc =
       <> metavar "CDF-OUTFILE"
       <> help desc
 
+optTextInputFile :: String -> String -> Parser TextInputFile
+optTextInputFile optname desc =
+  fmap TextInputFile $
+    Opt.option Opt.str
+      $ long optname
+      <> metavar "TEXT-INFILE"
+      <> help desc
+
 optTextOutputFile :: String -> String -> Parser TextOutputFile
 optTextOutputFile optname desc =
   fmap TextOutputFile $
@@ -279,56 +240,3 @@ optWord optname desc def =
     <> metavar "INT"
     <> help desc
     <> value def
-
-parseMachineTimelineOutputFiles :: Parser MachineTimelineOutputFiles
-parseMachineTimelineOutputFiles =
-  MachineTimelineOutputFiles
-    <$> optional
-        (optJsonOutputFile "slotstats-json"
-           "Per-slot performance summaries")
-    <*> optional
-        (optJsonOutputFile "analysis-json"
-           "Write analysis JSON to this file, if specified -- otherwise print to stdout.")
-    <*> optional
-        (optTextOutputFile "fullstats-text"
-           "Full performance statistics breakdown")
-    <*> optional
-        (optTextOutputFile "reportstats-text"
-           "Report performance statistics breakdown")
-    <*> optional
-        (optTextOutputFile "timeline-text"
-           "Dump pretty timeline of extracted slot leadership summaries, as a side-effect of log analysis")
-    <*> optional
-        (optCsvOutputFile "timeline-csv"
-           "Dump CSV of the timeline")
-    <*> optional
-        (optCsvOutputFile "stats-csv"
-           "Dump CSV of the timeline statistics")
-    <*> optional
-        (optCsvOutputFile "derived-vectors-0-csv"
-           "Dump CSV of vectors derived from the timeline")
-    <*> optional
-        (optCsvOutputFile "derived-vectors-1-csv"
-           "Dump CSV of vectors derived from the timeline")
-
-parseBlockPropOutputFiles :: Parser BlockPropOutputFiles
-parseBlockPropOutputFiles =
-  BlockPropOutputFiles
-    <$> optional
-        (optTextOutputFile "forger-text"       "Forger stats")
-    <*> optional
-        (optTextOutputFile "peers-text"        "Peers stats")
-    <*> optional
-        (optTextOutputFile "propagation-text"  "Propagation stats")
-    <*> optional
-        (optTextOutputFile "fullstats-text"    "Full (forger+peers+propagation) stats")
-    <*> optional
-        (optJsonOutputFile "mach-views-json"   "Machine chain views as JSON")
-    <*> optional
-        (optTextOutputFile "chain-text"        "Timeline of chain evolution, one line per block")
-    <*> optional
-        (optJsonOutputFile "chain-raw-json"    "Unfiltered chain as JSON")
-    <*> optional
-        (optJsonOutputFile "chain-json"        "Chain as JSON")
-    <*> optional
-        (optJsonOutputFile "analysis-json"     "Analysis as JSON")