add visualizations (#558) (#571)

* Initial ttft box plot implementation

* Add request latency chart

* Add heatmap graph

* Add scatter plot graph

* Add base class for plots

* Update requirements

* Add plot manager and refactor to use a base class

* Add parquet output to genai-perf

* Use new plot manager class and update gitignore to ignore artifacts directory

* Add compression to parquet files

* Remove unused imports

* Fix annotations for box plots

* Rename labels and update another feedback

* Fix title and formatting

* Add comment for deep copy

* Update plots to preprocess data and use a common API

* Uncomment run method and error logging

* Remove * in arg list for plots

* Disable failing test: test_random_synthetic

src/c++/perf_analyzer/genai-perf/.gitignore

@@ -0,0 +1 @@
+artifacts/

src/c++/perf_analyzer/genai-perf/genai_perf/constants.py

@@ -33,3 +33,7 @@
 OPEN_ORCA = "openorca"
 CNN_DAILY_MAIL = "cnn_dailymail"
 DEFAULT_INPUT_DATA_JSON = "llm_inputs.json"
+
+
+DEFAULT_ARTIFACT_DIR = "artifacts"
+DEFAULT_PARQUET_FILE = "all_data"

src/c++/perf_analyzer/genai-perf/genai_perf/graphs/base_plot.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+from copy import deepcopy
+from typing import Dict
+
+from genai_perf.constants import DEFAULT_ARTIFACT_DIR
+from genai_perf.exceptions import GenAIPerfException
+from genai_perf.llm_metrics import Statistics
+from pandas import DataFrame
+from plotly.graph_objects import Figure
+
+
+class BasePlot:
+    """
+    Base class for plots
+    """
+
+    def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
+        self._stats = stats
+        self._metrics_data = deepcopy(stats.metrics.data)
+        if extra_data:
+            self._metrics_data = self._metrics_data | extra_data
+
+    def create_plot(
+        self,
+        x_key: str,
+        y_key: str,
+        x_metric: str,
+        y_metric: str,
+        graph_title: str,
+        x_label: str,
+        y_label: str,
+        filename_root: str,
+    ) -> None:
+        """
+        Create plot for specific graph type
+        """
+        raise NotImplementedError
+
+    def _generate_parquet(self, dataframe: DataFrame, file: str):
+        dataframe.to_parquet(
+            f"{DEFAULT_ARTIFACT_DIR}/data/{file}.gzip", compression="gzip"
+        )
+
+    def _generate_graph_file(self, fig: Figure, file: str, title: str):
+        if file.endswith("jpeg"):
+            print(f"Generating '{title}' jpeg")
+            fig.write_image(f"{DEFAULT_ARTIFACT_DIR}/images/{file}")
+        elif file.endswith("html"):
+            print(f"Generating '{title}' html")
+            fig.write_html(f"{DEFAULT_ARTIFACT_DIR}/images/{file}")
+        else:
+            extension = file.split(".")[-1]
+            raise GenAIPerfException(f"image file type {extension} is not supported")

src/c++/perf_analyzer/genai-perf/genai_perf/graphs/box_plot.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import copy
+from typing import Dict
+
+import pandas as pd
+import plotly.express as px
+from genai_perf.graphs.base_plot import BasePlot
+from genai_perf.llm_metrics import Statistics
+from genai_perf.utils import scale
+from plotly.graph_objects import Figure
+
+
+class BoxPlot(BasePlot):
+    """
+    Generate a box plot in jpeg and html format.
+    """
+
+    def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
+        super().__init__(stats, extra_data)
+
+    def create_plot(
+        self,
+        x_key: str = "",
+        y_key: str = "",
+        x_metric: str = "",
+        y_metric: str = "",
+        graph_title: str = "",
+        x_label: str = "",
+        y_label: str = "",
+        filename_root: str = "",
+    ):
+        df = pd.DataFrame({y_metric: self._metrics_data[y_key]})
+        fig = px.box(
+            df,
+            y=y_metric,
+            points="all",
+            title=graph_title,
+        )
+        fig.update_layout(title_x=0.5)
+        fig.update_xaxes(title_text=x_label)
+
+        fig.update_yaxes(title_text="")
+
+        # create a copy to avoid annotations on html file
+        fig_jpeg = copy.deepcopy(fig)
+        self._add_annotations(fig_jpeg, y_metric)
+
+        self._generate_parquet(df, filename_root)
+        self._generate_graph_file(fig, filename_root + ".html", graph_title)
+        self._generate_graph_file(fig_jpeg, filename_root + ".jpeg", graph_title)
+
+    def _add_annotations(self, fig: Figure, y_metric: str) -> None:
+        """
+        Add annotations to the non html version of the box plot
+        to replace the missing hovertext
+        """
+        stat_root_name = self._stats.metrics.get_base_name(y_metric)
+
+        val = scale(self._stats.data[f"max_{stat_root_name}"], (1 / 1e9))
+        fig.add_annotation(
+            x=0.5,
+            y=val,
+            text=f"max: {round(val, 2)}",
+            showarrow=False,
+            yshift=10,
+        )
+
+        val = scale(self._stats.data[f"p75_{stat_root_name}"], (1 / 1e9))
+        fig.add_annotation(
+            x=0.5,
+            y=val,
+            text=f"q3: {round(val, 2)}",
+            showarrow=False,
+            yshift=10,
+        )
+
+        val = scale(self._stats.data[f"p50_{stat_root_name}"], (1 / 1e9))
+        fig.add_annotation(
+            x=0.5,
+            y=val,
+            text=f"median: {round(val, 2)}",
+            showarrow=False,
+            yshift=10,
+        )
+
+        val = scale(self._stats.data[f"p25_{stat_root_name}"], (1 / 1e9))
+        fig.add_annotation(
+            x=0.5,
+            y=val,
+            text=f"q1: {round(val, 2)}",
+            showarrow=False,
+            yshift=10,
+        )
+
+        val = scale(self._stats.data[f"min_{stat_root_name}"], (1 / 1e9))
+        fig.add_annotation(
+            x=0.5,
+            y=val,
+            text=f"min: {round(val, 2)}",
+            showarrow=False,
+            yshift=10,
+        )

src/c++/perf_analyzer/genai-perf/genai_perf/graphs/heat_map.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from typing import Dict
+
+import pandas as pd
+import plotly.express as px
+from genai_perf.graphs.base_plot import BasePlot
+from genai_perf.llm_metrics import Statistics
+
+
+class HeatMap(BasePlot):
+    """
+    Generate a heat map in jpeg and html format.
+    """
+
+    def __init__(self, stats: Statistics, extra_data: Dict | None = None) -> None:
+        super().__init__(stats, extra_data)
+
+    def create_plot(
+        self,
+        x_key: str = "",
+        y_key: str = "",
+        x_metric: str = "",
+        y_metric: str = "",
+        graph_title: str = "",
+        x_label: str = "",
+        y_label: str = "",
+        filename_root: str = "",
+    ):
+        x_values = self._metrics_data[x_key]
+        y_values = self._metrics_data[y_key]
+        df = pd.DataFrame(
+            {
+                x_metric: x_values,
+                y_metric: y_values,
+            }
+        )
+        fig = px.density_heatmap(
+            df,
+            x=x_metric,
+            y=y_metric,
+        )
+        fig.update_layout(
+            title={
+                "text": graph_title,
+                "xanchor": "center",
+                "x": 0.5,
+            }
+        )
+        fig.update_xaxes(title_text=x_label)
+        fig.update_yaxes(title_text=y_label)
+
+        self._generate_parquet(df, filename_root)
+        self._generate_graph_file(fig, filename_root + ".html", graph_title)
+        self._generate_graph_file(fig, filename_root + ".jpeg", graph_title)