diff --git a/tools/rw-heatmaps/cmd/root.go b/tools/rw-heatmaps/cmd/root.go new file mode 100644 index 000000000000..65ff794039af --- /dev/null +++ b/tools/rw-heatmaps/cmd/root.go @@ -0,0 +1,97 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "go.etcd.io/etcd/v3/tools/rw-heatmaps/pkg/chart" + "go.etcd.io/etcd/v3/tools/rw-heatmaps/pkg/dataset" +) + +var ( + // ErrMissingTitleArg is returned when the title argument is missing. + ErrMissingTitleArg = fmt.Errorf("missing title argument") + // ErrMissingOutputImageFileArg is returned when the output image file argument is missing. + ErrMissingOutputImageFileArg = fmt.Errorf("missing output image file argument") + // ErrMissingInputFileArg is returned when the input file argument is missing. + ErrMissingInputFileArg = fmt.Errorf("missing input file argument") + // ErrInvalidOutputFormat is returned when the output format is invalid. + ErrInvalidOutputFormat = fmt.Errorf("invalid output format, must be one of png, jpg, jpeg, tiff") +) + +// NewRootCommand returns the root command for the rw-heatmaps tool. +func NewRootCommand() *cobra.Command { + o := newOptions() + rootCmd := &cobra.Command{ + Use: "rw-heatmaps [input file in csv format]", + Short: "A tool to generate read/write heatmaps for etcd3", + Long: "rw-heatmaps is a tool to generate read/write heatmaps images for etcd3.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + dataset, err := dataset.LoadCSVData(args[0]) + if err != nil { + return err + } + + return chart.PlotHeatMaps(dataset, o.title, o.outputImageFile, o.outputFormat) + }, + } + + o.AddFlags(rootCmd.Flags()) + return rootCmd +} + +// options holds the options for the command. +type options struct { + title string + outputImageFile string + outputFormat string +} + +// Returns a new options for the command with the default values applied. +func newOptions() options { + return options{ + outputFormat: "png", + } +} + +// AddFlags sets the flags for the command. +func (o *options) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.title, "title", o.title, "plot graph title (required)") + fs.StringVar(&o.outputImageFile, "output-image-file", o.outputImageFile, "output image filename (required)") + fs.StringVar(&o.outputFormat, "output-format", o.outputFormat, "output image file format") +} + +// Validate returns an error if the options are invalid. +func (o *options) Validate() error { + if o.title == "" { + return ErrMissingTitleArg + } + if o.outputImageFile == "" { + return ErrMissingOutputImageFileArg + } + switch o.outputFormat { + case "png", "jpg", "jpeg", "tiff": + default: + return ErrInvalidOutputFormat + } + return nil +} diff --git a/tools/rw-heatmaps/main.go b/tools/rw-heatmaps/main.go index ebe751ee1c82..13cac4681849 100644 --- a/tools/rw-heatmaps/main.go +++ b/tools/rw-heatmaps/main.go @@ -1,372 +1,27 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( - "encoding/csv" - "flag" - "fmt" - "image/color" - "io" - "log" - "math" "os" - "sort" - "strconv" - "strings" - - "gonum.org/v1/plot" - "gonum.org/v1/plot/palette" - "gonum.org/v1/plot/palette/brewer" - "gonum.org/v1/plot/plotter" - "gonum.org/v1/plot/vg" - "gonum.org/v1/plot/vg/draw" - "gonum.org/v1/plot/vg/vgimg" -) -var ( - inputFileA string - inputFileB string - title string - zeroCentered bool - outputImage string - outputFormat string + "go.etcd.io/etcd/v3/tools/rw-heatmaps/cmd" ) -func init() { - log.SetFlags(0) - log.SetPrefix("[INFO] ") - - flag.StringVar(&inputFileA, "input_file_a", "", "first input data file in csv format. (required)") - flag.StringVar(&inputFileB, "input_file_b", "", "second input data file in csv format. (optional)") - flag.StringVar(&title, "title", "", "plot graph title string (required)") - flag.BoolVar(&zeroCentered, "zero-centered", true, "plot the improvement graph with white color represents 0.0") - flag.StringVar(&outputImage, "output-image-file", "", "output image filename (required)") - flag.StringVar(&outputFormat, "output-format", "png", "output image file format. default: png") - - flag.Parse() - - if inputFileA == "" || title == "" || outputImage == "" { - fmt.Println("Missing required arguments.") - flag.Usage() - os.Exit(2) - } -} - -type DataSet struct { - Records map[float64][]DataRecord - Param string -} - -type DataRecord struct { - ConnSize int64 - ValueSize int64 - Read float64 - Write float64 -} - -func loadCSVData(inputFile string) (*DataSet, error) { - file, err := os.Open(inputFile) - if err != nil { - return nil, err - } - defer file.Close() - - reader := csv.NewReader(file) - lines, err := reader.ReadAll() - if err != nil { - return nil, err - } - dataset := &DataSet{Records: make(map[float64][]DataRecord)} - records := dataset.Records - - iters := 0 - for _, header := range lines[0][4:] { - if strings.HasPrefix(header, "iter") { - iters++ - } - } - - for _, line := range lines[2:] { // Skip header line. - ratio, _ := strconv.ParseFloat(line[1], 64) - if _, ok := records[ratio]; !ok { - records[ratio] = make([]DataRecord, 0) - } - connSize, _ := strconv.ParseInt(line[2], 10, 64) - valueSize, _ := strconv.ParseInt(line[3], 10, 64) - - readSum := float64(0) - writeSum := float64(0) - - for _, v := range line[4 : 4+iters] { - splitted := strings.Split(v, ":") - readValue, _ := strconv.ParseFloat(splitted[0], 64) - readSum += readValue - - writeValue, _ := strconv.ParseFloat(splitted[1], 64) - writeSum += writeValue - } - - records[ratio] = append(records[ratio], DataRecord{ - ConnSize: connSize, - ValueSize: valueSize, - Read: readSum / float64(iters), - Write: writeSum / float64(iters), - }) - } - dataset.Param = lines[1][iters+4] - return dataset, nil -} - -// HeatMapGrid holds X, Y, Z values for a heatmap. -type HeatMapGrid struct { - x, y []float64 - z [][]float64 // The Z values should be arranged in a 2D slice. -} - -// Len implements the plotter.GridXYZ interface. -func (h *HeatMapGrid) Dims() (int, int) { - return len(h.x), len(h.y) -} - -// ValueAt returns the value of a grid cell at (c, r). -// It implements the plotter.GridXYZ interface. -func (h *HeatMapGrid) Z(c, r int) float64 { - return h.z[r][c] -} - -// X returns the coordinate for the column at index c. -// It implements the plotter.GridXYZ interface. -func (h *HeatMapGrid) X(c int) float64 { - if c >= len(h.x) { - panic("index out of range") - } - return h.x[c] -} - -// Y returns the coordinate for the row at index r. -// It implements the plotter.GridXYZ interface. -func (h *HeatMapGrid) Y(r int) float64 { - if r >= len(h.y) { - panic("index out of range") - } - return h.y[r] -} - -func uniqueSortedFloats(input []float64) []float64 { - unique := make([]float64, 0) - seen := make(map[float64]bool) - - for _, value := range input { - if !seen[value] { - seen[value] = true - unique = append(unique, value) - } - } - - sort.Float64s(unique) - return unique -} - -func populateGridAxes(records []DataRecord) ([]float64, []float64) { - var xslice, yslice []float64 - - for _, record := range records { - xslice = append(xslice, float64(record.ConnSize)) - yslice = append(yslice, float64(record.ValueSize)) - } - - // Sort and deduplicate the slices - xUnique := uniqueSortedFloats(xslice) - yUnique := uniqueSortedFloats(yslice) - - return xUnique, yUnique -} - -func plotHeatMaps(title, plotType string, dataset *DataSet) { - const rows, cols = 4, 2 - plots := make([][]*plot.Plot, rows) - legends := make([][]plot.Legend, rows) - for i := range plots { - plots[i] = make([]*plot.Plot, cols) - legends[i] = make([]plot.Legend, cols) - } - - row, col := 0, 0 - ratios := make([]float64, 0) - for ratio := range dataset.Records { - ratios = append(ratios, ratio) - } - sort.Float64s(ratios) - for _, ratio := range ratios { - records := dataset.Records[ratio] - p, l := plotHeatMap(fmt.Sprintf("R/W Ratio %0.04f", ratio), plotType, records) - plots[row][col] = p - legends[row][col] = l - - if col++; col == cols { - col = 0 - row++ - } - } - - const width, height = 12 * vg.Inch, 16 * vg.Inch - img := vgimg.New(width, height) - dc := draw.New(img) - - t := draw.Tiles{ - Rows: rows, - Cols: cols, - PadX: vg.Inch * 0.5, - PadY: vg.Inch * 0.5, - PadTop: vg.Inch * 0.5, - PadBottom: vg.Inch * 0.5, - PadLeft: vg.Inch * 0.25, - PadRight: vg.Inch * 0.25, - } - - canvases := plot.Align(plots, t, dc) - for i := 0; i < rows; i++ { - for j := 0; j < cols; j++ { - if plots[i][j] != nil { - l := legends[i][j] - r := l.Rectangle(canvases[i][j]) - legendWidth := r.Max.X - r.Min.X - l.YOffs = -plots[i][j].Title.TextStyle.FontExtents().Height // Adjust the legend down a little. - l.Draw(canvases[i][j]) - - c := draw.Crop(canvases[i][j], 0, -legendWidth-vg.Millimeter, 0, 0) // Make space for the legend. - plots[i][j].Draw(c) - } - } - } - - l := plot.NewLegend() - l.Add(title) - l.Add(dataset.Param) - l.Top = true - l.Left = true - l.Draw(dc) - - fh, err := os.Create(fmt.Sprintf("%s_%s_heatmap.%s", outputImage, plotType, outputFormat)) - if err != nil { - panic(err) - } - defer fh.Close() - - var w io.WriterTo - switch outputFormat { - case "png": - w = vgimg.PngCanvas{Canvas: img} - case "jpeg", "jpg": - w = vgimg.PngCanvas{Canvas: img} - case "tiff": - w = vgimg.TiffCanvas{Canvas: img} - } - if _, err := w.WriteTo(fh); err != nil { - panic(err) - } -} - -type pow2Ticks struct{} - -func (pow2Ticks) Ticks(min, max float64) []plot.Tick { - var t []plot.Tick - for i := math.Log2(min); math.Pow(2, i) <= max; i++ { - t = append(t, plot.Tick{ - Value: math.Pow(2, i), - Label: fmt.Sprintf("2^%d", int(i)), - }) - } - return t -} - -func plotHeatMap(title, plotType string, records []DataRecord) (*plot.Plot, plot.Legend) { - p := plot.New() - p.X.Scale = plot.LogScale{} - p.X.Tick.Marker = pow2Ticks{} - p.X.Label.Text = "Connections Amount" - p.Y.Scale = plot.LogScale{} - p.Y.Tick.Marker = pow2Ticks{} - p.Y.Label.Text = "Value Size" - - // Populate X and Y axis data from records - xCoords, yCoords := populateGridAxes(records) - - gridData := &HeatMapGrid{ - x: xCoords, - y: yCoords, - z: make([][]float64, len(yCoords)), - } - - for i := range gridData.z { - gridData.z[i] = make([]float64, len(xCoords)) - - for j := range gridData.z[i] { - recordIndex := i*len(gridData.x) + j - if recordIndex >= len(records) { - break - } - record := records[recordIndex] - if plotType == "read" { - gridData.z[i][j] = record.Read - } else { - gridData.z[i][j] = record.Write - } - } - } - - colors, _ := brewer.GetPalette(brewer.TypeAny, "YlGnBu", 9) - pal := invertedPalette{colors} - h := plotter.NewHeatMap(gridData, pal) - - p.Title.Text = title + fmt.Sprintf(" [%.2f, %.2f]", h.Min, h.Max) - p.Add(h) - - // Create a legend with the scale. - l := plot.NewLegend() - thumbs := plotter.PaletteThumbnailers(pal) - step := (h.Max - h.Min) / float64(len(thumbs)-1) - for i := len(thumbs) - 1; i >= 0; i-- { - t := thumbs[i] - l.Add(fmt.Sprintf("%.0f", h.Min+step*float64(i)), t) - } - l.Top = true - - return p, l -} - -// invertedPalette takes an existing palette and inverts it. -type invertedPalette struct { - Base palette.Palette -} - -// Colors returns the sequence of colors in reverse order from the base palette. -func (p invertedPalette) Colors() []color.Color { - baseColors := p.Base.Colors() - invertedColors := make([]color.Color, len(baseColors)) - for i, c := range baseColors { - invertedColors[len(baseColors)-i-1] = c - } - return invertedColors -} - func main() { - //var aRecords []DataRecord - //var bRecords []DataRecord - //var err error - - aRecords, err := loadCSVData(inputFileA) - if err != nil { - log.Fatalf("failed to load data from %s: %v\n", inputFileA, err) + if err := cmd.NewRootCommand().Execute(); err != nil { + os.Exit(1) } - - // if inputFileB != "" { - // bRecords, err = loadCSVData(inputFileB) - // if err != nil { - // log.Fatalf("failed to load data from %s: %v\n", inputFileB, err) - // } - // } - - //plotHeatMap(title+" Read Plot", "read", aRecords, maxRead) - plotHeatMaps(fmt.Sprintf("%s [READ]", title), "read", aRecords) - plotHeatMaps(fmt.Sprintf("%s [WRITE]", title), "write", aRecords) } diff --git a/tools/rw-heatmaps/pkg/chart/heatmap.go b/tools/rw-heatmaps/pkg/chart/heatmap.go new file mode 100644 index 000000000000..382e88f3a1fb --- /dev/null +++ b/tools/rw-heatmaps/pkg/chart/heatmap.go @@ -0,0 +1,147 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart + +import ( + "fmt" + "image/color" + "math" + "sort" + + "go.etcd.io/etcd/v3/tools/rw-heatmaps/pkg/dataset" + "gonum.org/v1/plot" + "gonum.org/v1/plot/palette" + "gonum.org/v1/plot/palette/brewer" + "gonum.org/v1/plot/plotter" +) + +// pow2Ticks is a type that implements the plot.Ticker interface for log2 scale. +type pow2Ticks struct{} + +// Ticks returns the ticks for the log2 scale. +func (pow2Ticks) Ticks(min, max float64) []plot.Tick { + var t []plot.Tick + for i := math.Log2(min); math.Pow(2, i) <= max; i++ { + t = append(t, plot.Tick{ + Value: math.Pow(2, i), + Label: fmt.Sprintf("2^%d", int(i)), + }) + } + return t +} + +// invertedPalette takes an existing palette and inverts it. +type invertedPalette struct { + base palette.Palette +} + +// Colors returns the sequence of colors in reverse order from the base palette. +func (p invertedPalette) Colors() []color.Color { + baseColors := p.base.Colors() + invertedColors := make([]color.Color, len(baseColors)) + for i, c := range baseColors { + invertedColors[len(baseColors)-i-1] = c + } + return invertedColors +} + +// plotIndividualHeatMap plots a heatmap for a given set of records. +func plotIndividualHeatMap(title, plotType string, records []dataset.DataRecord) (*plot.Plot, plot.Legend) { + p := plot.New() + p.X.Scale = plot.LogScale{} + p.X.Tick.Marker = pow2Ticks{} + p.X.Label.Text = "Connections Amount" + p.Y.Scale = plot.LogScale{} + p.Y.Tick.Marker = pow2Ticks{} + p.Y.Label.Text = "Value Size" + + // Populate X and Y axis data from records + xCoords, yCoords := populateGridAxes(records) + + gridData := &HeatMapGrid{ + x: xCoords, + y: yCoords, + z: make([][]float64, len(yCoords)), + } + + for i := range gridData.z { + gridData.z[i] = make([]float64, len(xCoords)) + + for j := range gridData.z[i] { + recordIndex := i*len(gridData.x) + j + if recordIndex >= len(records) { + break + } + record := records[recordIndex] + if plotType == "read" { + gridData.z[i][j] = record.Read + } else { + gridData.z[i][j] = record.Write + } + } + } + + // Use the YlGnBu color palette from ColorBrewer to match the original implementation. + colors, _ := brewer.GetPalette(brewer.TypeAny, "YlGnBu", 9) + pal := invertedPalette{colors} + h := plotter.NewHeatMap(gridData, pal) + + p.Title.Text = title + fmt.Sprintf(" [%.2f, %.2f]", h.Min, h.Max) + p.Add(h) + + // Create a legend with the scale. + l := plot.NewLegend() + thumbs := plotter.PaletteThumbnailers(pal) + step := (h.Max - h.Min) / float64(len(thumbs)-1) + for i := len(thumbs) - 1; i >= 0; i-- { + t := thumbs[i] + l.Add(fmt.Sprintf("%.0f", h.Min+step*float64(i)), t) + } + l.Top = true + + return p, l +} + +// populateGridAxes populates the X and Y axes for the heatmap grid. +func populateGridAxes(records []dataset.DataRecord) ([]float64, []float64) { + var xslice, yslice []float64 + + for _, record := range records { + xslice = append(xslice, float64(record.ConnSize)) + yslice = append(yslice, float64(record.ValueSize)) + } + + // Sort and deduplicate the slices + xUnique := uniqueSortedFloats(xslice) + yUnique := uniqueSortedFloats(yslice) + + return xUnique, yUnique +} + +// uniqueSortedFloats returns a sorted slice of unique float64 values. +func uniqueSortedFloats(input []float64) []float64 { + unique := make([]float64, 0) + seen := make(map[float64]bool) + + for _, value := range input { + if !seen[value] { + seen[value] = true + unique = append(unique, value) + } + } + + sort.Float64s(unique) + return unique +} diff --git a/tools/rw-heatmaps/pkg/chart/heatmap_grid.go b/tools/rw-heatmaps/pkg/chart/heatmap_grid.go new file mode 100644 index 000000000000..00412e31ee47 --- /dev/null +++ b/tools/rw-heatmaps/pkg/chart/heatmap_grid.go @@ -0,0 +1,51 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart + +// HeatMapGrid holds X, Y, Z values for a heatmap. +type HeatMapGrid struct { + x, y []float64 + z [][]float64 // The Z values should be arranged in a 2D slice. +} + +// Len returns the number of elements in the grid. +// It implements the plotter.GridXYZ interface. +func (h *HeatMapGrid) Dims() (int, int) { + return len(h.x), len(h.y) +} + +// ValueAt returns the value of a grid cell at (c, r). +// It implements the plotter.GridXYZ interface. +func (h *HeatMapGrid) Z(c, r int) float64 { + return h.z[r][c] +} + +// X returns the coordinate for the column at index c. +// It implements the plotter.GridXYZ interface. +func (h *HeatMapGrid) X(c int) float64 { + if c >= len(h.x) { + panic("index out of range") + } + return h.x[c] +} + +// Y returns the coordinate for the row at index r. +// It implements the plotter.GridXYZ interface. +func (h *HeatMapGrid) Y(r int) float64 { + if r >= len(h.y) { + panic("index out of range") + } + return h.y[r] +} diff --git a/tools/rw-heatmaps/pkg/chart/heatmaps.go b/tools/rw-heatmaps/pkg/chart/heatmaps.go new file mode 100644 index 000000000000..717b80ac7f90 --- /dev/null +++ b/tools/rw-heatmaps/pkg/chart/heatmaps.go @@ -0,0 +1,138 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package chart + +import ( + "fmt" + "io" + "os" + "strings" + + "go.etcd.io/etcd/v3/tools/rw-heatmaps/pkg/dataset" + "gonum.org/v1/plot" + "gonum.org/v1/plot/vg" + "gonum.org/v1/plot/vg/draw" + "gonum.org/v1/plot/vg/vgimg" +) + +// PlotHeatMaps plots, and saves the heatmaps for the given dataset. +func PlotHeatMaps(dataset *dataset.DataSet, title, outputImageFile, outputFormat string) error { + for _, plotType := range []string{"read", "write"} { + canvas := plotHeatMapGrid(dataset, title, plotType) + if err := saveCanvas(canvas, plotType, outputImageFile, outputFormat); err != nil { + return err + } + } + return nil +} + +// plotHeatMapGrid plots a grid of heatmaps for the given dataset. +func plotHeatMapGrid(dataset *dataset.DataSet, title, plotType string) *vgimg.Canvas { + // Make a 4x2 grid of heatmaps. + const rows, cols = 4, 2 + + // Store the plots and legends (scale label) in a grid. + plots := make([][]*plot.Plot, rows) + legends := make([][]plot.Legend, rows) + for i := range plots { + plots[i] = make([]*plot.Plot, cols) + legends[i] = make([]plot.Legend, cols) + } + + // Load records into the grid. + ratios := dataset.GetSortedRatios() + row, col := 0, 0 + for _, ratio := range ratios { + records := dataset.Records[ratio] + p, l := plotIndividualHeatMap(fmt.Sprintf("R/W Ratio %0.04f", ratio), plotType, records) + plots[row][col] = p + legends[row][col] = l + + if col++; col == cols { + col = 0 + row++ + } + } + + const width, height = 30 * vg.Centimeter, 40 * vg.Centimeter + canvas := vgimg.New(width, height) + dc := draw.New(canvas) + + // Create a tiled layout for the plots. + t := draw.Tiles{ + Rows: rows, + Cols: cols, + PadX: vg.Millimeter * 10, + PadY: vg.Millimeter * 10, + PadTop: vg.Millimeter * 10, + PadBottom: vg.Millimeter * 10, + PadLeft: vg.Millimeter * 5, + PadRight: vg.Millimeter * 5, + } + + // Fill the canvas with the plots and legends. + canvases := plot.Align(plots, t, dc) + for i := 0; i < rows; i++ { + for j := 0; j < cols; j++ { + // Continue if there is no plot in the current cell (incomplete data). + if plots[i][j] == nil { + continue + } + + l := legends[i][j] + r := l.Rectangle(canvases[i][j]) + legendWidth := r.Max.X - r.Min.X + // Adjust the legend down a little. + l.YOffs = -plots[i][j].Title.TextStyle.FontExtents().Height + l.Draw(canvases[i][j]) + + // Crop the plot to make space for the legend. + c := draw.Crop(canvases[i][j], 0, -legendWidth-vg.Millimeter, 0, 0) + plots[i][j].Draw(c) + } + } + + // Add the title and parameter legend. + l := plot.NewLegend() + l.Add(fmt.Sprintf("%s [%s]", title, strings.ToUpper(plotType))) + l.Add(dataset.Param) + l.Top = true + l.Left = true + l.Draw(dc) + + return canvas +} + +// saveCanvas saves the canvas to a file. +func saveCanvas(canvas *vgimg.Canvas, plotType, outputImageFile, outputFormat string) error { + f, err := os.Create(fmt.Sprintf("%s_%s.%s", outputImageFile, plotType, outputFormat)) + if err != nil { + return err + } + defer f.Close() + + var w io.WriterTo + switch outputFormat { + case "png": + w = vgimg.PngCanvas{Canvas: canvas} + case "jpeg", "jpg": + w = vgimg.PngCanvas{Canvas: canvas} + case "tiff": + w = vgimg.TiffCanvas{Canvas: canvas} + } + + _, err = w.WriteTo(f) + return err +} diff --git a/tools/rw-heatmaps/pkg/dataset/dataset.go b/tools/rw-heatmaps/pkg/dataset/dataset.go new file mode 100644 index 000000000000..d2a2bcd73603 --- /dev/null +++ b/tools/rw-heatmaps/pkg/dataset/dataset.go @@ -0,0 +1,119 @@ +// Copyright 2024 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dataset + +import ( + "encoding/csv" + "os" + "sort" + "strconv" + "strings" +) + +const ( + _ = iota + // fieldIndexRatio is the index of the ratio field in the CSV file. + fieldIndexRatio + // fieldIndexConnSize is the index of the connection size (connSize) field in the CSV file. + fieldIndexConnSize + // fieldIndexValueSize is the index of the value size (valueSize) field in the CSV file. + fieldIndexValueSize + // fieldIndexIterOffset is the index of the first iteration field in the CSV file. + fieldIndexIterOffset +) + +// DataSet holds the data for the heatmaps, including the parameter used for the run. +type DataSet struct { + // Records is a map from the ratio of read to write operations to the data for that ratio. + Records map[float64][]DataRecord + // Param is the parameter used for the run. + Param string +} + +// DataRecord holds the data for a single heatmap chart. +type DataRecord struct { + ConnSize int64 + ValueSize int64 + Read float64 + Write float64 +} + +// GetSortedRatios returns the sorted ratios of read to write operations in the dataset. +func (d *DataSet) GetSortedRatios() []float64 { + ratios := make([]float64, 0) + for ratio := range d.Records { + ratios = append(ratios, ratio) + } + sort.Float64s(ratios) + return ratios +} + +// LoadCSVData loads the data from a CSV file into a DataSet. +func LoadCSVData(inputFile string) (*DataSet, error) { + file, err := os.Open(inputFile) + if err != nil { + return nil, err + } + defer file.Close() + + reader := csv.NewReader(file) + lines, err := reader.ReadAll() + if err != nil { + return nil, err + } + + records := make(map[float64][]DataRecord) + + // Count the number of iterations. + iters := 0 + for _, header := range lines[0][fieldIndexIterOffset:] { + if strings.HasPrefix(header, "iter") { + iters++ + } + } + + // Running parameters are stored in the first line after the header, after the iteration fields. + param := lines[1][fieldIndexIterOffset+iters] + + for _, line := range lines[2:] { // Skip header line. + ratio, _ := strconv.ParseFloat(line[fieldIndexRatio], 64) + if _, ok := records[ratio]; !ok { + records[ratio] = make([]DataRecord, 0) + } + connSize, _ := strconv.ParseInt(line[fieldIndexConnSize], 10, 64) + valueSize, _ := strconv.ParseInt(line[fieldIndexValueSize], 10, 64) + + // Calculate the average read and write values for the iterations. + var readSum, writeSum float64 + for _, v := range line[fieldIndexIterOffset : fieldIndexIterOffset+iters] { + splitted := strings.Split(v, ":") + + readValue, _ := strconv.ParseFloat(splitted[0], 64) + readSum += readValue + + writeValue, _ := strconv.ParseFloat(splitted[1], 64) + writeSum += writeValue + } + + records[ratio] = append(records[ratio], DataRecord{ + ConnSize: connSize, + ValueSize: valueSize, + Read: readSum / float64(iters), + Write: writeSum / float64(iters), + }) + } + + return &DataSet{Records: records, Param: param}, nil +}