From 07abbaa564a65aeeabdfd03d4bffa9131ad12380 Mon Sep 17 00:00:00 2001 From: Rong Rong Date: Mon, 7 Nov 2022 08:50:17 -0800 Subject: [PATCH 1/4] add aggregated metrics for segment metadata --- .../restlet/resources/TableMetadataInfo.java | 9 ++++- .../util/ServerSegmentMetadataReader.java | 13 ++++++- .../server/api/resources/TablesResource.java | 11 +++++- .../pinot/server/api/TablesResourceTest.java | 34 +++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java index 81d86b6eb754..27e28ab376cc 100644 --- a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java +++ b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/TableMetadataInfo.java @@ -42,13 +42,15 @@ public class TableMetadataInfo { private final Map _columnLengthMap; private final Map _columnCardinalityMap; private final Map _maxNumMultiValuesMap; + private final Map> _columnIndexSizeMap; @JsonCreator public TableMetadataInfo(@JsonProperty("tableName") String tableName, @JsonProperty("diskSizeInBytes") long sizeInBytes, @JsonProperty("numSegments") long numSegments, @JsonProperty("numRows") long numRows, @JsonProperty("columnLengthMap") Map columnLengthMap, @JsonProperty("columnCardinalityMap") Map columnCardinalityMap, - @JsonProperty("maxNumMultiValuesMap") Map maxNumMultiValuesMap) { + @JsonProperty("maxNumMultiValuesMap") Map maxNumMultiValuesMap, + @JsonProperty("columnIndexSizeMap") Map> columnIndexSizeMap) { _tableName = tableName; _diskSizeInBytes = sizeInBytes; _numSegments = numSegments; @@ -56,6 +58,7 @@ public TableMetadataInfo(@JsonProperty("tableName") String tableName, _columnLengthMap = columnLengthMap; _columnCardinalityMap = columnCardinalityMap; _maxNumMultiValuesMap = maxNumMultiValuesMap; + _columnIndexSizeMap = columnIndexSizeMap; } public String getTableName() { @@ -85,4 +88,8 @@ public Map getColumnCardinalityMap() { public Map getMaxNumMultiValuesMap() { return _maxNumMultiValuesMap; } + + public Map> getColumnIndexSizeMap() { + return _columnIndexSizeMap; + } } diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java index 7afd0a744564..e860a4bf8f01 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java @@ -88,6 +88,7 @@ public TableMetadataInfo getAggregatedTableMetadataFromServer(String tableNameWi final Map columnLengthMap = new HashMap<>(); final Map columnCardinalityMap = new HashMap<>(); final Map maxNumMultiValuesMap = new HashMap<>(); + final Map> columnIndexSizeMap = new HashMap<>(); for (Map.Entry streamResponse : serviceResponse._httpResponses.entrySet()) { try { TableMetadataInfo tableMetadataInfo = @@ -98,6 +99,12 @@ public TableMetadataInfo getAggregatedTableMetadataFromServer(String tableNameWi tableMetadataInfo.getColumnLengthMap().forEach((k, v) -> columnLengthMap.merge(k, v, Double::sum)); tableMetadataInfo.getColumnCardinalityMap().forEach((k, v) -> columnCardinalityMap.merge(k, v, Double::sum)); tableMetadataInfo.getMaxNumMultiValuesMap().forEach((k, v) -> maxNumMultiValuesMap.merge(k, v, Double::sum)); + tableMetadataInfo.getColumnIndexSizeMap().forEach((k, v) -> columnIndexSizeMap.merge(k, v, (l, r) -> { + for (Map.Entry e : r.entrySet()) { + l.put(e.getKey(), l.getOrDefault(e.getKey(), 0d) + e.getValue()); + } + return l; + })); } catch (IOException e) { failedParses++; LOGGER.error("Unable to parse server {} response due to an error: ", streamResponse.getKey(), e); @@ -107,6 +114,10 @@ public TableMetadataInfo getAggregatedTableMetadataFromServer(String tableNameWi columnLengthMap.replaceAll((k, v) -> v / finalTotalNumSegments); columnCardinalityMap.replaceAll((k, v) -> v / finalTotalNumSegments); maxNumMultiValuesMap.replaceAll((k, v) -> v / finalTotalNumSegments); + columnIndexSizeMap.replaceAll((k, v) -> { + v.replaceAll((key, value) -> v.get(key) / finalTotalNumSegments); + return v; + }); // Since table segments may have multiple replicas, divide diskSizeInBytes, numRows and numSegments by numReplica // to avoid double counting, for columnAvgLengthMap, columnAvgCardinalityMap and maxNumMultiValuesMap, dividing by @@ -117,7 +128,7 @@ public TableMetadataInfo getAggregatedTableMetadataFromServer(String tableNameWi TableMetadataInfo aggregateTableMetadataInfo = new TableMetadataInfo(tableNameWithType, totalDiskSizeInBytes, totalNumSegments, totalNumRows, columnLengthMap, - columnCardinalityMap, maxNumMultiValuesMap); + columnCardinalityMap, maxNumMultiValuesMap, columnIndexSizeMap); if (failedParses != 0) { LOGGER.warn("Failed to parse {} / {} aggregated segment metadata responses from servers.", failedParses, serverUrls.size()); diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java index 2658a45d9966..2017dc40a58d 100644 --- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java +++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java @@ -80,6 +80,7 @@ import org.apache.pinot.segment.spi.ColumnMetadata; import org.apache.pinot.segment.spi.ImmutableSegment; import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl; +import org.apache.pinot.segment.spi.store.ColumnIndexType; import org.apache.pinot.server.access.AccessControl; import org.apache.pinot.server.access.AccessControlFactory; import org.apache.pinot.server.access.HttpRequesterIdentity; @@ -210,6 +211,7 @@ public String getSegmentMetadata( Map columnLengthMap = new HashMap<>(); Map columnCardinalityMap = new HashMap<>(); Map maxNumMultiValuesMap = new HashMap<>(); + Map> columnIndexSizesMap = new HashMap<>(); try { for (SegmentDataManager segmentDataManager : segmentDataManagers) { if (segmentDataManager instanceof ImmutableSegmentDataManager) { @@ -256,6 +258,13 @@ public String getSegmentMetadata( int maxNumMultiValues = columnMetadata.getMaxNumberOfMultiValues(); maxNumMultiValuesMap.merge(column, (double) maxNumMultiValues, Double::sum); } + for (Map.Entry entry : columnMetadata.getIndexSizeMap().entrySet()) { + Map columnIndexSizes = columnIndexSizesMap.getOrDefault(column, new HashMap<>()); + Double indexSize = columnIndexSizes.getOrDefault(entry.getKey().getIndexName(), 0d); + indexSize += entry.getValue(); + columnIndexSizes.put(entry.getKey().getIndexName(), indexSize); + columnIndexSizesMap.put(column, columnIndexSizes); + } } } } @@ -270,7 +279,7 @@ public String getSegmentMetadata( TableMetadataInfo tableMetadataInfo = new TableMetadataInfo(tableDataManager.getTableName(), totalSegmentSizeBytes, segmentDataManagers.size(), - totalNumRows, columnLengthMap, columnCardinalityMap, maxNumMultiValuesMap); + totalNumRows, columnLengthMap, columnCardinalityMap, maxNumMultiValuesMap, columnIndexSizesMap); return ResourceUtils.convertToJsonString(tableMetadataInfo); } diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java index b36874598f20..2acf42694ff9 100644 --- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java +++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java @@ -24,6 +24,7 @@ import java.util.List; import javax.ws.rs.core.Response; import org.apache.commons.io.FileUtils; +import org.apache.pinot.common.restlet.resources.TableMetadataInfo; import org.apache.pinot.common.restlet.resources.TableSegments; import org.apache.pinot.common.restlet.resources.TablesList; import org.apache.pinot.common.utils.TarGzCompressionUtils; @@ -32,6 +33,7 @@ import org.apache.pinot.segment.spi.SegmentMetadata; import org.apache.pinot.segment.spi.V1Constants; import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl; +import org.apache.pinot.segment.spi.store.ColumnIndexType; import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; @@ -99,6 +101,36 @@ public void getSegments() Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); } + @Test + public void getTableMetadata() + throws Exception { + String tableMetadataPath = "/tables/" + TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME) + "/metadata"; + + JsonNode jsonResponse = + JsonUtils.stringToJsonNode(_webTarget.path(tableMetadataPath).request().get(String.class)); + TableMetadataInfo metadataInfo = JsonUtils.jsonNodeToObject(jsonResponse, TableMetadataInfo.class); + Assert.assertNotNull(metadataInfo); + Assert.assertEquals(metadataInfo.getNumSegments(), 2); + Assert.assertEquals(metadataInfo.getTableName(), TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME)); + Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 0); + Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 0); + Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 0); + + jsonResponse = JsonUtils.stringToJsonNode(_webTarget.path(tableMetadataPath) + .queryParam("columns", "column1").queryParam("columns", "column2").request().get(String.class)); + metadataInfo = JsonUtils.jsonNodeToObject(jsonResponse, TableMetadataInfo.class); + Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 2); + Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 2); + Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 2); + Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().containsKey(ColumnIndexType.DICTIONARY.getIndexName())); + Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().containsKey(ColumnIndexType.FORWARD_INDEX.getIndexName())); + + // No such table + Response response = _webTarget.path("/tables/noSuchTable/metadata").request().get(Response.class); + Assert.assertNotNull(response); + Assert.assertEquals(response.getStatus(), Response.Status.NOT_FOUND.getStatusCode()); + } + @Test public void testSegmentMetadata() throws Exception { @@ -124,6 +156,8 @@ public void testSegmentMetadata() .get(String.class)); Assert.assertEquals(jsonResponse.get("columns").size(), 2); Assert.assertEquals(jsonResponse.get("indexes").size(), 2); + Assert.assertNotNull(jsonResponse.get("columns").get(0).get("indexSizeMap")); + Assert.assertNotNull(jsonResponse.get("columns").get(1).get("indexSizeMap")); jsonResponse = JsonUtils.stringToJsonNode( (_webTarget.path(segmentMetadataPath).queryParam("columns", "*").request().get(String.class))); From a7889bcd5608e2ef77facb701c9750144128a539 Mon Sep 17 00:00:00 2001 From: Rong Rong Date: Mon, 7 Nov 2022 09:41:10 -0800 Subject: [PATCH 2/4] modify test --- .../org/apache/pinot/server/api/TablesResourceTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java index 2acf42694ff9..4baa08bbdfe3 100644 --- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java +++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java @@ -122,8 +122,10 @@ public void getTableMetadata() Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 2); Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 2); Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 2); - Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().containsKey(ColumnIndexType.DICTIONARY.getIndexName())); - Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().containsKey(ColumnIndexType.FORWARD_INDEX.getIndexName())); + Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().get("column1") + .containsKey(ColumnIndexType.DICTIONARY.getIndexName())); + Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().get("column2") + .containsKey(ColumnIndexType.FORWARD_INDEX.getIndexName())); // No such table Response response = _webTarget.path("/tables/noSuchTable/metadata").request().get(Response.class); From d4b8b5266f3ab69bfe8df253dce23c99ec2c27cb Mon Sep 17 00:00:00 2001 From: Rong Rong Date: Mon, 7 Nov 2022 18:56:36 -0800 Subject: [PATCH 3/4] address comment --- .../apache/pinot/server/api/resources/TablesResource.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java index 2017dc40a58d..79df252dcdbc 100644 --- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java +++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java @@ -259,10 +259,10 @@ public String getSegmentMetadata( maxNumMultiValuesMap.merge(column, (double) maxNumMultiValues, Double::sum); } for (Map.Entry entry : columnMetadata.getIndexSizeMap().entrySet()) { + String indexName = entry.getKey().getIndexName(); Map columnIndexSizes = columnIndexSizesMap.getOrDefault(column, new HashMap<>()); - Double indexSize = columnIndexSizes.getOrDefault(entry.getKey().getIndexName(), 0d); - indexSize += entry.getValue(); - columnIndexSizes.put(entry.getKey().getIndexName(), indexSize); + Double indexSize = columnIndexSizes.getOrDefault(indexName, 0d) + entry.getValue(); + columnIndexSizes.put(indexName, indexSize); columnIndexSizesMap.put(column, columnIndexSizes); } } From 3e664088c52d4581796b15347f3eca3d3403a206 Mon Sep 17 00:00:00 2001 From: Rong Rong Date: Wed, 9 Nov 2022 21:16:12 -0800 Subject: [PATCH 4/4] add test --- .../pinot/server/api/TablesResourceTest.java | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java index 4baa08bbdfe3..c60ffd1b8482 100644 --- a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java +++ b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java @@ -34,6 +34,7 @@ import org.apache.pinot.segment.spi.V1Constants; import org.apache.pinot.segment.spi.index.metadata.SegmentMetadataImpl; import org.apache.pinot.segment.spi.store.ColumnIndexType; +import org.apache.pinot.spi.config.table.TableType; import org.apache.pinot.spi.utils.JsonUtils; import org.apache.pinot.spi.utils.builder.TableNameBuilder; import org.testng.Assert; @@ -104,28 +105,31 @@ public void getSegments() @Test public void getTableMetadata() throws Exception { - String tableMetadataPath = "/tables/" + TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME) + "/metadata"; - - JsonNode jsonResponse = - JsonUtils.stringToJsonNode(_webTarget.path(tableMetadataPath).request().get(String.class)); - TableMetadataInfo metadataInfo = JsonUtils.jsonNodeToObject(jsonResponse, TableMetadataInfo.class); - Assert.assertNotNull(metadataInfo); - Assert.assertEquals(metadataInfo.getNumSegments(), 2); - Assert.assertEquals(metadataInfo.getTableName(), TableNameBuilder.REALTIME.tableNameWithType(TABLE_NAME)); - Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 0); - Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 0); - Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 0); - - jsonResponse = JsonUtils.stringToJsonNode(_webTarget.path(tableMetadataPath) - .queryParam("columns", "column1").queryParam("columns", "column2").request().get(String.class)); - metadataInfo = JsonUtils.jsonNodeToObject(jsonResponse, TableMetadataInfo.class); - Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 2); - Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 2); - Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 2); - Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().get("column1") - .containsKey(ColumnIndexType.DICTIONARY.getIndexName())); - Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().get("column2") - .containsKey(ColumnIndexType.FORWARD_INDEX.getIndexName())); + for (TableType tableType : TableType.values()) { + String tableMetadataPath = "/tables/" + TableNameBuilder.forType(tableType).tableNameWithType(TABLE_NAME) + + "/metadata"; + + JsonNode jsonResponse = + JsonUtils.stringToJsonNode(_webTarget.path(tableMetadataPath).request().get(String.class)); + TableMetadataInfo metadataInfo = JsonUtils.jsonNodeToObject(jsonResponse, TableMetadataInfo.class); + Assert.assertNotNull(metadataInfo); + Assert.assertEquals(metadataInfo.getTableName(), TableNameBuilder.forType(tableType) + .tableNameWithType(TABLE_NAME)); + Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 0); + Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 0); + Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 0); + + jsonResponse = JsonUtils.stringToJsonNode(_webTarget.path(tableMetadataPath) + .queryParam("columns", "column1").queryParam("columns", "column2").request().get(String.class)); + metadataInfo = JsonUtils.jsonNodeToObject(jsonResponse, TableMetadataInfo.class); + Assert.assertEquals(metadataInfo.getColumnLengthMap().size(), 2); + Assert.assertEquals(metadataInfo.getColumnCardinalityMap().size(), 2); + Assert.assertEquals(metadataInfo.getColumnIndexSizeMap().size(), 2); + Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().get("column1") + .containsKey(ColumnIndexType.DICTIONARY.getIndexName())); + Assert.assertTrue(metadataInfo.getColumnIndexSizeMap().get("column2") + .containsKey(ColumnIndexType.FORWARD_INDEX.getIndexName())); + } // No such table Response response = _webTarget.path("/tables/noSuchTable/metadata").request().get(Response.class);