From 0ece49839a0c2335a398ee74d7b26c66a97d7c9f Mon Sep 17 00:00:00 2001 From: skrzypo987 Date: Tue, 20 Sep 2022 12:55:52 +0200 Subject: [PATCH] Extends flush_metadata_cache procedure Now it is possible to flush all caches related to a given table via a procedure --- docs/src/main/sphinx/connector/hive.rst | 5 +++++ .../procedure/FlushHiveMetastoreCacheProcedure.java | 10 ++++++++-- .../io/trino/plugin/hive/BaseTestHiveOnDataLake.java | 10 ++++++++++ .../cache/TestCachingHiveMetastoreWithQueryRunner.java | 7 ++++--- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/src/main/sphinx/connector/hive.rst b/docs/src/main/sphinx/connector/hive.rst index ab64b2c0d54a..02385442c1bc 100644 --- a/docs/src/main/sphinx/connector/hive.rst +++ b/docs/src/main/sphinx/connector/hive.rst @@ -1092,6 +1092,11 @@ The following procedures are available: Flush all Hive metadata caches. +* ``system.flush_metadata_cache(schema_name => ..., table_name => ...)`` + + Flush Hive metadata caches entries connected with selected table. + Procedure requires named parameters to be passed + * ``system.flush_metadata_cache(schema_name => ..., table_name => ..., partition_column => ARRAY[...], partition_value => ARRAY[...])`` Flush Hive metadata cache entries connected with selected partition. diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/procedure/FlushHiveMetastoreCacheProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/procedure/FlushHiveMetastoreCacheProcedure.java index a05bc739937c..156e50f8e55d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/procedure/FlushHiveMetastoreCacheProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/procedure/FlushHiveMetastoreCacheProcedure.java @@ -48,6 +48,7 @@ public class FlushHiveMetastoreCacheProcedure private static final String PROCEDURE_USAGE_EXAMPLES = format( "Valid usages:%n" + " - '%1$s()'%n" + + " - %1$s(%2$s => ..., %3$s => ...)" + " - %1$s(%2$s => ..., %3$s => ..., %4$s => ARRAY['...'], %5$s => ARRAY['...'])", PROCEDURE_NAME, // Use lowercase parameter names per convention. In the usage example the names are not delimited. @@ -113,8 +114,13 @@ private void doFlushMetadataCache(Optional schemaName, Optional if (schemaName.isEmpty() && tableName.isEmpty() && partitionColumns.isEmpty()) { cachingHiveMetastore.flushCache(); } - else if (schemaName.isPresent() && tableName.isPresent() && !partitionColumns.isEmpty()) { - cachingHiveMetastore.flushPartitionCache(schemaName.get(), tableName.get(), partitionColumns, partitionValues); + else if (schemaName.isPresent() && tableName.isPresent()) { + if (!partitionColumns.isEmpty()) { + cachingHiveMetastore.flushPartitionCache(schemaName.get(), tableName.get(), partitionColumns, partitionValues); + } + else { + cachingHiveMetastore.invalidateTable(schemaName.get(), tableName.get()); + } } else { throw new TrinoException( diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java index 67028d70f783..048283a452c9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseTestHiveOnDataLake.java @@ -243,6 +243,16 @@ public void testFlushPartitionCache() // Should return 0 rows as we left cache untouched assertQueryReturnsEmptyResult(queryUsingPartitionCacheForValue2); + // Refresh cache for schema_name => 'dummy_schema', table_name => 'dummy_table' + getQueryRunner().execute(format( + "CALL system.flush_metadata_cache(schema_name => '%s', table_name => '%s')", + HIVE_TEST_SCHEMA, + tableName)); + + // Should return expected rows for all partitions + assertQuery(queryUsingPartitionCacheForValue1, expectedQueryResultForValue1); + assertQuery(queryUsingPartitionCacheForValue2, expectedQueryResultForValue2); + computeActual(format("DROP TABLE %s", fullyQualifiedTestTableName)); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java index 459f91cbcdd3..96dea2ef16dd 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/cache/TestCachingHiveMetastoreWithQueryRunner.java @@ -143,15 +143,16 @@ public void testFlushHiveMetastoreCacheProcedureCallable() @Test public void testIllegalFlushHiveMetastoreCacheProcedureCalls() { - String illegalParameterMessage = "Illegal parameter set passed. Valid usages:\n - 'flush_metadata_cache()'\n - flush_metadata_cache(schema_name => ..., table_name => ..., partition_column => ARRAY['...'], partition_value => ARRAY['...'])"; + String illegalParameterMessage = "Illegal parameter set passed. Valid usages:\n" + + " - 'flush_metadata_cache()'\n" + + " - flush_metadata_cache(schema_name => ..., table_name => ...)" + + " - flush_metadata_cache(schema_name => ..., table_name => ..., partition_column => ARRAY['...'], partition_value => ARRAY['...'])"; assertThatThrownBy(() -> getQueryRunner().execute("CALL system.flush_metadata_cache('dummy_schema')")) .hasMessageContaining("Only named arguments are allowed for this procedure"); assertThatThrownBy(() -> getQueryRunner().execute("CALL system.flush_metadata_cache(schema_name => 'dummy_schema')")) .hasMessage(illegalParameterMessage); - assertThatThrownBy(() -> getQueryRunner().execute("CALL system.flush_metadata_cache(schema_name => 'dummy_schema', table_name => 'dummy_table')")) - .hasMessage(illegalParameterMessage); assertThatThrownBy(() -> getQueryRunner().execute("CALL system.flush_metadata_cache(schema_name => 'dummy_schema', table_name => 'dummy_table', partition_column => ARRAY['dummy_partition'])")) .hasMessage("Parameters partition_column and partition_value should have same length");