From 30b47b894b1eb64a23a33e48d54447763632c447 Mon Sep 17 00:00:00 2001 From: Rob Kiefer Date: Tue, 4 Feb 2025 13:00:09 -0500 Subject: [PATCH] Remove limitation of compression policy for continuous aggregates Before compressed chunks were mutable, adding a compression policy to a continuous aggregate that could include portions of the refrsh window were blocked. When a continuous aggregate is refreshed, the underlying chunks needed to allow DELETEs and INSERTs, so they could not be compressed. Now, compressed chunks allow both operations and there is no longer a need to prevent the refresh window and compression window from overlapping. --- .unreleased/pr_7656 | 1 + tsl/src/bgw_policy/compression_api.c | 15 ---------- tsl/test/expected/cagg_errors.out | 6 ---- tsl/test/expected/cagg_policy.out | 42 ++++++++++++++++++++-------- tsl/test/sql/cagg_errors.sql | 3 -- tsl/test/sql/cagg_policy.sql | 8 ++++-- 6 files changed, 36 insertions(+), 39 deletions(-) create mode 100644 .unreleased/pr_7656 diff --git a/.unreleased/pr_7656 b/.unreleased/pr_7656 new file mode 100644 index 00000000000..d02f8515a3f --- /dev/null +++ b/.unreleased/pr_7656 @@ -0,0 +1 @@ +Implements: #7656 Remove limitation of compression policy for continuous aggregates diff --git a/tsl/src/bgw_policy/compression_api.c b/tsl/src/bgw_policy/compression_api.c index ed5fcb9c8c0..954b67ce598 100644 --- a/tsl/src/bgw_policy/compression_api.c +++ b/tsl/src/bgw_policy/compression_api.c @@ -338,21 +338,6 @@ policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, format_type_be(compress_after_type)))); } - /* - * If this is a compression policy for a cagg, verify that - * compress_after > refresh_start of cagg policy. We do not want - * to compress regions that can be refreshed by the cagg policy. - */ - if (is_cagg && !policy_refresh_cagg_refresh_start_lt(hypertable->fd.id, - compress_after_type, - compress_after_datum)) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("compress_after value for compression policy should be greater than the " - "start of the refresh window of continuous aggregate policy for %s", - get_rel_name(user_rel_oid)))); - } JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); diff --git a/tsl/test/expected/cagg_errors.out b/tsl/test/expected/cagg_errors.out index 95b8698f839..64126a31bdd 100644 --- a/tsl/test/expected/cagg_errors.out +++ b/tsl/test/expected/cagg_errors.out @@ -595,14 +595,8 @@ ALTER MATERIALIZED VIEW i2980_cagg SET ( timescaledb.compress ); NOTICE: defaulting compress_orderby to time_bucket WARNING: there was some uncertainty picking the default segment by for the hypertable: You do not have any indexes on columns that can be used for segment_by and thus we are not using segment_by for compression. Please make sure you are not missing any indexes NOTICE: default segment by for hypertable "_materialized_hypertable_13" is set to "" -SELECT add_compression_policy('i2980_cagg', '8 day'::interval); -ERROR: compress_after value for compression policy should be greater than the start of the refresh window of continuous aggregate policy for i2980_cagg SELECT add_continuous_aggregate_policy('i2980_cagg2', '10 day'::interval, '6 day'::interval); ERROR: function add_continuous_aggregate_policy(unknown, interval, interval) does not exist at character 8 -SELECT add_compression_policy('i2980_cagg2', '3 day'::interval); -ERROR: compress_after value for compression policy should be greater than the start of the refresh window of continuous aggregate policy for i2980_cagg2 -SELECT add_compression_policy('i2980_cagg2', '1 day'::interval); -ERROR: compress_after value for compression policy should be greater than the start of the refresh window of continuous aggregate policy for i2980_cagg2 SELECT add_compression_policy('i2980_cagg2', '3'::integer); ERROR: unsupported compress_after argument type, expected type : interval SELECT add_compression_policy('i2980_cagg2', 13::integer); diff --git a/tsl/test/expected/cagg_policy.out b/tsl/test/expected/cagg_policy.out index e2b011a8d24..b5ba9a6beb9 100644 --- a/tsl/test/expected/cagg_policy.out +++ b/tsl/test/expected/cagg_policy.out @@ -1158,24 +1158,42 @@ ALTER MATERIALIZED VIEW mat_smallint SET (timescaledb.compress); NOTICE: defaulting compress_orderby to a WARNING: there was some uncertainty picking the default segment by for the hypertable: You do not have any indexes on columns that can be used for segment_by and thus we are not using segment_by for compression. Please make sure you are not missing any indexes NOTICE: default segment by for hypertable "_materialized_hypertable_11" is set to "" -\set ON_ERROR_STOP 0 -SELECT add_compression_policy('mat_smallint', 0::smallint); -ERROR: compress_after value for compression policy should be greater than the start of the refresh window of continuous aggregate policy for mat_smallint +-- With immutable compressed chunks, these policies would fail by overlapping the refresh window SELECT add_compression_policy('mat_smallint', -4::smallint); -ERROR: compress_after value for compression policy should be greater than the start of the refresh window of continuous aggregate policy for mat_smallint + add_compression_policy +------------------------ + 1053 +(1 row) + +SELECT remove_compression_policy('mat_smallint'); + remove_compression_policy +--------------------------- + t +(1 row) + SELECT add_compression_policy('mat_bigint', 0::bigint); -ERROR: compress_after value for compression policy should be greater than the start of the refresh window of continuous aggregate policy for mat_bigint -\set ON_ERROR_STOP 1 + add_compression_policy +------------------------ + 1054 +(1 row) + +SELECT remove_compression_policy('mat_bigint'); + remove_compression_policy +--------------------------- + t +(1 row) + +-- End previous limitation tests SELECT add_compression_policy('mat_smallint', 5::smallint); add_compression_policy ------------------------ - 1053 + 1055 (1 row) SELECT add_compression_policy('mat_bigint', 20::bigint); add_compression_policy ------------------------ - 1054 + 1056 (1 row) -- end of coverage tests @@ -1227,7 +1245,7 @@ WITH NO DATA; SELECT add_continuous_aggregate_policy('metrics_cagg', '7 day'::interval, NULL, '1 h'::interval, if_not_exists => true); add_continuous_aggregate_policy --------------------------------- - 1055 + 1057 (1 row) SELECT add_continuous_aggregate_policy('metrics_cagg', '7 day'::interval, '1 day'::interval, '1 h'::interval, if_not_exists => true); @@ -1246,7 +1264,7 @@ SELECT remove_continuous_aggregate_policy('metrics_cagg'); SELECT add_continuous_aggregate_policy('metrics_cagg', NULL, '1 day'::interval, '1h'::interval, if_not_exists=>true); add_continuous_aggregate_policy --------------------------------- - 1056 + 1058 (1 row) SELECT add_continuous_aggregate_policy('metrics_cagg', NULL, '1 day'::interval, '1h'::interval, if_not_exists=>true); -- same param values, so we get a NOTICE @@ -1287,7 +1305,7 @@ ERROR: cannot use "compress_created_before" with continuous aggregate "metrics_ SELECT add_compression_policy('metrics_cagg', '8 day'::interval) AS "COMP_JOB" ; COMP_JOB ---------- - 1058 + 1060 (1 row) SELECT remove_compression_policy('metrics_cagg'); @@ -1429,7 +1447,7 @@ SELECT set_integer_now_func('t', 'unix_now'); SELECT add_retention_policy('t', 20); add_retention_policy ---------------------- - 1060 + 1062 (1 row) CREATE MATERIALIZED VIEW cagg(a, sumb) WITH (timescaledb.continuous) diff --git a/tsl/test/sql/cagg_errors.sql b/tsl/test/sql/cagg_errors.sql index dca90177da7..3c0b8de7bae 100644 --- a/tsl/test/sql/cagg_errors.sql +++ b/tsl/test/sql/cagg_errors.sql @@ -490,11 +490,8 @@ ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress, timescaledb.comp select add_continuous_aggregate_policy('i2980_cagg2', interval '10 day', interval '2 day' ,'4h') AS job_id ; SELECT add_compression_policy('i2980_cagg', '8 day'::interval); ALTER MATERIALIZED VIEW i2980_cagg SET ( timescaledb.compress ); -SELECT add_compression_policy('i2980_cagg', '8 day'::interval); SELECT add_continuous_aggregate_policy('i2980_cagg2', '10 day'::interval, '6 day'::interval); -SELECT add_compression_policy('i2980_cagg2', '3 day'::interval); -SELECT add_compression_policy('i2980_cagg2', '1 day'::interval); SELECT add_compression_policy('i2980_cagg2', '3'::integer); SELECT add_compression_policy('i2980_cagg2', 13::integer); diff --git a/tsl/test/sql/cagg_policy.sql b/tsl/test/sql/cagg_policy.sql index 1e73ddcdec7..a3913770dcb 100644 --- a/tsl/test/sql/cagg_policy.sql +++ b/tsl/test/sql/cagg_policy.sql @@ -505,11 +505,13 @@ SELECT * FROM mat_bigint WHERE a>100 ORDER BY 1; ALTER MATERIALIZED VIEW mat_bigint SET (timescaledb.compress); ALTER MATERIALIZED VIEW mat_smallint SET (timescaledb.compress); -\set ON_ERROR_STOP 0 -SELECT add_compression_policy('mat_smallint', 0::smallint); +-- With immutable compressed chunks, these policies would fail by overlapping the refresh window SELECT add_compression_policy('mat_smallint', -4::smallint); +SELECT remove_compression_policy('mat_smallint'); SELECT add_compression_policy('mat_bigint', 0::bigint); -\set ON_ERROR_STOP 1 +SELECT remove_compression_policy('mat_bigint'); +-- End previous limitation tests + SELECT add_compression_policy('mat_smallint', 5::smallint); SELECT add_compression_policy('mat_bigint', 20::bigint);