From 2dc409ad8ce5b1ad8b7606a4dfbb744ef079f1a5 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Mon, 13 May 2024 11:12:15 -0700 Subject: [PATCH 1/6] add /timeplusd rest endpoints --- .../RestRouterHandlers/RestRouterFactory.h | 351 +++++++++--------- 1 file changed, 181 insertions(+), 170 deletions(-) diff --git a/src/Server/RestRouterHandlers/RestRouterFactory.h b/src/Server/RestRouterHandlers/RestRouterFactory.h index 8aa37de3b28..8ecbed4cad8 100644 --- a/src/Server/RestRouterHandlers/RestRouterFactory.h +++ b/src/Server/RestRouterHandlers/RestRouterFactory.h @@ -4,7 +4,6 @@ #include "ClusterInfoHandler.h" #include "ColumnRestRouterHandler.h" #include "DatabaseRestRouterHandler.h" -#include "StorageInfoHandler.h" #include "ExternalStreamRestRouterHandler.h" #include "IngestRawStoreHandler.h" #include "IngestRestRouterHandler.h" @@ -17,9 +16,10 @@ #include "SQLAnalyzerRestRouterHandler.h" #include "SQLFormatHandler.h" #include "SearchHandler.h" +#include "StorageInfoHandler.h" +#include "SystemCommandHandler.h" #include "TabularTableRestRouterHandler.h" #include "UDFHandler.h" -#include "SystemCommandHandler.h" #include #include @@ -32,7 +32,7 @@ using CompiledRegexPtr = std::shared_ptr; namespace ErrorCodes { - extern const int CANNOT_COMPILE_REGEXP; +extern const int CANNOT_COMPILE_REGEXP; } class RestRouterFactory final @@ -49,171 +49,180 @@ class RestRouterFactory final constexpr auto * stream_name_regex = "(?P[_%\\.\\-\\w]+)"; auto & factory = RestRouterFactory::instance(); - factory.registerRouterHandler( - fmt::format("/proton/v1/ingest/streams/{}(\\?mode=\\w+){{0,1}}", stream_name_regex), - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - fmt::format("/proton/v1/ingest/rawstores/{}(\\?mode=\\w+){{0,1}}", stream_name_regex), - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/search(\\?[\\w\\-=&#]+){0,1}", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/pipeline_metrics", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/ingest/statuses", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - /// PATH: '/proton/v1/ddl/streams [/{databse}] [/{key}] ...' - "/proton/v1/ddl/streams(/?(\\?[\\w\\-=&#]+){0,1}$|/(?P[%\\w]+)(/?(\\?[\\w\\-=&#]+){0,1}$|/(?P[%\\-\\.\\w]+)(\\?[\\w\\-=&#]+){0,1})(\\?[\\w\\-=&#]+){0,1})", - "GET", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/ddl/streams(?:\\?\?[^/]*)?", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - fmt::format("/proton/v1/ddl/streams/{}(\\?[\\w\\-=&#]+){{0,1}}", stream_name_regex), - "PATCH/DELETE", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/ddl/externalstreams(\\?[\\w\\-=&#]+){0,1}", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/ddl/rawstores(\\?[\\w\\-=&#]+){0,1}", - "GET/POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - fmt::format("/proton/v1/ddl/rawstores/{}(\\?[\\w\\-=&#]+){{0,1}}", stream_name_regex), - "DELETE" /* So far, not support PATCH */, - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - /// So far, we do not support ALTER TABLE ADD/UPDATE/DELETE COLUMN ... - // factory.registerRouterHandler( - // fmt::format("/proton/v1/ddl/{}/columns(\\?[\\w\\-=&#]+){{0,1}}", stream_name_regex), - // "GET/POST", - // [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - // return std::make_shared(query_context); - // }); - - // factory.registerRouterHandler( - // fmt::format("/proton/v1/ddl/{}/columns/(?P[%\\w]+)(\\?[\\w\\-=&#]+){{0,1}}", stream_name_regex}, - // "PATCH/DELETE", - // [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - // return std::make_shared(query_context); - // }); - - factory.registerRouterHandler( - "/proton/v1/ddl/databases(\\?[\\w\\-=&#]+){0,1}", - "GET/POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/ddl/databases/(?P\\w+)(\\?[\\w\\-=&#]+){0,1}", - "DELETE", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/sqlanalyzer", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/sqlformat", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); -#if USE_NURAFT - /// GET/POST/DELETE: /proton/v1/udfs[/{func}] - factory.registerRouterHandler( - "/proton/v1/udfs(/?$|/(?P[%\\w]+))", - "GET/POST/DELETE", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); -#endif - factory.registerRouterHandler( - "/proton/(?Pping|info)$", - "GET", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/v1/clusterinfo", - "GET", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - /// GET: /proton/v1/storageinfo[/{database}[/{stream}]] - factory.registerRouterHandler( - "/proton/v1/storageinfo(/?(\\?[\\w\\-=&#]+){0,1}$|/(?P[%\\w]+)(/?(\\?[\\w\\-=&#]+){0,1}$|/(?P[%\\-\\.\\w]+)(\\?[\\w\\-=&#]+){0,1})(\\?[\\w\\-=&#]+){0,1})", - "GET", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - /// POST /proton/v1/system? - factory.registerRouterHandler( - "/proton/v1/system(\\?[\\w\\-=&#]+){0,1}", - "POST", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); - - factory.registerRouterHandler( - "/proton/apis", - "GET", - [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA - return std::make_shared(query_context); - }); + for (const auto * prefix : {"proton", "timeplusd"}) + { + factory.registerRouterHandler( + fmt::format("/{}/v1/ingest/streams/{}(\\?mode=\\w+){{0,1}}", prefix, stream_name_regex), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ingest/rawstores/{}(\\?mode=\\w+){{0,1}}", prefix, stream_name_regex), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/search(\\?[\\w\\-=&#]+){{0,1}}", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/pipeline_metrics", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ingest/statuses", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + /// PATH: '/proton/v1/ddl/streams [/{databse}] [/{key}] ...' + fmt::format( + "/{}/v1/ddl/streams(/?(\\?[\\w\\-=&#]+){{0,1}}$|/(?P[%\\w]+)(/?(\\?[\\w\\-=&#]+){{0,1}}$|/" + "(?P[%\\-\\.\\w]+)(\\?[\\w\\-=&#]+){{0,1}})(\\?[\\w\\-=&#]+){{0,1}})", + prefix), + "GET", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/streams(?:\\?\?[^/]*)?", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/streams/{}(\\?[\\w\\-=&#]+){{0,1}}", prefix, stream_name_regex), + "PATCH/DELETE", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/externalstreams(\\?[\\w\\-=&#]+){{0,1}}", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/rawstores(\\?[\\w\\-=&#]+){{0,1}}", prefix), + "GET/POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/rawstores/{}(\\?[\\w\\-=&#]+){{0,1}}", prefix, stream_name_regex), + "DELETE" /* So far, not support PATCH */, + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + /// So far, we do not support ALTER TABLE ADD/UPDATE/DELETE COLUMN ... + // factory.registerRouterHandler( + // fmt::format("/proton/v1/ddl/{}/columns(\\?[\\w\\-=&#]+){{0,1}}", stream_name_regex), + // "GET/POST", + // [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + // return std::make_shared(query_context); + // }); + + // factory.registerRouterHandler( + // fmt::format("/proton/v1/ddl/{}/columns/(?P[%\\w]+)(\\?[\\w\\-=&#]+){{0,1}}", stream_name_regex}, + // "PATCH/DELETE", + // [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + // return std::make_shared(query_context); + // }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/databases(\\?[\\w\\-=&#]+){{0,1}}", prefix), + "GET/POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/ddl/databases/(?P\\w+)(\\?[\\w\\-=&#]+){{0,1}}", prefix), + "DELETE", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/sqlanalyzer", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/sqlformat", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + /// GET/POST/DELETE: /proton/v1/udfs[/{func}] + factory.registerRouterHandler( + fmt::format("/{}/v1/udfs(/?$|/(?P[%\\w]+))", prefix), + "GET/POST/DELETE", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/(?Pping|info)$", prefix), + "GET", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/v1/clusterinfo", prefix), + "GET", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + /// GET: /proton/v1/storageinfo[/{database}[/{stream}]] + factory.registerRouterHandler( + fmt::format( + "/{}/v1/storageinfo(/?(\\?[\\w\\-=&#]+){{0,1}}$|/(?P[%\\w]+)(/?(\\?[\\w\\-=&#]+){{0,1}}$|/" + "(?P[%\\-\\.\\w]+)(\\?[\\w\\-=&#]+){{0,1}})(\\?[\\w\\-=&#]+){{0,1}})", + prefix), + "GET", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + /// POST /proton/v1/system? + factory.registerRouterHandler( + fmt::format("/{}/v1/system(\\?[\\w\\-=&#]+){{0,1}}", prefix), + "POST", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + + factory.registerRouterHandler( + fmt::format("/{}/apis", prefix), + "GET", + [](ContextMutablePtr query_context) { /// STYLE_CHECK_ALLOW_BRACE_SAME_LINE_LAMBDA + return std::make_shared(query_context); + }); + } } static void registerMetaStoreRestRouterHandlers() @@ -283,8 +292,10 @@ class RestRouterFactory final { throw Exception( ErrorCodes::CANNOT_COMPILE_REGEXP, - "Cannot compile re2: '{}' for http handling rule, error: {}. Look at https://github.com/google/re2/wiki/Syntax for reference.", - expression, compiled_regex->error()); + "Cannot compile re2: '{}' for http handling rule, error: {}. Look at https://github.com/google/re2/wiki/Syntax for " + "reference.", + expression, + compiled_regex->error()); } return compiled_regex; From 033de162615cd137bf6478757e3e88ce3b1aa682 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Mon, 13 May 2024 11:31:51 -0700 Subject: [PATCH 2/6] more /timeplusd endpoints --- src/Server/HTTPHandlerFactory.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index ef778ad801b..03cc2e9afc8 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -154,6 +154,7 @@ HTTPRequestHandlerFactoryPtr createHandlerFactory(IServer & server, Asynchronous auto factory = std::make_shared(name); auto handler = std::make_shared>( server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics, server.context())); + handler->attachStrictPath("/timeplusd/metrics"); handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics")); handler->allowGetAndHeadRequest(); factory->addHandler(handler); @@ -168,6 +169,7 @@ HTTPRequestHandlerFactoryPtr createMetaStoreHandlerFactory(IServer & server, con { auto factory = std::make_shared(name); auto rest_handler = std::make_shared>(server, "metastore"); + rest_handler->attachNonStrictPath("/timeplusd/metastore"); rest_handler->attachNonStrictPath("/proton/metastore"); factory->addHandler(rest_handler); return factory; @@ -185,12 +187,13 @@ void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IS factory.addHandler(root_handler); auto ping_handler = std::make_shared>(server, ping_response_expression); + ping_handler->attachStrictPath("/timeplusd/ping"); ping_handler->attachStrictPath("/ping"); ping_handler->allowGetAndHeadRequest(); factory.addHandler(ping_handler); auto web_ui_handler = std::make_shared>(server, "play.html"); - web_ui_handler->attachNonStrictPath("/play"); + web_ui_handler->attachNonStrictPath("/timeplusd/play"); web_ui_handler->allowGetAndHeadRequest(); factory.addHandler(web_ui_handler); } @@ -202,9 +205,12 @@ void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer addCommonDefaultHandlersFactory(factory, server); /// proton: start. Add rest request process handler - auto rest_handler = std::make_shared>(server, "proton"); - rest_handler->attachNonStrictPath("/proton"); - factory.addHandler(rest_handler); + for (const auto * prefix : {"proton", "timeplusd"}) + { + auto rest_handler = std::make_shared>(server, prefix); + rest_handler->attachNonStrictPath(fmt::format("/{}", prefix)); + factory.addHandler(rest_handler); + } /// proton: end. /// proton: starts @@ -219,6 +225,7 @@ void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer { auto prometheus_handler = std::make_shared>( server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics, server.context())); + prometheus_handler->attachStrictPath("/timeplusd/metrics"); prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics")); prometheus_handler->allowGetAndHeadRequest(); factory.addHandler(prometheus_handler); From b6876c936bfbcd060f3397417c713cf32a58012b Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Mon, 13 May 2024 11:33:48 -0700 Subject: [PATCH 3/6] more comments regarding dynamic http handlers --- programs/server/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/server/config.yaml b/programs/server/config.yaml index 583ae7d084a..b063fe68346 100644 --- a/programs/server/config.yaml +++ b/programs/server/config.yaml @@ -902,6 +902,7 @@ query_masking_rules: # content_type - use with static type, response content-type # response_content - use with static type, Response content sent to client, when using the prefix 'file://' or 'config://', find the content from the file or configuration send to client. +# NOTE, if enable these dynamic handlers, please follow /timeplusd/{uri_path} convention, check createHandlersFactoryFromConfig() in HTTPHandlerFactory.cpp # http_handlers: # - rule: # url: / From 28a3d04df941848f6c8d63b4fa521206d5e91402 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Mon, 13 May 2024 11:45:08 -0700 Subject: [PATCH 4/6] cleanup --- programs/server/play.html | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/programs/server/play.html b/programs/server/play.html index f7154643504..1053b479126 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -2,7 +2,7 @@ - ClickHouse Query + Timeplus Query