From 50243c0354d401bc0013261342dbfedaeb4ba430 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sun, 12 Jan 2025 07:59:30 +0100 Subject: [PATCH 01/10] tests/copyright: add support for .S files Signed-off-by: Balazs Scheidler --- tests/copyright/check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/copyright/check.sh b/tests/copyright/check.sh index b0fdf21000..ba5e0f4a4f 100755 --- a/tests/copyright/check.sh +++ b/tests/copyright/check.sh @@ -301,7 +301,7 @@ extract_holder_license() { local EXT="`echo "$FILE" | sed -r "s~^.*\.([^.]+)$~\1~"`" case "$EXT" in - c|h|cpp|hpp|m|mm|ym|java|table) + c|h|cpp|hpp|m|mm|ym|java|table|S) extract_holder_license_c ;; ac|am|cmake|conf|sh|pl|py) From b069171c0ae3e3aca6027492809769e71ac4127c Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sun, 12 Jan 2025 07:59:46 +0100 Subject: [PATCH 02/10] tests/copyright: exclude tests/light/reports directory Signed-off-by: Balazs Scheidler --- tests/copyright/policy | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/copyright/policy b/tests/copyright/policy index 635e21a39e..68387a6e7a 100644 --- a/tests/copyright/policy +++ b/tests/copyright/policy @@ -59,6 +59,7 @@ cmake/.*$ tests/light/pytest.ini tests/light/tox.ini tests/light/.pre-commit-config.yaml +tests/light/reports .github/workflows/.*$ lib/logmsg/tests/messages/syslog-ng-(pe-)?[.0-9]*-msg.h$ modules/python-modules/requirements\.lock From 1e0d66b92e5f8b11fb7b018e8de94edc8bd2d31c Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Thu, 26 Dec 2024 21:29:23 +0100 Subject: [PATCH 03/10] lib/perf: add support for perf based profiling Signed-off-by: Balazs Scheidler --- cmake/syslog-ng-config.h.in | 1 + configure.ac | 18 ++ lib/Makefile.am | 2 + lib/gprocess.c | 16 +- lib/perf/Makefile.am | 18 ++ lib/perf/perf.c | 229 ++++++++++++++++++++++++++ lib/perf/perf.h | 47 ++++++ lib/perf/trampoline.S | 41 +++++ tests/build-log-cflags-propagation.sh | 7 + tests/copyright/policy | 1 + 10 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 lib/perf/Makefile.am create mode 100644 lib/perf/perf.c create mode 100644 lib/perf/perf.h create mode 100644 lib/perf/trampoline.S diff --git a/cmake/syslog-ng-config.h.in b/cmake/syslog-ng-config.h.in index 713a468833..b5b60e3903 100644 --- a/cmake/syslog-ng-config.h.in +++ b/cmake/syslog-ng-config.h.in @@ -106,3 +106,4 @@ #cmakedefine01 SYSLOG_NG_HAVE_SO_MEMINFO #cmakedefine01 SYSLOG_NG_ENABLE_AFSOCKET_MEMINFO_METRICS #cmakedefine01 SYSLOG_NG_HAVE_IV_WORK_POOL_SUBMIT_CONTINUATION +#cmakedefine01 SYSLOG_NG_ENABLE_PERF diff --git a/configure.ac b/configure.ac index f906a0dfa8..596eb046c0 100644 --- a/configure.ac +++ b/configure.ac @@ -266,6 +266,10 @@ AC_ARG_ENABLE(ebpf, [ --enable-ebpf Enable support for loading of eBPF programs (default: no)] ,,enable_ebpf="no") +AC_ARG_ENABLE(perf, + [ --enable-perf Enable support for the Linux perf tool (default: auto)] + ,,enable_perf="auto") + AC_ARG_ENABLE(gcov, [ --enable-gcov Enable coverage profiling (default: no)] ,,enable_gcov="no") @@ -466,6 +470,7 @@ dnl Checks for programs. AC_PROG_CC AC_PROG_CC_C99 AM_PROG_CC_C_O +AM_PROG_AS if test "x$ac_cv_prog_cc_c99" = "xno"; then AC_MSG_ERROR([C99 standard compliant C compiler required. Try GCC 3.x or later.]) fi @@ -2060,6 +2065,16 @@ if test "x$enable_ebpf" = "xyes"; then AC_SUBST(BPF_CC) fi +if test "x$enable_perf" = "xauto"; then + uname_s=`uname -s` + uname_m=`uname -m` + if test "$uname_s" = "Linux" -a "$uname_m" = "x86_64"; then + enable_perf="yes" + else + enable_perf="no" + fi +fi + dnl *************************************************************************** dnl check if we have timezone variable in dnl *************************************************************************** @@ -2243,6 +2258,7 @@ AC_DEFINE_UNQUOTED(ENABLE_IPV6, `enable_value $enable_ipv6`, [Enable IPv6 suppor AC_DEFINE_UNQUOTED(ENABLE_TCP_WRAPPER, `enable_value $enable_tcp_wrapper`, [Enable TCP wrapper support]) AC_DEFINE_UNQUOTED(ENABLE_LINUX_CAPS, `enable_value $enable_linux_caps`, [Enable Linux capability management support]) AC_DEFINE_UNQUOTED(ENABLE_EBPF, `enable_value $enable_ebpf`, [Enable Linux eBPF support]) +AC_DEFINE_UNQUOTED(ENABLE_PERF, `enable_value $enable_perf`, [Enable Linux perf support]) AC_DEFINE_UNQUOTED(ENABLE_ENV_WRAPPER, `enable_value $enable_env_wrapper`, [Enable environment wrapper support]) AC_DEFINE_UNQUOTED(ENABLE_SYSTEMD, `enable_value $enable_systemd`, [Enable systemd support]) AC_DEFINE_UNQUOTED(ENABLE_KAFKA, `enable_value $enable_kafka`, [Enable kafka support]) @@ -2259,6 +2275,7 @@ AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"]) AM_CONDITIONAL(ENABLE_DARWIN_OSL, [test "$enable_darwin_osl" = "yes"]) AM_CONDITIONAL(ENABLE_OPENBSD_SYSTEM_SOURCE, [test "$enable_openbsd_system_source" = "yes"]) AM_CONDITIONAL(ENABLE_EBPF, [test "$enable_ebpf" = "yes"]) +AM_CONDITIONAL(ENABLE_PERF, [test "$enable_perf" = "yes"]) AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"]) AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"]) AM_CONDITIONAL(ENABLE_SMTP, [test "$enable_smtp" = "yes"]) @@ -2414,6 +2431,7 @@ echo " Env wrapper support : ${enable_env_wrapper:=no}" echo " systemd support : ${enable_systemd:=no} (unit dir: ${systemdsystemunitdir:=none})" echo " systemd-journal support : ${with_systemd_journal:=no}" echo " JSON support : $with_jsonc" +echo " perf support : ${enable_perf:=no}" echo " Build options:" echo " Generate manual pages : ${enable_manpages:=no}" echo " Install manual pages : ${enable_manpages_install:=no}" diff --git a/lib/Makefile.am b/lib/Makefile.am index 29bf2e39f9..b510758468 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -29,6 +29,7 @@ include lib/logthrsource/Makefile.am include lib/logthrdest/Makefile.am include lib/signal-slot-connector/Makefile.am include lib/multi-line/Makefile.am +include lib/perf/Makefile.am include lib/adt/Makefile.am LSNG_RELEASE = $(shell echo @PACKAGE_VERSION@ | cut -d. -f1,2) @@ -309,6 +310,7 @@ lib_libsyslog_ng_la_SOURCES = \ $(logthrsource_sources) \ $(logthrdest_sources) \ $(signal_slot_connector_sources) \ + $(perf_sources) \ $(adt_sources) lib_libsyslog_ng_la_CFLAGS = \ diff --git a/lib/gprocess.c b/lib/gprocess.c index 2e7c99353e..a4fcb42790 100644 --- a/lib/gprocess.c +++ b/lib/gprocess.c @@ -27,6 +27,7 @@ #include "messages.h" #include "reloc.h" #include "console.h" +#include "perf/perf.h" #include #include @@ -126,6 +127,7 @@ static struct const gchar *cwd; const gchar *caps; gboolean enable_caps; + gboolean enable_perf; gint argc; gchar **argv; gchar *argv_start; @@ -719,6 +721,16 @@ g_process_enable_core(void) } } +static void +g_process_enable_perf(void) +{ + if (process_opts.enable_perf || perf_autodetect()) + { + if (!perf_enable()) + console_printf("Error enabling Linux perf profiling support, maybe not compiled in using --enable-perf?"); + } +} + /** * g_process_format_pidfile_name: * @buf: buffer to store the pidfile name @@ -1425,6 +1437,7 @@ g_process_start(void) } g_process_enable_core(); g_process_change_dir(); + g_process_enable_perf(); } @@ -1554,7 +1567,8 @@ static GOptionEntry g_process_option_entries[] = { "no-caps", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, g_process_process_no_caps, "Disable managing Linux capabilities", NULL }, { "pidfile", 'p', 0, G_OPTION_ARG_STRING, &process_opts.pidfile, "Set path to pid file", "" }, { "enable-core", 0, 0, G_OPTION_ARG_NONE, &process_opts.core, "Enable dumping core files", NULL }, - { "fd-limit", 0, 0, G_OPTION_ARG_INT, &process_opts.fd_limit_min, "The minimum required number of fds", NULL }, + { "fd-limit", 0, 0, G_OPTION_ARG_INT, &process_opts.fd_limit_min, "The minimum required number of fds", NULL }, + { "perf-profiling", 0, 0, G_OPTION_ARG_NONE, &process_opts.enable_perf, "Enable Linux perf based profiling", NULL }, { NULL, 0, 0, 0, NULL, NULL, NULL }, }; diff --git a/lib/perf/Makefile.am b/lib/perf/Makefile.am new file mode 100644 index 0000000000..0b80606231 --- /dev/null +++ b/lib/perf/Makefile.am @@ -0,0 +1,18 @@ +perfincludedir = ${pkgincludedir}/perf + +perfinclude_HEADERS = \ + lib/perf/perf.h + +_perf_sources = \ + lib/perf/perf.c \ + lib/perf/trampoline.S + +if ENABLE_PERF + +perf_sources = $(_perf_sources) + +else + +EXTRA_DIST += $(_perf_sources) + +endif diff --git a/lib/perf/perf.c b/lib/perf/perf.c new file mode 100644 index 0000000000..ffc9b8b15b --- /dev/null +++ b/lib/perf/perf.c @@ -0,0 +1,229 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 Balázs Scheidler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "perf.h" +#include "console.h" +#include +#include +#include +#include + +#if SYSLOG_NG_ENABLE_PERF + +static gboolean perf_enabled; + +typedef struct _PerfTrampolineArea +{ + guint8 *area; + gsize size; + gsize code_size; + struct + { + gint num; + gint alloc; + } trampolines; +} PerfTrampolineArea; + +extern void _perf_trampoline_func_start(void); +extern void _perf_trampoline_func_end(void); + +#define PAGE_SIZE 4096 +#define PAGE_MAX_INDEX ((PAGE_SIZE-1)) + +#define SIZE_TO_PAGES(s) (((s + PAGE_MAX_INDEX) & ~PAGE_MAX_INDEX) / PAGE_SIZE) +#define PAGE_TO_SIZE(p) (p * PAGE_SIZE) + +#define ROUND_TO_PAGE_BOUNDARY(p) ((gpointer) ((((uintptr_t) (p)) / PAGE_SIZE) * PAGE_SIZE)) + +#define MAX_TRAMPOLINES 16384 + +static gboolean +_allocate_trampoline_area(PerfTrampolineArea *self) +{ + guint8 *start = (guint8 *) &_perf_trampoline_func_start; + guint8 *end = (guint8 *) &_perf_trampoline_func_end; + + self->code_size = end - start; + gsize code_pages = SIZE_TO_PAGES(self->code_size * MAX_TRAMPOLINES); + + self->size = PAGE_TO_SIZE(code_pages); + self->area = mmap(NULL, + self->size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + if (self->area == MAP_FAILED) + return FALSE; + + self->trampolines.num = self->size / self->code_size; + self->trampolines.alloc = 0; + + for (gint i = 0; i < self->trampolines.num; i++) + { + memcpy(self->area + i * self->code_size, start, self->code_size * sizeof(gchar)); + } + return TRUE; +} + +static gpointer +_generate_trampoline(PerfTrampolineArea *self, gpointer target_address) +{ + if (self->trampolines.alloc >= self->trampolines.num) + return NULL; + + + guint8 *trampoline_start = self->area + (self->trampolines.alloc++) * self->code_size; + guint8 *trampoline_end = trampoline_start + self->code_size; + + guint8 *trampoline_page_start = ROUND_TO_PAGE_BOUNDARY(trampoline_start); + guint8 *trampoline_page_end = ROUND_TO_PAGE_BOUNDARY(trampoline_end); + gsize trampoline_page_len = (trampoline_page_end - trampoline_page_start) + PAGE_SIZE; + + gint res = mprotect(trampoline_page_start, trampoline_page_len, PROT_READ | PROT_WRITE); + if (res < 0) + return NULL; + + + uintptr_t *value_p = (uintptr_t *) (trampoline_end - sizeof(target_address)); + *value_p = (uintptr_t) target_address; + + __builtin___clear_cache((gpointer) trampoline_start, (gpointer) trampoline_end); + + res = mprotect(trampoline_page_start, trampoline_page_len, PROT_READ | PROT_EXEC); + if (res < 0) + return NULL; + return (gpointer) trampoline_start; +} + +static gboolean +_is_trampoline_address(PerfTrampolineArea *self, guint8 *address) +{ + return self->area <= address && self->area + self->size > address; +} + +static gboolean +_save_symbol(gpointer address, gsize size, const gchar *symbol_name) +{ + gchar filename[64]; + static FILE *mapfile = NULL; + + if (!mapfile) + { + g_snprintf(filename, sizeof(filename), "/tmp/perf-%d.map", (int) getpid()); + mapfile = fopen(filename, "a"); + if (!mapfile) + return FALSE; + } + fprintf(mapfile, "%0lx %lx %p:%s\n", (uintptr_t) address, size, address, symbol_name); + fflush(mapfile); + return TRUE; +} + +static PerfTrampolineArea trampolines; + +static gchar * +_sanitize_symbol(const gchar *symbol_name) +{ + GString *translated_text = g_string_sized_new(256); + gboolean prev_whitespace = FALSE; + + for (gsize i = 0; symbol_name[i]; i++) + { + gchar ch = symbol_name[i]; + switch (ch) + { + case ' ': + case '\t': + if (!prev_whitespace) + g_string_append_unichar(translated_text, 0xa0); + break; + case '(': + g_string_append_unichar(translated_text, 0x27EE); + break; + case ')': + g_string_append_unichar(translated_text, 0x27EF); + break; + case '"': + g_string_append_unichar(translated_text, 0x201D); + break; + default: + g_string_append_c(translated_text, ch); + break; + } + if (ch == ' ') + prev_whitespace = TRUE; + else + prev_whitespace = FALSE; + } + return g_string_free(translated_text, FALSE); +} + +gpointer +perf_generate_trampoline(gpointer target_address, const gchar *symbol_name) +{ + if (_is_trampoline_address(&trampolines, target_address)) + return target_address; + + gpointer t = _generate_trampoline(&trampolines, target_address); + + if (!t) + { + console_printf("WARNING: out of free trampoline slots, unable to instrument symbol '%s', " + "increase MAX_TRAMPOLINES", symbol_name); + return target_address; + } + + gchar *sanitized_symbol = _sanitize_symbol(symbol_name); + _save_symbol(t, trampolines.code_size, sanitized_symbol); + g_free(sanitized_symbol); + return t; +} + +gboolean +perf_is_enabled(void) +{ + return perf_enabled; +} + +gboolean +perf_autodetect(void) +{ + /* perf sets this environment variable, so with that set, we can assume we + * are running under perf record */ + + if (getenv("PERF_BUILDID_DIR") != NULL) + return TRUE; + return FALSE; +} + +gboolean +perf_enable(void) +{ + if (!_allocate_trampoline_area(&trampolines)) + return FALSE; + + console_printf("perf support enabled"); + perf_enabled = TRUE; + return TRUE; +} + +#endif diff --git a/lib/perf/perf.h b/lib/perf/perf.h new file mode 100644 index 0000000000..a0b66dbf8b --- /dev/null +++ b/lib/perf/perf.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 Balázs Scheidler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef SYSLOG_NG_PERF_H_INCLUDED +#define SYSLOG_NG_PERF_H_INCLUDED + +#include "syslog-ng.h" + +#if SYSLOG_NG_ENABLE_PERF + +gpointer perf_generate_trampoline(gpointer target_address, const gchar *symbol_name); + +gboolean perf_is_enabled(void); +gboolean perf_autodetect(void); +gboolean perf_enable(void); + +#else + +#define perf_generate_trampoline(addr, symbol) ({const gchar *__p G_GNUC_UNUSED = symbol; addr;}) +#define perf_is_enabled() FALSE +#define perf_autodetect() FALSE +#define perf_enable() FALSE + +#endif + +#endif diff --git a/lib/perf/trampoline.S b/lib/perf/trampoline.S new file mode 100644 index 0000000000..fb849565a1 --- /dev/null +++ b/lib/perf/trampoline.S @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 Axoflow + * Copyright (c) 2024 Balázs Scheidler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + .text + .global _perf_trampoline_func_start + .align 16 +_perf_trampoline_func_start: +#ifdef __x86_64__ + push %rbp + mov _perf_trampoline_target_address(%rip), %r11 + call *%r11 + pop %rbp + ret + .align 8 +_perf_trampoline_target_address: + .quad 0x0 +#endif // __x86_64__ + + .global _perf_trampoline_func_end +_perf_trampoline_func_end: + .section .note.GNU-stack,"",@progbits diff --git a/tests/build-log-cflags-propagation.sh b/tests/build-log-cflags-propagation.sh index 719a23735e..40f03fbbb9 100644 --- a/tests/build-log-cflags-propagation.sh +++ b/tests/build-log-cflags-propagation.sh @@ -79,6 +79,7 @@ find_not_prop() { { find_gcc "$@" | ignore_submodule_gcc | + ignore_trampoline_S | grep -vE -- " -Wshadow( |.* )-Werror " } 2>&1 } @@ -106,3 +107,9 @@ gcc( -std=gnu99)? -DHAVE_CONFIG_H -I\. -I(\.\./)+lib/ivykis/contrib/iv_getaddrin gcc( -std=gnu99)? -DHAVE_CONFIG_H -I\. -I(\.\./)+lib/ivykis/contrib/kojines \ )" "$@" } + +ignore_trampoline_S() { + grep -vE -- "\<(\ +gcc( -std=gnu99)? -DHAVE_CONFIG_H.*lib/perf/trampoline.S \ +)" "$@" +} diff --git a/tests/copyright/policy b/tests/copyright/policy index 68387a6e7a..ad20f8c1da 100644 --- a/tests/copyright/policy +++ b/tests/copyright/policy @@ -143,6 +143,7 @@ lib/filterx/filterx-grammar\.ym lib/logproto/logproto-auto-server\.[ch] lib/transport/transport-factory-haproxy\.[ch] lib/transport/tests/test_transport\.c +lib/perf/.*\.[chS] ########################################################################### # These tests are GPLd even though they reside under lib/ and are excluded From d079ead4a3c0deed1f186b4e13526b5088117037 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sat, 11 Jan 2025 16:09:45 +0100 Subject: [PATCH 04/10] logpipe: introduce log_pipe_{pre,post)_config_init_method These were originally optional methods, but that means that descendant classes will not call the one from LogPipe. Let's create that so we have a spot to initialize perf trampolines. Signed-off-by: Balazs Scheidler --- lib/afinter.c | 7 +++++-- lib/logpipe.c | 15 ++++++++++++++- lib/logpipe.h | 2 ++ lib/logthrdest/logthrdestdrv.c | 23 ++++++++++++++++------- lib/logthrdest/logthrdestdrv.h | 4 +++- lib/logthrsource/logthrsourcedrv.c | 23 ++++++++++++++++------- lib/logthrsource/logthrsourcedrv.h | 2 ++ 7 files changed, 58 insertions(+), 18 deletions(-) diff --git a/lib/afinter.c b/lib/afinter.c index 25f577cfe4..004269dcb1 100644 --- a/lib/afinter.c +++ b/lib/afinter.c @@ -414,7 +414,7 @@ static gboolean afinter_sd_pre_config_init(LogPipe *s) { main_loop_worker_allocate_thread_space(1); - return TRUE; + return log_pipe_pre_config_init_method(s); } static gboolean @@ -459,7 +459,10 @@ afinter_sd_post_config_init(LogPipe *s) { AFInterSourceDriver *self = (AFInterSourceDriver *) s; - return afinter_source_start_thread(self->source); + if (!afinter_source_start_thread(self->source)) + return FALSE; + + return log_pipe_post_config_init_method(s); } static gboolean diff --git a/lib/logpipe.c b/lib/logpipe.c index 2a3fbdec69..b43ba00828 100644 --- a/lib/logpipe.c +++ b/lib/logpipe.c @@ -146,6 +146,18 @@ log_pipe_clone_method(LogPipe *dst, const LogPipe *src) log_pipe_set_options(dst, &src->options); } +gboolean +log_pipe_pre_config_init_method(LogPipe *self) +{ + return TRUE; +} + +gboolean +log_pipe_post_config_init_method(LogPipe *self) +{ + return TRUE; +} + void log_pipe_init_instance(LogPipe *self, GlobalConfig *cfg) { @@ -154,7 +166,8 @@ log_pipe_init_instance(LogPipe *self, GlobalConfig *cfg) self->pipe_next = NULL; self->persist_name = NULL; self->plugin_name = NULL; - + self->pre_config_init = log_pipe_pre_config_init_method; + self->post_config_init = log_pipe_post_config_init_method; self->queue = log_pipe_forward_msg; self->free_fn = log_pipe_free_method; self->arcs = _arcs; diff --git a/lib/logpipe.h b/lib/logpipe.h index b15c8f7517..761ddee278 100644 --- a/lib/logpipe.h +++ b/lib/logpipe.h @@ -349,6 +349,8 @@ extern gboolean (*pipe_single_step_hook)(LogPipe *pipe, LogMessage *msg, const L LogPipe *log_pipe_ref(LogPipe *self); gboolean log_pipe_unref(LogPipe *self); LogPipe *log_pipe_new(GlobalConfig *cfg); +gboolean log_pipe_pre_config_init_method(LogPipe *self); +gboolean log_pipe_post_config_init_method(LogPipe *self); void log_pipe_init_instance(LogPipe *self, GlobalConfig *cfg); void log_pipe_clone_method(LogPipe *dst, const LogPipe *src); void log_pipe_forward_notify(LogPipe *self, gint notify_code, gpointer user_data); diff --git a/lib/logthrdest/logthrdestdrv.c b/lib/logthrdest/logthrdestdrv.c index 83a0d15c37..52745c3107 100644 --- a/lib/logthrdest/logthrdestdrv.c +++ b/lib/logthrdest/logthrdestdrv.c @@ -1372,12 +1372,12 @@ _create_workers(LogThreadedDestDriver *self, gint stats_level, StatsClusterKeyBu } gboolean -log_threaded_dest_driver_pre_config_init(LogPipe *s) +log_threaded_dest_driver_pre_config_init_method(LogPipe *s) { LogThreadedDestDriver *self = (LogThreadedDestDriver *)s; main_loop_worker_allocate_thread_space(self->num_workers); - return TRUE; + return log_pipe_pre_config_init_method(s); } gboolean @@ -1428,10 +1428,8 @@ log_threaded_dest_driver_init_method(LogPipe *s) * method, the caller is responsible for explicitly calling _start_workers() at * the end of post_config_init(). */ gboolean -log_threaded_dest_driver_start_workers(LogPipe *s) +log_threaded_dest_driver_start_workers(LogThreadedDestDriver *self) { - LogThreadedDestDriver *self = (LogThreadedDestDriver *) s; - for (gint worker_index = 0; worker_index < self->num_workers; worker_index++) { if (!log_threaded_dest_worker_start(self->workers[worker_index])) @@ -1440,6 +1438,17 @@ log_threaded_dest_driver_start_workers(LogPipe *s) return TRUE; } +gboolean +log_threaded_dest_driver_post_config_init_method(LogPipe *s) +{ + LogThreadedDestDriver *self = (LogThreadedDestDriver *) s; + + if (!log_threaded_dest_driver_start_workers(self)) + return FALSE; + + return log_pipe_post_config_init_method(s); +} + static void _destroy_worker(LogThreadedDestDriver *self, LogThreadedDestWorker *worker) { @@ -1495,8 +1504,8 @@ log_threaded_dest_driver_init_instance(LogThreadedDestDriver *self, GlobalConfig self->super.super.super.deinit = log_threaded_dest_driver_deinit_method; self->super.super.super.queue = log_threaded_dest_driver_queue; self->super.super.super.free_fn = log_threaded_dest_driver_free; - self->super.super.super.pre_config_init = log_threaded_dest_driver_pre_config_init; - self->super.super.super.post_config_init = log_threaded_dest_driver_start_workers; + self->super.super.super.pre_config_init = log_threaded_dest_driver_pre_config_init_method; + self->super.super.super.post_config_init = log_threaded_dest_driver_post_config_init_method; self->time_reopen = -1; self->batch_lines = -1; self->batch_timeout = -1; diff --git a/lib/logthrdest/logthrdestdrv.h b/lib/logthrdest/logthrdestdrv.h index 3ed6eab76c..956f619c43 100644 --- a/lib/logthrdest/logthrdestdrv.h +++ b/lib/logthrdest/logthrdestdrv.h @@ -306,10 +306,12 @@ void log_threaded_dest_driver_insert_msg_length_stats(LogThreadedDestDriver *sel void log_threaded_dest_driver_insert_batch_length_stats(LogThreadedDestDriver *self, gsize len); void log_threaded_dest_driver_register_aggregated_stats(LogThreadedDestDriver *self); void log_threaded_dest_driver_unregister_aggregated_stats(LogThreadedDestDriver *self); +gboolean log_threaded_dest_driver_start_workers(LogThreadedDestDriver *self); gboolean log_threaded_dest_driver_deinit_method(LogPipe *s); gboolean log_threaded_dest_driver_init_method(LogPipe *s); -gboolean log_threaded_dest_driver_start_workers(LogPipe *s); +gboolean log_threaded_dest_driver_pre_config_init_method(LogPipe *s); +gboolean log_threaded_dest_driver_post_config_init_method(LogPipe *s); void log_threaded_dest_driver_init_instance(LogThreadedDestDriver *self, GlobalConfig *cfg); void log_threaded_dest_driver_free(LogPipe *s); diff --git a/lib/logthrsource/logthrsourcedrv.c b/lib/logthrsource/logthrsourcedrv.c index e495fccf78..c2ffe4a8b0 100644 --- a/lib/logthrsource/logthrsourcedrv.c +++ b/lib/logthrsource/logthrsourcedrv.c @@ -248,11 +248,11 @@ _construct_worker(LogThreadedSourceDriver *self, gint worker_index) } gboolean -log_threaded_source_driver_pre_config_init(LogPipe *s) +log_threaded_source_driver_pre_config_init_method(LogPipe *s) { LogThreadedSourceDriver *self = (LogThreadedSourceDriver *) s; main_loop_worker_allocate_thread_space(self->num_workers); - return TRUE; + return log_pipe_pre_config_init_method(s); } static void @@ -352,16 +352,25 @@ log_threaded_source_driver_free_method(LogPipe *s) } gboolean -log_threaded_source_driver_start_workers(LogPipe *s) +log_threaded_source_driver_start_workers(LogThreadedSourceDriver *self) { - LogThreadedSourceDriver *self = (LogThreadedSourceDriver *) s; - for (size_t i = 0; i < self->num_workers; i++) g_assert(main_loop_threaded_worker_start(&self->workers[i]->thread)); return TRUE; } +gboolean +log_threaded_source_driver_post_config_init_method(LogPipe *s) +{ + LogThreadedSourceDriver *self = (LogThreadedSourceDriver *) s; + + if (!log_threaded_source_driver_start_workers(self)) + return FALSE; + + return log_pipe_post_config_init_method(s); +} + static gboolean _is_default_priority_or_facility_set(MsgFormatOptions *parse_options) { @@ -469,8 +478,8 @@ log_threaded_source_driver_init_instance(LogThreadedSourceDriver *self, GlobalCo self->super.super.super.init = log_threaded_source_driver_init_method; self->super.super.super.deinit = log_threaded_source_driver_deinit_method; self->super.super.super.free_fn = log_threaded_source_driver_free_method; - self->super.super.super.pre_config_init = log_threaded_source_driver_pre_config_init; - self->super.super.super.post_config_init = log_threaded_source_driver_start_workers; + self->super.super.super.pre_config_init = log_threaded_source_driver_pre_config_init_method; + self->super.super.super.post_config_init = log_threaded_source_driver_post_config_init_method; self->worker_construct = _construct_worker; diff --git a/lib/logthrsource/logthrsourcedrv.h b/lib/logthrsource/logthrsourcedrv.h index cf204426c5..e2fe524673 100644 --- a/lib/logthrsource/logthrsourcedrv.h +++ b/lib/logthrsource/logthrsourcedrv.h @@ -91,6 +91,8 @@ void log_threaded_source_driver_set_transport_name(LogThreadedSourceDriver *self void log_threaded_source_driver_init_instance(LogThreadedSourceDriver *self, GlobalConfig *cfg); gboolean log_threaded_source_driver_init_method(LogPipe *s); gboolean log_threaded_source_driver_deinit_method(LogPipe *s); +gboolean log_threaded_source_driver_pre_config_init_method(LogPipe *s); +gboolean log_threaded_source_driver_post_config_init_method(LogPipe *s); void log_threaded_source_driver_free_method(LogPipe *s); static inline void From 849960ce79d8b35151dc4de35ae8ff3e450c5c68 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sat, 11 Jan 2025 10:06:03 +0100 Subject: [PATCH 05/10] cfg-source: cache the result of g_strsplit() When extracting the source text, we need to split the input into lines. Previously this was done every time we extracted a line of text, causing the startup to be very slow when I enabled source extraction for every FilterXExpr instance. This patch causes the result of that g_strsplit() call to be saved, effectively using more memory startup, but being a lot more efficient (e.g. startup within a second instead of 20 seconds or so). Signed-off-by: Balazs Scheidler --- lib/cfg-lexer.c | 2 ++ lib/cfg-lexer.h | 1 + lib/cfg-source.c | 11 +++++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/cfg-lexer.c b/lib/cfg-lexer.c index e1aa3fc8a4..052ec740ab 100644 --- a/lib/cfg-lexer.c +++ b/lib/cfg-lexer.c @@ -393,6 +393,7 @@ cfg_lexer_init_include_level_buffer(CfgLexer *self, CfgIncludeLevel *level, level->buffer.content = lexer_buffer; level->buffer.content_length = lexer_buffer_len; level->buffer.original_content = g_strdup(lexer_buffer); + level->buffer.original_lines = NULL; } gboolean @@ -458,6 +459,7 @@ cfg_lexer_include_level_clear(CfgLexer *self, CfgIncludeLevel *level) { g_free(level->buffer.content); g_free(level->buffer.original_content); + g_strfreev(level->buffer.original_lines); } memset(level, 0, sizeof(*level)); } diff --git a/lib/cfg-lexer.h b/lib/cfg-lexer.h index 786a3275c0..8aabf33171 100644 --- a/lib/cfg-lexer.h +++ b/lib/cfg-lexer.h @@ -160,6 +160,7 @@ struct _CfgIncludeLevel { /* the lexer mutates content, so save it for error reporting */ gchar *original_content; + gchar **original_lines; /* buffer for the lexer */ gchar *content; gsize content_length; diff --git a/lib/cfg-source.c b/lib/cfg-source.c index d0cef36fd6..294d57bd7a 100644 --- a/lib/cfg-source.c +++ b/lib/cfg-source.c @@ -226,9 +226,13 @@ _extract_source_from_file_location(GString *result, const gchar *filename, const } static gboolean -_extract_source_from_buffer_location(GString *result, const gchar *buffer_content, const CFG_LTYPE *yylloc) +_extract_source_from_buffer_location(GString *result, CfgIncludeLevel *level, const CFG_LTYPE *yylloc) { - gchar **lines = g_strsplit(buffer_content, "\n", yylloc->last_line + 1); + const gchar *buffer_content = level->buffer.original_content; + gchar **lines = level->buffer.original_lines; + + if (!lines) + lines = level->buffer.original_lines = g_strsplit(buffer_content, "\n", 0); gint num_lines = g_strv_length(lines); if (num_lines <= yylloc->first_line) @@ -262,7 +266,6 @@ _extract_source_from_buffer_location(GString *result, const gchar *buffer_conten } exit: - g_strfreev(lines); return TRUE; } @@ -279,7 +282,7 @@ cfg_source_extract_source_text(CfgLexer *lexer, const CFG_LTYPE *yylloc, GString CFG_LTYPE buf_lloc = *yylloc; cfg_lexer_undo_set_file_location(lexer, &buf_lloc); - return _extract_source_from_buffer_location(result, level->buffer.original_content, &buf_lloc); + return _extract_source_from_buffer_location(result, level, &buf_lloc); } else g_assert_not_reached(); From c2368c596c55e3a8c7a6eecbda67a27ded85605c Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Thu, 9 Jan 2025 11:53:14 +0100 Subject: [PATCH 06/10] filterx/filterx-expr: fix potential memory leak for expr_text In case an expr is optimized we might be setting the location of exprs multiple times, prepare for this case by freeing expr_text before setting it first. Signed-off-by: Balazs Scheidler --- lib/filterx/filterx-expr.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/filterx/filterx-expr.c b/lib/filterx/filterx-expr.c index 1e039982ab..e16ab1e046 100644 --- a/lib/filterx/filterx-expr.c +++ b/lib/filterx/filterx-expr.c @@ -36,8 +36,11 @@ filterx_expr_set_location_with_text(FilterXExpr *self, CFG_LTYPE *lloc, const gc self->lloc = g_new0(CFG_LTYPE, 1); *self->lloc = *lloc; - if (debug_flag && text) - self->expr_text = g_strdup(text); + if (debug_flag && text && text != self->expr_text) + { + g_free(self->expr_text); + self->expr_text = g_strdup(text); + } } void @@ -48,6 +51,7 @@ filterx_expr_set_location(FilterXExpr *self, CfgLexer *lexer, CFG_LTYPE *lloc) *self->lloc = *lloc; if (debug_flag) { + g_free(self->expr_text); GString *res = g_string_sized_new(0); cfg_source_extract_source_text(lexer, lloc, res); self->expr_text = g_string_free(res, FALSE); From 01ca1b41b11d1fde6db82682c705c9d7af1ca13c Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sat, 28 Dec 2024 20:43:57 +0100 Subject: [PATCH 07/10] filterx/filterx-expr: add expr level detail to perf stacktraces Signed-off-by: Balazs Scheidler --- lib/filterx/filterx-expr.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/filterx/filterx-expr.c b/lib/filterx/filterx-expr.c index e16ab1e046..a1e9959ab0 100644 --- a/lib/filterx/filterx-expr.c +++ b/lib/filterx/filterx-expr.c @@ -28,6 +28,13 @@ #include "mainloop.h" #include "stats/stats-registry.h" #include "stats/stats-cluster-single.h" +#include "perf/perf.h" + +static inline gboolean +_extract_source_text(void) +{ + return debug_flag || perf_is_enabled(); +} void filterx_expr_set_location_with_text(FilterXExpr *self, CFG_LTYPE *lloc, const gchar *text) @@ -36,7 +43,7 @@ filterx_expr_set_location_with_text(FilterXExpr *self, CFG_LTYPE *lloc, const gc self->lloc = g_new0(CFG_LTYPE, 1); *self->lloc = *lloc; - if (debug_flag && text && text != self->expr_text) + if (_extract_source_text() && text && text != self->expr_text) { g_free(self->expr_text); self->expr_text = g_strdup(text); @@ -49,7 +56,7 @@ filterx_expr_set_location(FilterXExpr *self, CfgLexer *lexer, CFG_LTYPE *lloc) if (!self->lloc) self->lloc = g_new0(CFG_LTYPE, 1); *self->lloc = *lloc; - if (debug_flag) + if (_extract_source_text()) { g_free(self->expr_text); GString *res = g_string_sized_new(0); @@ -114,7 +121,7 @@ _init_sc_key_name(FilterXExpr *self, gchar *buf, gsize buf_len) gboolean filterx_expr_init_method(FilterXExpr *self, GlobalConfig *cfg) { - gchar buf[64]; + gchar buf[256]; _init_sc_key_name(self, buf, sizeof(buf)); stats_lock(); @@ -127,6 +134,16 @@ filterx_expr_init_method(FilterXExpr *self, GlobalConfig *cfg) stats_cluster_single_key_set(&sc_key, buf, labels, labels_len); stats_register_counter(STATS_LEVEL3, &sc_key, SC_TYPE_SINGLE_VALUE, &self->eval_count); stats_unlock(); + + if (perf_is_enabled()) + { + if (self->expr_text) + g_snprintf(buf, sizeof(buf), "filterx::%s", self->expr_text); + else + g_snprintf(buf, sizeof(buf), "filterx::@%s", self->type); + self->eval = perf_generate_trampoline(self->eval, buf); + } + return TRUE; } From 206541f22b1942722dab5d2ca46b19840b318dc4 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sat, 11 Jan 2025 16:32:36 +0100 Subject: [PATCH 08/10] logpipe: add LogPipe level detail to perf stackdumps Signed-off-by: Balazs Scheidler --- lib/logpipe.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/logpipe.c b/lib/logpipe.c index b43ba00828..ded69b5fe6 100644 --- a/lib/logpipe.c +++ b/lib/logpipe.c @@ -25,6 +25,7 @@ #include "logpipe.h" #include "cfg-tree.h" #include "cfg-walker.h" +#include "perf/perf.h" gboolean (*pipe_single_step_hook)(LogPipe *pipe, LogMessage *msg, const LogPathOptions *path_options); @@ -155,6 +156,12 @@ log_pipe_pre_config_init_method(LogPipe *self) gboolean log_pipe_post_config_init_method(LogPipe *self) { + if ((self->flags & PIF_CONFIG_RELATED) && perf_is_enabled()) + { + gchar buf[256]; + + self->queue = perf_generate_trampoline(self->queue, log_expr_node_format_location(self->expr_node, buf, sizeof(buf))); + } return TRUE; } From 710141a1c90746bc56879ac96119324a249cfba8 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sun, 12 Jan 2025 18:27:09 +0100 Subject: [PATCH 09/10] cfg-source: do proper boundary checking of yylloc values Sometimes location tracking is buggy, make sure we don't address outside of the source text. Signed-off-by: Balazs Scheidler --- lib/cfg-source.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/cfg-source.c b/lib/cfg-source.c index 294d57bd7a..1ac846e62a 100644 --- a/lib/cfg-source.c +++ b/lib/cfg-source.c @@ -248,10 +248,18 @@ _extract_source_from_buffer_location(GString *result, CfgIncludeLevel *level, co if (lineno == yylloc->first_line) { + gint token_start = MIN(linelen, yylloc->first_column - 1); + if (yylloc->first_line == yylloc->last_line) - g_string_append_len(result, &line[MIN(linelen, yylloc->first_column-1)], yylloc->last_column - yylloc->first_column); + { + /* both last_column & first_column are 1 based, they cancel that out */ + gint token_len = yylloc->last_column - yylloc->first_column; + if (token_start + token_len > linelen) + token_len = linelen - token_start; + g_string_append_len(result, &line[token_start], token_len); + } else - g_string_append(result, &line[MIN(linelen, yylloc->first_column-1)]); + g_string_append(result, &line[token_start]); } else if (lineno < yylloc->last_line) { @@ -260,8 +268,12 @@ _extract_source_from_buffer_location(GString *result, CfgIncludeLevel *level, co } else if (lineno == yylloc->last_line) { + /* last_column is 1 based */ + gint token_len = yylloc->last_column - 1; + if (token_len > linelen) + token_len = linelen; g_string_append_c(result, ' '); - g_string_append_len(result, line, yylloc->last_column); + g_string_append_len(result, line, token_len); } } From 45743a43dedd82974b410b4957cfbc0ef808ab8a Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sat, 25 Jan 2025 16:24:33 +0100 Subject: [PATCH 10/10] filterx/filterx-grammar: set location for generator expressions Signed-off-by: Balazs Scheidler --- lib/filterx/filterx-grammar.ym | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/filterx/filterx-grammar.ym b/lib/filterx/filterx-grammar.ym index 584b36c580..f8e04063d2 100644 --- a/lib/filterx/filterx-grammar.ym +++ b/lib/filterx/filterx-grammar.ym @@ -147,7 +147,7 @@ _assign_location(FilterXExpr *expr, CfgLexer *lexer, CFG_LTYPE *lloc) %type comparison_operator %type arithmetic_operator %type expr_generator -%type expr_generator_unchecked +%type __expr_generator %type expr_plus_generator %type generator_function_call %type function_call @@ -469,13 +469,13 @@ expr_value ; expr_generator - : expr_generator_unchecked { - $$ = $1; + : __expr_generator { CHECK_ERROR($1, @1, "error initializing generator expression"); + $$ = _assign_location($1, lexer, &@1); } ; -expr_generator_unchecked +__expr_generator : dict_generator | list_generator | generator_function_call