Skip to content

Commit

Permalink
test/fuzz: collect and print Lua metrics
Browse files Browse the repository at this point in the history
Fuzzing test for LuaJIT generates random Lua programs and executes them.
We want to build a fuzzing test that will produce Lua programs that will
not contain semantic errors and will trigger as much as possible
components in LuaJIT.

This proposed patch introduces metrics that gathered after running the
test. LuaJIT metrics gathered using LuaJIT getmetrics module [1]. All
gathered metrics test will output after running with a finite number of
runs or finite duration of time (options `-runs` and `-max_total_time`)
or after sending SIGUSR1 to a test process.

```
$ ./build/test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer -runs=1000

<snipped>

Done 1000 runs in 1 second(s)
Total number of samples: 1000
Total number of samples with errors: 438 (43%)
Total number of samples with recorded traces: 87 (8%)
Total number of samples with snap restores: 30 (3%)
Total number of samples with abort traces: 55 (5%)
```

1. https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api

NO_CHANGELOG=testing
NO_DOC=testing

(cherry picked from commit 430fa6a)
  • Loading branch information
ligurio committed Aug 17, 2023
1 parent a9b8d60 commit c6a8f23
Showing 1 changed file with 79 additions and 0 deletions.
79 changes: 79 additions & 0 deletions test/fuzz/luaL_loadbuffer/luaL_loadbuffer_fuzzer.cc
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
extern "C"
{
#include <signal.h>
#include <unistd.h>

#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
/* luaM_metrics */
#include <lmisclib.h>
}

#include "lua_grammar.pb.h"
Expand All @@ -11,13 +16,85 @@ extern "C"
#include <libprotobuf-mutator/port/protobuf.h>
#include <libprotobuf-mutator/src/libfuzzer/libfuzzer_macro.h>

#define PRINT_METRIC(desc, val, total) \
std::cerr << (desc) << (val) \
<< " (" << (val) * 100 / (total) << "%)" \
<< std::endl

struct metrics {
size_t total_num;
size_t total_num_with_errors;
size_t jit_snap_restore;
size_t jit_trace_abort;
size_t jit_trace_num;
};

static struct metrics metrics;

static inline void
print_metrics(struct metrics *metrics)
{
if (metrics->total_num == 0)
return;

std::cerr << "Total number of samples: "
<< metrics->total_num << std::endl;
PRINT_METRIC("Total number of samples with errors: ",
metrics->total_num_with_errors, metrics->total_num);
PRINT_METRIC("Total number of samples with recorded traces: ",
metrics->jit_trace_num, metrics->total_num);
PRINT_METRIC("Total number of samples with snap restores: ",
metrics->jit_snap_restore, metrics->total_num);
PRINT_METRIC("Total number of samples with abort traces: ",
metrics->jit_trace_abort, metrics->total_num);
}

/* https://www.tarantool.io/en/doc/latest/reference/tooling/luajit_getmetrics/#getmetrics-c-api */
static inline void
collect_lj_metrics(struct metrics *metrics, lua_State *L)
{
struct luam_Metrics lj_metrics;
luaM_metrics(L, &lj_metrics);
if (lj_metrics.jit_snap_restore != 0)
metrics->jit_snap_restore++;
if (lj_metrics.jit_trace_abort != 0)
metrics->jit_trace_abort++;
if (lj_metrics.jit_trace_num != 0)
metrics->jit_trace_num++;
}

void
sig_handler(int signo, siginfo_t *info, void *context)
{
print_metrics(&metrics);
}

__attribute__((constructor))
static void
setup(void)
{
metrics = {};
struct sigaction act = {};
act.sa_flags = SA_SIGINFO;
act.sa_sigaction = &sig_handler;
sigaction(SIGUSR1, &act, NULL);
}

__attribute__((destructor))
static void
teardown(void)
{
print_metrics(&metrics);
}

/**
* Get an error message from the stack, and report it to std::cerr.
* Remove the message from the stack.
*/
static inline void
report_error(lua_State *L, const std::string &prefix)
{
metrics.total_num_with_errors++;
const char *verbose = ::getenv("LUA_FUZZER_VERBOSE");
if (!verbose)
return;
Expand Down Expand Up @@ -67,6 +144,8 @@ DEFINE_PROTO_FUZZER(const lua_grammar::Block &message)
report_error(L, "lua_pcall()");

end:
metrics.total_num++;
collect_lj_metrics(&metrics, L);
lua_settop(L, 0);
lua_close(L);
}

0 comments on commit c6a8f23

Please sign in to comment.