Skip to content

Commit

Permalink
Add "run_oomer" tool to simulate various out-of-memory situations.
Browse files Browse the repository at this point in the history
Check run_oomer for flags. Flags are passed though environment
variables. This commit also fixes a few bugs found by the tool.
  • Loading branch information
iphydf committed May 2, 2020
1 parent 9be4dbb commit 589ddac
Show file tree
Hide file tree
Showing 10 changed files with 257 additions and 5 deletions.
1 change: 1 addition & 0 deletions auto_tests/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ flaky_tests = {
":check_compat",
":run_auto_test",
"//c-toxcore/testing:misc_tools",
"//c-toxcore/testing:oomer",
"//c-toxcore/toxav",
"//c-toxcore/toxcore",
"//c-toxcore/toxcore:DHT_srcs",
Expand Down
17 changes: 13 additions & 4 deletions auto_tests/run_auto_test.h
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ static void iterate_all_wait(uint32_t tox_count, Tox **toxes, State *state, uint
}

/* Also actually sleep a little, to allow for local network processing */
c_sleep(20);
c_sleep(5);
}

static uint64_t get_state_clock_callback(Mono_Time *mono_time, void *user_data)
Expand All @@ -63,6 +63,9 @@ static void run_auto_test(uint32_t tox_count, void test(Tox **toxes, State *stat
Tox **toxes = (Tox **)calloc(tox_count, sizeof(Tox *));
State *state = (State *)calloc(tox_count, sizeof(State));

ck_assert(toxes != nullptr);
ck_assert(state != nullptr);

for (uint32_t i = 0; i < tox_count; i++) {
state[i].index = i;
toxes[i] = tox_new_log(nullptr, nullptr, &state[i].index);
Expand All @@ -82,7 +85,9 @@ static void run_auto_test(uint32_t tox_count, void test(Tox **toxes, State *stat

uint8_t public_key[TOX_PUBLIC_KEY_SIZE];
tox_self_get_public_key(toxes[j], public_key);
tox_friend_add_norequest(toxes[i], public_key, nullptr);
Tox_Err_Friend_Add err;
tox_friend_add_norequest(toxes[i], public_key, &err);
ck_assert(err == TOX_ERR_FRIEND_ADD_OK);
}
}
} else {
Expand All @@ -93,7 +98,9 @@ static void run_auto_test(uint32_t tox_count, void test(Tox **toxes, State *stat
if (i != j) {
uint8_t public_key[TOX_PUBLIC_KEY_SIZE];
tox_self_get_public_key(toxes[j], public_key);
tox_friend_add_norequest(toxes[i], public_key, nullptr);
Tox_Err_Friend_Add err;
tox_friend_add_norequest(toxes[i], public_key, &err);
ck_assert(err == TOX_ERR_FRIEND_ADD_OK);
}
}
}
Expand All @@ -105,7 +112,9 @@ static void run_auto_test(uint32_t tox_count, void test(Tox **toxes, State *stat
const uint16_t dht_port = tox_self_get_udp_port(toxes[0], nullptr);

for (uint32_t i = 1; i < tox_count; i++) {
tox_bootstrap(toxes[i], "localhost", dht_port, dht_key, nullptr);
Tox_Err_Bootstrap err;
tox_bootstrap(toxes[i], "localhost", dht_port, dht_key, &err);
ck_assert(err == TOX_ERR_BOOTSTRAP_OK);
}

do {
Expand Down
3 changes: 3 additions & 0 deletions auto_tests/tox_one_test.c
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,10 @@ static void test_one(void)

uint32_t index[] = { 1, 2 };
Tox *tox1 = tox_new_log(nullptr, nullptr, &index[0]);
ck_assert(tox1 != nullptr);
set_random_name_and_status_message(tox1, name, status_message);
Tox *tox2 = tox_new_log(nullptr, nullptr, &index[1]);
ck_assert(tox2 != nullptr);
set_random_name_and_status_message(tox2, name2, status_message2);

uint8_t address[TOX_ADDRESS_SIZE];
Expand Down Expand Up @@ -85,6 +87,7 @@ static void test_one(void)
Tox_Err_New err_n;

struct Tox_Options *options = tox_options_new(nullptr);
ck_assert(options != nullptr);
tox_options_set_savedata_type(options, TOX_SAVEDATA_TYPE_TOX_SAVE);
tox_options_set_savedata_data(options, data, save_size);
tox2 = tox_new_log(options, &err_n, &index[1]);
Expand Down
6 changes: 6 additions & 0 deletions testing/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,9 @@ cc_binary(
"//c-toxcore/toxcore",
],
)

cc_library(
name = "oomer",
srcs = ["oomer.c"],
visibility = ["//visibility:public"],
)
146 changes: 146 additions & 0 deletions testing/oomer.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#include <fcntl.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

// The actual alloc functions. This only works on glibc.
void *__libc_malloc(size_t size);
void *__libc_calloc(size_t nmemb, size_t size);
void *__libc_realloc(void *ptr, size_t size);

typedef struct Oomer_Config {
/**
* OOMER_MAX_ALLOCS: the number of allocations to pass through until oomer
* starts running one of its strategies.
*/
long max_allocs;
/**
* OOMER_DONE_FILE: the file to write when all allocations have been allowed.
* This means any further tests are pointless because oomer will always allow
* all of them.
*/
const char *done_file;
/**
* OOMER_FLAKY: after max_allocs, start rejecting random allocations with the
* specified rate (between 0 and 1).
*/
double flaky;
/**
* Random seed for oomer's RNG. Not configurable through env vars for now.
*/
uint32_t seed;
/**
* OOMER_ONE_SHOT: reject only a single allocation after max_allocs
* allocations. All allocations after the one rejected one are passed through.
*/
bool one_shot;
/**
* OOMER_TRAP_ON_FAIL: cause a SIGTRAP when rejecting an allocation. Useful
* when running in a debugger to have a breakpoint at the rejected allocation.
*/
bool trap_on_fail;
} Oomer_Config;

static Oomer_Config config = { -1, 0, 0, 123456789, false, false };

uint32_t rand_u32(void) {
config.seed = (1664525 * config.seed + 1013904223);
return config.seed;
}

static void init_oomer(void) {
const char *env_max_allocs = getenv("OOMER_MAX_ALLOCS");
if (env_max_allocs != NULL) {
char *end;
config.max_allocs = strtol(env_max_allocs, &end, 10);
if (end != env_max_allocs + strlen(env_max_allocs)) {
fprintf(stderr, "invalid value for OOMER_MAX_ALLOCS: %s\n", env_max_allocs);
abort();
}
}

const char *env_flaky = getenv("OOMER_FLAKY");
if (env_flaky != NULL) {
char *end;
config.flaky = strtod(env_flaky, &end);
if (end != env_flaky + strlen(env_flaky) || config.flaky < 0 || config.flaky > 1) {
fprintf(stderr, "invalid value for OOMER_FLAKY: %s\n", env_flaky);
abort();
}
}

const char *env_one_shot = getenv("OOMER_ONE_SHOT");
config.one_shot = env_one_shot != NULL && *env_one_shot == '1';

const char *env_trap_on_fail = getenv("OOMER_TRAP_ON_FAIL");
config.trap_on_fail = env_trap_on_fail != NULL && *env_trap_on_fail == '1';

config.done_file = getenv("OOMER_DONE_FILE");

fprintf(stderr, "oomer: done_file = %s\n",
config.done_file != NULL ? config.done_file : "<unset>");
fprintf(stderr, "oomer: flaky = %f\n", config.flaky);
fprintf(stderr, "oomer: seed = %d\n", config.seed);
fprintf(stderr, "oomer: one_shot = %s\n", config.one_shot ? "true" : "false");
fprintf(stderr, "oomer: trap_on_fail = %s\n", config.trap_on_fail ? "true" : "false");

alarm(5);
}

static __attribute__((__destructor__)) void deinit_oomer(void) {
fprintf(stderr, "deinit_oomer: max_allocs = %ld\n", config.max_allocs);
if (config.done_file != NULL && config.max_allocs > 0) {
// Touch the done_file to signal to run_oomer that we've rejected at least
// one malloc call.
close(creat(config.done_file, 0644));
}
}

static bool can_alloc(void) {
if (config.max_allocs == -1) {
init_oomer();
}

if (config.max_allocs == 0) {
if (config.one_shot) {
// Allow all mallocs except this one.
config.max_allocs = -2;
}
if (config.trap_on_fail) {
__asm__("int $3");
}
if (config.flaky > 0) {
return rand_u32() > config.flaky * UINT32_MAX;
}
return false;
}

--config.max_allocs;
return true;
}

void *malloc(size_t size) {
if (can_alloc()) {
return __libc_malloc(size);
}
return NULL;
}

void *calloc(size_t nmemb, size_t size) {
if (can_alloc()) {
return __libc_calloc(nmemb, size);
}
return NULL;
}

void *realloc(void *ptr, size_t size) {
if (can_alloc()) {
return __libc_realloc(ptr, size);
}
return NULL;
}
74 changes: 74 additions & 0 deletions testing/run_oomer
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
"""Run an oomer-enabled program with an increasing number of max_allocs."""

import os
import subprocess
import sys

from typing import Dict, List


def run(args: List[str], env: Dict[str, str]) -> None:
"""Run a program with environment and print the env and args to stdout."""
print("-" * 60, "\x1b[0;32m")
for k in sorted(env.keys()):
print(f"{k}={env[k]} \\")
print(" ".join(args))
print("\x1b[0m", "-" * 60)
subprocess.run(args, env=env)


def run_oomer(exe: str, max_allocs: int) -> bool:
"""Run an oomer-enabled program with the provided max_allocs."""
print(f"\x1b[0;33mmax_allocs = {max_allocs}\x1b[0m")
done_file = f"oomer-{max_allocs}"
env = {
"OOMER_MAX_ALLOCS": str(max_allocs),
"OOMER_ONE_SHOT": "1",
"OOMER_FLAKY": "0",
"OOMER_DONE_FILE": done_file,
}
proc = subprocess.run([exe], env=env)

# The done_file is created by oomer.c when it considers the test complete.
if os.path.exists(done_file):
os.unlink(done_file)
return True

if proc.returncode >= 0:
# Process exited cleanly (success or failure).
pass
elif proc.returncode == -6:
# Assertion failed.
pass
elif proc.returncode == -14:
print(f"\x1b[1;31mProcess timed out at "
f"max_allocs = {max_allocs}\x1b[0m")
del env["OOMER_DONE_FILE"]
env["OOMER_TRAP_ON_FAIL"] = "1"
run(["gdb", "--eval-command=r", exe], env=env)
return True
else:
print(f"Process exited with signal {-proc.returncode} at "
f"max_allocs = {max_allocs}")
del env["OOMER_DONE_FILE"]
run(["gdb", "--eval-command=r", exe], env=env)
return True

return False


def main(args: List[str]) -> None:
"""Run an oomer-enabled program with an increasing number of max_allocs."""
if len(args) != 2:
print("Usage: oomer <exe>")
sys.exit(1)
exe = args[1]

for max_allocs in range(0, 1000):
if run_oomer(exe, max_allocs):
break


if __name__ == "__main__":
main(sys.argv)
5 changes: 5 additions & 0 deletions toxcore/DHT.c
Original file line number Diff line number Diff line change
Expand Up @@ -2721,6 +2721,11 @@ DHT *new_dht(const Logger *log, Mono_Time *mono_time, Networking_Core *net, bool
dht->dht_ping_array = ping_array_new(DHT_PING_ARRAY_SIZE, PING_TIMEOUT);
dht->dht_harden_ping_array = ping_array_new(DHT_PING_ARRAY_SIZE, PING_TIMEOUT);

if (dht->dht_ping_array == nullptr || dht->dht_harden_ping_array == nullptr) {
kill_dht(dht);
return nullptr;
}

for (uint32_t i = 0; i < DHT_FAKE_FRIEND_NUMBER; ++i) {
uint8_t random_public_key_bytes[CRYPTO_PUBLIC_KEY_SIZE];
uint8_t random_secret_key_bytes[CRYPTO_SECRET_KEY_SIZE];
Expand Down
2 changes: 1 addition & 1 deletion toxcore/Messenger.c
Original file line number Diff line number Diff line change
Expand Up @@ -2012,7 +2012,7 @@ Messenger *new_messenger(Mono_Time *mono_time, Messenger_Options *options, unsig
m->onion_c = new_onion_client(m->mono_time, m->net_crypto);
m->fr_c = new_friend_connections(m->mono_time, m->onion_c, options->local_discovery_enabled);

if (!(m->onion && m->onion_a && m->onion_c)) {
if (!(m->onion && m->onion_a && m->onion_c && m->fr_c)) {
kill_friend_connections(m->fr_c);
kill_onion(m->onion);
kill_onion_announce(m->onion_a);
Expand Down
4 changes: 4 additions & 0 deletions toxcore/ping.c
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,10 @@ Ping *ping_new(const Mono_Time *mono_time, DHT *dht)

void ping_kill(Ping *ping)
{
if (ping == nullptr) {
return;
}

networking_registerhandler(dht_get_net(ping->dht), NET_PACKET_PING_REQUEST, nullptr, nullptr);
networking_registerhandler(dht_get_net(ping->dht), NET_PACKET_PING_RESPONSE, nullptr, nullptr);
ping_array_kill(ping->ping_array);
Expand Down
4 changes: 4 additions & 0 deletions toxcore/ping_array.c
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ static void clear_entry(Ping_Array *array, uint32_t index)

void ping_array_kill(Ping_Array *array)
{
if (array == nullptr) {
return;
}

while (array->last_deleted != array->last_added) {
const uint32_t index = array->last_deleted % array->total_size;
clear_entry(array, index);
Expand Down

0 comments on commit 589ddac

Please sign in to comment.