feat: Add async event handling (callbacks) code.

Instead of synchronously handling events as they happen in
`tox_iterate`, this first collects all events in a structure and then
lets the client process them. This allows clients to process events in
parallel, since the data structure returned is mostly immutable.

This also makes toxcore compatible with languages that don't (easily)
support callbacks from C into the non-C language.

If we remove the callbacks, this allows us to add fields to the events
without breaking the API.

.circleci/config.yml

@@ -76,6 +76,7 @@ jobs:
           testing/misc_tools.c
           toxav/*.c
           toxcore/*.c
+          toxcore/*/*.c
           toxencryptsave/*.c
           -lpthread
           $(pkg-config --cflags --libs libsodium opus vpx)

.github/workflows/ci.yml

@@ -109,6 +109,7 @@ jobs:
             testing/misc_tools.c
             toxav/*.c
             toxcore/*.c
+            toxcore/*/*.c
             toxencryptsave/*.c
             $(pkg-config --cflags --libs libsodium opus vpx)
       - name: Run the test
@@ -146,6 +147,7 @@ jobs:
             testing/misc_tools.c
             toxav/*.c
             toxcore/*.c
+            toxcore/*/*.c
             toxencryptsave/*.c
             -D__COMPCERT__ -DDISABLE_VLA
             -lpthread $(pkg-config --cflags --libs libsodium opus vpx)

CMakeLists.txt

@@ -161,6 +161,7 @@ set(toxcore_PKGCONFIG_REQUIRES)
 # LAYER 1: Crypto core
 # --------------------
 set(toxcore_SOURCES ${toxcore_SOURCES}
+  toxcore/ccompat.c
   toxcore/ccompat.h
   toxcore/crypto_core.c
   toxcore/crypto_core.h)
@@ -240,10 +241,47 @@ set(toxcore_SOURCES ${toxcore_SOURCES}
 set(toxcore_SOURCES ${toxcore_SOURCES}
   toxcore/tox_api.c
   toxcore/tox.c
-  toxcore/tox_private.h
-  toxcore/tox.h)
+  toxcore/tox.h
+  toxcore/tox_private.h)
 set(toxcore_API_HEADERS ${toxcore_API_HEADERS} ${toxcore_SOURCE_DIR}/toxcore/tox.h^tox)
 
+# LAYER 9: New async events API
+# -------------------
+set(toxcore_SOURCES ${toxcore_SOURCES}
+  toxcore/events/conference_connected.c
+  toxcore/events/conference_invite.c
+  toxcore/events/conference_message.c
+  toxcore/events/conference_peer_list_changed.c
+  toxcore/events/conference_peer_name.c
+  toxcore/events/conference_title.c
+  toxcore/events/file_chunk_request.c
+  toxcore/events/file_recv.c
+  toxcore/events/file_recv_chunk.c
+  toxcore/events/file_recv_control.c
+  toxcore/events/friend_connection_status.c
+  toxcore/events/friend_lossless_packet.c
+  toxcore/events/friend_lossy_packet.c
+  toxcore/events/friend_message.c
+  toxcore/events/friend_name.c
+  toxcore/events/friend_read_receipt.c
+  toxcore/events/friend_request.c
+  toxcore/events/friend_status.c
+  toxcore/events/friend_status_message.c
+  toxcore/events/friend_typing.c
+  toxcore/events/events_alloc.c
+  toxcore/events/events_alloc.h
+  toxcore/events/self_connection_status.c
+  toxcore/tox_events.c
+  toxcore/tox_events.h)
+set(toxcore_API_HEADERS ${toxcore_API_HEADERS} ${toxcore_SOURCE_DIR}/toxcore/tox_events.h^tox)
+
+# LAYER 10: Dispatch recorded events to callbacks.
+# -------------------
+set(toxcore_SOURCES ${toxcore_SOURCES}
+  toxcore/tox_dispatch.c
+  toxcore/tox_dispatch.h)
+set(toxcore_API_HEADERS ${toxcore_API_HEADERS} ${toxcore_SOURCE_DIR}/toxcore/tox_dispatch.h^tox)
+
 ################################################################################
 #
 # :: Audio/Video Library
@@ -441,6 +479,8 @@ auto_test(save_load)
 auto_test(send_message)
 auto_test(set_name)
 auto_test(set_status_message)
+auto_test(tox_dispatch)
+auto_test(tox_events)
 auto_test(tox_many)
 auto_test(tox_many_tcp)
 auto_test(tox_one)
@@ -558,4 +598,3 @@ if (BUILD_FUZZ_TESTS)
   add_executable(bootstrap_fuzzer testing/fuzzing/bootstrap_harness.cc)
   target_link_libraries(bootstrap_fuzzer toxcore_static fuzz_adapter -fsanitize=fuzzer)
 endif()
-

auto_tests/BUILD.bazel

@@ -61,6 +61,8 @@ flaky_tests = {
         "//c-toxcore/toxcore:onion",
         "//c-toxcore/toxcore:onion_announce",
         "//c-toxcore/toxcore:onion_client",
+        "//c-toxcore/toxcore:tox_dispatch",
+        "//c-toxcore/toxcore:tox_events",
         "//c-toxcore/toxencryptsave",
         "@libsodium",
         "@libvpx",

auto_tests/tox_dispatch_test.c

@@ -0,0 +1,122 @@
+/* Auto Tests: Many clients.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "../testing/misc_tools.h"
+#include "../toxcore/tox.h"
+#include "../toxcore/tox_dispatch.h"
+#include "../toxcore/tox_events.h"
+#include "check_compat.h"
+
+static void handle_events_friend_message(Tox *tox, const Tox_Event_Friend_Message *event, void *user_data)
+{
+    bool *success = (bool *)user_data;
+
+    ck_assert(tox_event_friend_message_get_message_length(event) == sizeof("hello"));
+    const uint8_t *msg = tox_event_friend_message_get_message(event);
+    ck_assert_msg(memcmp(msg, "hello", sizeof("hello")) == 0,
+                  "message was not expected 'hello' but '%s'", (const char *)msg);
+
+    *success = true;
+}
+
+static bool await_message(Tox **toxes, const Tox_Dispatch *dispatch)
+{
+    for (uint32_t i = 0; i < 100; ++i) {
+        // Ignore events on tox 1.
+        tox_events_free(tox_events_iterate(toxes[0], nullptr));
+        // Check if tox 2 got the message from tox 1.
+        Tox_Events *events = tox_events_iterate(toxes[1], nullptr);
+
+        bool success = false;
+        tox_dispatch_invoke(dispatch, events, toxes[1], &success);
+        tox_events_free(events);
+
+        if (success) {
+            return true;
+        }
+
+        c_sleep(tox_iteration_interval(toxes[0]));
+    }
+
+    return false;
+}
+
+static void test_tox_events(void)
+{
+    uint8_t message[sizeof("hello")];
+    memcpy(message, "hello", sizeof(message));
+
+    Tox *toxes[2];
+    uint32_t index[2];
+
+    for (uint32_t i = 0; i < 2; ++i) {
+        index[i] = i + 1;
+        toxes[i] = tox_new_log(nullptr, nullptr, &index[i]);
+        tox_events_init(toxes[i]);
+        ck_assert_msg(toxes[i] != nullptr, "failed to create tox instances %u", i);
+    }
+
+    Tox_Dispatch *dispatch = tox_dispatch_new(nullptr);
+    ck_assert_msg(dispatch != nullptr, "failed to create event dispatcher");
+
+    tox_events_callback_friend_message(dispatch, handle_events_friend_message);
+
+    uint8_t pk[TOX_PUBLIC_KEY_SIZE];
+    tox_self_get_dht_id(toxes[0], pk);
+    tox_bootstrap(toxes[1], "localhost", tox_self_get_udp_port(toxes[0], nullptr), pk, nullptr);
+
+    tox_self_get_public_key(toxes[0], pk);
+    tox_friend_add_norequest(toxes[1], pk, nullptr);
+
+    tox_self_get_public_key(toxes[1], pk);
+    tox_friend_add_norequest(toxes[0], pk, nullptr);
+
+    printf("bootstrapping and connecting 2 toxes\n");
+
+    while (tox_self_get_connection_status(toxes[0]) == TOX_CONNECTION_NONE ||
+            tox_self_get_connection_status(toxes[1]) == TOX_CONNECTION_NONE) {
+        // Ignore connection events for now.
+        tox_events_free(tox_events_iterate(toxes[0], nullptr));
+        tox_events_free(tox_events_iterate(toxes[1], nullptr));
+
+        c_sleep(tox_iteration_interval(toxes[0]));
+    }
+
+    printf("toxes online, waiting for friend connection\n");
+
+    while (tox_friend_get_connection_status(toxes[0], 0, nullptr) == TOX_CONNECTION_NONE ||
+            tox_friend_get_connection_status(toxes[1], 0, nullptr) == TOX_CONNECTION_NONE) {
+        // Ignore connection events for now.
+        tox_events_free(tox_events_iterate(toxes[0], nullptr));
+        tox_events_free(tox_events_iterate(toxes[1], nullptr));
+
+        c_sleep(tox_iteration_interval(toxes[0]));
+    }
+
+    printf("friends are connected via %s, now sending message\n",
+           tox_friend_get_connection_status(toxes[0], 0, nullptr) == TOX_CONNECTION_TCP ? "TCP" : "UDP");
+
+    Tox_Err_Friend_Send_Message err;
+    tox_friend_send_message(toxes[0], 0, TOX_MESSAGE_TYPE_NORMAL, message, sizeof(message), &err);
+    ck_assert(err == TOX_ERR_FRIEND_SEND_MESSAGE_OK);
+
+    ck_assert(await_message(toxes, dispatch));
+
+    tox_dispatch_free(dispatch);
+
+    for (uint32_t i = 0; i < 2; ++i) {
+        tox_kill(toxes[i]);
+    }
+}
+
+int main(void)
+{
+    setvbuf(stdout, nullptr, _IONBF, 0);
+    test_tox_events();
+    return 0;
+}

auto_tests/tox_events_test.c

@@ -0,0 +1,105 @@
+/* Auto Tests: Many clients.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "../testing/misc_tools.h"
+#include "../toxcore/tox.h"
+#include "../toxcore/tox_events.h"
+#include "check_compat.h"
+
+static bool await_message(Tox **toxes)
+{
+    for (uint32_t i = 0; i < 100; ++i) {
+        // Ignore events on tox 1.
+        tox_events_free(tox_events_iterate(toxes[0], nullptr));
+        // Check if tox 2 got the message from tox 1.
+        Tox_Events *events = tox_events_iterate(toxes[1], nullptr);
+
+        if (events != nullptr) {
+            ck_assert(tox_events_get_friend_message_size(events) == 1);
+            const Tox_Event_Friend_Message *msg_event = tox_events_get_friend_message(events, 0);
+            ck_assert(tox_event_friend_message_get_message_length(msg_event) == sizeof("hello"));
+            const uint8_t *msg = tox_event_friend_message_get_message(msg_event);
+            ck_assert_msg(memcmp(msg, "hello", sizeof("hello")) == 0,
+                          "message was not expected 'hello' but '%s'", (const char *)msg);
+            tox_events_free(events);
+            return true;
+        }
+
+        c_sleep(tox_iteration_interval(toxes[0]));
+    }
+
+    return false;
+}
+
+static void test_tox_events(void)
+{
+    uint8_t message[sizeof("hello")];
+    memcpy(message, "hello", sizeof(message));
+
+    Tox *toxes[2];
+    uint32_t index[2];
+
+    for (uint32_t i = 0; i < 2; ++i) {
+        index[i] = i + 1;
+        toxes[i] = tox_new_log(nullptr, nullptr, &index[i]);
+        tox_events_init(toxes[i]);
+        ck_assert_msg(toxes[i] != nullptr, "failed to create tox instances %u", i);
+    }
+
+    uint8_t pk[TOX_PUBLIC_KEY_SIZE];
+    tox_self_get_dht_id(toxes[0], pk);
+    tox_bootstrap(toxes[1], "localhost", tox_self_get_udp_port(toxes[0], nullptr), pk, nullptr);
+
+    tox_self_get_public_key(toxes[0], pk);
+    tox_friend_add_norequest(toxes[1], pk, nullptr);
+
+    tox_self_get_public_key(toxes[1], pk);
+    tox_friend_add_norequest(toxes[0], pk, nullptr);
+
+    printf("bootstrapping and connecting 2 toxes\n");
+
+    while (tox_self_get_connection_status(toxes[0]) == TOX_CONNECTION_NONE ||
+            tox_self_get_connection_status(toxes[1]) == TOX_CONNECTION_NONE) {
+        // Ignore connection events for now.
+        tox_events_free(tox_events_iterate(toxes[0], nullptr));
+        tox_events_free(tox_events_iterate(toxes[1], nullptr));
+
+        c_sleep(tox_iteration_interval(toxes[0]));
+    }
+
+    printf("toxes online, waiting for friend connection\n");
+
+    while (tox_friend_get_connection_status(toxes[0], 0, nullptr) == TOX_CONNECTION_NONE ||
+            tox_friend_get_connection_status(toxes[1], 0, nullptr) == TOX_CONNECTION_NONE) {
+        // Ignore connection events for now.
+        tox_events_free(tox_events_iterate(toxes[0], nullptr));
+        tox_events_free(tox_events_iterate(toxes[1], nullptr));
+
+        c_sleep(tox_iteration_interval(toxes[0]));
+    }
+
+    printf("friends are connected via %s, now sending message\n",
+           tox_friend_get_connection_status(toxes[0], 0, nullptr) == TOX_CONNECTION_TCP ? "TCP" : "UDP");
+
+    Tox_Err_Friend_Send_Message err;
+    tox_friend_send_message(toxes[0], 0, TOX_MESSAGE_TYPE_NORMAL, message, sizeof(message), &err);
+    ck_assert(err == TOX_ERR_FRIEND_SEND_MESSAGE_OK);
+
+    ck_assert(await_message(toxes));
+
+    for (uint32_t i = 0; i < 2; ++i) {
+        tox_kill(toxes[i]);
+    }
+}
+
+int main(void)
+{
+    setvbuf(stdout, nullptr, _IONBF, 0);
+    test_tox_events();
+    return 0;
+}

other/analysis/gen-file.sh

@@ -10,6 +10,7 @@ CPPFLAGS+=("-Itesting")
 CPPFLAGS+=("-Itesting/fuzzing")
 CPPFLAGS+=("-Itesting/groupchats")
 CPPFLAGS+=("-Itoxcore")
+CPPFLAGS+=("-Itoxcore/events")
 CPPFLAGS+=("-Itoxav")
 CPPFLAGS+=("-Itoxencryptsave")
 

other/bootstrap_daemon/docker/tox-bootstrapd.sha256

@@ -1 +1 @@
-64e519c68f23246ea4f490bdc2973d8bc1217da802596ad550048d3767b7214a  /usr/local/bin/tox-bootstrapd
+21bbf4e49911d31cef0f50195affe86138630703c7072c67ffd7802f7e588190  /usr/local/bin/tox-bootstrapd

other/make_single_file

@@ -48,7 +48,7 @@ if (@ARGV and $ARGV[0] eq "-core") {
       emit(abs_path $fn);
    }
 } else {
-   for my $fn (<toxav/*.c>, <toxcore/*.c>, <toxencryptsave/*.c>) {
+   for my $fn (<toxav/*.c>, <toxcore/*.c>, <toxcore/*/*.c>, <toxencryptsave/*.c>) {
       emit(abs_path $fn);
    }
 }

toxcore/BUILD.bazel

@@ -10,6 +10,7 @@ exports_files(
 
 cc_library(
     name = "ccompat",
+    srcs = ["ccompat.c"],
     hdrs = ["ccompat.h"],
     visibility = ["//c-toxcore:__subpackages__"],
 )
@@ -459,9 +460,36 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "tox_events",
+    srcs = ["tox_events.c"] + glob([
+        "events/*.c",
+        "events/*.h",
+    ]),
+    hdrs = ["tox_events.h"],
+    visibility = ["//c-toxcore:__subpackages__"],
+    deps = [
+        ":ccompat",
+        ":toxcore",
+    ],
+)
+
+cc_library(
+    name = "tox_dispatch",
+    srcs = ["tox_dispatch.c"],
+    hdrs = ["tox_dispatch.h"],
+    visibility = ["//c-toxcore:__subpackages__"],
+    deps = [
+        ":ccompat",
+        ":tox_events",
+    ],
+)
+
 sh_library(
     name = "cimple_files",
     srcs = glob([
+        "events/*.c",
+        "events/*.h",
         "*.c",
         "*.h",
     ]),