From a04f8ef9fb09602b2c98aba4087dc52a718c1ec0 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 13 Jul 2023 15:53:09 -0700 Subject: [PATCH 1/3] test: Add Mongoose embedded HTTP server This adds Mongoose as a third-party library, and builds on top of that an embedded HTTP server for our unit tests. We are using a fork of Mongoose pending the merging of this PR: https://github.com/cesanta/mongoose/pull/2301 The embedded web server will make our HTTP-based tests independent of httpbin.org, which will make them quick and reliable. --- .gitignore | 1 + .gitmodules | 3 + packager/file/CMakeLists.txt | 3 +- packager/media/test/CMakeLists.txt | 10 +- packager/media/test/test_web_server.cc | 261 +++++++++++++++++++ packager/media/test/test_web_server.h | 67 +++++ packager/third_party/CMakeLists.txt | 1 + packager/third_party/mongoose/CMakeLists.txt | 15 ++ packager/third_party/mongoose/source | 1 + 9 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 packager/media/test/test_web_server.cc create mode 100644 packager/media/test/test_web_server.h create mode 100644 packager/third_party/mongoose/CMakeLists.txt create mode 160000 packager/third_party/mongoose/source diff --git a/.gitignore b/.gitignore index 5a7e5d5d9bf..1cca919155d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc *.sln +*.swp *.VC.db *.vcxproj* */.vs/* diff --git a/.gitmodules b/.gitmodules index c528969fd45..979da1c90b9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "packager/third_party/protobuf/source"] path = packager/third_party/protobuf/source url = https://github.com/protocolbuffers/protobuf +[submodule "packager/third_party/mongoose/source"] + path = packager/third_party/mongoose/source + url = https://github.com/joeyparrish/mongoose diff --git a/packager/file/CMakeLists.txt b/packager/file/CMakeLists.txt index 79396d36b09..9fa7de5fcb9 100644 --- a/packager/file/CMakeLists.txt +++ b/packager/file/CMakeLists.txt @@ -48,5 +48,6 @@ target_link_libraries(file_unittest gmock gtest gtest_main - nlohmann_json) + nlohmann_json + test_web_server) add_test(NAME file_unittest COMMAND file_unittest) diff --git a/packager/media/test/CMakeLists.txt b/packager/media/test/CMakeLists.txt index 41d890b4378..782879531f1 100644 --- a/packager/media/test/CMakeLists.txt +++ b/packager/media/test/CMakeLists.txt @@ -6,4 +6,12 @@ add_library(test_data_util STATIC test_data_util.cc) -target_link_libraries(test_data_util glog) +target_link_libraries(test_data_util + glog) +add_library(test_web_server STATIC + test_web_server.cc) +target_link_libraries(test_web_server + absl::str_format + absl::strings + mongoose + nlohmann_json) diff --git a/packager/media/test/test_web_server.cc b/packager/media/test/test_web_server.cc new file mode 100644 index 00000000000..944c76f12bc --- /dev/null +++ b/packager/media/test/test_web_server.cc @@ -0,0 +1,261 @@ +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#include "packager/media/test/test_web_server.h" + +#include +#include + +#include "absl/strings/numbers.h" +#include "absl/strings/str_format.h" +#include "nlohmann/json.hpp" + +// A full replacement for our former use of httpbin.org in tests. This +// embedded web server can: +// +// 1. Reflect the request method, body, and headers +// 2. Return a requested status code +// 3. Delay a response by a requested amount of time + +namespace { + +// Get a string_view on mongoose's mg_string, which may not be nul-terminated. +std::string_view view_mg_str(const mg_str& mg_string) { + return std::string_view(mg_string.ptr, mg_string.len); +} + +bool is_mg_str_null(const mg_str& mg_string) { + return mg_string.ptr == NULL; +} + +bool is_mg_str_null_or_blank(const mg_str& mg_string) { + return mg_string.ptr == NULL || mg_string.len == 0; +} + +// Get a string query parameter from a mongoose HTTP message. +bool get_string_query_parameter(struct mg_http_message* message, + const char* name, + std::string_view* str) { + struct mg_str value_mg_str = mg_http_var(message->query, mg_str(name)); + + if (!is_mg_str_null(value_mg_str)) { + *str = view_mg_str(value_mg_str); + return true; + } + + return false; +} + +// Get an integer query parameter from a mongoose HTTP message. +bool get_int_query_parameter(struct mg_http_message* message, + const char* name, + int* value) { + std::string_view str; + + if (get_string_query_parameter(message, name, &str)) { + return absl::SimpleAtoi(str, value); + } + + return false; +} + +} // namespace + +namespace shaka { +namespace media { + +TestWebServer::TestWebServer() : status_(kNew), stopped_(false) {} + +TestWebServer::~TestWebServer() { + { + absl::MutexLock lock(&mutex_); + stop_.Signal(); + stopped_ = true; + } + if (thread_) { + thread_->join(); + } + thread_.reset(); +} + +bool TestWebServer::Start(int port) { + thread_.reset(new std::thread(&TestWebServer::ThreadCallback, this, port)); + + absl::MutexLock lock(&mutex_); + while (status_ == kNew) { + started_.Wait(&mutex_); + } + + return status_ == kStarted; +} + +void TestWebServer::ThreadCallback(int port) { + // Mongoose needs an HTTP server address in string format. + // "127.0.0.1" is "localhost", and is not visible to other machines on the + // network. + std::string http_address = absl::StrFormat("http://127.0.0.1:%d", port); + + // Set up the manager structure to be automatically cleaned up when it leaves + // scope. + std::unique_ptr manager( + new struct mg_mgr, mg_mgr_free); + // Then initialize it. + mg_mgr_init(manager.get()); + + auto connection = + mg_http_listen(manager.get(), http_address.c_str(), + &TestWebServer::HandleEvent, this /* callback_data */); + if (connection == NULL) { + // Failed to listen to the requested port. Mongoose has already printed an + // error message. + absl::MutexLock lock(&mutex_); + status_ = kFailed; + started_.Signal(); + return; + } + + { + absl::MutexLock lock(&mutex_); + status_ = kStarted; + started_.Signal(); + } + + bool stopped = false; + while (!stopped) { + // Let Mongoose poll the sockets for 100ms. + mg_mgr_poll(manager.get(), 100); + + // Check for a stop signal from the test. + { + absl::MutexLock lock(&mutex_); + stopped = stopped_; + } + } + + { + absl::MutexLock lock(&mutex_); + status_ = kStopped; + } +} + +// static +void TestWebServer::HandleEvent(struct mg_connection* connection, + int event, + void* event_data, + void* callback_data) { + TestWebServer* instance = static_cast(callback_data); + + if (event == MG_EV_POLL) { + std::vector to_delete; + + // Check if it's time to re-handle any delayed connections. + for (const auto& pair : instance->delayed_connections_) { + const auto delayed_connection = pair.first; + const auto deadline = pair.second; + if (deadline <= absl::Now()) { + to_delete.push_back(delayed_connection); + instance->HandleDelay(NULL, delayed_connection); + } + } + + // Now that we're done iterating the map, delete any connections we are done + // responding to. + for (const auto& delayed_connection : to_delete) { + instance->delayed_connections_.erase(delayed_connection); + } + } else if (event == MG_EV_CLOSE) { + if (instance->delayed_connections_.count(connection)) { + // The client hung up before our delay expired. Remove this from our map. + instance->delayed_connections_.erase(connection); + } + } + + if (event != MG_EV_HTTP_MSG) + return; + + struct mg_http_message* message = + static_cast(event_data); + if (mg_http_match_uri(message, "/reflect")) { + if (instance->HandleReflect(message, connection)) + return; + } else if (mg_http_match_uri(message, "/status")) { + if (instance->HandleStatus(message, connection)) + return; + } else if (mg_http_match_uri(message, "/delay")) { + if (instance->HandleDelay(message, connection)) + return; + } + + mg_http_reply(connection, 400 /* bad request */, NULL /* headers */, + "Bad request!"); +} + +bool TestWebServer::HandleStatus(struct mg_http_message* message, + struct mg_connection* connection) { + int code = 0; + + if (get_int_query_parameter(message, "code", &code)) { + // Reply with the requested status code. + mg_http_reply(connection, code, NULL /* headers */, "%s", "{}"); + return true; + } + + return false; +} + +bool TestWebServer::HandleDelay(struct mg_http_message* message, + struct mg_connection* connection) { + if (delayed_connections_.count(connection)) { + // We're being called back after a delay has elapsed. + // Respond now. + mg_http_reply(connection, 200 /* OK */, NULL /* headers */, "%s", "{}"); + return true; + } + + int seconds = 0; + // Checking |message| here is a small safety measure, since we call this + // method back a second time with message set to NULL. That is supposed to + // be handled above, but this is defense in depth against a crash. + if (message && get_int_query_parameter(message, "seconds", &seconds)) { + // We can't block this thread, so compute the deadline and add the + // connection to a map. The main handler will call us back later if the + // client doesn't hang up first. + absl::Time deadline = absl::Now() + absl::Seconds(seconds); + delayed_connections_[connection] = deadline; + return true; + } + + return false; +} + +bool TestWebServer::HandleReflect(struct mg_http_message* message, + struct mg_connection* connection) { + // Serialize a reply in JSON that reflects the request method, body, and + // headers. + nlohmann::json reply; + reply["method"] = view_mg_str(message->method); + if (!is_mg_str_null(message->body)) { + reply["body"] = view_mg_str(message->body); + } + + nlohmann::json headers; + for (int i = 0; i < MG_MAX_HTTP_HEADERS; ++i) { + struct mg_http_header header = message->headers[i]; + if (is_mg_str_null_or_blank(header.name)) { + break; + } + + headers[view_mg_str(header.name)] = view_mg_str(header.value); + } + reply["headers"] = headers; + + mg_http_reply(connection, 200 /* OK */, NULL /* headers */, "%s\n", + reply.dump().c_str()); + return true; +} + +} // namespace media +} // namespace shaka diff --git a/packager/media/test/test_web_server.h b/packager/media/test/test_web_server.h new file mode 100644 index 00000000000..b49a051fa96 --- /dev/null +++ b/packager/media/test/test_web_server.h @@ -0,0 +1,67 @@ +// Copyright 2023 Google LLC. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +#ifndef PACKAGER_MEDIA_TEST_TEST_WEB_SERVER_H_ +#define PACKAGER_MEDIA_TEST_TEST_WEB_SERVER_H_ + +#include +#include +#include + +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" +#include "mongoose.h" + +namespace shaka { +namespace media { + +class TestWebServer { + public: + TestWebServer(); + ~TestWebServer(); + + bool Start(int port); + + private: + enum TestWebServerStatus { + kNew, + kFailed, + kStarted, + kStopped, + }; + + absl::Mutex mutex_; + TestWebServerStatus status_ GUARDED_BY(mutex_); + absl::CondVar started_ GUARDED_BY(mutex_); + absl::CondVar stop_ GUARDED_BY(mutex_); + bool stopped_ GUARDED_BY(mutex_); + + // Connections to be handled again later, mapped to the time at which we + // should handle them again. We can't block the server thread directly to + // simulate delays. Only ever accessed from |thread_|. + std::map delayed_connections_; + + std::unique_ptr thread_; + + void ThreadCallback(int port); + + static void HandleEvent(struct mg_connection* connection, + int event, + void* event_data, + void* callback_data); + + bool HandleStatus(struct mg_http_message* message, + struct mg_connection* connection); + bool HandleDelay(struct mg_http_message* message, + struct mg_connection* connection); + bool HandleReflect(struct mg_http_message* message, + struct mg_connection* connection); +}; + +} // namespace media +} // namespace shaka + +#endif // PACKAGER_MEDIA_TEST_TEST_WEB_SERVER_H_ diff --git a/packager/third_party/CMakeLists.txt b/packager/third_party/CMakeLists.txt index f19b3fcdeff..dba432e8a01 100644 --- a/packager/third_party/CMakeLists.txt +++ b/packager/third_party/CMakeLists.txt @@ -35,5 +35,6 @@ add_subdirectory(libpng EXCLUDE_FROM_ALL) add_subdirectory(libwebm EXCLUDE_FROM_ALL) add_subdirectory(libxml2 EXCLUDE_FROM_ALL) add_subdirectory(mbedtls EXCLUDE_FROM_ALL) +add_subdirectory(mongoose EXCLUDE_FROM_ALL) add_subdirectory(protobuf EXCLUDE_FROM_ALL) add_subdirectory(zlib EXCLUDE_FROM_ALL) diff --git a/packager/third_party/mongoose/CMakeLists.txt b/packager/third_party/mongoose/CMakeLists.txt new file mode 100644 index 00000000000..c590633167d --- /dev/null +++ b/packager/third_party/mongoose/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright 2023 Google LLC. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# CMake build file for the mongoose library, which is used as a built-in web +# server for testing certain HTTP client features of Packager. + +# Mongoose does not have its own CMakeLists.txt, but mongoose is very simple. + +add_library(mongoose STATIC + source/mongoose.c) +target_include_directories(mongoose + PUBLIC source/) diff --git a/packager/third_party/mongoose/source b/packager/third_party/mongoose/source new file mode 160000 index 00000000000..25650bd9be1 --- /dev/null +++ b/packager/third_party/mongoose/source @@ -0,0 +1 @@ +Subproject commit 25650bd9be1578d8cc28881ca1255392a53c6123 From d46ee2ca3d06c62e2bf84a0a0e27a878e3dbd779 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 13 Jul 2023 16:20:33 -0700 Subject: [PATCH 2/3] Hide Mongoose header from TestWebServer users --- packager/media/test/test_web_server.cc | 1 + packager/media/test/test_web_server.h | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packager/media/test/test_web_server.cc b/packager/media/test/test_web_server.cc index 944c76f12bc..557a514cbf5 100644 --- a/packager/media/test/test_web_server.cc +++ b/packager/media/test/test_web_server.cc @@ -11,6 +11,7 @@ #include "absl/strings/numbers.h" #include "absl/strings/str_format.h" +#include "mongoose.h" #include "nlohmann/json.hpp" // A full replacement for our former use of httpbin.org in tests. This diff --git a/packager/media/test/test_web_server.h b/packager/media/test/test_web_server.h index b49a051fa96..d6e0c5f20b1 100644 --- a/packager/media/test/test_web_server.h +++ b/packager/media/test/test_web_server.h @@ -13,7 +13,10 @@ #include "absl/synchronization/mutex.h" #include "absl/time/time.h" -#include "mongoose.h" + +// Forward declare mongoose struct types, used as pointers below. +struct mg_connection; +struct mg_http_message; namespace shaka { namespace media { From a03978595827f0e23e52b9367f05f62a45bff681 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Thu, 13 Jul 2023 16:33:55 -0700 Subject: [PATCH 3/3] Revise for style and other feedback --- packager/media/test/test_web_server.cc | 45 ++++++++++++-------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/packager/media/test/test_web_server.cc b/packager/media/test/test_web_server.cc index 557a514cbf5..1d4fb9ac0f2 100644 --- a/packager/media/test/test_web_server.cc +++ b/packager/media/test/test_web_server.cc @@ -24,26 +24,26 @@ namespace { // Get a string_view on mongoose's mg_string, which may not be nul-terminated. -std::string_view view_mg_str(const mg_str& mg_string) { +std::string_view MongooseStringView(const mg_str& mg_string) { return std::string_view(mg_string.ptr, mg_string.len); } -bool is_mg_str_null(const mg_str& mg_string) { +bool IsMongooseStringNull(const mg_str& mg_string) { return mg_string.ptr == NULL; } -bool is_mg_str_null_or_blank(const mg_str& mg_string) { +bool IsMongooseStringNullOrBlank(const mg_str& mg_string) { return mg_string.ptr == NULL || mg_string.len == 0; } // Get a string query parameter from a mongoose HTTP message. -bool get_string_query_parameter(struct mg_http_message* message, - const char* name, - std::string_view* str) { +bool GetStringQueryParameter(struct mg_http_message* message, + const char* name, + std::string_view* str) { struct mg_str value_mg_str = mg_http_var(message->query, mg_str(name)); - if (!is_mg_str_null(value_mg_str)) { - *str = view_mg_str(value_mg_str); + if (!IsMongooseStringNull(value_mg_str)) { + *str = MongooseStringView(value_mg_str); return true; } @@ -51,12 +51,12 @@ bool get_string_query_parameter(struct mg_http_message* message, } // Get an integer query parameter from a mongoose HTTP message. -bool get_int_query_parameter(struct mg_http_message* message, - const char* name, - int* value) { +bool GetIntQueryParameter(struct mg_http_message* message, + const char* name, + int* value) { std::string_view str; - if (get_string_query_parameter(message, name, &str)) { + if (GetStringQueryParameter(message, name, &str)) { return absl::SimpleAtoi(str, value); } @@ -133,13 +133,10 @@ void TestWebServer::ThreadCallback(int port) { { absl::MutexLock lock(&mutex_); stopped = stopped_; + if (stopped) + status_ = kStopped; } } - - { - absl::MutexLock lock(&mutex_); - status_ = kStopped; - } } // static @@ -198,7 +195,7 @@ bool TestWebServer::HandleStatus(struct mg_http_message* message, struct mg_connection* connection) { int code = 0; - if (get_int_query_parameter(message, "code", &code)) { + if (GetIntQueryParameter(message, "code", &code)) { // Reply with the requested status code. mg_http_reply(connection, code, NULL /* headers */, "%s", "{}"); return true; @@ -220,7 +217,7 @@ bool TestWebServer::HandleDelay(struct mg_http_message* message, // Checking |message| here is a small safety measure, since we call this // method back a second time with message set to NULL. That is supposed to // be handled above, but this is defense in depth against a crash. - if (message && get_int_query_parameter(message, "seconds", &seconds)) { + if (message && GetIntQueryParameter(message, "seconds", &seconds)) { // We can't block this thread, so compute the deadline and add the // connection to a map. The main handler will call us back later if the // client doesn't hang up first. @@ -237,19 +234,19 @@ bool TestWebServer::HandleReflect(struct mg_http_message* message, // Serialize a reply in JSON that reflects the request method, body, and // headers. nlohmann::json reply; - reply["method"] = view_mg_str(message->method); - if (!is_mg_str_null(message->body)) { - reply["body"] = view_mg_str(message->body); + reply["method"] = MongooseStringView(message->method); + if (!IsMongooseStringNull(message->body)) { + reply["body"] = MongooseStringView(message->body); } nlohmann::json headers; for (int i = 0; i < MG_MAX_HTTP_HEADERS; ++i) { struct mg_http_header header = message->headers[i]; - if (is_mg_str_null_or_blank(header.name)) { + if (IsMongooseStringNullOrBlank(header.name)) { break; } - headers[view_mg_str(header.name)] = view_mg_str(header.value); + headers[MongooseStringView(header.name)] = MongooseStringView(header.value); } reply["headers"] = headers;