Merge #2632

2632: Download image from mirror site r=townsend2010 a=rim99 Hi team, I would like to present a PR for issue: #717 **Feature: mirror of cloud images** By setting `local.image.mirror`, users can download images from mirror sites instead of the official one. The feature can be disabled by leaving the config empty. **Implementation** When updating product versions, it will choose the manifest from mirror site if possible. Since mirror sites is syncing with official site periodically, the latest version in official site may not exist in mirror sites. Only if the content of a version is *identical* to the same version from official site, it will be updated into the `manifests` cache of `UbuntuVMImageHost` instance. Co-authored-by: rim99 <[email protected]>
canonical · Jul 29, 2022 · 577da62 · 577da62
2 parents 87e6346 + 38078bf
commit 577da62
Show file tree

Hide file tree

Showing 17 changed files with 684 additions and 56 deletions.
diff --git a/include/multipass/constants.h b/include/multipass/constants.h
@@ -51,14 +51,15 @@ constexpr auto bridged_network_name = "bridged";
 constexpr auto settings_extension = ".conf";
 constexpr auto daemon_settings_root = "local";
 
-constexpr auto petenv_key = "client.primary-name";     // This will eventually be moved to some dynamic settings schema
-constexpr auto driver_key = "local.driver";            // idem
-constexpr auto passphrase_key = "local.passphrase";    // idem
-constexpr auto bridged_interface_key = "local.bridged-network"; // idem
-constexpr auto mounts_key = "local.privileged-mounts"; // idem
-constexpr auto autostart_key = "client.gui.autostart"; // idem
+constexpr auto petenv_key = "client.primary-name";  // This will eventually be moved to some dynamic settings schema
+constexpr auto driver_key = "local.driver";         // idem
+constexpr auto passphrase_key = "local.passphrase"; // idem
+constexpr auto bridged_interface_key = "local.bridged-network";       // idem
+constexpr auto mounts_key = "local.privileged-mounts";                // idem
+constexpr auto autostart_key = "client.gui.autostart";                // idem
 constexpr auto winterm_key = "client.apps.windows-terminal.profiles"; // idem
 constexpr auto hotkey_key = "client.gui.hotkey";                      // idem
+constexpr auto mirror_key = "local.image.mirror";                     // idem; this defines the mirror of simple streams
 
 [[maybe_unused]] // hands off clang-format
 constexpr auto key_examples = {autostart_key, driver_key, mounts_key};

diff --git a/include/multipass/simple_streams_manifest.h b/include/multipass/simple_streams_manifest.h
@@ -28,18 +28,20 @@
 #include <QString>
 
 #include <memory>
+#include <optional>
 #include <vector>
 
 namespace multipass
 {
 
 struct SimpleStreamsManifest
 {
-    static std::unique_ptr<SimpleStreamsManifest> fromJson(const QByteArray& json, const QString& host_url);
+    static std::unique_ptr<SimpleStreamsManifest>
+    fromJson(const QByteArray& json, const std::optional<QByteArray>& json_from_mirror, const QString& host_url);
 
     const QString updated_at;
     const std::vector<VMImageInfo> products;
     const QMap<QString, const VMImageInfo*> image_records;
 };
-}
+} // namespace multipass
 #endif // MULTIPASS_SIMPLE_STREAMS_MANIFEST_H
diff --git a/src/daemon/daemon_config.cpp b/src/daemon/daemon_config.cpp
@@ -38,6 +38,7 @@
 
 #include <chrono>
 #include <memory>
+#include <optional>
 
 namespace mp = multipass;
 namespace mpl = multipass::logging;
@@ -134,10 +135,12 @@ std::unique_ptr<const mp::DaemonConfig> mp::DaemonConfigBuilder::build()
         image_hosts.push_back(std::make_unique<mp::CustomVMImageHost>(QSysInfo::currentCpuArchitecture(),
                                                                       url_downloader.get(), manifest_ttl));
         image_hosts.push_back(std::make_unique<mp::UbuntuVMImageHost>(
-            std::vector<std::pair<std::string, std::string>>{
-                {mp::release_remote, "https://cloud-images.ubuntu.com/releases/"},
-                {mp::daily_remote, "https://cloud-images.ubuntu.com/daily/"},
-                {mp::appliance_remote, "https://cdimage.ubuntu.com/ubuntu-core/appliances/"}},
+            std::vector<std::pair<std::string, UbuntuVMImageRemote>>{
+                {mp::release_remote, UbuntuVMImageRemote{"https://cloud-images.ubuntu.com/", "releases/",
+                                                         std::make_optional<QString>(mp::mirror_key)}},
+                {mp::daily_remote, UbuntuVMImageRemote{"https://cloud-images.ubuntu.com/", "daily/",
+                                                       std::make_optional<QString>(mp::mirror_key)}},
+                {mp::appliance_remote, UbuntuVMImageRemote{"https://cdimage.ubuntu.com/", "ubuntu-core/appliances/"}}},
             url_downloader.get(), manifest_ttl));
     }
     if (vault == nullptr)

diff --git a/src/daemon/daemon_init_settings.cpp b/src/daemon/daemon_init_settings.cpp
@@ -59,6 +59,26 @@ QString driver_interpreter(QString val)
     return val;
 }
 
+QString image_mirror_interpreter(QString val)
+{
+    if (val.size() == 0)
+    {
+        return val;
+    }
+
+    if (!val.startsWith("https://"))
+    {
+        throw mp::InvalidSettingException(mp::mirror_key, val,
+                                          "The hostname of mirror must contain protocol name: https");
+    }
+
+    if (!val.endsWith("/"))
+    {
+        val.append("/");
+    }
+    return val;
+}
+
 } // namespace
 
 void mp::daemon::monitor_and_quit_on_settings_change() // temporary
@@ -79,6 +99,7 @@ void mp::daemon::register_global_settings_handlers()
     settings.insert(std::make_unique<CustomSettingSpec>(mp::passphrase_key, "", [](QString val) {
         return val.isEmpty() ? val : MP_UTILS.generate_scrypt_hash_for(val);
     }));
+    settings.insert(std::make_unique<CustomSettingSpec>(mp::mirror_key, "", image_mirror_interpreter));
 
     MP_SETTINGS.register_handler(
         std::make_unique<PersistentSettingsHandler>(persistent_settings_filename(), std::move(settings)));

diff --git a/src/daemon/ubuntu_image_host.cpp b/src/daemon/ubuntu_image_host.cpp
@@ -17,8 +17,10 @@
 
 #include "ubuntu_image_host.h"
 
+#include <multipass/constants.h>
 #include <multipass/platform.h>
 #include <multipass/query.h>
+#include <multipass/settings/settings.h>
 #include <multipass/simple_streams_index.h>
 #include <multipass/url_downloader.h>
 
@@ -27,6 +29,8 @@
 #include <multipass/exceptions/unsupported_image_exception.h>
 #include <multipass/exceptions/unsupported_remote_exception.h>
 
+#include <QJsonDocument>
+#include <QJsonObject>
 #include <QUrl>
 
 #include <algorithm>
@@ -44,7 +48,7 @@ auto download_manifest(const QString& host_url, mp::URLDownloader* url_downloade
     auto index = mp::SimpleStreamsIndex::fromJson(json_index);
 
     auto json_manifest = url_downloader->download({host_url + index.manifest_path});
-    return mp::SimpleStreamsManifest::fromJson(json_manifest, host_url);
+    return json_manifest;
 }
 
 mp::VMImageInfo with_location_fully_resolved(const QString& host_url, const mp::VMImageInfo& info)
@@ -73,7 +77,7 @@ auto key_from(const std::string& search_string)
 }
 } // namespace
 
-mp::UbuntuVMImageHost::UbuntuVMImageHost(std::vector<std::pair<std::string, std::string>> remotes,
+mp::UbuntuVMImageHost::UbuntuVMImageHost(std::vector<std::pair<std::string, UbuntuVMImageRemote>> remotes,
                                          URLDownloader* downloader, std::chrono::seconds manifest_time_to_live)
     : CommonVMImageHost{manifest_time_to_live}, url_downloader{downloader}, remotes{std::move(remotes)}
 {
@@ -220,28 +224,39 @@ std::vector<std::string> mp::UbuntuVMImageHost::supported_remotes()
 {
     std::vector<std::string> supported_remotes;
 
-    for (const auto& remote : remotes)
+    for (const auto& [remote_name, _] : remotes)
     {
-        supported_remotes.push_back(remote.first);
+        supported_remotes.push_back(remote_name);
     }
 
     return supported_remotes;
 }
 
 void mp::UbuntuVMImageHost::fetch_manifests()
 {
-    for (const auto& remote : remotes)
+    for (const auto& [remote_name, remote_info] : remotes)
     {
         try
         {
-            check_remote_is_supported(remote.first);
+            check_remote_is_supported(remote_name);
+            auto official_site = remote_info.get_official_url();
+            auto manifest_bytes_from_official = download_manifest(official_site, url_downloader);
 
-            manifests.emplace_back(
-                std::make_pair(remote.first, download_manifest(QString::fromStdString(remote.second), url_downloader)));
+            auto mirror_site = remote_info.get_mirror_url();
+            std::optional<QByteArray> manifest_bytes_from_mirror = std::nullopt;
+            if (mirror_site)
+            {
+                auto bytes = download_manifest(mirror_site.value(), url_downloader);
+                manifest_bytes_from_mirror = std::make_optional(bytes);
+            }
+
+            auto manifest = mp::SimpleStreamsManifest::fromJson(
+                manifest_bytes_from_official, manifest_bytes_from_mirror, mirror_site.value_or(official_site));
+            manifests.emplace_back(std::make_pair(remote_name, std::move(manifest)));
         }
         catch (mp::EmptyManifestException& /* e */)
         {
-            on_manifest_empty(fmt::format("Did not find any supported products in \"{}\"", remote.first));
+            on_manifest_empty(fmt::format("Did not find any supported products in \"{}\"", remote_name));
         }
         catch (mp::GenericManifestException& e)
         {
@@ -275,7 +290,9 @@ mp::SimpleStreamsManifest* mp::UbuntuVMImageHost::manifest_from(const std::strin
                            });
 
     if (it == manifests.cend())
-        throw std::runtime_error(fmt::format("Remote \"{}\" is unknown or unreachable.", remote));
+        throw std::runtime_error(fmt::format(
+            "Remote \"{}\" is unknown or unreachable. If image mirror is enabled, please confirm it is valid.",
+            remote));
 
     return it->second.get();
 }
@@ -296,12 +313,49 @@ std::string mp::UbuntuVMImageHost::remote_url_from(const std::string& remote_nam
 {
     std::string url;
 
-    auto it = std::find_if(
-        remotes.cbegin(), remotes.cend(),
-        [&remote_name](const std::pair<std::string, std::string>& element) { return element.first == remote_name; });
+    auto it = std::find_if(remotes.cbegin(), remotes.cend(),
+                           [&remote_name](const std::pair<std::string, UbuntuVMImageRemote>& element) {
+                               return element.first == remote_name;
+                           });
 
     if (it != remotes.cend())
-        url = it->second;
+    {
+        url = it->second.get_url().toStdString();
+    }
 
     return url;
 }
+
+mp::UbuntuVMImageRemote::UbuntuVMImageRemote(std::string official_host, std::string uri,
+                                             std::optional<QString> mirror_key)
+    : official_host(std::move(official_host)), uri(std::move(uri)), mirror_key(std::move(mirror_key))
+{
+}
+
+const QString mp::UbuntuVMImageRemote::get_url() const
+{
+    return get_mirror_url().value_or(get_official_url());
+}
+
+const QString mp::UbuntuVMImageRemote::get_official_url() const
+{
+    auto host = official_host;
+    host.append(uri);
+    return QString::fromStdString(host);
+}
+
+const std::optional<QString> mp::UbuntuVMImageRemote::get_mirror_url() const
+{
+    if (mirror_key)
+    {
+        auto mirror = MP_SETTINGS.get(mirror_key.value());
+        if (!mirror.isEmpty())
+        {
+            auto url = mirror.toStdString();
+            url.append(uri);
+            return std::make_optional(QString::fromStdString(url));
+        }
+    }
+
+    return std::nullopt;
+}
diff --git a/src/daemon/ubuntu_image_host.h b/src/daemon/ubuntu_image_host.h
@@ -34,10 +34,11 @@ constexpr auto daily_remote = "daily";
 constexpr auto appliance_remote = "appliance";
 
 class URLDownloader;
+class UbuntuVMImageRemote;
 class UbuntuVMImageHost final : public CommonVMImageHost
 {
 public:
-    UbuntuVMImageHost(std::vector<std::pair<std::string, std::string>> remotes, URLDownloader* downloader,
+    UbuntuVMImageHost(std::vector<std::pair<std::string, UbuntuVMImageRemote>> remotes, URLDownloader* downloader,
                       std::chrono::seconds manifest_time_to_live);
 
     std::optional<VMImageInfo> info_for(const Query& query) override;
@@ -56,9 +57,22 @@ class UbuntuVMImageHost final : public CommonVMImageHost
     const VMImageInfo* match_alias(const QString& key, const SimpleStreamsManifest& manifest) const;
     std::vector<std::pair<std::string, std::unique_ptr<SimpleStreamsManifest>>> manifests;
     URLDownloader* const url_downloader;
-    std::vector<std::pair<std::string, std::string>> remotes;
+    std::vector<std::pair<std::string, UbuntuVMImageRemote>> remotes;
     std::string remote_url_from(const std::string& remote_name);
     QString index_path;
 };
+class UbuntuVMImageRemote
+{
+public:
+    UbuntuVMImageRemote(std::string official_host, std::string uri, std::optional<QString> mirror_key = std::nullopt);
+    const QString get_url() const;
+    const QString get_official_url() const;
+    const std::optional<QString> get_mirror_url() const;
+
+private:
+    const std::string official_host;
+    const std::string uri;
+    const std::optional<QString> mirror_key;
+};
 } // namespace multipass
 #endif // MULTIPASS_UBUNTU_IMAGE_HOST_H
diff --git a/src/simplestreams/simple_streams_manifest.cpp b/src/simplestreams/simple_streams_manifest.cpp
@@ -74,27 +74,39 @@ QString derive_unpacked_file_path_prefix_from(const QString& image_location)
 
     return prefix;
 }
-}
+} // namespace
 
-std::unique_ptr<mp::SimpleStreamsManifest> mp::SimpleStreamsManifest::fromJson(const QByteArray& json,
-                                                                               const QString& host_url)
+std::unique_ptr<mp::SimpleStreamsManifest>
+mp::SimpleStreamsManifest::fromJson(const QByteArray& json_from_official,
+                                    const std::optional<QByteArray>& json_from_mirror, const QString& host_url)
 {
-    const auto manifest = parse_manifest(json);
-    const auto updated = manifest["updated"].toString();
+    const auto manifest_from_official = parse_manifest(json_from_official);
+    const auto updated = manifest_from_official["updated"].toString();
 
-    const auto manifest_products = manifest["products"].toObject();
-    if (manifest_products.isEmpty())
+    const auto manifest_products_from_official = manifest_from_official["products"].toObject();
+    if (manifest_products_from_official.isEmpty())
         throw mp::GenericManifestException("No products found");
 
     auto arch = arch_to_manifest.value(QSysInfo::currentCpuArchitecture());
 
     if (arch.isEmpty())
         throw mp::GenericManifestException("Unsupported cloud image architecture");
 
+    std::optional<QJsonObject> manifest_products_from_mirror = std::nullopt;
+    if (json_from_mirror)
+    {
+        const auto manifest_from_mirror = parse_manifest(json_from_mirror.value());
+        const auto products_from_mirror = manifest_from_mirror["products"].toObject();
+        manifest_products_from_mirror = std::make_optional(products_from_mirror);
+    }
+
+    const QJsonObject manifest_products = manifest_products_from_mirror.value_or(manifest_products_from_official);
+
     std::vector<VMImageInfo> products;
-    for (const auto& value : manifest_products)
+    for (auto it = manifest_products.constBegin(); it != manifest_products.constEnd(); ++it)
     {
-        const auto product = value.toObject();
+        const auto product_key = it.key();
+        const auto product = it.value();
 
         if (product["arch"].toString() != arch)
             continue;
@@ -115,6 +127,14 @@ std::unique_ptr<mp::SimpleStreamsManifest> mp::SimpleStreamsManifest::fromJson(c
         {
             const auto version_string = it.key();
             const auto version = versions[version_string].toObject();
+            const auto version_from_official = manifest_products_from_official[product_key]
+                                                   .toObject()["versions"]
+                                                   .toObject()[version_string]
+                                                   .toObject();
+
+            if (version != version_from_official)
+                continue;
+
             const auto items = version["items"].toObject();
             if (items.isEmpty())
                 continue;

diff --git a/tests/path.cpp b/tests/path.cpp
@@ -18,6 +18,7 @@
  */
 
 #include "path.h"
+#include <multipass/format.h>
 
 #include <QCoreApplication>
 #include <QDir>
@@ -39,6 +40,14 @@ QString mpt::test_data_path_for(const char* file_name)
     return dir.filePath(file_name);
 }
 
+QString mpt::test_data_sub_dir_path(const char* dir_name)
+{
+    QDir dir{test_data_path()};
+    if (!dir.cd(dir_name))
+        throw std::runtime_error(fmt::format("could not find sub dir '{}' under test_data directory", dir_name));
+    return QDir::toNativeSeparators(dir.path()) + QDir::separator();
+}
+
 std::string mpt::mock_bin_path()
 {
     QDir dir{QCoreApplication::applicationDirPath()};

diff --git a/tests/path.h b/tests/path.h
@@ -28,7 +28,8 @@ namespace test
 {
 QString test_data_path();
 QString test_data_path_for(const char* file_name);
+QString test_data_sub_dir_path(const char* dir_name);
 std::string mock_bin_path();
-}
-}
+} // namespace test
+} // namespace multipass
 #endif // MULTIPASS_TEST_DATA_PATH_H