Merge #579

579: custom_image_host: Update custom images information when requested r=Saviq a=townsend2010

Fixes #569

Co-authored-by: Chris Townsend <[email protected]>
Co-authored-by: Ricardo Abreu <[email protected]>

src/daemon/CMakeLists.txt

@@ -18,6 +18,7 @@ set(CMAKE_AUTOMOC ON)
 
 add_library(daemon STATIC
   cli.cpp
+  common_image_host.cpp
   custom_image_host.cpp
   daemon.cpp
   daemon_config.cpp

src/daemon/common_image_host.cpp

@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2019 Canonical, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 3.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "common_image_host.h"
+
+namespace mp = multipass;
+
+mp::CommonVMImageHost::CommonVMImageHost(std::chrono::seconds manifest_time_to_live)
+  : manifest_time_to_live{manifest_time_to_live}, last_update{}
+{
+}
+
+void mp::CommonVMImageHost::for_each_entry_do(const Action& action)
+{
+    update_manifests();
+
+    for_each_entry_do_impl(action);
+}
+
+auto mp::CommonVMImageHost::info_for_full_hash(const std::string& full_hash) -> VMImageInfo
+{
+    update_manifests();
+
+    return info_for_full_hash_impl(full_hash);
+}
+
+void mp::CommonVMImageHost::update_manifests()
+{
+    const auto now = std::chrono::steady_clock::now();
+    if ((now - last_update) > manifest_time_to_live || empty())
+    {
+        clear();
+        fetch_manifests();
+
+        last_update = now;
+    }
+}

src/daemon/common_image_host.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2019 Canonical, Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 3.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef MULTIPASS_COMMON_IMAGE_HOST_H_
+#define MULTIPASS_COMMON_IMAGE_HOST_H_
+
+#include "multipass/vm_image_host.h"
+
+#include <chrono>
+
+namespace multipass
+{
+
+class CommonVMImageHost : public VMImageHost
+{
+public:
+    CommonVMImageHost(std::chrono::seconds manifest_time_to_live);
+    void for_each_entry_do(const Action& action) final;
+    VMImageInfo info_for_full_hash(const std::string& full_hash) final;
+
+protected:
+    void update_manifests();
+
+    virtual void for_each_entry_do_impl(const Action& action) = 0;
+    virtual VMImageInfo info_for_full_hash_impl(const std::string& full_hash) = 0;
+    virtual bool empty() const = 0;
+    virtual void clear() = 0;
+    virtual void fetch_manifests() = 0;
+
+private:
+    std::chrono::seconds manifest_time_to_live;
+    std::chrono::steady_clock::time_point last_update;
+
+};
+
+}
+
+
+#endif /* MULTIPASS_COMMON_IMAGE_HOST_H_ */

src/daemon/custom_image_host.cpp

@@ -161,12 +161,16 @@ auto custom_aliases(mp::URLDownloader* url_downloader, const QString& path_prefi
 }
 } // namespace
 
-mp::CustomVMImageHost::CustomVMImageHost(URLDownloader* downloader) : CustomVMImageHost{downloader, ""}
+mp::CustomVMImageHost::CustomVMImageHost(URLDownloader* downloader, std::chrono::seconds manifest_time_to_live)
+    : CustomVMImageHost{downloader, manifest_time_to_live, ""}
 {
 }
 
-mp::CustomVMImageHost::CustomVMImageHost(URLDownloader* downloader, const QString& path_prefix)
-    : url_downloader{downloader},
+mp::CustomVMImageHost::CustomVMImageHost(URLDownloader* downloader, std::chrono::seconds manifest_time_to_live,
+                                         const QString& path_prefix)
+    : CommonVMImageHost{manifest_time_to_live},
+      url_downloader{downloader},
+      path_prefix{path_prefix},
       custom_image_info{custom_aliases(url_downloader, path_prefix)},
       remotes{no_remote, snapcraft_remote}
 {
@@ -195,7 +199,7 @@ std::vector<mp::VMImageInfo> mp::CustomVMImageHost::all_info_for(const Query& qu
     return images;
 }
 
-mp::VMImageInfo mp::CustomVMImageHost::info_for_full_hash(const std::string& full_hash)
+mp::VMImageInfo mp::CustomVMImageHost::info_for_full_hash_impl(const std::string& full_hash)
 {
     return {};
 }
@@ -213,7 +217,7 @@ std::vector<mp::VMImageInfo> mp::CustomVMImageHost::all_images_for(const std::st
     return images;
 }
 
-void mp::CustomVMImageHost::for_each_entry_do(const Action& action)
+void mp::CustomVMImageHost::for_each_entry_do_impl(const Action& action)
 {
     for (const auto& manifest : custom_image_info)
     {
@@ -229,8 +233,26 @@ std::vector<std::string> mp::CustomVMImageHost::supported_remotes()
     return remotes;
 }
 
+void mp::CustomVMImageHost::fetch_manifests()
+{
+    custom_image_info = custom_aliases(url_downloader, path_prefix);
+}
+
+
+bool mp::CustomVMImageHost::empty() const
+{
+    return custom_image_info.empty();
+}
+
+void mp::CustomVMImageHost::clear()
+{
+    custom_image_info.clear();
+}
+
 mp::CustomManifest* mp::CustomVMImageHost::manifest_from(const std::string& remote_name)
 {
+    update_manifests();
+
     auto it = custom_image_info.find(remote_name);
     if (it == custom_image_info.end())
         throw std::runtime_error(fmt::format("Remote \"{}\" is unknown.", remote_name));

src/daemon/custom_image_host.h

@@ -18,7 +18,7 @@
 #ifndef MULTIPASS_CUSTOM_IMAGE_HOST
 #define MULTIPASS_CUSTOM_IMAGE_HOST
 
-#include <multipass/vm_image_host.h>
+#include "common_image_host.h"
 
 #include <QString>
 
@@ -37,24 +37,30 @@ struct CustomManifest
     const std::unordered_map<std::string, const VMImageInfo*> image_records;
 };
 
-class CustomVMImageHost final : public VMImageHost
+class CustomVMImageHost final : public CommonVMImageHost
 {
 public:
-    CustomVMImageHost(URLDownloader* downloader);
+    CustomVMImageHost(URLDownloader* downloader, std::chrono::seconds manifest_time_to_live);
     // For testing
-    CustomVMImageHost(URLDownloader* downloader, const QString& path_prefix);
+    CustomVMImageHost(URLDownloader* downloader, std::chrono::seconds manifest_time_to_live, const QString& path_prefix);
 
     optional<VMImageInfo> info_for(const Query& query) override;
     std::vector<VMImageInfo> all_info_for(const Query& query) override;
-    VMImageInfo info_for_full_hash(const std::string& full_hash) override;
     std::vector<VMImageInfo> all_images_for(const std::string& remote_name) override;
-    void for_each_entry_do(const Action& action) override;
     std::vector<std::string> supported_remotes() override;
 
+protected:
+    void for_each_entry_do_impl(const Action& action) override;
+    VMImageInfo info_for_full_hash_impl(const std::string& full_hash) override;
+    void fetch_manifests() override;
+    bool empty() const override;
+    void clear() override;
+
 private:
     CustomManifest* manifest_from(const std::string& remote_name);
 
     URLDownloader* const url_downloader;
+    const QString path_prefix;
     std::unordered_map<std::string, std::unique_ptr<CustomManifest>> custom_image_info;
     std::vector<std::string> remotes;
 };

src/daemon/daemon_config.cpp

@@ -32,13 +32,16 @@
 
 #include <QStandardPaths>
 
+#include <chrono>
 #include <memory>
 
 namespace mp = multipass;
 namespace mpl = multipass::logging;
 
 namespace
 {
+constexpr auto manifest_ttl = std::chrono::minutes{5};
+
 std::string server_name_from(const std::string& server_address)
 {
     auto tokens = mp::utils::split(server_address, ":");
@@ -77,12 +80,12 @@ std::unique_ptr<const mp::DaemonConfig> mp::DaemonConfigBuilder::build()
         factory = platform::vm_backend(data_directory);
     if (image_hosts.empty())
     {
-        image_hosts.push_back(std::make_unique<mp::CustomVMImageHost>(url_downloader.get()));
+        image_hosts.push_back(std::make_unique<mp::CustomVMImageHost>(url_downloader.get(), manifest_ttl));
         image_hosts.push_back(std::make_unique<mp::UbuntuVMImageHost>(
             std::vector<std::pair<std::string, std::string>>{
                 {mp::release_remote, "http://cloud-images.ubuntu.com/releases/"},
                 {mp::daily_remote, "http://cloud-images.ubuntu.com/daily/"}},
-            url_downloader.get(), std::chrono::minutes{5}));
+            url_downloader.get(), manifest_ttl));
     }
     if (vault == nullptr)
     {

src/daemon/ubuntu_image_host.cpp

@@ -74,12 +74,12 @@ auto key_from(const std::string& search_string)
 
 mp::UbuntuVMImageHost::UbuntuVMImageHost(std::vector<std::pair<std::string, std::string>> remotes,
                                          URLDownloader* downloader, std::chrono::seconds manifest_time_to_live)
-    : manifest_time_to_live{manifest_time_to_live}, url_downloader{downloader}, remotes{std::move(remotes)}
+    : CommonVMImageHost{manifest_time_to_live}, url_downloader{downloader}, remotes{std::move(remotes)}
 {
     manifest_single_shot.singleShot(0, [this]() {
         try
         {
-            update_manifest();
+            update_manifests();
         }
         catch (const std::exception& e)
         {
@@ -170,10 +170,8 @@ std::vector<mp::VMImageInfo> mp::UbuntuVMImageHost::all_info_for(const Query& qu
     return images;
 }
 
-mp::VMImageInfo mp::UbuntuVMImageHost::info_for_full_hash(const std::string& full_hash)
+mp::VMImageInfo mp::UbuntuVMImageHost::info_for_full_hash_impl(const std::string& full_hash)
 {
-    update_manifest();
-
     for (const auto& manifest : manifests)
     {
         for (const auto& product : manifest.second->products)
@@ -209,10 +207,8 @@ std::vector<mp::VMImageInfo> mp::UbuntuVMImageHost::all_images_for(const std::st
     return images;
 }
 
-void mp::UbuntuVMImageHost::for_each_entry_do(const Action& action)
+void mp::UbuntuVMImageHost::for_each_entry_do_impl(const Action& action)
 {
-    update_manifest();
-
     for (const auto& manifest : manifests)
     {
         for (const auto& product : manifest.second->products)
@@ -235,25 +231,28 @@ std::vector<std::string> mp::UbuntuVMImageHost::supported_remotes()
     return supported_remotes;
 }
 
-void mp::UbuntuVMImageHost::update_manifest()
+void mp::UbuntuVMImageHost::fetch_manifests()
 {
-    const auto now = std::chrono::steady_clock::now();
-    if ((now - last_update) > manifest_time_to_live || manifests.empty())
+    for (const auto& remote : remotes)
     {
-        manifests.clear();
-
-        for (const auto& remote : remotes)
-        {
-            manifests.emplace_back(
-                std::make_pair(remote.first, download_manifest(QString::fromStdString(remote.second), url_downloader)));
-        }
-        last_update = now;
+        manifests.emplace_back(
+            std::make_pair(remote.first, download_manifest(QString::fromStdString(remote.second), url_downloader)));
     }
 }
 
+bool mp::UbuntuVMImageHost::empty() const
+{
+    return manifests.empty();
+}
+
+void mp::UbuntuVMImageHost::clear()
+{
+    manifests.clear();
+}
+
 mp::SimpleStreamsManifest* mp::UbuntuVMImageHost::manifest_from(const std::string& remote)
 {
-    update_manifest();
+    update_manifests();
 
     auto it = std::find_if(manifests.begin(), manifests.end(),
                            [&remote](const std::pair<std::string, std::unique_ptr<SimpleStreamsManifest>>& element) {

src/daemon/ubuntu_image_host.h

@@ -18,13 +18,12 @@
 #ifndef MULTIPASS_UBUNTU_IMAGE_HOST_H
 #define MULTIPASS_UBUNTU_IMAGE_HOST_H
 
-#include <multipass/simple_streams_manifest.h>
-#include <multipass/vm_image_host.h>
+#include "common_image_host.h"
+#include "multipass/simple_streams_manifest.h"
 
 #include <QString>
 #include <QTimer>
 
-#include <chrono>
 #include <string>
 #include <vector>
 
@@ -34,24 +33,27 @@ constexpr auto release_remote = "release";
 constexpr auto daily_remote = "daily";
 
 class URLDownloader;
-class UbuntuVMImageHost final : public VMImageHost
+class UbuntuVMImageHost final : public CommonVMImageHost
 {
 public:
     UbuntuVMImageHost(std::vector<std::pair<std::string, std::string>> remotes, URLDownloader* downloader,
                       std::chrono::seconds manifest_time_to_live);
+
     optional<VMImageInfo> info_for(const Query& query) override;
     std::vector<VMImageInfo> all_info_for(const Query& query) override;
-    VMImageInfo info_for_full_hash(const std::string& full_hash) override;
     std::vector<VMImageInfo> all_images_for(const std::string& remote_name) override;
-    void for_each_entry_do(const Action& action) override;
     std::vector<std::string> supported_remotes() override;
 
+protected:
+    void for_each_entry_do_impl(const Action& action) override;
+    VMImageInfo info_for_full_hash_impl(const std::string& full_hash) override;
+    void fetch_manifests() override;
+    bool empty() const override;
+    void clear() override;
+
 private:
-    void update_manifest();
     SimpleStreamsManifest* manifest_from(const std::string& remote);
     void match_alias(const QString& key, const VMImageInfo** info, const SimpleStreamsManifest& manifest);
-    std::chrono::seconds manifest_time_to_live;
-    std::chrono::steady_clock::time_point last_update;
     std::vector<std::pair<std::string, std::unique_ptr<SimpleStreamsManifest>>> manifests;
     URLDownloader* const url_downloader;
     std::vector<std::pair<std::string, std::string>> remotes;