Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make image downloads and prepare operations asynchronous #724

Merged
merged 5 commits into from
Apr 19, 2019
Merged

Make image downloads and prepare operations asynchronous #724

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7493f5e
backends: Misc changes not captured in last merge
Apr 5, 2019
6bc4b60
daemon/create: Make image download and prep asynchronous
Apr 8, 2019
f4ee636
url_downloader: Allocate QNetworkAccessManager object as needed
Apr 16, 2019
30475a3
tests: Fix build failures
Apr 16, 2019
bae45c8
review: Add explicit namespace for the type in qRegisterMetaType
Apr 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions include/multipass/url_downloader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2017 Canonical, Ltd.
* Copyright (C) 2017-2019 Canonical, Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
Expand All @@ -26,8 +26,6 @@

#include <QByteArray>
#include <QDateTime>
#include <QNetworkAccessManager>
#include <QNetworkDiskCache>

#include <chrono>

Expand All @@ -50,8 +48,7 @@ class URLDownloader
URLDownloader(const URLDownloader&) = delete;
URLDownloader& operator=(const URLDownloader&) = delete;

QNetworkAccessManager manager;
QNetworkDiskCache network_cache;
const Path cache_dir_path;
std::chrono::milliseconds timeout;
};
}
Expand Down
6 changes: 3 additions & 3 deletions include/multipass/virtual_machine.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,15 @@ class VirtualMachine
virtual void update_state() = 0;

VirtualMachine::State state;
const SSHKeyProvider& key_provider;
const SSHKeyProvider* key_provider;
const std::string vm_name;
std::condition_variable state_wait;
std::mutex state_mutex;

protected:
VirtualMachine(VirtualMachine::State state, const SSHKeyProvider& key_provider, const std::string& vm_name)
VirtualMachine(VirtualMachine::State state, const SSHKeyProvider* key_provider, const std::string& vm_name)
: state{state}, key_provider{key_provider}, vm_name{vm_name} {};
VirtualMachine(const SSHKeyProvider& key_provider, const std::string& vm_name)
VirtualMachine(const SSHKeyProvider* key_provider, const std::string& vm_name)
: VirtualMachine(State::off, key_provider, vm_name){};
VirtualMachine(const VirtualMachine&) = delete;
VirtualMachine& operator=(const VirtualMachine&) = delete;
Expand Down
7 changes: 6 additions & 1 deletion include/multipass/virtual_machine_description.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
#include <multipass/vm_image.h>
#include <string>

#include <QMetaType>

namespace multipass
{
class SSHKeyProvider;
Expand All @@ -40,7 +42,10 @@ class VirtualMachineDescription
std::string ssh_username;
VMImage image;
Path cloud_init_iso;
const SSHKeyProvider& key_provider;
const SSHKeyProvider* key_provider;
};
}

Q_DECLARE_METATYPE(multipass::VirtualMachineDescription)

#endif // MULTIPASS_VIRTUAL_MACHINE_DESCRIPTION_H
3 changes: 2 additions & 1 deletion src/daemon/custom_image_host.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (C) 2018 Canonical, Ltd.
* Copyright (C) 2018-2019 Canonical, Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
Expand All @@ -25,6 +25,7 @@
#include <fmt/format.h>

#include <QMap>
#include <QUrl>
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Needed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, because some other includes that pulled in QUrl in url_downloader.h were removed and now an explicit QUrl include is needed here.


#include <utility>

Expand Down
162 changes: 86 additions & 76 deletions src/daemon/daemon.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ mp::VirtualMachineDescription to_machine_desc(const mp::LaunchRequest* request,
const std::string& mac_addr, const std::string& ssh_username,
const mp::VMImage& image, YAML::Node& meta_data_config,
YAML::Node& user_data_config, YAML::Node& vendor_data_config,
const mp::SSHKeyProvider& key_provider)
const mp::SSHKeyProvider* key_provider)
{
const auto num_cores = request->num_cores() < 1 ? 1 : request->num_cores();
const auto instance_dir = mp::utils::base_dir(image.image_path);
Expand Down Expand Up @@ -550,6 +550,7 @@ mp::Daemon::Daemon(std::unique_ptr<const DaemonConfig> the_config)
get_unique_id(config->data_directory), config->data_directory},
metrics_opt_in{get_metrics_opt_in(config->data_directory)}
{
qRegisterMetaType<mp::VirtualMachineDescription>();
connect_rpc(daemon_rpc, *this);
std::vector<std::string> invalid_specs;
bool mac_addr_missing{false};
Expand Down Expand Up @@ -578,7 +579,7 @@ mp::Daemon::Daemon(std::unique_ptr<const DaemonConfig> the_config)
const auto cloud_init_iso = instance_dir.filePath("cloud-init-config.iso");
mp::VirtualMachineDescription vm_desc{spec.num_cores, spec.mem_size, spec.disk_space,
name, mac_addr, spec.ssh_username,
vm_image, cloud_init_iso, *config->ssh_key_provider};
vm_image, cloud_init_iso, config->ssh_key_provider.get()};

try
{
Expand Down Expand Up @@ -1863,93 +1864,102 @@ void mp::Daemon::create_vm(const CreateRequest* request, grpc::ServerWriter<Crea
create_error.SerializeAsString()));
}

auto query = query_from(request, name);

config->factory->check_hypervisor_support();

auto progress_monitor = [server](int progress_type, int percentage) {
CreateReply create_reply;
create_reply.mutable_launch_progress()->set_percent_complete(std::to_string(percentage));
create_reply.mutable_launch_progress()->set_type((CreateProgress::ProgressTypes)progress_type);
return server->Write(create_reply);
};
QObject::connect(this, &mp::Daemon::on_prepare_finished, this,
[this, server, status_promise, name, start](const mp::VirtualMachineDescription& vm_desc) {
vm_instances[name] = config->factory->create_virtual_machine(vm_desc, *this);
vm_instance_specs[name] = {vm_desc.num_cores,
vm_desc.mem_size,
vm_desc.disk_space,
vm_desc.mac_addr,
config->ssh_username,
VirtualMachine::State::off,
{},
false,
QJsonObject()};
persist_instances();

if (start)
{
LaunchReply reply;
reply.set_create_message("Starting " + name);
server->Write(reply);

auto& vm = vm_instances[name];
vm->start();

reply.set_vm_instance_name(name);
config->update_prompt->populate_if_time_to_show(reply.mutable_update_info());
server->Write(reply);

auto future_watcher = create_future_watcher();
future_watcher->setFuture(QtConcurrent::run(this, &Daemon::async_wait_for_ready_all<LaunchReply>, server,
std::vector<std::string>{name}, status_promise));

}
else
{
status_promise->set_value(grpc::Status::OK);
}
},
Qt::QueuedConnection);

auto prepare_action = [this, server, &name](const VMImage& source_image) -> VMImage {
CreateReply reply;
reply.set_create_message("Preparing image for " + name);
server->Write(reply);
QtConcurrent::run([this, server, request, name, checked_args] {
auto query = query_from(request, name);

return config->factory->prepare_source_image(source_image);
};
auto progress_monitor = [server](int progress_type, int percentage) {
CreateReply create_reply;
create_reply.mutable_launch_progress()->set_percent_complete(std::to_string(percentage));
create_reply.mutable_launch_progress()->set_type((CreateProgress::ProgressTypes)progress_type);
return server->Write(create_reply);
};

auto fetch_type = config->factory->fetch_type();
auto prepare_action = [this, server, &name](const VMImage& source_image) -> VMImage {
CreateReply reply;
reply.set_create_message("Preparing image for " + name);
server->Write(reply);

CreateReply reply;
reply.set_create_message("Creating " + name);
server->Write(reply);
auto vm_image = config->vault->fetch_image(fetch_type, query, prepare_action, progress_monitor);
return config->factory->prepare_source_image(source_image);
};

reply.set_create_message("Configuring " + name);
server->Write(reply);
auto vendor_data_cloud_init_config =
make_cloud_init_vendor_config(*config->ssh_key_provider, request->time_zone(), config->ssh_username);
auto meta_data_cloud_init_config = make_cloud_init_meta_config(name);
auto user_data_cloud_init_config = YAML::Load(request->cloud_init_user_data());
prepare_user_data(user_data_cloud_init_config, vendor_data_cloud_init_config);
config->factory->configure(name, meta_data_cloud_init_config, vendor_data_cloud_init_config);

std::string mac_addr;
while (true)
{
mac_addr = mp::utils::generate_mac_address();
auto fetch_type = config->factory->fetch_type();

auto it = allocated_mac_addrs.find(mac_addr);
if (it == allocated_mac_addrs.end())
{
allocated_mac_addrs.insert(mac_addr);
break;
}
}
auto vm_desc =
to_machine_desc(request, name, checked_args.mem_size, checked_args.disk_space, mac_addr, config->ssh_username,
vm_image, meta_data_cloud_init_config, user_data_cloud_init_config,
vendor_data_cloud_init_config, *config->ssh_key_provider);

config->factory->prepare_instance_image(vm_image, vm_desc);

vm_instances[name] = config->factory->create_virtual_machine(vm_desc, *this);
vm_instance_specs[name] = {vm_desc.num_cores,
vm_desc.mem_size,
vm_desc.disk_space,
vm_desc.mac_addr,
config->ssh_username,
VirtualMachine::State::off,
{},
false,
QJsonObject()};
persist_instances();
CreateReply reply;
reply.set_create_message("Creating " + name);
server->Write(reply);
auto vm_image = config->vault->fetch_image(fetch_type, query, prepare_action, progress_monitor);

if (start)
{
LaunchReply reply;
reply.set_create_message("Starting " + name);
reply.set_create_message("Configuring " + name);
server->Write(reply);
auto vendor_data_cloud_init_config =
make_cloud_init_vendor_config(*config->ssh_key_provider, request->time_zone(), config->ssh_username);
auto meta_data_cloud_init_config = make_cloud_init_meta_config(name);
auto user_data_cloud_init_config = YAML::Load(request->cloud_init_user_data());
prepare_user_data(user_data_cloud_init_config, vendor_data_cloud_init_config);
config->factory->configure(name, meta_data_cloud_init_config, vendor_data_cloud_init_config);

std::string mac_addr;
while (true)
{
mac_addr = mp::utils::generate_mac_address();

auto& vm = vm_instances[name];
vm->start();
auto it = allocated_mac_addrs.find(mac_addr);
if (it == allocated_mac_addrs.end())
{
allocated_mac_addrs.insert(mac_addr);
break;
}
}
auto vm_desc =
to_machine_desc(request, name, checked_args.mem_size, checked_args.disk_space, mac_addr,
config->ssh_username, vm_image, meta_data_cloud_init_config, user_data_cloud_init_config,
vendor_data_cloud_init_config, config->ssh_key_provider.get());

reply.set_vm_instance_name(name);
config->update_prompt->populate_if_time_to_show(reply.mutable_update_info());
server->Write(reply);
config->factory->prepare_instance_image(vm_image, vm_desc);

auto future_watcher = create_future_watcher();
future_watcher->setFuture(QtConcurrent::run(this, &Daemon::async_wait_for_ready_all<LaunchReply>, server,
std::vector<std::string>{name}, status_promise));
}
else
{
status_promise->set_value(grpc::Status::OK);
}
emit on_prepare_finished(vm_desc);
});
}

grpc::Status mp::Daemon::reboot_vm(VirtualMachine& vm)
Expand Down
3 changes: 3 additions & 0 deletions src/daemon/daemon.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ public slots:
virtual void version(const VersionRequest* request, grpc::ServerWriter<VersionReply>* response,
std::promise<grpc::Status>* status_promise);

signals:
void on_prepare_finished(const VirtualMachineDescription& vm_desc);

private:
void persist_instances();
void start_mount(const VirtualMachine::UPtr& vm, const std::string& name, const std::string& source_path,
Expand Down
Loading