From c2a1bcf1fe2fec17bc4a8411923e0c83e7302ad6 Mon Sep 17 00:00:00 2001 From: Tatsuyuki Ishi Date: Thu, 22 Mar 2018 11:26:07 +0900 Subject: [PATCH] Promisify docker --- api/hosts-api.js | 92 +++++++------ lib/docker.js | 328 ++++++++++++++++------------------------------- lib/machines.js | 166 ++++++++++-------------- 3 files changed, 222 insertions(+), 364 deletions(-) diff --git a/api/hosts-api.js b/api/hosts-api.js index d08b60c3..d7aa462e 100644 --- a/api/hosts-api.js +++ b/api/hosts-api.js @@ -292,7 +292,7 @@ hostAPI.get('/images', { title: 'List host images', description: 'List the Docker images available on this host.', - handler: (request, response) => { + handler: async (request, response) => { const { user } = request; if (!users.isAdmin(user)) { response.statusCode = 404; // Not Found @@ -307,15 +307,15 @@ hostAPI.get('/images', { return; } - docker.listImages({ host: hostname }, (error, images) => { - if (error) { - log('[fail] host images', error); - response.statusCode = 503; // Service Unavailable - response.json({ error: 'Host unreachable' }, null, 2); - return; - } + try { + const images = await docker.listImages({ host: hostname }); + response.json(images, null, 2); - }); + } catch (error) { + log('[fail] host images', error); + response.statusCode = 503; // Service Unavailable + response.json({ error: 'Host unreachable' }, null, 2); + } }, examples: [], @@ -324,7 +324,7 @@ hostAPI.get('/images', { hostAPI.get('/version', { title: 'Show host version', - handler: (request, response) => { + handler: async (request, response) => { const { user } = request; if (!users.isAdmin(user)) { response.statusCode = 404; // Not Found @@ -339,15 +339,14 @@ hostAPI.get('/version', { return; } - docker.version({ host: hostname }, (error, version) => { - if (error) { - log('[fail] host version', error); - response.statusCode = 503; // Service Unavailable - response.json({ error: 'Host unreachable' }, null, 2); - return; - } + try { + const version = await docker.version({ host: hostname }); response.json({ docker: version }, null, 2); - }); + } catch (error) { + log('[fail] host version', error); + response.statusCode = 503; // Service Unavailable + response.json({ error: 'Host unreachable' }, null, 2); + } }, examples: [{ @@ -375,7 +374,7 @@ containersAPI.get({ title: 'List containers', description: 'List all Docker containers on this host.', - handler: (request, response) => { + handler: async (request, response) => { const { user } = request; if (!users.isAdmin(user)) { response.statusCode = 404; // Not Found @@ -390,15 +389,14 @@ containersAPI.get({ return; } - docker.listContainers({ host: hostname }, (error, containers) => { - if (error) { - log('[fail] host containers', error); - response.statusCode = 503; // Service Unavailable - response.json({ error: 'Host unreachable' }, null, 2); - return; - } + try { + const containers = await docker.listContainers({ host: hostname }); response.json(containers, null, 2); - }); + } catch (error) { + log('[fail] host containers', error); + response.statusCode = 503; // Service Unavailable + response.json({ error: 'Host unreachable' }, null, 2); + } }, examples: [], @@ -407,7 +405,7 @@ containersAPI.get({ containersAPI.put({ title: 'Create a container', - handler (request, response) { + async handler (request, response) { const { user } = request; if (!user) { response.statusCode = 403; // Forbidden @@ -422,17 +420,15 @@ containersAPI.put({ return; } - machines.spawn(user, projectId, (error, machine) => { - if (error) { - log('[fail] could not spawn machine', error); - response.statusCode = 500; // Internal Server Error - response.json({ error: 'Could not create new container' }, null, 2); - return; - } - + try { + const machine = await machines.spawn(user, projectId); response.statusCode = 201; // Created response.json({ container: machine.docker.container }, null, 2); - }); + } catch (error) { + log('[fail] could not spawn machine', error); + response.statusCode = 500; // Internal Server Error + response.json({ error: 'Could not create new container' }, null, 2); + } }, examples: [] @@ -581,10 +577,10 @@ containerAPI.delete({ containerAPI.get('/changes', { title: 'List changed files in a container', description: - 'List all files that were modified (Kind: 0), added (1) or deleted (2) ' + - 'in a given Docker container.', + 'List all files that were modified (Kind: 0), added (1) or deleted (2) ' + + 'in a given Docker container.', - handler: (request, response) => { + handler: async (request, response) => { const { user } = request; if (!user) { response.statusCode = 403; // Forbidden @@ -609,16 +605,14 @@ containerAPI.get('/changes', { } const parameters = { host: hostname, container }; - docker.listChangedFilesInContainer(parameters, (error, changedFiles) => { - if (error) { - log('[fail] container changes', error); - response.statusCode = 503; // Service Unavailable - response.json({ error: 'Host unreachable' }, null, 2); - return; - } - + try { + const changedFiles = await docker.listChangedFilesInContainer(parameters); response.json(changedFiles, null, 2); - }); + } catch (error) { + log('[fail] container changes', error); + response.statusCode = 503; // Service Unavailable + response.json({ error: 'Host unreachable' }, null, 2); + } }, examples: [{ diff --git a/lib/docker.js b/lib/docker.js index b0b505f9..1575be32 100644 --- a/lib/docker.js +++ b/lib/docker.js @@ -9,18 +9,16 @@ const util = require('util'); const db = require('./db'); const hosts = require('./hosts'); -const log = require('./log'); // Get client access to a given Docker host. -function getDocker (hostname, callback) { +function getDocker (hostname) { const host = hosts.get(hostname); if (!host) { - callback(new Error('Unknown Docker host: ' + hostname)); - return; + throw new Error('Unknown Docker host: ' + hostname); } const { ca, client } = db.get('tls'); - const docker = new Dockerode({ + return new Dockerode({ protocol: 'https', host: hostname, port: Number(host.properties.port), @@ -28,302 +26,196 @@ function getDocker (hostname, callback) { cert: host.properties.cert || host.properties.crt || client.crt, key: host.properties.key || client.key }); - - callback(null, docker); } // List all Docker images on a given host. -exports.listImages = function (parameters, callback) { - const { host } = parameters; - - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } - - docker.listImages({ all: 1 }, (error, images) => { - callback(error, images); - }); - }); +exports.listImages = function ({ host }) { + const docker = getDocker(host); + return docker.listImages({ all: 1 }); }; // Build a Docker image from a given Dockerfile. -exports.buildImage = function (parameters, callback) { +exports.buildImage = function (parameters) { const { host, tag, dockerfile } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); + + // Add the Dockerfile to a tar stream for Docker's Remote API. + const pack = tar.pack(); + pack.entry({ name: 'Dockerfile' }, dockerfile); + pack.finalize(); - // Add the Dockerfile to a tar stream for Docker's Remote API. - const pack = tar.pack(); - pack.entry({ name: 'Dockerfile' }, dockerfile); - pack.finalize(); - - // FIXME: If `docker.buildImage()` ever supports streams, use the tar stream - // directly instead of flushing it into a Buffer. - const chunks = []; - pack.on('data', chunk => { chunks.push(chunk); }); - pack.on('end', () => { - const buffer = Buffer.concat(chunks); - const options = { - t: tag, - nocache: true - }; - - docker.buildImage(buffer, options, (error, response) => { - if (error) { - callback(error); - return; - } + // FIXME: If `docker.buildImage()` ever supports streams, use the tar stream + // directly instead of flushing it into a Buffer. + const chunks = []; + pack.on('data', chunk => { chunks.push(chunk); }); + return new Promise((resolve, reject) => { + pack.on('end', async () => { + try { + const buffer = Buffer.concat(chunks); + const options = { + t: tag, + nocache: true + }; + + const response = await docker.buildImage(buffer, options); // Transform Docker's response into a proper Node.js Stream. const dockerResponse = new DockerResponse(); response.pipe(dockerResponse); - callback(null, dockerResponse); - }); + resolve(dockerResponse); + } catch (error) { + reject(error); + } }); }); }; // Pull a Docker image into a given host. -exports.pullImage = function (parameters, callback) { +exports.pullImage = function (parameters) { const { host, image: imageId } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } - - docker.pull(imageId, function (error, stream) { - if (error) { - callback(error); - return; - } - - callback(null, stream); - }); - }); + const docker = getDocker(host); + return docker.pull(imageId); }; // Get low-level information on a Docker image from a given host. -exports.inspectImage = function (parameters, callback) { +exports.inspectImage = function (parameters) { const { host, image: imageId } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } - - const image = docker.getImage(imageId); - image.inspect((error, data) => { - callback(error, data); - }); - }); + const docker = getDocker(host); + const image = docker.getImage(imageId); + return image.inspect(); }; // Tag a Docker image in a given host. -exports.tagImage = function (parameters, callback) { +exports.tagImage = function (parameters) { const { host, image: imageId, tag: tagId } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - const image = docker.getImage(imageId); - const [ repo, tag = 'latest' ] = tagId.split(':'); - image.tag({ repo, tag }, (error, data) => { - callback(error); - }); - }); + const image = docker.getImage(imageId); + const [ repo, tag = 'latest' ] = tagId.split(':'); + return image.tag({ repo, tag }); }; // Delete a Docker image from a given host. -exports.removeImage = function (parameters, callback) { +exports.removeImage = function (parameters) { const { host, image: imageId } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - const image = docker.getImage(imageId); - image.remove((error, data) => { - callback(error); - }); - }); + const image = docker.getImage(imageId); + return image.remove(); }; // List all Docker containers on a given host. -exports.listContainers = function (parameters, callback) { +exports.listContainers = function (parameters) { const { host } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - docker.listContainers({ all: 1 }, (error, containers) => { - callback(error, containers); - }); - }); + return docker.listContainers({ all: 1 }); }; // Spawn a new Docker container from a given image. -exports.runContainer = function (parameters, callback) { +exports.runContainer = async function (parameters) { const { host, image, ports } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - const options = { - Image: image, - ExposedPorts: {}, - HostConfig: { PortBindings: {} } - }; - - for (const port in ports) { - options.ExposedPorts[port + '/tcp'] = {}; - options.HostConfig.PortBindings[port + '/tcp'] = [{ - HostIp: ports[port].publish ? '0.0.0.0' : '127.0.0.1', - HostPort: String(ports[port].hostPort) - }]; - } + const options = { + Image: image, + ExposedPorts: {}, + HostConfig: { PortBindings: {} } + }; - docker.createContainer(options, (error, container) => { - if (error) { - callback(error, container); - return; - } + for (const port in ports) { + options.ExposedPorts[port + '/tcp'] = {}; + options.HostConfig.PortBindings[port + '/tcp'] = [{ + HostIp: ports[port].publish ? '0.0.0.0' : '127.0.0.1', + HostPort: String(ports[port].hostPort) + }]; + } - container.start((error, logs) => { - callback(error, container, logs); - }); - }); - }); + const container = await docker.createContainer(options); + return { container, logs: await container.start() }; }; // Copy files into a given Docker container. -exports.copyIntoContainer = function (parameters, callback) { +exports.copyIntoContainer = function (parameters) { const { host, container: containerId, files, path } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - // Add the files to a tar stream for Docker's Remote API. - const pack = tar.pack(); - for (const name in files) { - pack.entry({ name }, files[name]); - } - pack.finalize(); - - // FIXME: If `container.putArchive()` ever supports streams, use the tar - // stream directly instead of flushing it into a Buffer. - const chunks = []; - pack.on('data', chunk => { chunks.push(chunk); }); - pack.on('end', () => { - const buffer = Buffer.concat(chunks); - const container = docker.getContainer(containerId); - - container.putArchive(buffer, { path }, (error, response) => { - callback(error); - }); + // Add the files to a tar stream for Docker's Remote API. + const pack = tar.pack(); + for (const name in files) { + pack.entry({ name }, files[name]); + } + pack.finalize(); + + // FIXME: If `container.putArchive()` ever supports streams, use the tar + // stream directly instead of flushing it into a Buffer. + const chunks = []; + pack.on('data', chunk => { chunks.push(chunk); }); + return new Promise((resolve, reject) => { + pack.on('end', async () => { + try { + const buffer = Buffer.concat(chunks); + const container = docker.getContainer(containerId); + + resolve(await container.putArchive(buffer, { path })); + } catch (error) { + reject(error); + } }); }); }; // Execute a specific command inside a given Docker container. -exports.execInContainer = function (parameters, callback) { +exports.execInContainer = async function (parameters) { const { host, container: containerId, command } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); + const container = docker.getContainer(containerId); + const options = { + Cmd: [ '/bin/bash', '-c', command ], + AttachStdout: true, + AttachStderr: true + }; - const container = docker.getContainer(containerId); - const options = { - Cmd: [ '/bin/bash', '-c', command ], - AttachStdout: true, - AttachStderr: true - }; - - container.exec(options, (error, exec) => { - if (error) { - callback(error); - return; - } - - exec.start((error, stream) => { - callback(error, stream); - }); - }); - }); + const exec = await container.exec(options); + return exec.start(); }; // List all files that were modified, added or deleted in a Docker container. -exports.listChangedFilesInContainer = function (parameters, callback) { +exports.listChangedFilesInContainer = function (parameters) { const { host, container: containerId } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - const container = docker.getContainer(containerId); - container.changes((error, changedFiles) => { - callback(error, changedFiles); - }); - }); + const container = docker.getContainer(containerId); + return container.changes(); }; // Kill and delete a Docker container from a given host. -exports.removeContainer = function (parameters, callback) { +exports.removeContainer = function (parameters) { const { host, container: containerId } = parameters; - getDocker(host, (error, docker) => { - if (error) { - callback(error); - return; - } + const docker = getDocker(host); - const container = docker.getContainer(containerId); - container.remove({ force: true }, (error, data) => { - callback(error); - }); - }); + const container = docker.getContainer(containerId); + return container.remove({ force: true }); }; // Get the Docker version of a given host. -exports.version = function (parameters, callback) { +exports.version = function (parameters) { const { host } = parameters; - getDocker(host, (error, docker) => { - if (error) { - log('[fail] could not get the docker client', error); - } - docker.version((error, data) => { - callback(error, data); - }); - }); + const docker = getDocker(host); + return docker.version(); }; // Docker Remote API response stream. diff --git a/lib/machines.js b/lib/machines.js index 30db0093..6ba82977 100644 --- a/lib/machines.js +++ b/lib/machines.js @@ -74,13 +74,8 @@ exports.pull = function (projectId, callback) { const { host, _baseImage: image } = project.docker; const time = Date.now(); - docker.pullImage({ host, image }, (error, stream) => { - if (error) { - log('pull', image, error); - callback(new Error('Could not pull project')); - return; - } - + docker.pullImage({ host, image }).then((stream) => { + let error; log('pull', image, 'started'); streams.set(project.docker, 'logs', stream); @@ -96,13 +91,7 @@ exports.pull = function (projectId, callback) { } // Inspect the pulled image to check its creation time. - docker.inspectImage({ host, image }, (error, data) => { - if (error) { - log('pull-inspect', image, error); - callback(new Error('Problem while inspecting image')); - return; - } - + docker.inspectImage({ host, image }).then(data => { const imageCreated = new Date(data.Created).getTime(); if (imageCreated <= project.data.updated) { // If the pulled image is as old as, or older than the Docker image @@ -115,21 +104,24 @@ exports.pull = function (projectId, callback) { // The pulled image is more recent than the one we currently use in // production. Let's use the newer image, by tagging it appropriately. const { _productionImage: tag } = project.docker; - docker.tagImage({ host, image, tag }, error => { - if (error) { - log('pull-tag', image, tag, error); - callback(new Error('Problem while tagging project')); - return; - } - + docker.tagImage({ host, image, tag }).then(() => { log('pull-tag', image, tag, 'success'); const now = Date.now(); metrics.set(project, 'updated', imageCreated); - metrics.push(project, 'pull-time', [ now, now - time ]); + metrics.push(project, 'pull-time', [now, now - time]); callback(null, { image, created: imageCreated }); + }, (error) => { + log('pull-tag', image, tag, error); + callback(new Error('Problem while tagging project')); }); + }, (error) => { + log('pull-inspect', image, error); + callback(new Error('Problem while inspecting image')); }); }); + }, (error) => { + log('pull', image, error); + callback(new Error('Could not pull project')); }); }; @@ -144,15 +136,10 @@ exports.update = function (projectId, callback) { const { host, update: dockerfile, _productionImage: image } = project.docker; const time = Date.now(); - docker.buildImage({ host, tag: image, dockerfile }, (error, stream) => { - if (error) { - log('update', image, error); - callback(new Error('Unable to update project: ' + projectId)); - return; - } - + docker.buildImage({ host, tag: image, dockerfile }).then(stream => { log('update', image, 'started'); streams.set(project.docker, 'logs', stream); + let error; stream.on('error', err => { log('update', image, err); @@ -169,18 +156,20 @@ exports.update = function (projectId, callback) { log('update', image, 'success'); const now = Date.now(); metrics.set(project, 'updated', now); - metrics.push(project, 'update-time', [ now, now - time ]); + metrics.push(project, 'update-time', [now, now - time]); callback(); }); + }, function (error) { + log('update', image, error); + callback(new Error('Unable to update project: ' + projectId)); }); }; // Instantiate a user machine for a project. (Fast!) -exports.spawn = function (user, projectId, callback) { +exports.spawn = async function (user, projectId) { const project = getProject(projectId); if (!project) { - callback(new Error('Unknown project: ' + projectId)); - return; + throw new Error('Unknown project: ' + projectId); } const machine = getOrCreateNewMachine(user, projectId); @@ -202,35 +191,34 @@ exports.spawn = function (user, projectId, callback) { const time = Date.now(); log('spawn', image, 'started'); - docker.runContainer({ host, image, ports }, (error, container, logs) => { - if (error) { - log('spawn', image, error); - callback(new Error('Unable to start machine for project: ' + projectId)); - return; - } - + let container; + try { + ({ container } = await docker.runContainer({ host, image, ports })); log('spawn', image, 'success', container.id.slice(0, 16)); machine.docker.container = container.id; machine.status = 'started'; + } catch (error) { + log('spawn', image, error); + throw new Error('Unable to start machine for project: ' + projectId); + } - const now = Date.now(); - metrics.push(project, 'spawn-time', [ now, now - time ]); - db.save(); + const now = Date.now(); + metrics.push(project, 'spawn-time', [now, now - time]); + db.save(); - // Install all non-empty user configuration files into this container. - Object.keys(user.configurations).forEach(file => { - if (!user.configurations[file]) { - return; - } - exports.deployConfiguration(user, machine, file).then(() => { - log('spawn-config', file, container.id.slice(0, 16), 'success'); - }).catch(error => { - log('spawn-config', file, container.id.slice(0, 16), error); - }); + // Install all non-empty user configuration files into this container. + Object.keys(user.configurations).forEach(file => { + if (!user.configurations[file]) { + return; + } + exports.deployConfiguration(user, machine, file).then(() => { + log('spawn-config', file, container.id.slice(0, 16), 'success'); + }).catch(error => { + log('spawn-config', file, container.id.slice(0, 16), error); }); - - callback(null, machine); }); + + return machine; }; // Destroy a given user machine and recycle its ports. @@ -258,13 +246,7 @@ exports.destroy = function (user, projectId, machineId, callback) { } log('destroy', containerId.slice(0, 16), 'started'); - docker.removeContainer({ host, container: containerId }, error => { - if (error) { - log('destroy', containerId.slice(0, 16), error); - callback(error); - return; - } - + docker.removeContainer({ host, container: containerId }, () => { // Recycle the machine's name and ports. machine.status = 'new'; machine.docker.container = ''; @@ -272,6 +254,9 @@ exports.destroy = function (user, projectId, machineId, callback) { log('destroy', containerId.slice(0, 16), 'success'); callback(); + }, (error) => { + log('destroy', containerId.slice(0, 16), error); + callback(error); }); }; @@ -296,46 +281,33 @@ exports.deployConfigurationInAllContainers = function (user, file) { }; // Install or overwrite a configuration file in a given user container. -exports.deployConfiguration = function (user, machine, file) { +exports.deployConfiguration = async function (user, machine, file) { const { host, container: containerId } = machine.docker; if (containerId.length < 16 || !/^[0-9a-f]+$/.test(containerId)) { - return Promise.reject(new Error('Invalid container ID: ' + containerId)); + throw new Error('Invalid container ID: ' + containerId); } - return new Promise((resolve, reject) => { - docker.copyIntoContainer({ - host, - container: containerId, - path: '/home/user/', - files: { - [file]: user.configurations[file], - } - }, error => { - if (error) { - reject(error); - return; - } + await docker.copyIntoContainer({ + host, + container: containerId, + path: '/home/user/', + files: { + [file]: user.configurations[file], + } + }); - // FIXME: Remove this workaround when the following Docker bug is fixed: - // https://github.com/docker/docker/issues/21651. - const command = `sudo chown user:user /home/user/${file}`; - docker.execInContainer({ - host, - container: containerId, - command - }, error => { - if (error) { - reject(error); - return; - } - resolve(); - }); - }); + // FIXME: Remove this workaround when the following Docker bug is fixed: + // https://github.com/docker/docker/issues/21651. + const command = `sudo chown user:user /home/user/${file}`; + return docker.execInContainer({ + host, + container: containerId, + command }); }; // Get an available user machine for a project, or create a new one. -function getOrCreateNewMachine (user, projectId) { +function getOrCreateNewMachine(user, projectId) { const project = getProject(projectId); if (!project) { log(new Error('Unknown project: ' + projectId + @@ -389,7 +361,7 @@ function getOrCreateNewMachine (user, projectId) { } // Get a unique available port starting from 42000. -function getPort () { +function getPort() { const ports = db.get('ports'); const port = ports.next || 42000; @@ -400,7 +372,7 @@ function getPort () { } // Find an existing project, or optionally create a new one for that ID. -function getProject (projectId, create) { +function getProject(projectId, create) { const projects = db.get('projects'); let project = projects[projectId]; @@ -446,7 +418,7 @@ function getProject (projectId, create) { // Get a full Docker image ID, with an explicit tag (by default ':latest'). if (!project.docker.hasOwnProperty('_baseImage')) { Object.defineProperty(project.docker, '_baseImage', { - get () { + get() { return this.image + (this.image.includes(':') ? '' : ':latest'); } }); @@ -455,7 +427,7 @@ function getProject (projectId, create) { // Get a hidden internal Docker image tagged with ':janitor-production'. if (!project.docker.hasOwnProperty('_productionImage')) { Object.defineProperty(project.docker, '_productionImage', { - get () { + get() { return this.image.split(':')[0] + ':janitor-production'; } });