Skip to content

Commit af5eec1

Browse files
catsbycatsby
committed
Merge pull request #381 from hashicorp/f-getter-refactor
client/drivers: Refactor to use Getter wrapper
2 parents cb811dd + 0fe2ea9 commit af5eec1

14 files changed

+260
-113
lines changed

‎client/driver/exec.go

+15-22
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,15 @@ package driver
22

33
import (
44
"fmt"
5-
"log"
6-
"path"
75
"path/filepath"
86
"runtime"
97
"syscall"
108
"time"
119

12-
"github.com/hashicorp/go-getter"
1310
"github.com/hashicorp/nomad/client/allocdir"
1411
"github.com/hashicorp/nomad/client/config"
1512
"github.com/hashicorp/nomad/client/driver/executor"
13+
"github.com/hashicorp/nomad/client/getter"
1614
"github.com/hashicorp/nomad/nomad/structs"
1715
)
1816

@@ -55,29 +53,24 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
5553
return nil, fmt.Errorf("missing command for exec driver")
5654
}
5755

56+
// Create a location to download the artifact.
57+
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
58+
if !ok {
59+
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
60+
}
61+
5862
// Check if an artificat is specified and attempt to download it
5963
source, ok := task.Config["artifact_source"]
6064
if ok && source != "" {
6165
// Proceed to download an artifact to be executed.
62-
// We use go-getter to support a variety of protocols, but need to change
63-
// file permissions of the resulted download to be executable
64-
65-
// Create a location to download the artifact.
66-
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
67-
if !ok {
68-
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
69-
}
70-
destDir := filepath.Join(taskDir, allocdir.TaskLocal)
71-
72-
artifactName := path.Base(source)
73-
artifactFile := filepath.Join(destDir, artifactName)
74-
if err := getter.GetFile(artifactFile, source); err != nil {
75-
return nil, fmt.Errorf("Error downloading artifact for Exec driver: %s", err)
76-
}
77-
78-
// Add execution permissions to the newly downloaded artifact
79-
if err := syscall.Chmod(artifactFile, 0755); err != nil {
80-
log.Printf("[ERR] driver.exec: Error making artifact executable: %s", err)
66+
_, err := getter.GetArtifact(
67+
filepath.Join(taskDir, allocdir.TaskLocal),
68+
task.Config["artifact_source"],
69+
task.Config["checksum"],
70+
d.logger,
71+
)
72+
if err != nil {
73+
return nil, err
8174
}
8275
}
8376

‎client/driver/exec_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -123,11 +123,12 @@ func TestExecDriver_Start_Wait(t *testing.T) {
123123
func TestExecDriver_Start_Artifact_basic(t *testing.T) {
124124
ctestutils.ExecCompatible(t)
125125
file := "hi_linux_amd64"
126+
checksum := "sha256:6f99b4c5184726e601ecb062500aeb9537862434dfe1898dbe5c68d9f50c179c"
126127

127128
task := &structs.Task{
128129
Name: "sleep",
129130
Config: map[string]string{
130-
"artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file),
131+
"artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s?checksum=%s", file, checksum),
131132
"command": filepath.Join("$NOMAD_TASK_DIR", file),
132133
},
133134
Resources: basicResources,

‎client/driver/java.go

+12-15
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,16 @@ import (
44
"bytes"
55
"fmt"
66
"os/exec"
7-
"path"
87
"path/filepath"
98
"runtime"
109
"strings"
1110
"syscall"
1211
"time"
1312

14-
"github.com/hashicorp/go-getter"
1513
"github.com/hashicorp/nomad/client/allocdir"
1614
"github.com/hashicorp/nomad/client/config"
1715
"github.com/hashicorp/nomad/client/driver/executor"
16+
"github.com/hashicorp/nomad/client/getter"
1817
"github.com/hashicorp/nomad/nomad/structs"
1918
)
2019

@@ -89,26 +88,24 @@ func (d *JavaDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool,
8988
}
9089

9190
func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
92-
// Get the jar source
93-
source, ok := task.Config["jar_source"]
94-
if !ok || source == "" {
95-
return nil, fmt.Errorf("missing jar source for Java Jar driver")
96-
}
97-
9891
taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
9992
if !ok {
10093
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
10194
}
10295

103-
destDir := filepath.Join(taskDir, allocdir.TaskLocal)
104-
105-
// Create a location to download the binary.
106-
jarName := path.Base(source)
107-
jarPath := filepath.Join(destDir, jarName)
108-
if err := getter.GetFile(jarPath, source); err != nil {
109-
return nil, fmt.Errorf("Error downloading source for Java driver: %s", err)
96+
// Proceed to download an artifact to be executed.
97+
path, err := getter.GetArtifact(
98+
filepath.Join(taskDir, allocdir.TaskLocal),
99+
task.Config["artifact_source"],
100+
task.Config["checksum"],
101+
d.logger,
102+
)
103+
if err != nil {
104+
return nil, err
110105
}
111106

107+
jarName := filepath.Base(path)
108+
112109
// Get the environment variables.
113110
envVars := TaskEnvironmentVariables(ctx, task)
114111

‎client/driver/java_test.go

+4-7
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,9 @@ func TestJavaDriver_Start_Wait(t *testing.T) {
9797
task := &structs.Task{
9898
Name: "demo-app",
9999
Config: map[string]string{
100-
"jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
101-
// "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar",
102-
// "args": "-d64",
103-
"jvm_options": "-Xmx2048m -Xms256m",
100+
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
101+
"jvm_options": "-Xmx2048m -Xms256m",
102+
"checksum": "sha256:58d6e8130308d32e197c5108edd4f56ddf1417408f743097c2e662df0f0b17c8",
104103
},
105104
Resources: basicResources,
106105
}
@@ -145,9 +144,7 @@ func TestJavaDriver_Start_Kill_Wait(t *testing.T) {
145144
task := &structs.Task{
146145
Name: "demo-app",
147146
Config: map[string]string{
148-
"jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
149-
// "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar",
150-
// "args": "-d64",
147+
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
151148
},
152149
Resources: basicResources,
153150
}

‎client/driver/qemu.go

+12-31
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@ package driver
22

33
import (
44
"bytes"
5-
"crypto/sha256"
6-
"encoding/hex"
75
"encoding/json"
86
"fmt"
9-
"io"
107
"log"
118
"os"
129
"os/exec"
@@ -17,9 +14,9 @@ import (
1714
"strings"
1815
"time"
1916

20-
"github.com/hashicorp/go-getter"
2117
"github.com/hashicorp/nomad/client/allocdir"
2218
"github.com/hashicorp/nomad/client/config"
19+
"github.com/hashicorp/nomad/client/getter"
2320
"github.com/hashicorp/nomad/nomad/structs"
2421
)
2522

@@ -82,7 +79,7 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool,
8279
// image and save it to the Drivers Allocation Dir
8380
func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
8481
// Get the image source
85-
source, ok := task.Config["image_source"]
82+
source, ok := task.Config["artifact_source"]
8683
if !ok || source == "" {
8784
return nil, fmt.Errorf("Missing source image Qemu driver")
8885
}
@@ -99,34 +96,18 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
9996
return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
10097
}
10198

102-
// Create a location to download the binary.
103-
destDir := filepath.Join(taskDir, allocdir.TaskLocal)
104-
vmID := fmt.Sprintf("qemu-vm-%s-%s", structs.GenerateUUID(), filepath.Base(source))
105-
vmPath := filepath.Join(destDir, vmID)
106-
if err := getter.GetFile(vmPath, source); err != nil {
107-
return nil, fmt.Errorf("Error downloading artifact for Qemu driver: %s", err)
99+
// Proceed to download an artifact to be executed.
100+
vmPath, err := getter.GetArtifact(
101+
filepath.Join(taskDir, allocdir.TaskLocal),
102+
task.Config["artifact_source"],
103+
task.Config["checksum"],
104+
d.logger,
105+
)
106+
if err != nil {
107+
return nil, err
108108
}
109109

110-
// compute and check checksum
111-
if check, ok := task.Config["checksum"]; ok {
112-
d.logger.Printf("[DEBUG] Running checksum on (%s)", vmID)
113-
hasher := sha256.New()
114-
file, err := os.Open(vmPath)
115-
if err != nil {
116-
return nil, fmt.Errorf("Failed to open file for checksum")
117-
}
118-
119-
defer file.Close()
120-
io.Copy(hasher, file)
121-
122-
sum := hex.EncodeToString(hasher.Sum(nil))
123-
if sum != check {
124-
return nil, fmt.Errorf(
125-
"Error in Qemu: checksums did not match.\nExpected (%s), got (%s)",
126-
check,
127-
sum)
128-
}
129-
}
110+
vmID := filepath.Base(vmPath)
130111

131112
// Parse configuration arguments
132113
// Create the base arguments

‎client/driver/qemu_test.go

+9-9
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,10 @@ func TestQemuDriver_Start(t *testing.T) {
5454
task := &structs.Task{
5555
Name: "linux",
5656
Config: map[string]string{
57-
"image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
58-
"checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
59-
"accelerator": "tcg",
60-
"guest_ports": "22,8080",
57+
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
58+
"checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
59+
"accelerator": "tcg",
60+
"guest_ports": "22,8080",
6161
},
6262
Resources: &structs.Resources{
6363
MemoryMB: 512,
@@ -103,11 +103,11 @@ func TestQemuDriver_RequiresMemory(t *testing.T) {
103103
task := &structs.Task{
104104
Name: "linux",
105105
Config: map[string]string{
106-
"image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
107-
"accelerator": "tcg",
108-
"host_port": "8080",
109-
"guest_port": "8081",
110-
"checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
106+
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img",
107+
"accelerator": "tcg",
108+
"host_port": "8080",
109+
"guest_port": "8081",
110+
"checksum": "sha256:a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544",
111111
// ssh u/p would be here
112112
},
113113
}

‎client/driver/raw_exec.go

+9-21
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,18 @@ package driver
22

33
import (
44
"fmt"
5-
"log"
65
"os"
76
"os/exec"
8-
"path"
97
"path/filepath"
108
"runtime"
119
"strconv"
1210
"strings"
13-
"syscall"
1411
"time"
1512

16-
"github.com/hashicorp/go-getter"
1713
"github.com/hashicorp/nomad/client/allocdir"
1814
"github.com/hashicorp/nomad/client/config"
1915
"github.com/hashicorp/nomad/client/driver/args"
16+
"github.com/hashicorp/nomad/client/getter"
2017
"github.com/hashicorp/nomad/nomad/structs"
2118
)
2219

@@ -83,23 +80,14 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
8380
source, ok := task.Config["artifact_source"]
8481
if ok && source != "" {
8582
// Proceed to download an artifact to be executed.
86-
// We use go-getter to support a variety of protocols, but need to change
87-
// file permissions of the resulted download to be executable
88-
89-
// Create a location to download the artifact.
90-
destDir := filepath.Join(taskDir, allocdir.TaskLocal)
91-
92-
artifactName := path.Base(source)
93-
artifactFile := filepath.Join(destDir, artifactName)
94-
if err := getter.GetFile(artifactFile, source); err != nil {
95-
return nil, fmt.Errorf("Error downloading artifact for Raw Exec driver: %s", err)
96-
}
97-
98-
// Add execution permissions to the newly downloaded artifact
99-
if runtime.GOOS != "windows" {
100-
if err := syscall.Chmod(artifactFile, 0755); err != nil {
101-
log.Printf("[ERR] driver.raw_exec: Error making artifact executable: %s", err)
102-
}
83+
_, err := getter.GetArtifact(
84+
filepath.Join(taskDir, allocdir.TaskLocal),
85+
task.Config["artifact_source"],
86+
task.Config["checksum"],
87+
d.logger,
88+
)
89+
if err != nil {
90+
return nil, err
10391
}
10492
}
10593

‎client/driver/raw_exec_test.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,22 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
9494
}
9595

9696
func TestRawExecDriver_Start_Artifact_basic(t *testing.T) {
97-
var file string
97+
var file, checksum string
9898
switch runtime.GOOS {
9999
case "darwin":
100100
file = "hi_darwin_amd64"
101+
checksum = "md5:d7f2fdb13b36dcb7407721d78926b335"
101102
default:
102103
file = "hi_linux_amd64"
104+
checksum = "md5:a9b14903a8942748e4f8474e11f795d3"
103105
}
104106

105107
task := &structs.Task{
106108
Name: "sleep",
107109
Config: map[string]string{
108110
"artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file),
109111
"command": filepath.Join("$NOMAD_TASK_DIR", file),
112+
"checksum": checksum,
110113
},
111114
}
112115
driverCtx := testDriverContext(task.Name)

‎client/getter/getter.go

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package getter
2+
3+
import (
4+
"fmt"
5+
"log"
6+
"net/url"
7+
"path"
8+
"path/filepath"
9+
"runtime"
10+
"strings"
11+
"syscall"
12+
13+
gg "github.com/hashicorp/go-getter"
14+
)
15+
16+
func GetArtifact(destDir, source, checksum string, logger *log.Logger) (string, error) {
17+
if source == "" {
18+
return "", fmt.Errorf("Source url is empty in Artifact Getter")
19+
}
20+
u, err := url.Parse(source)
21+
if err != nil {
22+
return "", err
23+
}
24+
25+
// if checksum is seperate, apply to source
26+
if checksum != "" {
27+
source = strings.Join([]string{source, fmt.Sprintf("checksum=%s", checksum)}, "?")
28+
logger.Printf("[DEBUG] client.getter: Applying checksum to Artifact Source URL, new url: %s", source)
29+
}
30+
31+
artifactFile := filepath.Join(destDir, path.Base(u.Path))
32+
if err := gg.GetFile(artifactFile, source); err != nil {
33+
return "", fmt.Errorf("Error downloading artifact: %s", err)
34+
}
35+
36+
// Add execution permissions to the newly downloaded artifact
37+
if runtime.GOOS != "windows" {
38+
if err := syscall.Chmod(artifactFile, 0755); err != nil {
39+
logger.Printf("[ERR] driver.raw_exec: Error making artifact executable: %s", err)
40+
}
41+
}
42+
return artifactFile, nil
43+
}

0 commit comments

Comments
 (0)