Merge pull request #1943 from buildkite/jobs-api

Add current-job api

.gitignore

@@ -1,3 +1,5 @@
+*.DS_STORE
+
 /tmp
 /pkg
 /releases

ACKNOWLEDGEMENTS.md

@@ -1637,6 +1637,35 @@ SOFTWARE.
 ```
 
 
+---
+
+## github.com/go-chi/chi/v5/LICENSE
+
+```
+Copyright (c) 2015-present Peter Kieltyka (https://github.com/pkieltyka), Google Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+```
+
+
 ---
 
 ## github.com/go-logr/logr/LICENSE
@@ -7387,4 +7416,4 @@ limitations under the License.
 ---
 
 File generated using ./scripts/generate-acknowledgements.sh
-Tue 14 Mar 2023 11:57:14 NZDT
+Wed 15 Mar 2023 14:20:09 NZDT

EXPERIMENTS.md

@@ -68,10 +68,19 @@ This will result in errors unless orchestrated in a similar manner to that proje
 
 **Status**: Being used in a preview release of agent-stack-k8s. As it has little applicability outside of Kubernetes, this will not be the default behaviour.
 
-### `descending-spawn-priority`
+### `job-api`
 
-Changes the priority numbering when using `--spawn-with-priority`. By default, priorities start at 1 and increase. Using this experiment, priorities start at -1 and decrease. (Yes, negative priorities are allowed!) This experiment fixes imbalanced work assignment among different hosts with agents that have different values for `--spawn`. 
+Exposes a local API for the currently running job to introspect and mutate its state in the form of environment variables. This allows you to write scripts, hooks and plugins in languages other than bash, using them to interact with the agent.
 
-For example, without this experiment and all other things being equal, a host with `--spawn=3` would normally need to be running at least two jobs before a host with `--spawn=1` would see any work, because the two extra spawn would have higher priorities. With this experiment, one job would be running on both hosts before the additional spawn on the first host are assigned work.
+The API is exposed via a Unix Domain Socket, whose path is exposed to running jobs with the `BUILDKITE_AGENT_JOB_API_SOCKET` envar, and authenticated with a token exposed using the `BUILDKITE_AGENT_JOB_API_TOKEN` envar, using the `Bearer` HTTP Authorization scheme.
 
-**Status**: Likely to become the default in a release soon.
+The API exposes the following endpoints:
+- `GET /api/current-job/v0/env` - returns a JSON object of all environment variables for the current job
+- `PATCH /api/current-job/v0/env` - accepts a JSON object of environment variables to set for the current job
+- `DELETE /api/current-job/v0/env` - accepts a JSON array of environment variable names to unset for the current job
+
+See [jobapi/payloads.go](./jobapi/payloads.go) for the full API request/response definitions.
+
+The Job API is unavailable on windows agents running versions of windows prior to build 17063, as this was when windows added Unix Domain Socket support. Using this experiment on such agents will output a warning, and the API will be unavailable.
+
+**Status:** Experimental while we iron out the API and test it out in the wild. We'll probably promote this to non-experiment soon™️.

agent/agent_configuration.go

@@ -7,6 +7,7 @@ type AgentConfiguration struct {
 	BootstrapScript            string
 	BuildPath                  string
 	HooksPath                  string
+	SocketsPath                string
 	GitMirrorsPath             string
 	GitMirrorsLockTimeout      int
 	GitMirrorsSkipUpdate       bool

agent/job_runner.go

@@ -44,6 +44,33 @@ const (
 	BuildkiteMessageName = "BUILDKITE_MESSAGE"
 )
 
+// Certain env can only be set by agent configuration.
+// We show the user a warning in the bootstrap if they use any of these at a job level.
+var ProtectedEnv = map[string]struct{}{
+	"BUILDKITE_AGENT_ENDPOINT":           {},
+	"BUILDKITE_AGENT_ACCESS_TOKEN":       {},
+	"BUILDKITE_AGENT_DEBUG":              {},
+	"BUILDKITE_AGENT_PID":                {},
+	"BUILDKITE_BIN_PATH":                 {},
+	"BUILDKITE_CONFIG_PATH":              {},
+	"BUILDKITE_BUILD_PATH":               {},
+	"BUILDKITE_GIT_MIRRORS_PATH":         {},
+	"BUILDKITE_GIT_MIRRORS_SKIP_UPDATE":  {},
+	"BUILDKITE_HOOKS_PATH":               {},
+	"BUILDKITE_PLUGINS_PATH":             {},
+	"BUILDKITE_SSH_KEYSCAN":              {},
+	"BUILDKITE_GIT_SUBMODULES":           {},
+	"BUILDKITE_COMMAND_EVAL":             {},
+	"BUILDKITE_PLUGINS_ENABLED":          {},
+	"BUILDKITE_LOCAL_HOOKS_ENABLED":      {},
+	"BUILDKITE_GIT_CLONE_FLAGS":          {},
+	"BUILDKITE_GIT_FETCH_FLAGS":          {},
+	"BUILDKITE_GIT_CLONE_MIRROR_FLAGS":   {},
+	"BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT": {},
+	"BUILDKITE_GIT_CLEAN_FLAGS":          {},
+	"BUILDKITE_SHELL":                    {},
+}
+
 type JobRunnerConfig struct {
 	// The configuration of the agent from the CLI
 	AgentConfiguration AgentConfiguration
@@ -534,41 +561,12 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 		env["BUILDKITE_ENV_FILE"] = r.envFile.Name()
 	}
 
-	// Certain env can only be set by agent configuration.
-	// We show the user a warning in the bootstrap if they use any of these at a job level.
-
-	var protectedEnv = []string{
-		"BUILDKITE_AGENT_ENDPOINT",
-		"BUILDKITE_AGENT_ACCESS_TOKEN",
-		"BUILDKITE_AGENT_DEBUG",
-		"BUILDKITE_AGENT_PID",
-		"BUILDKITE_BIN_PATH",
-		"BUILDKITE_CONFIG_PATH",
-		"BUILDKITE_BUILD_PATH",
-		"BUILDKITE_GIT_MIRRORS_PATH",
-		"BUILDKITE_GIT_MIRRORS_SKIP_UPDATE",
-		"BUILDKITE_HOOKS_PATH",
-		"BUILDKITE_PLUGINS_PATH",
-		"BUILDKITE_SSH_KEYSCAN",
-		"BUILDKITE_GIT_SUBMODULES",
-		"BUILDKITE_COMMAND_EVAL",
-		"BUILDKITE_PLUGINS_ENABLED",
-		"BUILDKITE_LOCAL_HOOKS_ENABLED",
-		"BUILDKITE_GIT_CHECKOUT_FLAGS",
-		"BUILDKITE_GIT_CLONE_FLAGS",
-		"BUILDKITE_GIT_FETCH_FLAGS",
-		"BUILDKITE_GIT_CLONE_MIRROR_FLAGS",
-		"BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT",
-		"BUILDKITE_GIT_CLEAN_FLAGS",
-		"BUILDKITE_SHELL",
-	}
-
 	var ignoredEnv []string
 
 	// Check if the user has defined any protected env
-	for _, p := range protectedEnv {
-		if _, exists := r.job.Env[p]; exists {
-			ignoredEnv = append(ignoredEnv, p)
+	for k := range ProtectedEnv {
+		if _, exists := r.job.Env[k]; exists {
+			ignoredEnv = append(ignoredEnv, k)
 		}
 	}
 
@@ -602,6 +600,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) {
 	// Add options from the agent configuration
 	env["BUILDKITE_CONFIG_PATH"] = r.conf.AgentConfiguration.ConfigPath
 	env["BUILDKITE_BUILD_PATH"] = r.conf.AgentConfiguration.BuildPath
+	env["BUILDKITE_SOCKETS_PATH"] = r.conf.AgentConfiguration.SocketsPath
 	env["BUILDKITE_GIT_MIRRORS_PATH"] = r.conf.AgentConfiguration.GitMirrorsPath
 	env["BUILDKITE_GIT_MIRRORS_SKIP_UPDATE"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.GitMirrorsSkipUpdate)
 	env["BUILDKITE_HOOKS_PATH"] = r.conf.AgentConfiguration.HooksPath

bootstrap/api.go

@@ -0,0 +1,50 @@
+package bootstrap
+
+import (
+	"fmt"
+
+	"github.com/buildkite/agent/v3/experiments"
+	"github.com/buildkite/agent/v3/jobapi"
+)
+
+// startJobAPI starts the job API server, iff the job API experiment is enabled, and the OS of the box supports it
+// otherwise it returns a noop cleanup function
+// It also sets the BUILDKITE_AGENT_JOB_API_SOCKET and BUILDKITE_AGENT_JOB_API_TOKEN environment variables
+func (b *Bootstrap) startJobAPI() (cleanup func(), err error) {
+	cleanup = func() {}
+
+	if !experiments.IsEnabled("job-api") {
+		return cleanup, nil
+	}
+
+	if !jobapi.Available() {
+		b.shell.Warningf("The Job API isn't available on this machine, as it's running an unsupported version of Windows")
+		b.shell.Warningf("The Job API is available on Unix agents, and agents running Windows versions after build 17063")
+		b.shell.Warningf("We'll continue to run your job, but you won't be able to use the Job API")
+		return cleanup, nil
+	}
+
+	socketPath, err := jobapi.NewSocketPath(b.Config.SocketsPath)
+	if err != nil {
+		return cleanup, fmt.Errorf("creating job API socket path: %v", err)
+	}
+
+	srv, token, err := jobapi.NewServer(b.shell.Logger, socketPath, b.shell.Env)
+	if err != nil {
+		return cleanup, fmt.Errorf("creating job API server: %v", err)
+	}
+
+	b.shell.Env.Set("BUILDKITE_AGENT_JOB_API_SOCKET", socketPath)
+	b.shell.Env.Set("BUILDKITE_AGENT_JOB_API_TOKEN", token)
+
+	if err := srv.Start(); err != nil {
+		return cleanup, fmt.Errorf("starting Job API server: %v", err)
+	}
+
+	return func() {
+		err = srv.Stop()
+		if err != nil {
+			b.shell.Errorf("Error stopping Job API server: %v", err)
+		}
+	}, nil
+}

bootstrap/bootstrap.go

@@ -109,6 +109,18 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) {
 		}
 	}()
 
+	// Create an empty env for us to keep track of our env changes in
+	b.shell.Env = env.FromSlice(os.Environ())
+
+	// Initialize the job API, iff the experiment is enabled. Noop otherwise
+	cleanup, err := b.startJobAPI()
+	if err != nil {
+		b.shell.Errorf("Error setting up job API: %v", err)
+		return 1
+	}
+
+	defer cleanup()
+
 	// Tear down the environment (and fire pre-exit hook) before we exit
 	defer func() {
 		if err = b.tearDown(ctx); err != nil {
@@ -516,9 +528,6 @@ func (b *Bootstrap) setUp(ctx context.Context) error {
 	var err error
 	defer func() { span.FinishWithError(err) }()
 
-	// Create an empty env for us to keep track of our env changes in
-	b.shell.Env = env.FromSlice(os.Environ())
-
 	// Add the $BUILDKITE_BIN_PATH to the $PATH if we've been given one
 	if b.BinPath != "" {
 		path, _ := b.shell.Env.Get("PATH")

bootstrap/config.go

@@ -110,6 +110,9 @@ type Config struct {
 	// Path where the builds will be run
 	BuildPath string
 
+	// Path where the sockets are stored
+	SocketsPath string
+
 	// Path where the repository mirrors are stored
 	GitMirrorsPath string
 

bootstrap/integration/bootstrap_tester.go

@@ -44,7 +44,15 @@ type BootstrapTester struct {
 }
 
 func NewBootstrapTester() (*BootstrapTester, error) {
-	homeDir, err := os.MkdirTemp("", "home")
+	// The job API experiment adds a unix domain socket to a directory in the home directory
+	// UDS names are limited to 108 characters, so we need to use a shorter home directory
+	// Who knows what's going on in windowsland
+	tmpHomeDir := "/tmp"
+	if runtime.GOOS == "windows" {
+		tmpHomeDir = ""
+	}
+
+	homeDir, err := os.MkdirTemp(tmpHomeDir, "home")
 	if err != nil {
 		return nil, fmt.Errorf("making home directory: %w", err)
 	}