From 0b16d0eb681d10497f0c96dec5e86252510574cc Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 2 Mar 2023 14:50:19 +1300 Subject: [PATCH 01/15] Add .DS_STORE files to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d76a82c430..40b22a80c0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.DS_STORE + /tmp /pkg /releases From f637da906654b456890d53bae6fb0d621a6edb47 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 2 Mar 2023 14:51:20 +1300 Subject: [PATCH 02/15] Export protected environment variables --- agent/job_runner.go | 62 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/agent/job_runner.go b/agent/job_runner.go index 23ee4dc422..759d4505a2 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -44,6 +44,33 @@ const ( BuildkiteMessageName = "BUILDKITE_MESSAGE" ) +// Certain env can only be set by agent configuration. +// We show the user a warning in the bootstrap if they use any of these at a job level. +var ProtectedEnv = map[string]struct{}{ + "BUILDKITE_AGENT_ENDPOINT": {}, + "BUILDKITE_AGENT_ACCESS_TOKEN": {}, + "BUILDKITE_AGENT_DEBUG": {}, + "BUILDKITE_AGENT_PID": {}, + "BUILDKITE_BIN_PATH": {}, + "BUILDKITE_CONFIG_PATH": {}, + "BUILDKITE_BUILD_PATH": {}, + "BUILDKITE_GIT_MIRRORS_PATH": {}, + "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE": {}, + "BUILDKITE_HOOKS_PATH": {}, + "BUILDKITE_PLUGINS_PATH": {}, + "BUILDKITE_SSH_KEYSCAN": {}, + "BUILDKITE_GIT_SUBMODULES": {}, + "BUILDKITE_COMMAND_EVAL": {}, + "BUILDKITE_PLUGINS_ENABLED": {}, + "BUILDKITE_LOCAL_HOOKS_ENABLED": {}, + "BUILDKITE_GIT_CLONE_FLAGS": {}, + "BUILDKITE_GIT_FETCH_FLAGS": {}, + "BUILDKITE_GIT_CLONE_MIRROR_FLAGS": {}, + "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT": {}, + "BUILDKITE_GIT_CLEAN_FLAGS": {}, + "BUILDKITE_SHELL": {}, +} + type JobRunnerConfig struct { // The configuration of the agent from the CLI AgentConfiguration AgentConfiguration @@ -534,41 +561,12 @@ func (r *JobRunner) createEnvironment() ([]string, error) { env["BUILDKITE_ENV_FILE"] = r.envFile.Name() } - // Certain env can only be set by agent configuration. - // We show the user a warning in the bootstrap if they use any of these at a job level. - - var protectedEnv = []string{ - "BUILDKITE_AGENT_ENDPOINT", - "BUILDKITE_AGENT_ACCESS_TOKEN", - "BUILDKITE_AGENT_DEBUG", - "BUILDKITE_AGENT_PID", - "BUILDKITE_BIN_PATH", - "BUILDKITE_CONFIG_PATH", - "BUILDKITE_BUILD_PATH", - "BUILDKITE_GIT_MIRRORS_PATH", - "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE", - "BUILDKITE_HOOKS_PATH", - "BUILDKITE_PLUGINS_PATH", - "BUILDKITE_SSH_KEYSCAN", - "BUILDKITE_GIT_SUBMODULES", - "BUILDKITE_COMMAND_EVAL", - "BUILDKITE_PLUGINS_ENABLED", - "BUILDKITE_LOCAL_HOOKS_ENABLED", - "BUILDKITE_GIT_CHECKOUT_FLAGS", - "BUILDKITE_GIT_CLONE_FLAGS", - "BUILDKITE_GIT_FETCH_FLAGS", - "BUILDKITE_GIT_CLONE_MIRROR_FLAGS", - "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT", - "BUILDKITE_GIT_CLEAN_FLAGS", - "BUILDKITE_SHELL", - } - var ignoredEnv []string // Check if the user has defined any protected env - for _, p := range protectedEnv { - if _, exists := r.job.Env[p]; exists { - ignoredEnv = append(ignoredEnv, p) + for k := range ProtectedEnv { + if _, exists := r.job.Env[k]; exists { + ignoredEnv = append(ignoredEnv, k) } } From a0ff9a0f5ac3a32374d31dcc1c5fa0f0d2d52abd Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 2 Mar 2023 14:56:55 +1300 Subject: [PATCH 03/15] Add Job API This is an API that allows the currently-running job to observe and mutate its own state in the form of environment variables. It's exposed via a Unix Domain Socket on supported OSes, and unsupported ones (Windows prior to build 17063) may not use it --- env/environment.go | 10 + go.mod | 1 + go.sum | 2 + jobapi/available.go | 7 + jobapi/available_windows.go | 38 ++++ jobapi/doc.go | 3 + jobapi/middleware.go | 51 +++++ jobapi/middleware_test.go | 100 ++++++++++ jobapi/payloads.go | 43 ++++ jobapi/routes.go | 154 +++++++++++++++ jobapi/server.go | 145 ++++++++++++++ jobapi/server_test.go | 383 ++++++++++++++++++++++++++++++++++++ jobapi/socket.go | 40 ++++ 13 files changed, 977 insertions(+) create mode 100644 jobapi/available.go create mode 100644 jobapi/available_windows.go create mode 100644 jobapi/doc.go create mode 100644 jobapi/middleware.go create mode 100644 jobapi/middleware_test.go create mode 100644 jobapi/payloads.go create mode 100644 jobapi/routes.go create mode 100644 jobapi/server.go create mode 100644 jobapi/server_test.go create mode 100644 jobapi/socket.go diff --git a/env/environment.go b/env/environment.go index 241164bfa9..33021b28cc 100644 --- a/env/environment.go +++ b/env/environment.go @@ -84,6 +84,16 @@ func (e *Environment) Get(key string) (string, bool) { return v, ok } +// Dump returns a copy of the environment with all keys normalized +func (e Environment) Dump() Environment { + d := make(Environment, len(e)) + for k, v := range e { + d.Set(k, v) + } + + return d +} + // Get a boolean value from environment, with a default for empty. Supports true|false, on|off, 1|0 func (e *Environment) GetBool(key string, defaultValue bool) bool { v, _ := e.Get(key) diff --git a/go.mod b/go.mod index 3cfa3b9ee9..313508e564 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf github.com/buildkite/roko v1.0.3-0.20221121010703-599521c80157 github.com/gliderlabs/ssh v0.3.5 + github.com/go-chi/chi/v5 v5.0.8 github.com/google/go-cmp v0.5.9 github.com/puzpuzpuz/xsync/v2 v2.4.0 go.opentelemetry.io/contrib/propagators/aws v1.15.0 diff --git a/go.sum b/go.sum index 714a973e3e..fb37f3d747 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= +github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= diff --git a/jobapi/available.go b/jobapi/available.go new file mode 100644 index 0000000000..d817aafb85 --- /dev/null +++ b/jobapi/available.go @@ -0,0 +1,7 @@ +//go:build unix && !windows + +package jobapi + +func Available() bool { + return true +} diff --git a/jobapi/available_windows.go b/jobapi/available_windows.go new file mode 100644 index 0000000000..f4e775ca42 --- /dev/null +++ b/jobapi/available_windows.go @@ -0,0 +1,38 @@ +//go:build windows + +package jobapi + +import ( + "strconv" + + "golang.org/x/sys/windows/registry" +) + +// Available returns true if the job api is available on this machine, which is determined by the OS and OS version running the agent +// The job API uses unix domain sockets, which are only available on unix machines, and on windows machines after build 17063 +// So: +// On all unices, this function will return true +// On windows, it will return true if and only if the build is after 17063 (the first build to support unix sockets) +func Available() bool { + return isAfterBuild17063() +} + +// isAfterBuild17063 returns true if the current build (of windows, this file is only compiled for windows) is after 17063 +// stolen from: https://github.com/golang/go/blob/76c45877c9e72ccc84db787dc08299e0182e0efb/src/net/unixsock_windows_test.go#L17 +func isAfterBuild17063() bool { + k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.READ) + if err != nil { + return false + } + defer k.Close() + + s, _, err := k.GetStringValue("CurrentBuild") + if err != nil { + return false + } + ver, err := strconv.Atoi(s) + if err != nil { + return false + } + return ver >= 17063 +} diff --git a/jobapi/doc.go b/jobapi/doc.go new file mode 100644 index 0000000000..11ec707c37 --- /dev/null +++ b/jobapi/doc.go @@ -0,0 +1,3 @@ +// jobapi provides an API with which to interact with and mutate the currently executing job +// Pronunciation: /ˈdʒɑbapi/ /joh-bah-pee/ +package jobapi diff --git a/jobapi/middleware.go b/jobapi/middleware.go new file mode 100644 index 0000000000..f338a6d388 --- /dev/null +++ b/jobapi/middleware.go @@ -0,0 +1,51 @@ +package jobapi + +import ( + "errors" + "net/http" + "strings" +) + +// AuthMiddleware is a middleware that checks the Authorization header of an incoming request for a Bearer token +// and checks that that token is the correct one. +func AuthMiddleware(token string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + if auth == "" { + writeError(w, errors.New("authorization header is required"), http.StatusUnauthorized) + return + } + + authType, reqToken, found := strings.Cut(auth, " ") + if !found { + writeError(w, errors.New("invalid authorization header: must be in the form `Bearer `"), http.StatusUnauthorized) + return + } + + if authType != "Bearer" { + writeError(w, errors.New("invalid authorization header: type must be Bearer"), http.StatusUnauthorized) + return + } + + if reqToken != token { + writeError(w, errors.New("invalid authorization token"), http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) + } +} + +// HeadersMiddleware is a middleware that sets the common headers for all responses. At the moment, this is just +// Content-Type: application/json. +func HeadersMiddleware() func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer next.ServeHTTP(w, r) + + w.Header().Set("Content-Type", "application/json") + }) + } +} diff --git a/jobapi/middleware_test.go b/jobapi/middleware_test.go new file mode 100644 index 0000000000..b6e648ab56 --- /dev/null +++ b/jobapi/middleware_test.go @@ -0,0 +1,100 @@ +package jobapi_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/buildkite/agent/v3/jobapi" + "github.com/google/go-cmp/cmp" +) + +func testHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("{}")) +} + +func TestAuthMdlw(t *testing.T) { + t.Parallel() + + token := "llamas" + cases := []struct { + title string + auth string + wantCode int + wantBody map[string]string + }{ + { + title: "valid token", + auth: fmt.Sprintf("Bearer %s", token), + wantCode: http.StatusOK, + wantBody: map[string]string{}, + }, + { + title: "invalid token", + auth: "Bearer alpacas", + wantCode: http.StatusUnauthorized, + wantBody: map[string]string{"error": "invalid authorization token"}, + }, + { + title: "non-bearer auth", + auth: fmt.Sprintf("Basic %s", token), + wantCode: http.StatusUnauthorized, + wantBody: map[string]string{"error": "invalid authorization header: type must be Bearer"}, + }, + { + title: "no auth", + auth: "", + wantCode: http.StatusUnauthorized, + wantBody: map[string]string{"error": "authorization header is required"}, + }, + } + + for _, c := range cases { + c := c + t.Run(c.title, func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequest("GET", "/", nil) + req.Header.Add("Authorization", c.auth) + + w := httptest.NewRecorder() + + mdlw := jobapi.AuthMiddleware(token) + wrapped := mdlw(http.HandlerFunc(testHandler)) + wrapped.ServeHTTP(w, req) + + gotCode := w.Result().StatusCode + if c.wantCode != gotCode { + t.Errorf("w.Result().StatusCode = %d (wanted %d)", gotCode, c.wantCode) + } + + var gotBody map[string]string + if err := json.NewDecoder(w.Body).Decode(&gotBody); err != nil { + t.Errorf("json.NewDecoder(w.Body).Decode(&gotBody) = %v", err) + } + + if diff := cmp.Diff(c.wantBody, gotBody); diff != "" { + t.Errorf("cmp.Diff(c.wantBody, gotBody) = %s (-want +got)", diff) + } + }) + } +} + +func TestHeadersMdlw(t *testing.T) { + t.Parallel() + + mdlw := jobapi.HeadersMiddleware() + req := httptest.NewRequest("GET", "/", nil) + w := httptest.NewRecorder() + + wrapped := mdlw(http.HandlerFunc(testHandler)) + wrapped.ServeHTTP(w, req) + + gotHeader := w.Header().Get("Content-Type") + if gotHeader != "application/json" { + t.Errorf("w.Header().Get(\"Content-Type\") = %s (wanted %s)", gotHeader, "application/json") + } +} diff --git a/jobapi/payloads.go b/jobapi/payloads.go new file mode 100644 index 0000000000..979e3cbff1 --- /dev/null +++ b/jobapi/payloads.go @@ -0,0 +1,43 @@ +package jobapi + +import "sort" + +// Error response is the response body for any errors that occur +type ErrorResponse struct { + Error string `json:"error"` +} + +// EnvGetResponse is the response body for the GET /env endpoint +type EnvGetResponse struct { + Env map[string]string `json:"env"` // Different to EnvUpdateRequest because we don't want to send nulls +} + +// EnvUpdateRequest is the request body for the GET /env endpoint +type EnvUpdateRequest struct { + Env map[string]*string `json:"env"` +} + +// EnvUpdateResponse is the response body for the PATCH /env endpoint +type EnvUpdateResponse struct { + Added []string `json:"added"` + Updated []string `json:"updated"` +} + +func (e EnvUpdateResponse) Normalize() { + sort.Strings(e.Added) + sort.Strings(e.Updated) +} + +// EnvDeleteRequest is the request body for the DELETE /env endpoint +type EnvDeleteRequest struct { + Keys []string `json:"keys"` +} + +// EnvDeleteResponse is the response body for the DELETE /env endpoint +type EnvDeleteResponse struct { + Deleted []string `json:"deleted"` +} + +func (e EnvDeleteResponse) Normalize() { + sort.Strings(e.Deleted) +} diff --git a/jobapi/routes.go b/jobapi/routes.go new file mode 100644 index 0000000000..6e263ebc08 --- /dev/null +++ b/jobapi/routes.go @@ -0,0 +1,154 @@ +package jobapi + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/buildkite/agent/v3/agent" + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "golang.org/x/exp/maps" +) + +// router returns a chi router with the jobapi routes and appropriate middlewares mounted +func (s *Server) router() chi.Router { + r := chi.NewRouter() + r.Use( + middleware.Recoverer, + // middleware.Logger, // REVIEW: Should we log requests to this API? If so, where should we log them to? The job logs? + HeadersMiddleware(), + AuthMiddleware(s.token), + ) + + r.Route("/api/current-job/v0", func(r chi.Router) { + r.Get("/env", s.getEnv) + r.Patch("/env", s.patchEnv) + r.Delete("/env", s.deleteEnv) + }) + + return r +} + +func (s *Server) getEnv(w http.ResponseWriter, _ *http.Request) { + s.mtx.RLock() + normalizedEnv := s.environ.Dump() + s.mtx.RUnlock() + + resp := EnvGetResponse{Env: normalizedEnv} + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) +} + +func (s *Server) patchEnv(w http.ResponseWriter, r *http.Request) { + var req EnvUpdateRequest + err := json.NewDecoder(r.Body).Decode(&req) + defer r.Body.Close() + if err != nil { + writeError(w, fmt.Errorf("failed to decode request body: %w", err), http.StatusBadRequest) + return + } + + added := make([]string, 0, len(req.Env)) + updated := make([]string, 0, len(req.Env)) + protected := checkProtected(maps.Keys(req.Env)) + + if len(protected) > 0 { + writeError( + w, + fmt.Sprintf("the following environment variables are protected, and cannot be modified: % v", protected), + http.StatusUnprocessableEntity, + ) + return + } + + nils := make([]string, 0, len(req.Env)) + + for k, v := range req.Env { + if v == nil { + nils = append(nils, k) + } + } + + if len(nils) > 0 { + writeError( + w, + fmt.Sprintf("removing environment variables (ie setting them to null) is not permitted on this endpoint. The following keys were set to null: % v", nils), + http.StatusUnprocessableEntity, + ) + return + } + + s.mtx.Lock() + for k, v := range req.Env { + if _, ok := s.environ.Get(k); ok { + updated = append(updated, k) + } else { + added = append(added, k) + } + s.environ.Set(k, *v) + } + s.mtx.Unlock() + + resp := EnvUpdateResponse{ + Added: added, + Updated: updated, + } + + resp.Normalize() + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) +} + +func (s *Server) deleteEnv(w http.ResponseWriter, r *http.Request) { + var req EnvDeleteRequest + err := json.NewDecoder(r.Body).Decode(&req) + defer r.Body.Close() + if err != nil { + err = fmt.Errorf("failed to decode request body: %w", err) + writeError(w, err, http.StatusBadRequest) + return + } + + protected := checkProtected(req.Keys) + if len(protected) > 0 { + writeError( + w, + fmt.Sprintf("the following environment variables are protected, and cannot be modified: % v", protected), + http.StatusUnprocessableEntity, + ) + return + } + + s.mtx.Lock() + deleted := make([]string, 0, len(req.Keys)) + for _, k := range req.Keys { + if _, ok := s.environ.Get(k); ok { + deleted = append(deleted, k) + delete(s.environ, k) + } + } + s.mtx.Unlock() + + resp := EnvDeleteResponse{Deleted: deleted} + resp.Normalize() + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) +} + +func checkProtected(candidates []string) []string { + protected := make([]string, 0, len(candidates)) + for _, c := range candidates { + if _, ok := agent.ProtectedEnv[c]; ok { + protected = append(protected, c) + } + } + return protected +} + +func writeError(w http.ResponseWriter, err any, code int) { + w.WriteHeader(code) + json.NewEncoder(w).Encode(ErrorResponse{Error: fmt.Sprint(err)}) +} diff --git a/jobapi/server.go b/jobapi/server.go new file mode 100644 index 0000000000..8d3746e02c --- /dev/null +++ b/jobapi/server.go @@ -0,0 +1,145 @@ +package jobapi + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "net" + "net/http" + "os" + "runtime" + "strings" + "sync" + "time" + + "github.com/buildkite/agent/v3/env" +) + +// Server is a Job API server. It provides an HTTP API with which to interact with the job currently running in the buildkite agent +// and allows jobs to introspect and mutate their own state +type Server struct { + // SocketPath is the path to the socket that the server is (or will be) listening on + SocketPath string + + environ env.Environment + token string + httpSvr *http.Server + started bool + mtx sync.RWMutex +} + +// NewServer creates a new Job API server +// socketPath is the path to the socket on which the server will listen +// environ is the environment which the server will mutate and inspect as part of its operation +func NewServer(socketPath string, environ env.Environment) (server *Server, token string, err error) { + if len(socketPath) >= socketPathLength() { + return nil, "", fmt.Errorf("socket path %s is too long (path length: %d, max %d characters). This is a limitation of your host OS", socketPath, len(socketPath), socketPathLength()) + } + + exists, err := socketExists(socketPath) + if err != nil { + return nil, "", err + } + + if exists { + return nil, "", fmt.Errorf("file already exists at socket path %s", socketPath) + } + + token, err = generateToken(32) + if err != nil { + return nil, "", fmt.Errorf("generating token: %w", err) + } + + return &Server{ + SocketPath: socketPath, + environ: environ, + token: token, + }, token, nil +} + +// Start starts the server in a goroutine, returning an error if the server can't be started +func (s *Server) Start() error { + if s.started { + return errors.New("server already started") + } + + r := s.router() + l, err := net.Listen("unix", s.SocketPath) + if err != nil { + return fmt.Errorf("listening on socket: %w", err) + } + + s.httpSvr = &http.Server{Handler: r} + go func() { + _ = s.httpSvr.Serve(l) + }() + s.started = true + + return nil +} + +// Stop gracefully shuts the server down, blocking until all requests have been served or the grace period has expired +// It returns an error if the server has not been started +func (s *Server) Stop() error { + if !s.started { + return errors.New("server not started") + } + + serverCtx, serverStopCtx := context.WithCancel(context.Background()) + defer serverStopCtx() + + // Shutdown signal with grace period of 30 seconds + shutdownCtx, _ := context.WithTimeout(serverCtx, 10*time.Second) + + go func() { + <-shutdownCtx.Done() + if shutdownCtx.Err() == context.DeadlineExceeded { + // What should we do in this situation? Force a return? Log something? + } + }() + + // Trigger graceful shutdown + err := s.httpSvr.Shutdown(shutdownCtx) + if err != nil { + return fmt.Errorf("shutting down Job API server: %w", err) + } + + return nil +} + +func socketExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + + return false, fmt.Errorf("checking if socket exists: %w", err) +} + +func generateToken(len int) (string, error) { + b := make([]byte, len) + _, err := rand.Read(b) + if err != nil { + return "", fmt.Errorf("reading from random: %w", err) + } + + withEqualses := base64.URLEncoding.EncodeToString(b) + return strings.TrimRight(withEqualses, "="), nil // Trim the equals signs because they're not valid in env vars +} + +func socketPathLength() int { + switch runtime.GOOS { + case "darwin", "freebsd", "openbsd", "netbsd", "dragonfly", "solaris": + return 104 + case "linux": + fallthrough + default: + return 108 + } +} diff --git a/jobapi/server_test.go b/jobapi/server_test.go new file mode 100644 index 0000000000..4d53f08848 --- /dev/null +++ b/jobapi/server_test.go @@ -0,0 +1,383 @@ +package jobapi_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "path" + "testing" + "time" + + "github.com/buildkite/agent/v3/env" + "github.com/buildkite/agent/v3/jobapi" + "github.com/google/go-cmp/cmp" +) + +func pt(s string) *string { + return &s +} + +func testEnviron() env.Environment { + e := env.Environment{} + e.Set("MOUNTAIN", "cotopaxi") + e.Set("CAPITAL", "quito") + + return e +} + +func testServer(e env.Environment) (*jobapi.Server, string, error) { + sockName, err := jobapi.NewSocketPath() + if err != nil { + return nil, "", fmt.Errorf("creating socket path: %w", err) + } + + return jobapi.NewServer(sockName, e) +} + +func testSocketClient(socketPath string) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + DialContext: func(context.Context, string, string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + }, + } +} + +func TestServerStartStop(t *testing.T) { + t.Parallel() + + env := testEnviron() + srv, _, err := testServer(env) + if err != nil { + t.Fatal(err) + } + + err = srv.Start() + if err != nil { + t.Fatal(err) + } + + info, err := os.Stat(srv.SocketPath) + if err != nil { + t.Fatal(err) + } + + isSocket := info.Mode()&os.ModeSocket != 0 + if !isSocket { + t.Fatalf("expected server socket file %s to be socket mode, got %v", srv.SocketPath, info.Mode()) + } + + err = srv.Stop() + if err != nil { + t.Fatal(err) + } + + time.Sleep(100 * time.Millisecond) // Wait for the socket file to be unlinked + info, err = os.Stat(srv.SocketPath) + if err == nil { + t.Fatalf("expected server socket file %s to be removed, got mode %s", srv.SocketPath, info.Mode()) + } + + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected os.Stat(%s) = _, os.ErrNotExist, got %v", srv.SocketPath, err) + } +} + +func TestServerStartStop_WithPreExistingSocket(t *testing.T) { + sockName := path.Join(os.TempDir(), "test-socket-collision.sock") + srv1, _, err := jobapi.NewServer(sockName, env.Environment{}) + if err != nil { + t.Fatalf("expected initial server creation to succeed, got %v", err) + } + + err = srv1.Start() + if err != nil { + t.Fatalf("expected initial server start to succeed, got %v", err) + } + defer srv1.Stop() + + expectedErr := fmt.Sprintf("creating server: file already exists at socket path %s", sockName) + _, _, err = jobapi.NewServer(sockName, env.Environment{}) + if err == nil { + t.Fatalf("expected second server creation to fail with %s, got nil", expectedErr) + } + + if err.Error() != expectedErr { + t.Fatalf("expected second server start to fail with %v, got %v", expectedErr, err) + } +} + +type apiTestCase[Req, Resp any] struct { + name string + requestBody *Req + expectedStatus int + expectedResponseBody *Resp + expectedEnv env.Environment + expectedError *jobapi.ErrorResponse +} + +func TestDeleteEnv(t *testing.T) { + t.Parallel() + + cases := []apiTestCase[jobapi.EnvDeleteRequest, jobapi.EnvDeleteResponse]{ + { + name: "happy case", + requestBody: &jobapi.EnvDeleteRequest{Keys: []string{"MOUNTAIN"}}, + expectedStatus: http.StatusOK, + expectedResponseBody: &jobapi.EnvDeleteResponse{Deleted: []string{"MOUNTAIN"}}, + expectedEnv: env.Environment{"CAPITAL": "quito"}.Dump(), + }, + { + name: "deleting a non-existent key is a no-op", + requestBody: &jobapi.EnvDeleteRequest{Keys: []string{"NATIONAL_PARKS"}}, + expectedStatus: http.StatusOK, + expectedResponseBody: &jobapi.EnvDeleteResponse{Deleted: []string{}}, + expectedEnv: testEnviron(), // ie no change + }, + { + name: "deleting protected keys returns a 422", + requestBody: &jobapi.EnvDeleteRequest{ + Keys: []string{"MOUNTAIN", "CAPITAL", "BUILDKITE_AGENT_PID"}, + }, + expectedStatus: http.StatusUnprocessableEntity, + expectedError: &jobapi.ErrorResponse{ + Error: "the following environment variables are protected, and cannot be modified: [BUILDKITE_AGENT_PID]", + }, + expectedEnv: testEnviron(), // ie no change + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + environ := testEnviron() + srv, token, err := testServer(environ) + if err != nil { + t.Fatalf("creating server: %v", err) + } + + err = srv.Start() + if err != nil { + t.Fatalf("starting server: %v", err) + } + + client := testSocketClient(srv.SocketPath) + + defer func() { + err := srv.Stop() + if err != nil { + t.Fatalf("stopping server: %v", err) + } + }() + + buf := bytes.NewBuffer(nil) + err = json.NewEncoder(buf).Encode(c.requestBody) + if err != nil { + t.Fatal() + } + + req, err := http.NewRequest(http.MethodDelete, "http://bootstrap/api/current-job/v0/env", buf) + if err != nil { + t.Fatalf("creating request: %v", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + testAPI(t, environ, req, client, c) // Ignore arguments, dial socket + }) + } +} + +func TestPatchEnv(t *testing.T) { + t.Parallel() + + cases := []apiTestCase[jobapi.EnvUpdateRequest, jobapi.EnvUpdateResponse]{ + { + name: "happy case", + requestBody: &jobapi.EnvUpdateRequest{ + Env: map[string]*string{ + "MOUNTAIN": pt("chimborazo"), + "CAPITAL": pt("quito"), + "NATIONAL_PARKS": pt("cayambe-coca,el-cajas,galápagos"), + }, + }, + expectedStatus: http.StatusOK, + expectedResponseBody: &jobapi.EnvUpdateResponse{ + Added: []string{"NATIONAL_PARKS"}, + Updated: []string{"CAPITAL", "MOUNTAIN"}, + }, + expectedEnv: env.Environment{ + "MOUNTAIN": "chimborazo", + "NATIONAL_PARKS": "cayambe-coca,el-cajas,galápagos", + "CAPITAL": "quito", + }.Dump(), + }, + { + name: "setting to nil returns a 422", + requestBody: &jobapi.EnvUpdateRequest{ + Env: map[string]*string{ + "NATIONAL_PARKS": nil, + "MOUNTAIN": pt("chimborazo"), + }, + }, + expectedStatus: http.StatusUnprocessableEntity, + expectedError: &jobapi.ErrorResponse{ + Error: "removing environment variables (ie setting them to null) is not permitted on this endpoint. The following keys were set to null: [NATIONAL_PARKS]", + }, + expectedEnv: testEnviron(), // ie no changes + }, + { + name: "setting protected variables returns a 422", + requestBody: &jobapi.EnvUpdateRequest{ + Env: map[string]*string{ + "BUILDKITE_AGENT_PID": pt("12345"), + "MOUNTAIN": pt("antisana"), + }, + }, + expectedStatus: http.StatusUnprocessableEntity, + expectedError: &jobapi.ErrorResponse{ + Error: "the following environment variables are protected, and cannot be modified: [BUILDKITE_AGENT_PID]", + }, + expectedEnv: testEnviron(), // ie no changes + }, + } + + for _, c := range cases { + c := c + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + environ := testEnviron() + srv, token, err := testServer(environ) + if err != nil { + t.Fatalf("creating server: %v", err) + } + + err = srv.Start() + if err != nil { + t.Fatalf("starting server: %v", err) + } + + client := testSocketClient(srv.SocketPath) + + defer func() { + err := srv.Stop() + if err != nil { + t.Fatalf("stopping server: %v", err) + } + }() + + buf := bytes.NewBuffer(nil) + err = json.NewEncoder(buf).Encode(c.requestBody) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest(http.MethodPatch, "http://bootstrap/api/current-job/v0/env", buf) + if err != nil { + t.Fatalf("creating request: %v", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + testAPI(t, environ, req, client, c) + }) + } + +} + +func TestGetEnv(t *testing.T) { + t.Parallel() + + env := testEnviron() + srv, token, err := testServer(env) + if err != nil { + t.Fatalf("creating server: %v", err) + } + + err = srv.Start() + if err != nil { + t.Fatalf("starting server: %v", err) + } + + client := testSocketClient(srv.SocketPath) + + defer func() { + err := srv.Stop() + if err != nil { + t.Fatalf("stopping server: %v", err) + } + }() + + req, err := http.NewRequest(http.MethodGet, "http://bootstrap/api/current-job/v0/env", nil) + if err != nil { + t.Fatalf("creating request: %v", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + testAPI(t, env, req, client, apiTestCase[any, jobapi.EnvGetResponse]{ + expectedStatus: http.StatusOK, + expectedResponseBody: &jobapi.EnvGetResponse{ + Env: testEnviron(), + }, + }) + + env.Set("MOUNTAIN", "chimborazo") + env.Set("NATIONAL_PARKS", "cayambe-coca,el-cajas,galápagos") + + expectedEnv := map[string]string{ + "NATIONAL_PARKS": "cayambe-coca,el-cajas,galápagos", + "MOUNTAIN": "chimborazo", + "CAPITAL": "quito", + } + + // It responds to out-of-band changes to the environment + testAPI(t, env, req, client, apiTestCase[any, jobapi.EnvGetResponse]{ + expectedStatus: http.StatusOK, + expectedResponseBody: &jobapi.EnvGetResponse{ + Env: expectedEnv, + }, + }) +} + +func testAPI[Req, Resp any](t *testing.T, env env.Environment, req *http.Request, client *http.Client, testCase apiTestCase[Req, Resp]) { + resp, err := client.Do(req) + if err != nil { + t.Fatalf("expected no error for client.Do(req) (got %v)", err) + } + + if resp.StatusCode != testCase.expectedStatus { + t.Fatalf("expected status code %d (got %d)", testCase.expectedStatus, resp.StatusCode) + } + + if testCase.expectedResponseBody != nil { + var got Resp + json.NewDecoder(resp.Body).Decode(&got) + if !cmp.Equal(testCase.expectedResponseBody, &got) { + t.Fatalf("\n\texpected response: % #v\n\tgot: % #v\n\tdiff = %s)", *testCase.expectedResponseBody, got, cmp.Diff(testCase.expectedResponseBody, &got)) + } + } + + if testCase.expectedError != nil { + var got jobapi.ErrorResponse + json.NewDecoder(resp.Body).Decode(&got) + if got.Error != testCase.expectedError.Error { + t.Fatalf("expected error %q (got %q)", testCase.expectedError.Error, got.Error) + } + } + + if testCase.expectedEnv != nil { + if !cmp.Equal(testCase.expectedEnv, env) { + t.Fatalf("\n\texpected env: % #v\n\tgot: % #v\n\tdiff = %s)", testCase.expectedEnv, env, cmp.Diff(testCase.expectedEnv, env)) + } + } +} diff --git a/jobapi/socket.go b/jobapi/socket.go new file mode 100644 index 0000000000..25f2d402f7 --- /dev/null +++ b/jobapi/socket.go @@ -0,0 +1,40 @@ +package jobapi + +import ( + "fmt" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/mitchellh/go-homedir" +) + +var random = rand.New(rand.NewSource(time.Now().UnixNano())) + +// NewSocketPath generates a path to a socket file (without actually creating the file itself) that can be used with the +// job api. +// These files are located in a subdirectory of the home dir (`~/.buildkite-agent/sockets/job-api` on unix machines), +// and are named after the PID of the process that runs this function, with a random number appended to the end. +// +// We use the home directory because we want to have strong file permissioning on the sockets - the traditional place to +// put sockets is in /run or /var/run, but these directories require the user to be root to create files in them, which +// isn't something that we can guarantee (or recommend) that users do with the agent. +// +// The other option is to use /tmp, but this is not a great option because files in /tmp are world-writable by default, +// and we don't want other users to be snooping on our socket if we can avoid it +func NewSocketPath() (string, error) { + home, err := homedir.Dir() + if err != nil { + return "", fmt.Errorf("finding home directory: %w", err) + } + + path := filepath.Join(home, ".buildkite-agent", "sockets", "job-api") + err = os.MkdirAll(path, 0700) + if err != nil { + return "", fmt.Errorf("creating socket directory: %w", err) + } + + sockNum := random.Int63() % 100_000 + return filepath.Join(path, fmt.Sprintf("%d-%d.sock", os.Getpid(), sockNum)), nil +} From 3d3074bf9f7670ec14242dca68394db11db6da6a Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 2 Mar 2023 14:57:40 +1300 Subject: [PATCH 04/15] Integrate Job API into the bootstrap under the job-api experiment --- EXPERIMENTS.md | 17 +- bootstrap/api.go | 52 +++++++ bootstrap/bootstrap.go | 15 +- bootstrap/integration/bootstrap_tester.go | 2 +- .../integration/job_api_integration_test.go | 146 ++++++++++++++++++ bootstrap/integration/main_test.go | 4 +- 6 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 bootstrap/api.go create mode 100644 bootstrap/integration/job_api_integration_test.go diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index 049ea90f24..a4fa61dd84 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -68,10 +68,19 @@ This will result in errors unless orchestrated in a similar manner to that proje **Status**: Being used in a preview release of agent-stack-k8s. As it has little applicability outside of Kubernetes, this will not be the default behaviour. -### `descending-spawn-priority` +### `job-api` -Changes the priority numbering when using `--spawn-with-priority`. By default, priorities start at 1 and increase. Using this experiment, priorities start at -1 and decrease. (Yes, negative priorities are allowed!) This experiment fixes imbalanced work assignment among different hosts with agents that have different values for `--spawn`. +Exposes a local API for the currently running job to introspect and mutate its state in the form of environment variables. This allows you to write scripts, hooks and plugins in languages other than bash, using them to interact with the agent. -For example, without this experiment and all other things being equal, a host with `--spawn=3` would normally need to be running at least two jobs before a host with `--spawn=1` would see any work, because the two extra spawn would have higher priorities. With this experiment, one job would be running on both hosts before the additional spawn on the first host are assigned work. +The API is exposed via a Unix Domain Socket, whose path is exposed to running jobs with the `BUILDKITE_AGENT_JOB_API_SOCKET` envar, and authenticated with a token exposed using the `BUILDKITE_AGENT_JOB_API_TOKEN` envar, using the `Bearer` HTTP Authorization scheme. -**Status**: Likely to become the default in a release soon. \ No newline at end of file +The API exposes the following endpoints: +- `GET /api/current-job/v0/env` - returns a JSON object of all environment variables for the current job +- `PATCH /api/current-job/v0/env` - accepts a JSON object of environment variables to set for the current job +- `DELETE /api/current-job/v0/env` - accepts a JSON array of environment variable names to unset for the current job + +See [jobapi/payloads.go](./jobapi/payloads.go) for the full API request/response definitions. + +The Job API is unavailable on windows agents running versions of windows prior to build 17063, as this was when windows added Unix Domain Socket support. Using this experiment on such agents will output a warning, and the API will be unavailable. + +**Status:** Experimental while we iron out the API and test it out in the wild. We'll probably promote this to non-experiment soon™️. diff --git a/bootstrap/api.go b/bootstrap/api.go new file mode 100644 index 0000000000..cfaf7701b7 --- /dev/null +++ b/bootstrap/api.go @@ -0,0 +1,52 @@ +package bootstrap + +import ( + "fmt" + + "github.com/buildkite/agent/v3/experiments" + "github.com/buildkite/agent/v3/jobapi" +) + +// startJobAPI starts the job API server, iff the job API experiment is enabled, and the OS of the box supports it +// otherwise it returns a noop cleanup function +// It also sets the BUILDKITE_AGENT_JOB_API_SOCKET and BUILDKITE_AGENT_JOB_API_TOKEN environment variables +func (b *Bootstrap) startJobAPI() (cleanup func(), err error) { + cleanup = func() {} + + if !experiments.IsEnabled("job-api") { + return cleanup, nil + } + + if !jobapi.Available() { + b.shell.Warningf("The Job API isn't available on this machine, as it's running an unsupported version of Windows") + b.shell.Warningf("The Job API is available on Unix agents, and agents running Windows versions after build 17063") + b.shell.Warningf("We'll continue to run your job, but you won't be able to use the Job API") + return cleanup, nil + } + + socketPath, err := jobapi.NewSocketPath() + if err != nil { + return cleanup, fmt.Errorf("creating job API socket path: %v", err) + } + + srv, token, err := jobapi.NewServer(socketPath, b.shell.Env) + if err != nil { + return cleanup, fmt.Errorf("creating job API server: %v", err) + } + + b.shell.Env.Set("BUILDKITE_AGENT_JOB_API_SOCKET", socketPath) + b.shell.Env.Set("BUILDKITE_AGENT_JOB_API_TOKEN", token) + + if err := srv.Start(); err != nil { + return cleanup, fmt.Errorf("starting Job API server: %v", err) + } + + return func() { + err = srv.Stop() + if err != nil { + b.shell.Errorf("Error stopping Job API server: %v", err) + } else { + b.shell.Commentf("Stopped Job API server") + } + }, nil +} diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 1d0d04a5d1..934ebb3adf 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -109,6 +109,18 @@ func (b *Bootstrap) Run(ctx context.Context) (exitCode int) { } }() + // Create an empty env for us to keep track of our env changes in + b.shell.Env = env.FromSlice(os.Environ()) + + // Initialize the job API, iff the experiment is enabled. Noop otherwise + cleanup, err := b.startJobAPI() + if err != nil { + b.shell.Errorf("Error setting up job API: %v", err) + return 1 + } + + defer cleanup() + // Tear down the environment (and fire pre-exit hook) before we exit defer func() { if err = b.tearDown(ctx); err != nil { @@ -516,9 +528,6 @@ func (b *Bootstrap) setUp(ctx context.Context) error { var err error defer func() { span.FinishWithError(err) }() - // Create an empty env for us to keep track of our env changes in - b.shell.Env = env.FromSlice(os.Environ()) - // Add the $BUILDKITE_BIN_PATH to the $PATH if we've been given one if b.BinPath != "" { path, _ := b.shell.Env.Get("PATH") diff --git a/bootstrap/integration/bootstrap_tester.go b/bootstrap/integration/bootstrap_tester.go index 9fead11df7..f97ac08785 100644 --- a/bootstrap/integration/bootstrap_tester.go +++ b/bootstrap/integration/bootstrap_tester.go @@ -44,7 +44,7 @@ type BootstrapTester struct { } func NewBootstrapTester() (*BootstrapTester, error) { - homeDir, err := os.MkdirTemp("", "home") + homeDir, err := os.MkdirTemp("/tmp", "home") if err != nil { return nil, fmt.Errorf("making home directory: %w", err) } diff --git a/bootstrap/integration/job_api_integration_test.go b/bootstrap/integration/job_api_integration_test.go new file mode 100644 index 0000000000..bfed4d7430 --- /dev/null +++ b/bootstrap/integration/job_api_integration_test.go @@ -0,0 +1,146 @@ +package integration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "testing" + + "github.com/buildkite/agent/v3/jobapi" + "github.com/buildkite/bintest/v3" +) + +func TestBootstrapRunsJobAPI(t *testing.T) { + defer experimentWithUndo("job-api")() + + tester, err := NewBootstrapTester() + if err != nil { + t.Fatalf("NewBootstrapTester() error = %v", err) + } + defer tester.Close() + + tester.ExpectGlobalHook("command").Once().AndCallFunc(func(c *bintest.Call) { + socketPath := c.GetEnv("BUILDKITE_AGENT_JOB_API_SOCKET") + if socketPath == "" { + t.Errorf("Expected BUILDKITE_AGENT_JOB_API_SOCKET to be set") + c.Exit(1) + return + } + + socketToken := c.GetEnv("BUILDKITE_AGENT_JOB_API_TOKEN") + if socketToken == "" { + t.Errorf("Expected BUILDKITE_AGENT_JOB_API_TOKEN to be set") + c.Exit(1) + return + } + + client := &http.Client{ + Transport: &http.Transport{ + DialContext: func(context.Context, string, string) (net.Conn, error) { + return net.Dial("unix", socketPath) + }, + }, + } + + req, err := http.NewRequest(http.MethodGet, "http://bootstrap/api/current-job/v0/env", nil) + if err != nil { + t.Errorf("creating request: %v", err) + c.Exit(1) + return + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", socketToken)) + + resp, err := client.Do(req) + if err != nil { + t.Errorf("sending request: %v", err) + c.Exit(1) + return + } + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Errorf("expected status 200, got %d, body: %s", resp.StatusCode, body) + c.Exit(1) + return + } + + var envResp jobapi.EnvGetResponse + err = json.NewDecoder(resp.Body).Decode(&envResp) + if err != nil { + t.Errorf("decoding env get response: %v", err) + c.Exit(1) + return + } + + for name, val := range envResp.Env { + if val != c.GetEnv(name) { + t.Errorf("expected c.GetEnv(%q) = %s, got %s", name, c.GetEnv(name), val) + c.Exit(1) + return + } + } + + mtn := "chimborazo" + b, err := json.Marshal(jobapi.EnvUpdateRequest{Env: map[string]*string{"MOUNTAIN": &mtn}}) + if err != nil { + t.Errorf("marshaling env update request: %v", err) + c.Exit(1) + return + } + + req, err = http.NewRequest(http.MethodPatch, "http://bootstrap/api/current-job/v0/env", bytes.NewBuffer(b)) + if err != nil { + t.Errorf("creating patch request: %v", err) + c.Exit(1) + return + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", socketToken)) + + resp, err = client.Do(req) + if err != nil { + t.Errorf("sending patch request: %v", err) + c.Exit(1) + return + } + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + t.Errorf("expected status 200, got %d, body: %s", resp.StatusCode, body) + c.Exit(1) + return + } + + var patchResp jobapi.EnvUpdateResponse + err = json.NewDecoder(resp.Body).Decode(&patchResp) + if err != nil { + t.Errorf("decoding env get response: %v", err) + c.Exit(1) + return + } + + if patchResp.Added[0] != "MOUNTAIN" { + t.Errorf("expected patchResp.Added[0] = %q, got %s", mtn, patchResp.Added[0]) + c.Exit(1) + return + } + + c.Exit(0) + }) + + tester.ExpectGlobalHook("post-command").Once().AndExitWith(0).AndCallFunc(func(c *bintest.Call) { + if got, want := c.GetEnv("MOUNTAIN"), "chimborazo"; got != want { + fmt.Fprintf(c.Stderr, "MOUNTAIN = %q, want %q\n", got, want) + c.Exit(1) + } else { + c.Exit(0) + } + }) + + tester.RunAndCheck(t) +} diff --git a/bootstrap/integration/main_test.go b/bootstrap/integration/main_test.go index 86c01a46f7..b55c57b0e9 100644 --- a/bootstrap/integration/main_test.go +++ b/bootstrap/integration/main_test.go @@ -30,12 +30,12 @@ func TestMain(m *testing.M) { os.Exit(0) } - if os.Getenv(`BINTEST_DEBUG`) == "1" { + if os.Getenv("BINTEST_DEBUG") == "1" { bintest.Debug = true } // Support running the test suite against a given experiment - if exp := os.Getenv(`TEST_EXPERIMENT`); exp != "" { + if exp := os.Getenv("TEST_EXPERIMENT"); exp != "" { experiments.Enable(exp) fmt.Printf("!!! Enabling experiment %q for test suite\n", exp) } From 5929a2d02f8a1c2574521ccb77502fea8f0443f0 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Thu, 2 Mar 2023 19:30:53 +1300 Subject: [PATCH 05/15] Add super basic logging middleware to the Job API --- bootstrap/api.go | 4 +--- jobapi/middleware.go | 25 ++++++++++++++++++------- jobapi/routes.go | 4 ++-- jobapi/server.go | 11 +++++++++-- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/bootstrap/api.go b/bootstrap/api.go index cfaf7701b7..b500729ad7 100644 --- a/bootstrap/api.go +++ b/bootstrap/api.go @@ -29,7 +29,7 @@ func (b *Bootstrap) startJobAPI() (cleanup func(), err error) { return cleanup, fmt.Errorf("creating job API socket path: %v", err) } - srv, token, err := jobapi.NewServer(socketPath, b.shell.Env) + srv, token, err := jobapi.NewServer(b.shell.Logger, socketPath, b.shell.Env) if err != nil { return cleanup, fmt.Errorf("creating job API server: %v", err) } @@ -45,8 +45,6 @@ func (b *Bootstrap) startJobAPI() (cleanup func(), err error) { err = srv.Stop() if err != nil { b.shell.Errorf("Error stopping Job API server: %v", err) - } else { - b.shell.Commentf("Stopped Job API server") } }, nil } diff --git a/jobapi/middleware.go b/jobapi/middleware.go index f338a6d388..3368b793a3 100644 --- a/jobapi/middleware.go +++ b/jobapi/middleware.go @@ -4,8 +4,21 @@ import ( "errors" "net/http" "strings" + "time" + + "github.com/buildkite/agent/v3/bootstrap/shell" ) +func LoggerMiddleware(l shell.Logger) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t := time.Now() + defer l.Commentf("Job API:\t%s\t%s\t%s", r.Method, r.URL.Path, time.Since(t)) + next.ServeHTTP(w, r) + }) + } +} + // AuthMiddleware is a middleware that checks the Authorization header of an incoming request for a Bearer token // and checks that that token is the correct one. func AuthMiddleware(token string) func(http.Handler) http.Handler { @@ -40,12 +53,10 @@ func AuthMiddleware(token string) func(http.Handler) http.Handler { // HeadersMiddleware is a middleware that sets the common headers for all responses. At the moment, this is just // Content-Type: application/json. -func HeadersMiddleware() func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - defer next.ServeHTTP(w, r) +func HeadersMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer next.ServeHTTP(w, r) - w.Header().Set("Content-Type", "application/json") - }) - } + w.Header().Set("Content-Type", "application/json") + }) } diff --git a/jobapi/routes.go b/jobapi/routes.go index 6e263ebc08..3a8375fa7d 100644 --- a/jobapi/routes.go +++ b/jobapi/routes.go @@ -15,9 +15,9 @@ import ( func (s *Server) router() chi.Router { r := chi.NewRouter() r.Use( + LoggerMiddleware(s.Logger), middleware.Recoverer, - // middleware.Logger, // REVIEW: Should we log requests to this API? If so, where should we log them to? The job logs? - HeadersMiddleware(), + HeadersMiddleware, AuthMiddleware(s.token), ) diff --git a/jobapi/server.go b/jobapi/server.go index 8d3746e02c..52ce479d1a 100644 --- a/jobapi/server.go +++ b/jobapi/server.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" ) @@ -22,6 +23,7 @@ import ( type Server struct { // SocketPath is the path to the socket that the server is (or will be) listening on SocketPath string + Logger shell.Logger environ env.Environment token string @@ -33,7 +35,7 @@ type Server struct { // NewServer creates a new Job API server // socketPath is the path to the socket on which the server will listen // environ is the environment which the server will mutate and inspect as part of its operation -func NewServer(socketPath string, environ env.Environment) (server *Server, token string, err error) { +func NewServer(logger shell.Logger, socketPath string, environ env.Environment) (server *Server, token string, err error) { if len(socketPath) >= socketPathLength() { return nil, "", fmt.Errorf("socket path %s is too long (path length: %d, max %d characters). This is a limitation of your host OS", socketPath, len(socketPath), socketPathLength()) } @@ -54,6 +56,7 @@ func NewServer(socketPath string, environ env.Environment) (server *Server, toke return &Server{ SocketPath: socketPath, + Logger: logger, environ: environ, token: token, }, token, nil @@ -77,6 +80,8 @@ func (s *Server) Start() error { }() s.started = true + s.Logger.Commentf("Job API server listening on %s", s.SocketPath) + return nil } @@ -96,7 +101,7 @@ func (s *Server) Stop() error { go func() { <-shutdownCtx.Done() if shutdownCtx.Err() == context.DeadlineExceeded { - // What should we do in this situation? Force a return? Log something? + s.Logger.Warningf("Job API server shutdown timed out, forcing server shutdown") } }() @@ -106,6 +111,8 @@ func (s *Server) Stop() error { return fmt.Errorf("shutting down Job API server: %w", err) } + s.Logger.Commentf("Successfully shut down Job API server") + return nil } From db957ecf558cc75e810457aaeadb7220557311c0 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Fri, 3 Mar 2023 14:50:41 +1300 Subject: [PATCH 06/15] Test cleanups Co-authored-by: Josh Deprez --- jobapi/middleware_test.go | 29 +++++++++++++++++++++++++---- jobapi/server_test.go | 37 +++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/jobapi/middleware_test.go b/jobapi/middleware_test.go index b6e648ab56..590e027471 100644 --- a/jobapi/middleware_test.go +++ b/jobapi/middleware_test.go @@ -16,7 +16,23 @@ func testHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte("{}")) } -func TestAuthMdlw(t *testing.T) { +func shouldCall(t *testing.T) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + }) + } +} + +func shouldNotCall(t *testing.T) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Error("next.ServeHTTP should not be called") + }) + } +} + +func TestAuthMiddleware(t *testing.T) { t.Parallel() token := "llamas" @@ -25,30 +41,35 @@ func TestAuthMdlw(t *testing.T) { auth string wantCode int wantBody map[string]string + next func(http.Handler) http.Handler }{ { title: "valid token", auth: fmt.Sprintf("Bearer %s", token), wantCode: http.StatusOK, wantBody: map[string]string{}, + next: shouldCall(t), }, { title: "invalid token", auth: "Bearer alpacas", wantCode: http.StatusUnauthorized, wantBody: map[string]string{"error": "invalid authorization token"}, + next: shouldNotCall(t), }, { title: "non-bearer auth", auth: fmt.Sprintf("Basic %s", token), wantCode: http.StatusUnauthorized, wantBody: map[string]string{"error": "invalid authorization header: type must be Bearer"}, + next: shouldNotCall(t), }, { title: "no auth", auth: "", wantCode: http.StatusUnauthorized, wantBody: map[string]string{"error": "authorization header is required"}, + next: shouldNotCall(t), }, } @@ -63,7 +84,7 @@ func TestAuthMdlw(t *testing.T) { w := httptest.NewRecorder() mdlw := jobapi.AuthMiddleware(token) - wrapped := mdlw(http.HandlerFunc(testHandler)) + wrapped := mdlw(c.next(http.HandlerFunc(testHandler))) wrapped.ServeHTTP(w, req) gotCode := w.Result().StatusCode @@ -83,10 +104,10 @@ func TestAuthMdlw(t *testing.T) { } } -func TestHeadersMdlw(t *testing.T) { +func TestHeadersMiddleware(t *testing.T) { t.Parallel() - mdlw := jobapi.HeadersMiddleware() + mdlw := jobapi.HeadersMiddleware req := httptest.NewRequest("GET", "/", nil) w := httptest.NewRecorder() diff --git a/jobapi/server_test.go b/jobapi/server_test.go index 4d53f08848..9e1aa44693 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/buildkite/agent/v3/bootstrap/shell" "github.com/buildkite/agent/v3/env" "github.com/buildkite/agent/v3/jobapi" "github.com/google/go-cmp/cmp" @@ -30,13 +31,13 @@ func testEnviron() env.Environment { return e } -func testServer(e env.Environment) (*jobapi.Server, string, error) { +func testServer(t *testing.T, e env.Environment) (*jobapi.Server, string, error) { sockName, err := jobapi.NewSocketPath() if err != nil { return nil, "", fmt.Errorf("creating socket path: %w", err) } - return jobapi.NewServer(sockName, e) + return jobapi.NewServer(shell.TestingLogger{T: t}, sockName, e) } func testSocketClient(socketPath string) *http.Client { @@ -53,19 +54,19 @@ func TestServerStartStop(t *testing.T) { t.Parallel() env := testEnviron() - srv, _, err := testServer(env) + srv, _, err := testServer(t, env) if err != nil { - t.Fatal(err) + t.Fatalf("testServer(t, env) error = %v", err) } err = srv.Start() if err != nil { - t.Fatal(err) + t.Fatalf("srv.Start() = %v", err) } info, err := os.Stat(srv.SocketPath) if err != nil { - t.Fatal(err) + t.Fatalf("os.Stat(%q) error = %v", srv.SocketPath, err) } isSocket := info.Mode()&os.ModeSocket != 0 @@ -75,15 +76,11 @@ func TestServerStartStop(t *testing.T) { err = srv.Stop() if err != nil { - t.Fatal(err) + t.Fatalf("srv.Stop() = %v", err) } time.Sleep(100 * time.Millisecond) // Wait for the socket file to be unlinked - info, err = os.Stat(srv.SocketPath) - if err == nil { - t.Fatalf("expected server socket file %s to be removed, got mode %s", srv.SocketPath, info.Mode()) - } - + _, err = os.Stat(srv.SocketPath) if !errors.Is(err, os.ErrNotExist) { t.Fatalf("expected os.Stat(%s) = _, os.ErrNotExist, got %v", srv.SocketPath, err) } @@ -91,7 +88,7 @@ func TestServerStartStop(t *testing.T) { func TestServerStartStop_WithPreExistingSocket(t *testing.T) { sockName := path.Join(os.TempDir(), "test-socket-collision.sock") - srv1, _, err := jobapi.NewServer(sockName, env.Environment{}) + srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.Environment{}) if err != nil { t.Fatalf("expected initial server creation to succeed, got %v", err) } @@ -102,14 +99,14 @@ func TestServerStartStop_WithPreExistingSocket(t *testing.T) { } defer srv1.Stop() - expectedErr := fmt.Sprintf("creating server: file already exists at socket path %s", sockName) - _, _, err = jobapi.NewServer(sockName, env.Environment{}) + expectedErr := fmt.Sprintf("file already exists at socket path %s", sockName) + _, _, err = jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.Environment{}) if err == nil { t.Fatalf("expected second server creation to fail with %s, got nil", expectedErr) } if err.Error() != expectedErr { - t.Fatalf("expected second server start to fail with %v, got %v", expectedErr, err) + t.Fatalf("expected second server start to fail with %q, got %q", expectedErr, err) } } @@ -159,7 +156,7 @@ func TestDeleteEnv(t *testing.T) { t.Parallel() environ := testEnviron() - srv, token, err := testServer(environ) + srv, token, err := testServer(t, environ) if err != nil { t.Fatalf("creating server: %v", err) } @@ -181,7 +178,7 @@ func TestDeleteEnv(t *testing.T) { buf := bytes.NewBuffer(nil) err = json.NewEncoder(buf).Encode(c.requestBody) if err != nil { - t.Fatal() + t.Fatalf("JSON-encoding c.requestBody into buf: %v", err) } req, err := http.NewRequest(http.MethodDelete, "http://bootstrap/api/current-job/v0/env", buf) @@ -256,7 +253,7 @@ func TestPatchEnv(t *testing.T) { t.Parallel() environ := testEnviron() - srv, token, err := testServer(environ) + srv, token, err := testServer(t, environ) if err != nil { t.Fatalf("creating server: %v", err) } @@ -298,7 +295,7 @@ func TestGetEnv(t *testing.T) { t.Parallel() env := testEnviron() - srv, token, err := testServer(env) + srv, token, err := testServer(t, env) if err != nil { t.Fatalf("creating server: %v", err) } From 1dcc1fb5b6419466ea920c9435ca8c087f61f319 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Fri, 3 Mar 2023 15:31:12 +1300 Subject: [PATCH 07/15] Clean up server startup/shutdown contextx Co-authored-by: Josh Deprez --- jobapi/server.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/jobapi/server.go b/jobapi/server.go index 52ce479d1a..97b5cc4e60 100644 --- a/jobapi/server.go +++ b/jobapi/server.go @@ -92,22 +92,16 @@ func (s *Server) Stop() error { return errors.New("server not started") } - serverCtx, serverStopCtx := context.WithCancel(context.Background()) + // Shutdown signal with grace period of 10 seconds + shutdownCtx, serverStopCtx := context.WithTimeout(context.Background(), 10*time.Second) defer serverStopCtx() - // Shutdown signal with grace period of 30 seconds - shutdownCtx, _ := context.WithTimeout(serverCtx, 10*time.Second) - - go func() { - <-shutdownCtx.Done() - if shutdownCtx.Err() == context.DeadlineExceeded { - s.Logger.Warningf("Job API server shutdown timed out, forcing server shutdown") - } - }() - // Trigger graceful shutdown err := s.httpSvr.Shutdown(shutdownCtx) if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + s.Logger.Warningf("Job API server shutdown timed out, server shutdown forced") + } return fmt.Errorf("shutting down Job API server: %w", err) } From 58e995abd8b3a6bef3d86004023a289f10680654 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Fri, 3 Mar 2023 15:38:34 +1300 Subject: [PATCH 08/15] Fix home dir on windows --- bootstrap/integration/bootstrap_tester.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bootstrap/integration/bootstrap_tester.go b/bootstrap/integration/bootstrap_tester.go index f97ac08785..a75766a424 100644 --- a/bootstrap/integration/bootstrap_tester.go +++ b/bootstrap/integration/bootstrap_tester.go @@ -44,7 +44,15 @@ type BootstrapTester struct { } func NewBootstrapTester() (*BootstrapTester, error) { - homeDir, err := os.MkdirTemp("/tmp", "home") + // The job API experiment adds a unix domain socket to a directory in the home directory + // UDS names are limited to 108 characters, so we need to use a shorter home directory + // Who knows what's going on in windowsland + tmpHomeDir := "/tmp" + if runtime.GOOS == "windows" { + tmpHomeDir = "" + } + + homeDir, err := os.MkdirTemp(tmpHomeDir, "home") if err != nil { return nil, fmt.Errorf("making home directory: %w", err) } From 3e29f1d9f92d6d91ef90064209a3e6a1a6e44a08 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Fri, 3 Mar 2023 16:24:56 +1300 Subject: [PATCH 09/15] Make socket paths configurable --- agent/agent_configuration.go | 1 + agent/job_runner.go | 1 + bootstrap/api.go | 2 +- bootstrap/config.go | 3 +++ clicommand/agent_start.go | 18 ++++++++++++++++++ clicommand/bootstrap.go | 8 ++++++++ jobapi/server_test.go | 2 +- jobapi/socket.go | 22 +++------------------- 8 files changed, 36 insertions(+), 21 deletions(-) diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index 07d407c0ea..6fe70880b2 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -7,6 +7,7 @@ type AgentConfiguration struct { BootstrapScript string BuildPath string HooksPath string + SocketsPath string GitMirrorsPath string GitMirrorsLockTimeout int GitMirrorsSkipUpdate bool diff --git a/agent/job_runner.go b/agent/job_runner.go index 759d4505a2..eeb13ad2de 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -600,6 +600,7 @@ func (r *JobRunner) createEnvironment() ([]string, error) { // Add options from the agent configuration env["BUILDKITE_CONFIG_PATH"] = r.conf.AgentConfiguration.ConfigPath env["BUILDKITE_BUILD_PATH"] = r.conf.AgentConfiguration.BuildPath + env["BUILDKITE_SOCKETS_PATH"] = r.conf.AgentConfiguration.SocketsPath env["BUILDKITE_GIT_MIRRORS_PATH"] = r.conf.AgentConfiguration.GitMirrorsPath env["BUILDKITE_GIT_MIRRORS_SKIP_UPDATE"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.GitMirrorsSkipUpdate) env["BUILDKITE_HOOKS_PATH"] = r.conf.AgentConfiguration.HooksPath diff --git a/bootstrap/api.go b/bootstrap/api.go index b500729ad7..75dd9d626a 100644 --- a/bootstrap/api.go +++ b/bootstrap/api.go @@ -24,7 +24,7 @@ func (b *Bootstrap) startJobAPI() (cleanup func(), err error) { return cleanup, nil } - socketPath, err := jobapi.NewSocketPath() + socketPath, err := jobapi.NewSocketPath(b.Config.SocketsPath) if err != nil { return cleanup, fmt.Errorf("creating job API socket path: %v", err) } diff --git a/bootstrap/config.go b/bootstrap/config.go index c2e2197f3a..112743c1d0 100644 --- a/bootstrap/config.go +++ b/bootstrap/config.go @@ -110,6 +110,9 @@ type Config struct { // Path where the builds will be run BuildPath string + // Path where the sockets are stored + SocketsPath string + // Path where the repository mirrors are stored GitMirrorsPath string diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index d39e3734a4..f6691a0129 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -30,6 +30,7 @@ import ( "github.com/buildkite/agent/v3/utils" "github.com/buildkite/agent/v3/version" "github.com/buildkite/shellwords" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli" "golang.org/x/exp/maps" ) @@ -69,6 +70,7 @@ type AgentStartConfig struct { EnableJobLogTmpfile bool `cli:"enable-job-log-tmpfile"` BuildPath string `cli:"build-path" normalize:"filepath" validate:"required"` HooksPath string `cli:"hooks-path" normalize:"filepath"` + SocketsPath string `cli:"sockets-path" normalize:"filepath"` PluginsPath string `cli:"plugins-path" normalize:"filepath"` Shell string `cli:"shell"` Tags []string `cli:"tags" normalize:"list"` @@ -432,6 +434,12 @@ var AgentStartCommand = cli.Command{ Usage: "Directory where the hook scripts are found", EnvVar: "BUILDKITE_HOOKS_PATH", }, + cli.StringFlag{ + Name: "sockets-path", + Value: defaultSocketsPath(), + Usage: "Directory where the agent will place sockets", + EnvVar: "BUILDKITE_SOCKETS_PATH", + }, cli.StringFlag{ Name: "plugins-path", Value: "", @@ -747,6 +755,7 @@ var AgentStartCommand = cli.Command{ agentConf := agent.AgentConfiguration{ BootstrapScript: cfg.BootstrapScript, BuildPath: cfg.BuildPath, + SocketsPath: cfg.SocketsPath, GitMirrorsPath: cfg.GitMirrorsPath, GitMirrorsLockTimeout: cfg.GitMirrorsLockTimeout, GitMirrorsSkipUpdate: cfg.GitMirrorsSkipUpdate, @@ -1076,3 +1085,12 @@ func agentLifecycleHook(hookName string, log logger.Logger, cfg AgentStartConfig wg.Wait() return nil } + +func defaultSocketsPath() string { + home, err := homedir.Dir() + if err != nil { + return filepath.Join(os.TempDir(), "buildkite-sockets") + } + + return filepath.Join(home, ".buildkite-agent", "sockets") +} diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go index ba170525f6..a257ef7036 100644 --- a/clicommand/bootstrap.go +++ b/clicommand/bootstrap.go @@ -74,6 +74,7 @@ type BootstrapConfig struct { BinPath string `cli:"bin-path" normalize:"filepath"` BuildPath string `cli:"build-path" normalize:"filepath"` HooksPath string `cli:"hooks-path" normalize:"filepath"` + SocketsPath string `cli:"sockets-path" normalize:"filepath"` PluginsPath string `cli:"plugins-path" normalize:"filepath"` CommandEval bool `cli:"command-eval"` PluginsEnabled bool `cli:"plugins-enabled"` @@ -270,6 +271,12 @@ var BootstrapCommand = cli.Command{ Usage: "Directory where the hook scripts are found", EnvVar: "BUILDKITE_HOOKS_PATH", }, + cli.StringFlag{ + Name: "sockets-path", + Value: defaultSocketsPath(), + Usage: "Directory where the agent will place sockets", + EnvVar: "BUILDKITE_SOCKETS_PATH", + }, cli.StringFlag{ Name: "plugins-path", Value: "", @@ -411,6 +418,7 @@ var BootstrapCommand = cli.Command{ BinPath: cfg.BinPath, Branch: cfg.Branch, BuildPath: cfg.BuildPath, + SocketsPath: cfg.SocketsPath, CancelSignal: cancelSig, CleanCheckout: cfg.CleanCheckout, Command: cfg.Command, diff --git a/jobapi/server_test.go b/jobapi/server_test.go index 9e1aa44693..c03527efcd 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -32,7 +32,7 @@ func testEnviron() env.Environment { } func testServer(t *testing.T, e env.Environment) (*jobapi.Server, string, error) { - sockName, err := jobapi.NewSocketPath() + sockName, err := jobapi.NewSocketPath(os.TempDir()) if err != nil { return nil, "", fmt.Errorf("creating socket path: %w", err) } diff --git a/jobapi/socket.go b/jobapi/socket.go index 25f2d402f7..13407231e3 100644 --- a/jobapi/socket.go +++ b/jobapi/socket.go @@ -6,31 +6,15 @@ import ( "os" "path/filepath" "time" - - "github.com/mitchellh/go-homedir" ) var random = rand.New(rand.NewSource(time.Now().UnixNano())) // NewSocketPath generates a path to a socket file (without actually creating the file itself) that can be used with the // job api. -// These files are located in a subdirectory of the home dir (`~/.buildkite-agent/sockets/job-api` on unix machines), -// and are named after the PID of the process that runs this function, with a random number appended to the end. -// -// We use the home directory because we want to have strong file permissioning on the sockets - the traditional place to -// put sockets is in /run or /var/run, but these directories require the user to be root to create files in them, which -// isn't something that we can guarantee (or recommend) that users do with the agent. -// -// The other option is to use /tmp, but this is not a great option because files in /tmp are world-writable by default, -// and we don't want other users to be snooping on our socket if we can avoid it -func NewSocketPath() (string, error) { - home, err := homedir.Dir() - if err != nil { - return "", fmt.Errorf("finding home directory: %w", err) - } - - path := filepath.Join(home, ".buildkite-agent", "sockets", "job-api") - err = os.MkdirAll(path, 0700) +func NewSocketPath(base string) (string, error) { + path := filepath.Join(base, "job-api") + err := os.MkdirAll(path, 0700) if err != nil { return "", fmt.Errorf("creating socket directory: %w", err) } From 4e5ef434b6c0e15c72168312481b98c5c5d209cd Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Mon, 13 Mar 2023 14:23:25 +1300 Subject: [PATCH 10/15] Better socket tests on windows --- jobapi/server_test.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/jobapi/server_test.go b/jobapi/server_test.go index c03527efcd..a45c5d9565 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "path" + "runtime" "testing" "time" @@ -64,16 +65,28 @@ func TestServerStartStop(t *testing.T) { t.Fatalf("srv.Start() = %v", err) } - info, err := os.Stat(srv.SocketPath) - if err != nil { - t.Fatalf("os.Stat(%q) error = %v", srv.SocketPath, err) + // Check the socket path exists and is a socket. + // Note that os.ModeSocket might not be set on Windows. + // (https://github.com/golang/go/issues/33357) + if runtime.GOOS != "windows" { + fi, err := os.Stat(srv.SocketPath) + if err != nil { + t.Fatalf("os.Stat(%q) = %v", srv.SocketPath, err) + } + + if fi.Mode()&os.ModeSocket == 0 { + t.Fatalf("%q is not a socket", srv.SocketPath) + } } - isSocket := info.Mode()&os.ModeSocket != 0 - if !isSocket { - t.Fatalf("expected server socket file %s to be socket mode, got %v", srv.SocketPath, info.Mode()) + // Try to connect to the socket. + test, err := net.Dial("unix", srv.SocketPath) + if err != nil { + t.Fatalf("socket test connection: %v", err) } + test.Close() + err = srv.Stop() if err != nil { t.Fatalf("srv.Stop() = %v", err) From c9f08483ad0b76dcd9c9632ba23b892dba4c05b7 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Mon, 13 Mar 2023 14:44:29 +1300 Subject: [PATCH 11/15] Fix incorrect filepath joining silly me, filepath.Join is OS-aware, but path.Join just puts in a slash! how could i have been so boneheaded! --- jobapi/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobapi/server_test.go b/jobapi/server_test.go index a45c5d9565..648bb0082d 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -9,7 +9,7 @@ import ( "net" "net/http" "os" - "path" + "path/filepath" "runtime" "testing" "time" @@ -100,7 +100,7 @@ func TestServerStartStop(t *testing.T) { } func TestServerStartStop_WithPreExistingSocket(t *testing.T) { - sockName := path.Join(os.TempDir(), "test-socket-collision.sock") + sockName := filepath.Join(os.TempDir(), "test-socket-collision.sock") srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.Environment{}) if err != nil { t.Fatalf("expected initial server creation to succeed, got %v", err) From 44c346c2e521165d5eac00fa41d93fee8aeedeae Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Mon, 13 Mar 2023 14:52:33 +1300 Subject: [PATCH 12/15] Don't stat sockets on windows --- jobapi/server.go | 18 ++++++++++++------ jobapi/server_test.go | 4 ++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/jobapi/server.go b/jobapi/server.go index 97b5cc4e60..d14a75b119 100644 --- a/jobapi/server.go +++ b/jobapi/server.go @@ -110,17 +110,23 @@ func (s *Server) Stop() error { return nil } +// socketExists returns true if the socket path exists on linux and darwin +// on windows it always returns false, because of https://github.com/golang/go/issues/33357 (stat on sockets is broken on windows) func socketExists(path string) (bool, error) { - _, err := os.Stat(path) - if err == nil { - return true, nil + if runtime.GOOS == "windows" { + return false, nil } - if errors.Is(err, os.ErrNotExist) { - return false, nil + _, err := os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + + return false, fmt.Errorf("stat socket: %w", err) } - return false, fmt.Errorf("checking if socket exists: %w", err) + return true, nil } func generateToken(len int) (string, error) { diff --git a/jobapi/server_test.go b/jobapi/server_test.go index 648bb0082d..90e692993d 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -100,6 +100,10 @@ func TestServerStartStop(t *testing.T) { } func TestServerStartStop_WithPreExistingSocket(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("socket collision detection isn't support on windows. If the current go version is >1.23, it might be worth re-enabling this test, because hopefully the bug (https://github.com/golang/go/issues/33357) is fixed") + } + sockName := filepath.Join(os.TempDir(), "test-socket-collision.sock") srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.Environment{}) if err != nil { From a320389e948bc6690daa3abb1bd0ff10995ebdb2 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Mon, 13 Mar 2023 15:37:32 +1300 Subject: [PATCH 13/15] Avoid race condition in random library --- jobapi/socket.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jobapi/socket.go b/jobapi/socket.go index 13407231e3..10b1fb41a4 100644 --- a/jobapi/socket.go +++ b/jobapi/socket.go @@ -8,7 +8,9 @@ import ( "time" ) -var random = rand.New(rand.NewSource(time.Now().UnixNano())) +func init() { + rand.Seed(time.Now().UnixNano()) +} // NewSocketPath generates a path to a socket file (without actually creating the file itself) that can be used with the // job api. @@ -19,6 +21,6 @@ func NewSocketPath(base string) (string, error) { return "", fmt.Errorf("creating socket directory: %w", err) } - sockNum := random.Int63() % 100_000 + sockNum := rand.Int63() % 100_000 return filepath.Join(path, fmt.Sprintf("%d-%d.sock", os.Getpid(), sockNum)), nil } From 008c805c00fd0d884ed045417ee470a97d296093 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Wed, 15 Mar 2023 14:13:27 +1300 Subject: [PATCH 14/15] Update job-api to use threadsafe environments --- env/environment.go | 10 ---------- jobapi/routes.go | 2 +- jobapi/server.go | 4 ++-- jobapi/server_test.go | 32 ++++++++++++++++---------------- 4 files changed, 19 insertions(+), 29 deletions(-) diff --git a/env/environment.go b/env/environment.go index 33021b28cc..241164bfa9 100644 --- a/env/environment.go +++ b/env/environment.go @@ -84,16 +84,6 @@ func (e *Environment) Get(key string) (string, bool) { return v, ok } -// Dump returns a copy of the environment with all keys normalized -func (e Environment) Dump() Environment { - d := make(Environment, len(e)) - for k, v := range e { - d.Set(k, v) - } - - return d -} - // Get a boolean value from environment, with a default for empty. Supports true|false, on|off, 1|0 func (e *Environment) GetBool(key string, defaultValue bool) bool { v, _ := e.Get(key) diff --git a/jobapi/routes.go b/jobapi/routes.go index 3a8375fa7d..1d54988fd9 100644 --- a/jobapi/routes.go +++ b/jobapi/routes.go @@ -126,7 +126,7 @@ func (s *Server) deleteEnv(w http.ResponseWriter, r *http.Request) { for _, k := range req.Keys { if _, ok := s.environ.Get(k); ok { deleted = append(deleted, k) - delete(s.environ, k) + s.environ.Remove(k) } } s.mtx.Unlock() diff --git a/jobapi/server.go b/jobapi/server.go index d14a75b119..0882a132ad 100644 --- a/jobapi/server.go +++ b/jobapi/server.go @@ -25,7 +25,7 @@ type Server struct { SocketPath string Logger shell.Logger - environ env.Environment + environ *env.Environment token string httpSvr *http.Server started bool @@ -35,7 +35,7 @@ type Server struct { // NewServer creates a new Job API server // socketPath is the path to the socket on which the server will listen // environ is the environment which the server will mutate and inspect as part of its operation -func NewServer(logger shell.Logger, socketPath string, environ env.Environment) (server *Server, token string, err error) { +func NewServer(logger shell.Logger, socketPath string, environ *env.Environment) (server *Server, token string, err error) { if len(socketPath) >= socketPathLength() { return nil, "", fmt.Errorf("socket path %s is too long (path length: %d, max %d characters). This is a limitation of your host OS", socketPath, len(socketPath), socketPathLength()) } diff --git a/jobapi/server_test.go b/jobapi/server_test.go index 90e692993d..e8acb42e3f 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -24,15 +24,15 @@ func pt(s string) *string { return &s } -func testEnviron() env.Environment { - e := env.Environment{} +func testEnviron() *env.Environment { + e := env.New() e.Set("MOUNTAIN", "cotopaxi") e.Set("CAPITAL", "quito") return e } -func testServer(t *testing.T, e env.Environment) (*jobapi.Server, string, error) { +func testServer(t *testing.T, e *env.Environment) (*jobapi.Server, string, error) { sockName, err := jobapi.NewSocketPath(os.TempDir()) if err != nil { return nil, "", fmt.Errorf("creating socket path: %w", err) @@ -105,7 +105,7 @@ func TestServerStartStop_WithPreExistingSocket(t *testing.T) { } sockName := filepath.Join(os.TempDir(), "test-socket-collision.sock") - srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.Environment{}) + srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.New()) if err != nil { t.Fatalf("expected initial server creation to succeed, got %v", err) } @@ -117,7 +117,7 @@ func TestServerStartStop_WithPreExistingSocket(t *testing.T) { defer srv1.Stop() expectedErr := fmt.Sprintf("file already exists at socket path %s", sockName) - _, _, err = jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.Environment{}) + _, _, err = jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.New()) if err == nil { t.Fatalf("expected second server creation to fail with %s, got nil", expectedErr) } @@ -132,7 +132,7 @@ type apiTestCase[Req, Resp any] struct { requestBody *Req expectedStatus int expectedResponseBody *Resp - expectedEnv env.Environment + expectedEnv map[string]string expectedError *jobapi.ErrorResponse } @@ -145,14 +145,14 @@ func TestDeleteEnv(t *testing.T) { requestBody: &jobapi.EnvDeleteRequest{Keys: []string{"MOUNTAIN"}}, expectedStatus: http.StatusOK, expectedResponseBody: &jobapi.EnvDeleteResponse{Deleted: []string{"MOUNTAIN"}}, - expectedEnv: env.Environment{"CAPITAL": "quito"}.Dump(), + expectedEnv: env.FromMap(map[string]string{"CAPITAL": "quito"}).Dump(), }, { name: "deleting a non-existent key is a no-op", requestBody: &jobapi.EnvDeleteRequest{Keys: []string{"NATIONAL_PARKS"}}, expectedStatus: http.StatusOK, expectedResponseBody: &jobapi.EnvDeleteResponse{Deleted: []string{}}, - expectedEnv: testEnviron(), // ie no change + expectedEnv: testEnviron().Dump(), // ie no change }, { name: "deleting protected keys returns a 422", @@ -163,7 +163,7 @@ func TestDeleteEnv(t *testing.T) { expectedError: &jobapi.ErrorResponse{ Error: "the following environment variables are protected, and cannot be modified: [BUILDKITE_AGENT_PID]", }, - expectedEnv: testEnviron(), // ie no change + expectedEnv: testEnviron().Dump(), // ie no change }, } @@ -228,11 +228,11 @@ func TestPatchEnv(t *testing.T) { Added: []string{"NATIONAL_PARKS"}, Updated: []string{"CAPITAL", "MOUNTAIN"}, }, - expectedEnv: env.Environment{ + expectedEnv: env.FromMap(map[string]string{ "MOUNTAIN": "chimborazo", "NATIONAL_PARKS": "cayambe-coca,el-cajas,galápagos", "CAPITAL": "quito", - }.Dump(), + }).Dump(), }, { name: "setting to nil returns a 422", @@ -246,7 +246,7 @@ func TestPatchEnv(t *testing.T) { expectedError: &jobapi.ErrorResponse{ Error: "removing environment variables (ie setting them to null) is not permitted on this endpoint. The following keys were set to null: [NATIONAL_PARKS]", }, - expectedEnv: testEnviron(), // ie no changes + expectedEnv: testEnviron().Dump(), // ie no changes }, { name: "setting protected variables returns a 422", @@ -260,7 +260,7 @@ func TestPatchEnv(t *testing.T) { expectedError: &jobapi.ErrorResponse{ Error: "the following environment variables are protected, and cannot be modified: [BUILDKITE_AGENT_PID]", }, - expectedEnv: testEnviron(), // ie no changes + expectedEnv: testEnviron().Dump(), // ie no changes }, } @@ -341,7 +341,7 @@ func TestGetEnv(t *testing.T) { testAPI(t, env, req, client, apiTestCase[any, jobapi.EnvGetResponse]{ expectedStatus: http.StatusOK, expectedResponseBody: &jobapi.EnvGetResponse{ - Env: testEnviron(), + Env: testEnviron().Dump(), }, }) @@ -363,7 +363,7 @@ func TestGetEnv(t *testing.T) { }) } -func testAPI[Req, Resp any](t *testing.T, env env.Environment, req *http.Request, client *http.Client, testCase apiTestCase[Req, Resp]) { +func testAPI[Req, Resp any](t *testing.T, env *env.Environment, req *http.Request, client *http.Client, testCase apiTestCase[Req, Resp]) { resp, err := client.Do(req) if err != nil { t.Fatalf("expected no error for client.Do(req) (got %v)", err) @@ -390,7 +390,7 @@ func testAPI[Req, Resp any](t *testing.T, env env.Environment, req *http.Request } if testCase.expectedEnv != nil { - if !cmp.Equal(testCase.expectedEnv, env) { + if !cmp.Equal(testCase.expectedEnv, env.Dump()) { t.Fatalf("\n\texpected env: % #v\n\tgot: % #v\n\tdiff = %s)", testCase.expectedEnv, env, cmp.Diff(testCase.expectedEnv, env)) } } From 62812685334e4673f891012adbd5959b67c981e4 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Wed, 15 Mar 2023 14:20:26 +1300 Subject: [PATCH 15/15] Add acknowledgements entry for chi --- ACKNOWLEDGEMENTS.md | 31 +++++++++++++++++++++++++++++- clicommand/ACKNOWLEDGEMENTS.md.gz | Bin 17318 -> 17383 bytes 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index 066ef5b785..f9fdc38ca8 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -1637,6 +1637,35 @@ SOFTWARE. ``` +--- + +## github.com/go-chi/chi/v5/LICENSE + +``` +Copyright (c) 2015-present Peter Kieltyka (https://github.com/pkieltyka), Google Inc. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +``` + + --- ## github.com/go-logr/logr/LICENSE @@ -7387,4 +7416,4 @@ limitations under the License. --- File generated using ./scripts/generate-acknowledgements.sh -Tue 14 Mar 2023 11:57:14 NZDT +Wed 15 Mar 2023 14:20:09 NZDT diff --git a/clicommand/ACKNOWLEDGEMENTS.md.gz b/clicommand/ACKNOWLEDGEMENTS.md.gz index 7d3a4d302e2062e7149f3d72c9915ffcc5eb2c26..9052d3c3022282469b64cbc72de4ad0b0a30274d 100644 GIT binary patch delta 8732 zcmaiZWmr^EyEY+R(hMnrfOLa^AR;Io3PXcZGITcsQX-N=2uSw;GYAYZGzv&4-Q78K zNY24{zw0~ipL5;U+E3i~T5Iq9W9_}3RTzP}9)XF+5`~Yi%tZbLuZ$Ti_QqWhdW4qs zZ(7U_g~#%-sP8@!V7_@s*j3ea797}PtMX9Wv~~FG)JXX5obGeEu5N9~T+-(wi|>)( za<;AS{L#zWIw|NU8+~$jr4%-ZbE^0l@iSQ6!v_FQZT=5u?5-5}cNdY+pMU%QaT-W5 z|0|=gaTP_7v3c1!O8t1~rq|0jtkUoa&B@>rku0)nURj5^vWfj-6X>5~E%adt^vaxf%@SYkZn0?$8h>uGfcDI{s^K3VNt_`95v(lPY*tiZJ58LPTGzw#V=epTtQm5I zOC$*HNy`hwyqwDx)|~dxvFTVf7hRn(`Kw4$-urU(EAUU#>;Bi(AcI+2_6~aN57~L2NnCw8D1n@8N zCx@JP?C91fhp`tC)VJ5)%pM8*JcsSICnMRMlT7V(M-^>92+1~| z*FF{B#)?*3bvs)hE81QFV#gema&pqX3s~P661Tq@0&v-?H|nJaq)laDfu{wj6k_55i45@Y~oWkaz(^8v;y%uhERzMR*b?~Y&U zeO@r_G8eStpN&M|erQ>c&1AW==jd_R_s)yMzkEgOSK3TZ#g$?A=6*ulk8u-sBk1IA zx9x8E8nkMW*3C8=nAx@C@4fmn!l|Z#Pj)GT*zj~3ki=N9ms{uu29{TeqlYe3qlAYy zhS=vrC+)7!)0lGwlBA7pHpB1dNIh!U?U}+%QP{hv9(Ou}U-AXaZHlfwyEHAm3|G_L zJG;moh73^2K%7 z=jx9RP~B%LfhUIc>L8t3TW8y5TenKo*H>pvH4dCQmNxJm?VD{rIWoSJK}lw6KJ>c* zOGKJw9>`Lz`+I`v%3T$5fFM~m`J@q(j=!GnP}oi>zQF+aU9m~^y;9wKP9cWa!#51}JvcgE_NZLHDIM>QGcYQT#=#3PB&`FiH)=8E!6xJ*CD!Nr>aw z;(V;wAzw{}r>)KSLXOpOvbiVn=`1%~rqqr@ujIiiE5tL!jL@lkh-B!~cB9dGT?TC} z#bOfZ`;!Fx0;u*GlEMkDcXTQpciO{-I(zwK2Oa?Cf-;YYLnFizBFxk`_ez|#a-AiE zYTBMeg2;3gQZwasM?_xf3K#3Dggb~4gOtBkr1-uzn|HBD+Y_M`c*_~^NrEAdRxfbdXIH4B0nX9D+NU?{pFJN&A6|ONO7z6UWzBajgjU$a@ z`=mJtrIx(8VQki1C^f;q8r^AvcFqBdcEO{<9xW_RS!eOw`PaC8rX`@y0aop%<&PQP z7}c1Tr(@YGjIsn3o#`mmnA0W%QY*k!02lY4^aCM#&jwN|jzC79fjDCDn!GjO{vn9R zG*FB;OKjCd|Fe#UDfOfCYy5NUwnt(JJnF2YeGndVAQJ%`8o$3y=dBwZ41dv53XVR$ zq(3$(0qywdZatpfG|HayTAoenYG4U+m%~`z9Y%p9kO5-jFsKWP`n@M2`R#nL6_B~N znWf_=Q6|VQFhJVdq(s=aY3Nx~s1Y#SXg*6nMJf~Z6)Ny@3#6(NfSm;sKiVAacGrDc zxYw1>5~SqCh@6U6FeA#@Q_6-s>#Cyl-Bnlz*=_jaIKsG>)+i$I(#}#O-4Nj-^WlQP zsL*p$`}#f?ue_dKAJ-F~qm}G#Ir6j_!7yc+EYdpqtz}D6h&u--?$d!VAL?L8Fb6&4 z0|yCs->rbCB;?0Z`Z)@E1TvHHyHP`K`IIr>GDr?yP|88|Qu9Fd2$Y|(gEfcOBhFJx zMGEdrU}+E8k6CeQ=uWla`cr$EQa$QgX|?d4n9|3w%j75DPc5BweBYKEdb~x6KOch* zvDtx3u^P|k`qaZ)i@{7cur6x8v^|eE2rr~^1pA~+Myx^x0fs9pF=k=)N0Z*c{0)**~Dc+hW&kqAJ_0Rhc7B zL;K!$IXugSWeRuC4!+3Ji5vhKnHqabfmgimb;f1t4dKYG=x8ZDUu(ICJUA8k)?zL! zPlvZmK6ztU*|C~5Y#R}snbXcsWV?D0OlWQ!)@ctvG(F+}Funt8phk{^wwTgOKWqU_ zV1XXv;*RCPqnWtiF@eywHaXHUHjYoMXDLm8|v8gan`e>UU0L#uAEj_Nf@qnl6R*O&j+1fhXQ~hArl#wf5 zMa1Ba@I}DJaK9jsW6A9|^hRC;Higxr4crbUw7i4UTjk)BnsddXRj5%@(w2COnP=0f z9&lg|d1GJ@TsqF>Obb_?%l?W$G&j(XM>%Oj8eR}N4y+*K#JSwu$_iekZWFpm2ka&! z4}Ii4_vn`{aVHAx2CWxNy@fw6hmmaG{a^wgI2%PtO?%uuf!Va5jqcO%PuG;U@}Ra< z=K3CRje?K1dVJ@03U|1nrcKJD4HIEr;-);!a8MNf_8X1@AnFN~}l?bo4%QGG$+Ll)9 z@1Wp!Js&OKAMo1N^#1)x5SJq25Ly-$;z;>Cp~$NPDn75Z250+-+<8~BVxYWmZ^1J!hf@zOV zxxrQfOTtZ+uzcVtRtePjG^%B-sih&jJ`N%rS!(w7D_c`(#bi{G@shDJ0sqM=4#ITD z7)Ipr#zx?igH?ZEy>HtNThQsdFYq0%uu~&lq&SJ)v%x?Ihq&kKZ0030zV(Sb1O-(( zNRy3+ekz42Q9gJF?m6x>mLv7cb;?SpFtCoImdP87vh1n;;nSh5GAUYzshIoQZ!V2k zYir0*aHOY#F|28<;Vt~Apve{IG=Tn?mQ z@x9VROxLrfSzA55Y3=PJ$^lYB6yeJ%2luJ9YwTh~6Sqs6LsOLXbe1nKPi@LNGuTrg zn`>@N8dN(g-kI>u$f!sUv`N#}aA4uNof!N1wtWn)rWbu~i z=iJ>~l(_uP!%r@dm!(HzejANP(KvahW75S<4s>>+^T@Qd>Cpku>Y0i>Nc$Ys=kegW zcCHv*m_7EQ)k6%&yLn{EvvO>+6yX{XQIE5me?m&V4aZiMFL`=6WV)%`Mmq@6;trE8^Vb8Vd6tXO?zl^@3EZGUie zhsXNn9#iNdP4iUFPOW)B!za1ov+7f)TE%{P{t_8TyRTNDllJ>v>qUAIrq__cQk%X* zu3z3VAGTRc%gD>YFOMD&fT6G8SoMS}+;OE(U|97%ocuuL!VnABbmAQAoBU11Uw37E zLt49cmhHnVukGlG+q=17LaeL7@9inHKA--_-{R_j7(O;t2~RZI-D zg5NhjlIIX|nd1YFLV`KMOitCBN5T0R$D5MGc%NJ5R^smix47pq3o4J>Oy=+`*7TIr zdzYk^BGTC4l*)ebVG6dwJq{$6*ip03TpJWSN|JNt)ZBI_B{CV6*}n?Xm42Ksperag zEpA6u1~$i5G-`9g8PKY!4)4?QtE|sT?8^EV?FphX=0?GUGu>D*KNMv1TA|*(A5=38 zh70sRvROxQiRIhl)=hpM!1tFfI*m8n9FMaKqR5LihquyQ@-Q)P4^|lR!gVioJIQ|$ zR=4(Q+^47g?=brD#0waLJPOu#D5G^1YABxKhMmaGa!!A&NwxSn+MjNP94=Th zTKNTKReOXi+^HCI`MPe^8``c|P9)a8T*Po?AB3C)8Efgw1gX-@e%;<7Q4L7&c}zUi z)uN+1=d3p_aUu2?Je0BR)~t9@;iO@>~tW+^TIDtxO~N8s!DF;E4}m?SCEGP?K9J&^W+P-=#O zRMDm*F%TCE7?W8k)Zzik^<42X!1hekEppA=B__IacY|5~+uUSPac zx;Om?|8N1P{lJGdGL`1u$Av76Qy&zJT;7%rhb#=}F;0OJVt$7_qz)Z9<&xE6siR3srm!XHH03To0C^;OIwǞwUhPei75Wqha3>SuXj7nzM%a z2BZ1m5BTnzw7a8DEW*oKOp!J0UYA=Fj6=SPaK)F0&Q4@}RUHP3V98Q^8|rK5Pi{j8 zdp2H+?EWKQx*-;{P@J3w$KuXq7V6#x_M_C(dm9Q&SSXs% z8aa%MiU>KD!-aJdvH;lymu`@alfyxsB?40G`Dr;pogdg>2Tf?u+ibK%bec>bHd^ZL z!ceZJm4>e4r;h}{1_oa0XsH>*x#pVLwZ(-bfOod6@|1X*w%@}-SnYeOX;?_ZncJ4H ziYR~j9d_G~ejOgZ?+ zxyK|UWAE|#B%`iN^G53DL~HOFCNJq9{Oq@ON$^P%26yYV0PsT2YL!KeU0zPw-dr!y zxHOMXK}<}@B{ELaerjl2+{{h)q2u z0-4*rUgQ40jF6hgewkg0&e&&%Zk_^P-k+X>>r;|w2Zy8&-!J{j&BXP;H>A0^FBj6i zz^`!wi7e(D+sy)(r;o5Dyg1|fZhrM1;rEH@78yEKB2gT%J3BK)c_rulzBfyy_Xj*y z=?8?-JaiX6I~#7-qAw9Y8GkKmv30aopWk1qw^Ov6Hkebo{k zt-!((j&b9H=xm)gCau7y{N>+aP!z560T#dM_x^WnkaH2n1W|l!HHSqArYKc{=v}+9 z+~uXP{*9$DmcVZvL|HQ(L~=4i{jnE#Ue)LA{M)g0{+KBOuaN#ZiO({_EYf0rCOi+1 zN5M2QF7E+!X#__?BC2nM2$-YtE9Z`}wE)?|PpsClF<;>p+b59z4TE|xl8z9RK|PR6 zQ3kxp5t=7fe|!MDJ)hKXA0ufM@3!BnnMiRXTZ#YC13Ig7%sgTxz{lIl9b`uo0O1uQ z!ZuG7gS3fJVvn=7LJmZ!m~sCo4mCa=nuR!1aV#XR;(ZpDv~upLac+GW0%4$HVp>RS zdl>R^?61KsBGv%>RV*Z$L|{Oes2IqlU3w0PKUQ7^Y5yK>l7{ql3IK##jD&{%qcEM7 zV2^oX8&}95^7d}F??QG(X&HV?ec19Cht2LKtl~Wres6iVyTrxb=M1Oaj3#DD4ioeORQ#iTiuZP#j|SNzA-5AD?st@H%2A_ zfM`k(F+E8V)3VC$d6E>&jjfCOhZfed8fOjq9YW+JXlM>Ii(OBrxl&0n_@{ z3G$Fh3D##~#g>192O%oBe^}vS%W>8x|9|+uc~>#+$!^)9(qL@**utrU57Mr!rM(RW8MU+i{ z%i#EN%x*+TgR)G_b870jv$Mk8(JyF!tE!lU|c=-Z(c-Q%MT&o0HrOTid^@#;PiIb?JM)vz?v_azcssgan zDP5Ty*GKN@sE&}N!V>AOGf>nG3IVl0zu3316BuL8xdvZz8)Y7}%366T?u^8-elRef zaq!ICaaCvC9C>pAH$;NReI|?eM{?&1;K&RKly&K*uaUzSNE44Ibj%EflnLH(*S$O$ zRcuhqLJ0Kp!_q|$Mv9J^)zK%4mmUq8!Kcz`Y{!?|y#+oc+-fSSg?Yjo!&hTjb3;3* z1li$oiQ|afCXlhQ^DC%}ddZ$)gYyY&S{6Jy4ws_+K@;FT1lsV5 z^5@^JPI5Ck=FZbSlNvQH;}N75+c7>mVAjqGbIZEaMgFz+i7^e-sobO9=1^QMf$(&- zwo4$h$(t0+hW!Usliw>W*u~c@0^(#eY+ST85pT^h19%{DNBR=PbpE*vBgH#?pX&!# zLKf~TYJDsFp4wqc6m-Z*Oz{BMQawe83{2Gh)?UpF(yb7@u0QJc>wyQ$PvY+f9av5G zqi4IcA1|=Zw}1D`aCMX4bp2SBHu;Qs+OjS#5&XJFCr~HzcWwVp3Rc&UWu#V=3Apx| zB&AVx@7RQaen<|6lXqez8W%qYq)5%QS!C&? zqGrC+XMa*k^BXG4nPshy1zZSAKd_a5n*9MgmVsC}BFj3J89QV8xzJn74i&zsP3cY5 zUa)m5|0}u&%mAX*LXj&Rqp+FpwstErlxvd>X(Ha-65e8w?&k=KUd{ubPR%3+xgNc$ zSxVJW5Uj$_hyS+1^#DL3a^6#^|K=hMPztF&SRrI*a{Hd zHkN*T2@VW-fr6ZzM|)856zixmF7#(g#SNW^?zH_YzNrO{=<#UnQ90r2LA$W|54T;T*@x$ z_4$7wN7;)OQlr>%-A3KGO$7YIwHab|o!NIO6>U!MAb8bJbWN|s@(*aBn_QrbUY*r@zNy*8@3YeM`AjFn4s_#^m(|pq)_b= z&Bqq6m0urNX5L3JxVW&vr0ppsmX?%^A@2V&{Epu`qU+9CXCWrCOoacDmGD7?Wbbp= zk&hexM+m!GP%@r*e<|R>wo8%tSq%>(L4D@;0UGv;-|O=0djQC;EFs|bRPMiy-hQ9O z{a%@KWEsFPEcSH3@b69fLFg52e1UMFwE^AaGZIJBBtXF{QedU+Ok0pNoiiTxuK6$O`8L4-IjP$ng0&z~+Ci}_BlD5r ztJ|u9rmTzrDdutGOVNi04arHQ%q;6|zr2l)g%r!cZM%0oT82%JdS{4U;LNN{#%ENi zp;W$qixm6#McTSli4~cX1(|OE-K|?B&1U{B?9+*!cEh;0n?ES{ z+KN)BgytE|9)12U&U1~o+lLks1%%FzLMGU7*>_~*RlvwTN;s=qMbZ}Mrc+gPn@9{I9u>=Bv#L6 zO{k6~%m0__VlG<>;iM34S~RYD!`bm&0Vk^bd0SBUn`hC(?(JW+WUfjWAm`YmQxcWd z;}^m5w3$@&44%vk%kI`K990rAj*g2?CD)TOQEvebgyCpb_6WaG4H2; zEN;J1e)HKGmbu<|e?>CjC76&ne}diBKN$zVaB#u$nKe*p`dJ|7o(Q7lP^0u(Z1tMu zsG+f2X*96oQG;8Hn$;H5;e0Qwv&t#vp-2C5R&J=ZHi)?~kKlu2(;UB7{reanRPY(NlQBRV z>+tW3r*d)(1w8w3&yCIQM6FiLSA4z)X>MP14ztud{_HSNd%^qZTQ1QMZ569VerqI^ zokqU?zOsKTbm6h`@I~Q7weu9g)YylblY~-1yyCu^RpnV;y+V((A1;F4a*-^Ziv#y+ xHv8N|38rj0WHSLY6|m0 z#l;(6(Z-F-Y_Z5Y<@r(GiMGJ{J_9T9l+mNB6ZS^6ipR390?oC^7pF=8d|%|NhN|~k zA62Uh(|k|D!Q0yD*qA@yvQ*$a?w-FnWZ?5H=QWkDm!J6721V(m3qd4?-zFcf%<~!j z*v{rU07OT4K?$~YihJ1byYCMxK1@!15_0hIshH6nkuP7Tt%y)d97_i7MyF;_bh0|D zCFbpl6jUr771|1o^-P)zjB>g1%ZtORm6=BSDlrK6`@7u>ksgIA=3PQ0H`fZ1H$yvBv7)SF}`;TQJ#*Xi4|@7l8}uv~j%Pj3dfMKy7UlU)qJMFh zVg+EwP!g|S>#$#wR`%$lPbQ?$;O+aK#B9l$@vY&J3#$uA-TqPLPF8397Z)AyLwF@o zDC1BUnzwch(^Jowdne5`lFPAmjXU*I%y^qUd@8SCmi3_ezUB*3p(P2o?c8g{;A9En z-J=%2KGE-1iFBL?9(A4>Kx0Y#DFyEc%_|_*gX|?+<1D)*`}Ij`ibkXFkGiMp_qzr< zOF{M-zXPLp3L17ijgrT-J9#ZC2=XN-1nw@3w^6|lXlZmFz4-x>ndzt zB#e8mdiANs>F#NrS=CQ31uk}HST?1qcdMUs+0}U;Zv8FC&h1luD~~S^CRo2s+t3v`|vvFF0E7l5^fQ3R?e@_ z_^GdgUn@(isSi(_Tx`0Ra0ZPxG7J{^__dRte|~=*ADRmrpgOVBRIGMm)ysXf^>HMl7ojA93TU=)m3xLgmnq zaE?t`qD<1Sn8z4YUlB;Uc57`$!>-2AXl%~UC|QDXee5i#YbMz#E!5zqED$)7@#=|^ zv5_FkKu;R3FS3eh2?TXYS3aeYMkGVc;b}52#XD&fBQ5WTpkB4Azd>Z`6bM1L1ksuf z9QFcZVEu@Gw)=4X+%km3GDZ*KTSz7tX%P#3F*a@&juat;#}S!V+>y~%GjYg7sM%Li zw3~sl>n3HK>0CM3ILE+>Kv)D7fdB|rHmPHnsl&5dzs08YNsaqnZI=Itrq+|!O0Eyc z+CQC;EXD7_Qdq`{j(RHUtjyx{Uuo+oX^p+fR}uLl!@|&@ePsxLh^_q(DkJnA_ZKxN zm3s;MA=*yBxe3d*L~O)g61(jh6_T|j?U?QTj^67RGWI$RAD0n71i-SPCXCHTs|mzz zQfkSH%V;wO>gQTa>a=FW^Aq|MP@s{VpK&wL>Kq()WMew|LH%slaGl)JAPG~wl#DU( z);|%`i*^VuQ+C+^eGLYOvF8p&Qtz@w)q3j!{n~Jp9$~>>nU#hVOC@}5gvKOgln4L# zS-}2q&%OqoOV=6u11Ns}iEQNK>L(I6-fFDigCIhc9cxhNb;j8d>7U;>CgDixagl#59t$%osCEn30auV!mkUr zUlN~6z;p+?6+fX=KE|nay1ipp{tbR}^3Z5BHIGj&)D<0f5AcDbT|PTbdBdIXOYds~ z7-@4#lv4qQ3g_B{2sX=&Fbr{Hn?@38J|geM{7Q1tGu1F z5p3M+-N^NRNFdYAYn%72q)dmtA`5o;QIMCA1}0ia4-)~_n)Y2t4HG@^-~-)8%U~Q= z4v}D(^IA1*0OohNbH=v`T`zNb?6B~9`|V}EzGWnIb-Zfd6`q(lvzjsqA$ybWX;j1} z$?q|(m0vz;mr77}Sg5Z(v&TcpCR&UZi}RV#3O+;IS^jXS=@d7DVBPS_rRA>#&M+S<3;{#H+?>cV;lVcMc8y0C;> z-{)5Edcn6eMM}V_XX|r{Hy+B6k(ICY!Vx$}7s1)RO_kzTD-(S9m&drD0Qd_G8H`{I zLd>RMI+Ia6o^<)vQ}UasY60F0V=`|cWy#+FZkL;qi4)vWmZFe8Z;hrmt;^EiCgD8? z-|FT&JI_t}+KgR49B`Ew9acP3G(29hLu^5X1h?L8CSK(flKzNELbmzgslYr;moQ_C zf^M8%D4K4~JqlsIhzPInr>t1`gop z_)xuTvOsILpgy7}T2U=idSuClw!4*ObxDHb9X@zL4*pNq>`ZO3Qw!?{T;lX~MB#-h z=X*(d6VeoM72X7^h4mIS0scBABZKD?2q&Fkl*4{*F8mV*tKBI$6xGM&FofJf*6CBX zjc@6L@GtBgW*JiwM|Ad~mF|Y6z-wn%_4xD}MQS|E=~M?ged#}SiUJu*`)r?ETVfX* zW&bd2;PMJ2nixOdvoqjslvUY#&RHW3*?xt}=<6?w*RU*R@7q18Rnk}FD_@xq-_IH0 ze82=tO|Z*GpeySH3uRcqh*$Hv)!Pme(qr{$_vUq5K{XE3Rd-4%VeDQ2UqXh~AdG$6 zt0K7^mhXMaq`hxOF=sfPJkONorY_3rd#w6x(|1B~H=ItN`xJ?+{H`Yi%MeDw$npfg z?YifoMpL;W)A04HzpB}sqxBu|tWPsSbtR6^lEZ7JWOgo zlLV~W$_)#0mxY(!&&&d>-j$3H))PABJsDT(S#;p4toNTpcZh4Z)xLee0kx|dd9=Rx zE7S}Tmo*t&rmkB!u@yS22GTQBCgvsx=c_ADYMiW=w{H+b7+LN-$>gl;AgJf_8b##@ zSFJr49s$`$AuL;vY%8xKD7^~9jyuBKaHY#BuCn;`N}`$Szu^K#ebca5#OL0Vp1K&L zDS{m6`zh}0q@UM&RBNwPdl-uSGw(;86t`G$r@jB;#>iMRmOwG)UFKHHHTESHVNw+* zezCfRT@i(WBIsx=&n87VYj2|X3m?i}z~Ry~sZk$uk2Y2{iWy=D6gR<{c{{9$MqF!8 z+eU*%^rF?*BbNatN;rignY8mhDocs7RI)4gR}I@H|0K>BM?>3et4_I+9D%K}LSV)y&1VwvG0PLQpFazpeyoJ= zgeVFSHFjqkcHQ3nFf)~tt=wN!t>LE01H+z;8{;V^(9(6^&Vu^5hSr5uP2VlcGwD!K zEgAsrXC4m-k%yx|`x)pe)52ljZM!6AT^U&1JUe!Q>}oaNMBK@|clLF1fIdCu1d?>& zOe1U>=+@7}8U)u>It&tQ`YN7gi^s0%dCciLr&%lH zAnOYTSUQPSGd_SQ8Nf7oVJzf8)_vu@?lC&4S*$c~uEX9ClnOx;H zpwgefkgtKjUtx=eNfqCzy*1Kn3>XuMBD)=V|4BN7e5Y8HIoP;=hq)rEf-L5=#Fht- zIlHFHibwc|joQ1fB?X4^SbjC5N-2zJip**nQk|TfrBaunwvF;;x^xy@s^=eEKbO|B zKV>)))Ra&V-Un_nThdK$DsTvQ??1K!@-Sse9`lUpV99NkC5f5jbK(#hn*LCu#wA4b zf6P?GAw+C{%)G>u*%voa(2-2Ul$pkqxl+dJFfNEv4V-#P>l|~{=v{mOzs^3{qM_t7 z@=sIFQY-MwgZT_-=8Gkhg%H7VpZvo?**^3KGs?ma(~XgQKg zZE(8ewydJFiy26o%_}aG2&rHMIKbYz>@Lg%!h*zS;dfw5oa|igRw@Z*FCxEjW{pLc zQ!ZG!y`n$ZO+m@qSxV=9JA1R(C1@05J-ER=7xD{Ii%)uohQU>~}}E&65Xy5h$U z!693d8`f1$7}gLQ?TuqKjI$UA!459Yy{M71!EMi;TljfaJiO+_^>7j}~`WFV2 zHM}mC3q7%kDK>I$X@~E=Z;l1Dn#S%}c{YxfIKDgAj-Hd{5lgQJ7S(SojB$h46NJQ| z&=S7IU;+PdW>dc>#B*-evpiAWnbQ#ULeU1$-PSvQpkelI$@MXIYiZg))#g6`%oh`> z-0a<4dlU}-(hBNsPH-HPz7@J#$-EUB@;rwrIa4C|dU=G%&c8BL9&s)k@3>^kazx45 z*wB2Dde{}PIGpV7V#7}4*q&LWN*o{qa$MwevFF~ zJEYFJFeiE|BDAgMeYdyiwZYwLlkUaxMuZo}#I z1`Eqsfvatoi8aYY;52G;*`oAAMGuP4!B}h}qML^f%ZH3;Z*H()d+~GZ;MwO`AQ6ke z<#o>@vEK#g(NlTX)WPD^j;!(N)57jih7`iUFMF34wvU2vB^a;;*C#C9TsTJ8t=+WR z&$6uFYb0IG;+6{E0=QiyliS~{+4eAiuB@IA&doCoyR@rV=AHcv1GSwQ*}nz5%g%g2 z*^7wEMx6OL^WEHB3Qgy)*D0xQbG56?$8$ORy)}Mf6144uxS^uJjwyQYTAr4^;U}co zB($(#1cIi-QZT+LsDqs&+Lgd_AUK`FyDMlV+A>hz2y+=wrZMY1x-I>_`V+QyAJ+Il z&a@DOKi%Nkomy~YHo1jR!nc~(ZoORP!MEBcXltAY-t1hgiuV;!wC8{+JYG~Akeh?5 z(9g>v2+T=aoRrLMg}*T`mWf+0dp?;tuw7l#yjT>jDNHZR!FZS*YWo;N(+bX7*7 zcf4pQnt}T>y@OMTw&$qzbt4_4*Gl_c^r6dP-c$#l6eNRMZwQ$J>xIwWGG$f0s|T*( zP8v}P&u6or1s>9m@0o&4ZlTyez>y?YUg4G=`F%}t`z)*^RB9s5UaI%9#AIW=EPL>D zR2eDaCj67d$|zBNkKX6G8d#q}?L~%}RQtzJ6HZW}*L!MTyzM~O$_UMqh--Si=nMH- zf|l;L=fHVS3F-C2OZs2(2T{ECBbIRAepDq4o z0~LOEOcSJF`pwm)k751FzYz4^LMKGoRD3b-2YWBEr{zJs5Iid)6a=T)=J_`OyBLe$Uy*ivzy z*s!$6|0*0Pp%&#|j#S+5y;BmRCiBVanR3py5W%S7kir+nBTqVgsC4P^DW%KyfqfrmdwkT78?$|E_hM3MRM&6Pt^gYwIde+ah|-WrR)#rMdm3~t%~xBT~b_lGbg zK*;6wRVfYfmYI=e=q?9U&5=Z+UrJ)$u}&9ki#tvHM$ULtw2qc&kLggDMs_C^&WweB z#g_g~9My zcYl4&OBh(zL;lLq#3?|=UHa4kkvk&}fYNTdwyiF8I_hsvLwoSKvH8xI3P#Y(reRTO zN_A;(e8wa+`Be;q3$|x3HX7#O)q!;pJ4;rkDHF_M^~tq*%B#FrASg#OY_rL3(`X&AOs`W5 z^G;~o0?$<@7k%G$gtx1rizf^8f!eL3yXB+WyA!$;yX1c3jbimRP9tra=_5Y;R}&Pw z6T1@FsC}Erq5Tr>gtc5K4=ZP4WKYK1V-Vo2R8g&Tu3SAgL|gY0>gw98UeF01j*!SBJcP#-Z>Ed-_N#~? z`pcj0C2`pK$weL5L9$3Uz5GV__-b1BG|wBh$%J{DCQtOz%Wy&keTp)AnLoeQ$@L8WNOd)l3 z?32!w^^x<{C(vZ$)Xu}tRwQC`2f=P=!z~xwC_hEZDmQxjk7iW1dvDT%!67RWRvW!L zu{qfHKCuyrIq`Ho@lUe!Cj8(;+n({2j2&*1x1Yn!nT!e4tC_}nT5!ZB4;)Z9kiDnR zEBbn@{LOmbZeXEcW&z1CvO1y2k>0jWDXHyr$pY#Jzva+|)wb-d)@t1jv@H1hR@i&Z zzHHi8)pJ%ES&UcA59BYfcu@!->+MqEn-pwe--0uyti*{J~q}9Y>~Czg!dDL(KIJ|Rb8C&_w{MX@8hl0POmyA zpF{j~y=YD?w=c8&=IX@k$qKL4txhfTJ>}!xx>5n7&JlGn#t0k`By`M?$}HI#mEv&0 zqwQDXnEZuwYKk2+v)wPDQeb-gA0$bVCGT06p8bdPFa1$^MVEK1X#d3x-;0Y)bN%$U zSy4Qf{~`U$qP041xWvD>wl+VJG_$0Y18C)bx&oIH4C{YOdkRo4?FU@z+e(B*x!g z9m-NU{=|JsyL{qAv*FM4c^@mB@G&J1uZhKvF2MuX?a95^HWLcsxt{yF%d52OTdg%mhSA|M5+IY^gbx0C-hwAIT_e*#ivr`@boMz^mQVYTP} zc=T~kblpg_jdWJ7kW%H{bR&&6UpL=Xm*>-kC$u$uH=RS5?mw$Zpy+)hb1`H%3$iE& zuU73iFS~$Rf)je5@Nz#HS8ohj?0NzphO0_QtpW+tc7k|9`R;t1Ox9`vB_nRplcne$!ojl~0(|9q)O!4B=yFcM zn`MEYzz+4>NQM|w1#L2w*eQL($D#QNL+`97jE}I>SIM*sIJOP@%@{$NDYv5&wfq+f z{Zo%@xDYThK|G3@moy`ZQ{^?G<=53psgA*A?iCIXb8&2Y6z(RIq+#`SFDr9qONMJx zDTPb0R|g1fa*Ogi$ys>c`8oP~b}|(l#Z9&Vkg?0pGXz);+^P83*^lz_@*lB`b&exG z0&f&^dLQnG8u>_Mv5WHiw0d?wW0LdQT`<&4mUp6>;}%W04miWVSNLao0$wWHDB;*P zZl}z*q|y=MC^$MSmcDsi*E!XND@n7CPp{8OJ!}LPEwS$*vU)0HV^pQgVlY;xMh}SD z)zJ;&{5_A`TdORF#T}H7mY#o}v`G7=@be*SKwm^5l5ci=W6GC(JD@Cetv*I?*sV%> zeyhUdQ5ctkabK|T*E`G_g)hIFH8UTILj!&?sB9;jjkee_+kn_n<4VOXPcnF?