Skip to content

Commit bdead31

Browse files
shoenigshoenig
committed
api: enable selecting subset of services using rendezvous hashing
This PR adds the 'choose' query parameter to the '/v1/service/<service>' endpoint. The value of 'choose' is in the form '<number>|<key>', number is the number of desired services and key is a value unique but consistent to the requester (e.g. allocID). Folks aren't really expected to use this API directly, but rather through consul-template which will soon be getting a new helper function making use of this query parameter. Example, curl 'localhost:4646/v1/service/redis?choose=2|abc123' Note: consul-templte v0.29.1 includes the necessary nomadServices functionality.
1 parent f5b8506 commit bdead31

11 files changed

+470
-15
lines changed

‎.changelog/12862.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:improvement
2+
api: enable setting ?choose parameter when querying services
3+
```

‎command/agent/service_registration_endpoint.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,10 @@ func (s *HTTPServer) ServiceRegistrationRequest(resp http.ResponseWriter, req *h
9090
func (s *HTTPServer) serviceGetRequest(
9191
resp http.ResponseWriter, req *http.Request, serviceName string) (interface{}, error) {
9292

93-
args := structs.ServiceRegistrationByNameRequest{ServiceName: serviceName}
93+
args := structs.ServiceRegistrationByNameRequest{
94+
ServiceName: serviceName,
95+
Choose: req.URL.Query().Get("choose"),
96+
}
9497
if s.parse(resp, req, &args.Region, &args.QueryOptions) {
9598
return nil, nil
9699
}

‎command/agent/service_registration_endpoint_test.go

+98-7
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/hashicorp/nomad/ci"
1111
"github.com/hashicorp/nomad/nomad/mock"
1212
"github.com/hashicorp/nomad/nomad/structs"
13+
"github.com/shoenig/test/must"
1314
"github.com/stretchr/testify/require"
1415
)
1516

@@ -157,6 +158,7 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
157158
name string
158159
}{
159160
{
161+
name: "delete by ID",
160162
testFn: func(s *TestAgent) {
161163

162164
// Grab the state, so we can manipulate it and test against it.
@@ -186,9 +188,9 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
186188
require.Nil(t, out)
187189
require.NoError(t, err)
188190
},
189-
name: "delete by ID",
190191
},
191192
{
193+
name: "get service by name",
192194
testFn: func(s *TestAgent) {
193195

194196
// Grab the state, so we can manipulate it and test against it.
@@ -214,9 +216,99 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
214216
require.NotZero(t, respW.Header().Get("X-Nomad-Index"))
215217
require.Equal(t, serviceReg, obj.([]*structs.ServiceRegistration)[0])
216218
},
217-
name: "get service by name",
218219
},
219220
{
221+
name: "get service using choose",
222+
testFn: func(s *TestAgent) {
223+
// Grab the state so we can manipulate and test against it.
224+
testState := s.Agent.server.State()
225+
226+
err := testState.UpsertServiceRegistrations(
227+
structs.MsgTypeTestSetup, 10,
228+
[]*structs.ServiceRegistration{{
229+
ID: "978d519a-46ad-fb04-966b-000000000001",
230+
ServiceName: "redis",
231+
Namespace: "default",
232+
NodeID: "node1",
233+
Datacenter: "dc1",
234+
JobID: "job1",
235+
AllocID: "8b83191f-cb29-e23a-d955-220b65ef676d",
236+
Tags: nil,
237+
Address: "10.0.0.1",
238+
Port: 8080,
239+
CreateIndex: 10,
240+
ModifyIndex: 10,
241+
}, {
242+
ID: "978d519a-46ad-fb04-966b-000000000002",
243+
ServiceName: "redis",
244+
Namespace: "default",
245+
NodeID: "node2",
246+
Datacenter: "dc1",
247+
JobID: "job1",
248+
AllocID: "df6de93c-9376-a774-bcdf-3bd817e18078",
249+
Tags: nil,
250+
Address: "10.0.0.2",
251+
Port: 8080,
252+
CreateIndex: 10,
253+
ModifyIndex: 10,
254+
}, {
255+
ID: "978d519a-46ad-fb04-966b-000000000003",
256+
ServiceName: "redis",
257+
Namespace: "default",
258+
NodeID: "node3",
259+
Datacenter: "dc1",
260+
JobID: "job1",
261+
AllocID: "df6de93c-9376-a774-bcdf-3bd817e18078",
262+
Tags: nil,
263+
Address: "10.0.0.3",
264+
Port: 8080,
265+
CreateIndex: 10,
266+
ModifyIndex: 10,
267+
}},
268+
)
269+
must.NoError(t, err)
270+
271+
// Build the HTTP request for 1 instance of the service, using key=abc123
272+
req, err := http.NewRequest(http.MethodGet, "/v1/service/redis?choose=1|abc123", nil)
273+
must.NoError(t, err)
274+
respW := httptest.NewRecorder()
275+
276+
// Send the HTTP request.
277+
obj, err := s.Server.ServiceRegistrationRequest(respW, req)
278+
must.NoError(t, err)
279+
280+
// Check we got the correct type back.
281+
services, ok := (obj).([]*structs.ServiceRegistration)
282+
must.True(t, ok)
283+
284+
// Check we got the expected number of services back.
285+
must.Len(t, 1, services)
286+
287+
// Build the HTTP request for 2 instances of the service, still using key=abc123
288+
req2, err := http.NewRequest(http.MethodGet, "/v1/service/redis?choose=2|abc123", nil)
289+
must.NoError(t, err)
290+
respW2 := httptest.NewRecorder()
291+
292+
// Send the 2nd HTTP request.
293+
obj2, err := s.Server.ServiceRegistrationRequest(respW2, req2)
294+
must.NoError(t, err)
295+
296+
// Check we got the correct type back.
297+
services2, ok := (obj2).([]*structs.ServiceRegistration)
298+
must.True(t, ok)
299+
300+
// Check we got the expected number of services back.
301+
must.Len(t, 2, services2)
302+
303+
// Check the first service is the same as the previous service.
304+
must.Eq(t, services[0], services2[0])
305+
306+
// Check the second service is not the same as the first service.
307+
must.NotEq(t, services2[0], services2[1])
308+
},
309+
},
310+
{
311+
name: "incorrect URI format",
220312
testFn: func(s *TestAgent) {
221313

222314
// Build the HTTP request.
@@ -230,9 +322,9 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
230322
require.Contains(t, err.Error(), "invalid URI")
231323
require.Nil(t, obj)
232324
},
233-
name: "incorrect URI format",
234325
},
235326
{
327+
name: "get service empty name",
236328
testFn: func(s *TestAgent) {
237329

238330
// Build the HTTP request.
@@ -246,9 +338,9 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
246338
require.Contains(t, err.Error(), "missing service name")
247339
require.Nil(t, obj)
248340
},
249-
name: "get service empty name",
250341
},
251342
{
343+
name: "get service incorrect method",
252344
testFn: func(s *TestAgent) {
253345

254346
// Build the HTTP request.
@@ -262,9 +354,9 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
262354
require.Contains(t, err.Error(), "Invalid method")
263355
require.Nil(t, obj)
264356
},
265-
name: "get service incorrect method",
266357
},
267358
{
359+
name: "delete service empty id",
268360
testFn: func(s *TestAgent) {
269361

270362
// Build the HTTP request.
@@ -278,9 +370,9 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
278370
require.Contains(t, err.Error(), "missing service id")
279371
require.Nil(t, obj)
280372
},
281-
name: "delete service empty id",
282373
},
283374
{
375+
name: "delete service incorrect method",
284376
testFn: func(s *TestAgent) {
285377

286378
// Build the HTTP request.
@@ -294,7 +386,6 @@ func TestHTTPServer_ServiceRegistrationRequest(t *testing.T) {
294386
require.Contains(t, err.Error(), "Invalid method")
295387
require.Nil(t, obj)
296388
},
297-
name: "delete service incorrect method",
298389
},
299390
}
300391

‎go.mod

+3-2
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ require (
4343
github.com/gosuri/uilive v0.0.4
4444
github.com/grpc-ecosystem/go-grpc-middleware v1.2.1-0.20200228141219-3ce3d519df39
4545
github.com/hashicorp/consul v1.7.8
46-
github.com/hashicorp/consul-template v0.29.0
46+
github.com/hashicorp/consul-template v0.29.1
4747
github.com/hashicorp/consul/api v1.13.0
4848
github.com/hashicorp/consul/sdk v0.8.0
4949
github.com/hashicorp/cronexpr v1.1.1
@@ -117,7 +117,7 @@ require (
117117
github.com/zclconf/go-cty-yaml v1.0.2
118118
go.etcd.io/bbolt v1.3.5
119119
go.uber.org/goleak v1.1.12
120-
golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167
120+
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
121121
golang.org/x/exp v0.0.0-20220609121020-a51bd0440498
122122
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
123123
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
@@ -210,6 +210,7 @@ require (
210210
github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 // indirect
211211
github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.1 // indirect
212212
github.com/hashicorp/mdns v1.0.4 // indirect
213+
github.com/hashicorp/vault/api/auth/kubernetes v0.1.0 // indirect
213214
github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 // indirect
214215
github.com/huandu/xstrings v1.3.2 // indirect
215216
github.com/imdario/mergo v0.3.12 // indirect

‎go.sum

+8-3
Original file line numberDiff line numberDiff line change
@@ -655,8 +655,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
655655
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
656656
github.com/hashicorp/consul v1.7.8 h1:hp308KxAf3zWoGuwp2e+0UUhrm6qHjeBQk3jCZ+bjcY=
657657
github.com/hashicorp/consul v1.7.8/go.mod h1:urbfGaVZDmnXC6geg0LYPh/SRUk1E8nfmDHpz+Q0nLw=
658-
github.com/hashicorp/consul-template v0.29.0 h1:rDmF3Wjqp5ztCq054MruzEpi9ArcyJ/Rp4eWrDhMldM=
659-
github.com/hashicorp/consul-template v0.29.0/go.mod h1:p1A8Z6Mz7gbXu38SI1c9nt5ItBK7ACWZG4ZE1A5Tr2M=
658+
github.com/hashicorp/consul-template v0.29.1 h1:icm/H7klHYlxpUoWqSmTIWaSLEfGqUJJBsZA/2JhTLU=
659+
github.com/hashicorp/consul-template v0.29.1/go.mod h1:QIohwBuXlKXtsmGGQdWrISlUy4E6LFg5tLZyrw4MyoU=
660660
github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU=
661661
github.com/hashicorp/consul/api v1.13.0 h1:2hnLQ0GjQvw7f3O61jMO8gbasZviZTrt9R8WzgiirHc=
662662
github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ=
@@ -801,9 +801,13 @@ github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
801801
github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY=
802802
github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
803803
github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q=
804+
github.com/hashicorp/vault/api v1.3.0/go.mod h1:EabNQLI0VWbWoGlA+oBLC8PXmR9D60aUVgQGvangFWQ=
804805
github.com/hashicorp/vault/api v1.4.1 h1:mWLfPT0RhxBitjKr6swieCEP2v5pp/M//t70S3kMLRo=
805806
github.com/hashicorp/vault/api v1.4.1/go.mod h1:LkMdrZnWNrFaQyYYazWVn7KshilfDidgVBq6YiTq/bM=
807+
github.com/hashicorp/vault/api/auth/kubernetes v0.1.0 h1:6BtyahbF4aQp8gg3ww0A/oIoqzbhpNP1spXU3nHE0n0=
808+
github.com/hashicorp/vault/api/auth/kubernetes v0.1.0/go.mod h1:Pdgk78uIs0mgDOLvc3a+h/vYIT9rznw2sz+ucuH9024=
806809
github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
810+
github.com/hashicorp/vault/sdk v0.3.0/go.mod h1:aZ3fNuL5VNydQk8GcLJ2TV8YCRVvyaakYkhZRoVuhj0=
807811
github.com/hashicorp/vault/sdk v0.4.1 h1:3SaHOJY687jY1fnB61PtL0cOkKItphrbLmux7T92HBo=
808812
github.com/hashicorp/vault/sdk v0.4.1/go.mod h1:aZ3fNuL5VNydQk8GcLJ2TV8YCRVvyaakYkhZRoVuhj0=
809813
github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 h1:O/pT5C1Q3mVXMyuqg7yuAWUg/jMZR1/0QTzTRdNR6Uw=
@@ -1329,8 +1333,9 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh
13291333
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
13301334
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
13311335
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
1332-
golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167 h1:O8uGbHCqlTp2P6QJSLmCojM4mN6UemYv8K+dCnmHmu0=
13331336
golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
1337+
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
1338+
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
13341339
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
13351340
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
13361341
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

‎nomad/service_registration_endpoint.go

+78-1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ package nomad
22

33
import (
44
"net/http"
5+
"sort"
6+
"strconv"
7+
"strings"
58
"time"
69

710
"github.com/armon/go-metrics"
@@ -423,6 +426,16 @@ func (s *ServiceRegistration) GetService(
423426
http.StatusBadRequest, "failed to read result page: %v", err)
424427
}
425428

429+
// Select which subset and the order of services to return if using ?choose
430+
if args.Choose != "" {
431+
chosen, chooseErr := s.choose(services, args.Choose)
432+
if chooseErr != nil {
433+
return structs.NewErrRPCCodedf(
434+
http.StatusBadRequest, "failed to choose services: %v", chooseErr)
435+
}
436+
services = chosen
437+
}
438+
426439
// Populate the reply.
427440
reply.Services = services
428441
reply.NextToken = nextToken
@@ -434,6 +447,70 @@ func (s *ServiceRegistration) GetService(
434447
})
435448
}
436449

450+
// choose uses rendezvous hashing to make a stable selection of a subset of services
451+
// to return.
452+
//
453+
// parameter must in the form "<number>|<key>", where number is the number of services
454+
// to select, and key is incorporated in the hashing function with each service -
455+
// creating a unique yet consistent priority distribution pertaining to the requester.
456+
// In practice (i.e. via consul-template), the key is the AllocID generating a request
457+
// for upstream services.
458+
//
459+
// https://en.wikipedia.org/wiki/Rendezvous_hashing
460+
// w := priority (i.e. hash value)
461+
// h := hash function
462+
// O := object - (i.e. requesting service - using key (allocID) as a proxy)
463+
// S := site (i.e. destination service)
464+
func (*ServiceRegistration) choose(services []*structs.ServiceRegistration, parameter string) ([]*structs.ServiceRegistration, error) {
465+
// extract the number of services
466+
tokens := strings.SplitN(parameter, "|", 2)
467+
if len(tokens) != 2 {
468+
return nil, structs.ErrMalformedChooseParameter
469+
}
470+
n, err := strconv.Atoi(tokens[0])
471+
if err != nil {
472+
return nil, structs.ErrMalformedChooseParameter
473+
}
474+
475+
// extract the hash key
476+
key := tokens[1]
477+
if key == "" {
478+
return nil, structs.ErrMalformedChooseParameter
479+
}
480+
481+
// if there are fewer services than requested, go with the number of services
482+
if l := len(services); l < n {
483+
n = l
484+
}
485+
486+
type pair struct {
487+
hash string
488+
service *structs.ServiceRegistration
489+
}
490+
491+
// associate hash for each service
492+
priorities := make([]*pair, len(services))
493+
for i, service := range services {
494+
priorities[i] = &pair{
495+
hash: service.HashWith(key),
496+
service: service,
497+
}
498+
}
499+
500+
// sort by the hash; creating random distribution of priority
501+
sort.SliceStable(priorities, func(i, j int) bool {
502+
return priorities[i].hash < priorities[j].hash
503+
})
504+
505+
// choose top n services
506+
chosen := make([]*structs.ServiceRegistration, n)
507+
for i := 0; i < n; i++ {
508+
chosen[i] = priorities[i].service
509+
}
510+
511+
return chosen, nil
512+
}
513+
437514
// handleMixedAuthEndpoint is a helper to handle auth on RPC endpoints that can
438515
// either be called by Nomad nodes, or by external clients.
439516
func (s *ServiceRegistration) handleMixedAuthEndpoint(args structs.QueryOptions, cap string) error {
@@ -451,7 +528,7 @@ func (s *ServiceRegistration) handleMixedAuthEndpoint(args structs.QueryOptions,
451528
}
452529
}
453530
default:
454-
// In the event we got any error other than notfound, consider this
531+
// In the event we got any error other than ErrTokenNotFound, consider this
455532
// terminal.
456533
if err != structs.ErrTokenNotFound {
457534
return err

0 commit comments

Comments
 (0)