Skip to content

Commit 25996f7

Browse files
nddqnddq
authored
refactor: SWIFT v2 Middlewares (#2390)
* refractor: swift v2 middleware * revert: unexport getTestService * fix: linter * fix: linter issue * refactor: generic renaming of middlewares-related code in cns * refactor: change type of swiftv2middleware * address feedbacks * rename for clarity * missed some renaming * renaming * addressed comments * add UT for no-op releasing ip config for v2 pod even if its mtpnc is not ready * pass in SWIFTv2Mode as string * typed SWIFTV2Mode
1 parent 0cb52df commit 25996f7

File tree

12 files changed

+374
-408
lines changed

12 files changed

+374
-408
lines changed

‎cns/api.go

+7-8
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,16 @@ type HTTPService interface {
4949
GetPendingReleaseIPConfigs() []IPConfigurationStatus
5050
GetPodIPConfigState() map[string]IPConfigurationStatus
5151
MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error)
52-
AttachSWIFTv2Middleware(middleware SWIFTv2Middleware)
52+
AttachIPConfigsHandlerMiddleware(IPConfigsHandlerMiddleware)
5353
}
5454

55-
// Middleware interface for testing later on
56-
type SWIFTv2Middleware interface {
57-
ValidateIPConfigsRequest(context.Context, *IPConfigsRequest) (types.ResponseCode, string)
58-
GetIPConfig(context.Context, PodInfo) (PodIpInfo, error)
59-
SetRoutes(*PodIpInfo) error
60-
}
55+
// IPConfigsHandlerFunc
56+
type IPConfigsHandlerFunc func(context.Context, IPConfigsRequest) (*IPConfigsResponse, error)
6157

62-
type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string)
58+
// IPConfigsHandlerMiddleware
59+
type IPConfigsHandlerMiddleware interface {
60+
IPConfigsRequestHandlerWrapper(defaultHandler IPConfigsHandlerFunc, failureHandler IPConfigsHandlerFunc) IPConfigsHandlerFunc
61+
}
6362

6463
// This is used for KubernetesCRD orchestrator Type where NC has multiple ips.
6564
// This struct captures the state for SecondaryIPs associated to a given NC

‎cns/configuration/configuration.go

+7
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,24 @@ import (
1313
"github.com/pkg/errors"
1414
)
1515

16+
type SWIFTV2Mode string
17+
1618
const (
1719
// EnvCNSConfig is the CNS_CONFIGURATION_PATH env var key
1820
EnvCNSConfig = "CNS_CONFIGURATION_PATH"
1921
defaultConfigName = "cns_config.json"
22+
// Service Fabric SWIFTV2 mode
23+
SFSWIFTV2 SWIFTV2Mode = "SFSWIFTV2"
24+
// K8s SWIFTV2 mode
25+
K8sSWIFTV2 SWIFTV2Mode = "K8sSWIFTV2"
2026
)
2127

2228
type CNSConfig struct {
2329
ChannelMode string
2430
EnablePprof bool
2531
EnableSubnetScarcity bool
2632
EnableSwiftV2 bool
33+
SWIFTV2Mode SWIFTV2Mode
2734
InitializeFromCNI bool
2835
ManagedSettings ManagedSettings
2936
MetricsBindAddress string

‎cns/fakes/cnsfake.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -277,4 +277,4 @@ func (fake *HTTPServiceFake) Init(*common.ServiceConfig) error {
277277

278278
func (fake *HTTPServiceFake) Stop() {}
279279

280-
func (fake *HTTPServiceFake) AttachSWIFTv2Middleware(cns.SWIFTv2Middleware) {}
280+
func (fake *HTTPServiceFake) AttachIPConfigsHandlerMiddleware(cns.IPConfigsHandlerMiddleware) {}

‎cns/middlewares/swiftV2.go ‎cns/middlewares/k8sSwiftV2.go

+77-12
Original file line numberDiff line numberDiff line change
@@ -30,25 +30,89 @@ const (
3030
overlayGatewayV6 = "fe80::1234:5678:9abc"
3131
)
3232

33-
type SWIFTv2Middleware struct {
33+
type K8sSWIFTv2Middleware struct {
3434
Cli client.Client
3535
}
3636

37-
// ValidateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario.
37+
// Verify interface compliance at compile time
38+
var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)
39+
40+
// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
41+
// and release IP configs handlers.
42+
func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
43+
return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
44+
podInfo, respCode, message := m.validateIPConfigsRequest(ctx, &req)
45+
46+
if respCode != types.Success {
47+
return &cns.IPConfigsResponse{
48+
Response: cns.Response{
49+
ReturnCode: respCode,
50+
Message: message,
51+
},
52+
}, errors.New("failed to validate ip configs request")
53+
}
54+
ipConfigsResp, err := defaultHandler(ctx, req)
55+
// If the pod is not v2, return the response from the handler
56+
if !req.SecondaryInterfacesExist {
57+
return ipConfigsResp, err
58+
}
59+
// If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config
60+
defer func() {
61+
// Release the default IP config if there is an error
62+
if err != nil {
63+
_, err = failureHandler(ctx, req)
64+
if err != nil {
65+
logger.Errorf("failed to release default IP config : %v", err)
66+
}
67+
}
68+
}()
69+
if err != nil {
70+
return ipConfigsResp, err
71+
}
72+
SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo)
73+
if err != nil {
74+
return &cns.IPConfigsResponse{
75+
Response: cns.Response{
76+
ReturnCode: types.FailedToAllocateIPConfig,
77+
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
78+
},
79+
PodIPInfo: []cns.PodIpInfo{},
80+
}, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req)
81+
}
82+
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfo)
83+
// Set routes for the pod
84+
for i := range ipConfigsResp.PodIPInfo {
85+
ipInfo := &ipConfigsResp.PodIPInfo[i]
86+
err = m.setRoutes(ipInfo)
87+
if err != nil {
88+
return &cns.IPConfigsResponse{
89+
Response: cns.Response{
90+
ReturnCode: types.FailedToAllocateIPConfig,
91+
Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req),
92+
},
93+
PodIPInfo: []cns.PodIpInfo{},
94+
}, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name())
95+
}
96+
}
97+
return ipConfigsResp, nil
98+
}
99+
}
100+
101+
// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
38102
// nolint
39-
func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) {
103+
func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
40104
// Retrieve the pod from the cluster
41105
podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
42106
if err != nil {
43107
errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req)
44-
return types.UnexpectedError, errBuf.Error()
108+
return nil, types.UnexpectedError, errBuf.Error()
45109
}
46110
logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
47111
podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
48112
pod := v1.Pod{}
49113
if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
50114
errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
51-
return types.UnexpectedError, errBuf.Error()
115+
return nil, types.UnexpectedError, errBuf.Error()
52116
}
53117

54118
// check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready
@@ -58,19 +122,20 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c
58122
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
59123
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
60124
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
61-
return types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
125+
return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
62126
}
63127
// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
64128
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
65-
return types.UnexpectedError, errMTPNCNotReady.Error()
129+
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
66130
}
67131
}
68132
logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist)
69-
return types.Success, ""
133+
// retrieve podinfo from orchestrator context
134+
return podInfo, types.Success, ""
70135
}
71136

72-
// GetIPConfig returns the pod's SWIFT V2 IP configuration.
73-
func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
137+
// getIPConfig returns the pod's SWIFT V2 IP configuration.
138+
func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
74139
// Check if the MTPNC CRD exists for the pod, if not, return error
75140
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
76141
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
@@ -109,8 +174,8 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo
109174
return podIPInfo, nil
110175
}
111176

112-
// SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
113-
func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error {
177+
// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
178+
func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
114179
logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType)
115180
podIPInfo.Routes = []cns.Route{}
116181
switch podIPInfo.NICType {

0 commit comments

Comments
 (0)