Skip to content

Commit b0c5a8d

Browse files
matmerrmatmerr
committed
Merge branch 'master' into matmerr/e2eframework-scenarios
2 parents b44781e + 8b5a956 commit b0c5a8d

31 files changed

+1138
-366
lines changed

‎.pipelines/singletenancy/aks-swift/e2e-step-template.yaml

+3-28
Original file line numberDiff line numberDiff line change
@@ -52,29 +52,13 @@ steps:
5252
pwd
5353
kubectl cluster-info
5454
kubectl get po -owide -A
55-
sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true
55+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
5656
retryCountOnTaskFailure: 3
5757
name: "aksswifte2e"
5858
displayName: "Run AKS Swift E2E"
5959
6060
- script: |
61-
echo "Logs will be available as a build artifact"
62-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
63-
echo $ARTIFACT_DIR
64-
sudo rm -rf $ARTIFACT_DIR
65-
sudo mkdir $ARTIFACT_DIR
66-
sudo cp test/integration/logs/* $ARTIFACT_DIR
67-
name: "GetLogs"
68-
displayName: "Get logs"
69-
condition: always()
70-
71-
- task: PublishBuildArtifacts@1
72-
inputs:
73-
artifactName: aks-swift-output
74-
pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output"
75-
condition: always()
76-
77-
- script: |
61+
kubectl get po -owide -A
7862
echo "Run Service Conformance E2E"
7963
export PATH=${PATH}:/usr/local/bin/gsutil
8064
KUBECONFIG=~/.kube/config kubetest2 noop \
@@ -98,7 +82,7 @@ steps:
9882
retryCountOnTaskFailure: 3
9983
name: "WireserverMetadataConnectivityTests"
10084
displayName: "Run Wireserver and Metadata Connectivity Tests"
101-
85+
10286
- script: |
10387
cd hack/scripts
10488
chmod +x async-delete-test.sh
@@ -108,12 +92,3 @@ steps:
10892
fi
10993
name: "testAsyncDelete"
11094
displayName: "Verify Async Delete when CNS is down"
111-
112-
- script: |
113-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
114-
echo $ARTIFACT_DIR
115-
sudo rm -rf $ARTIFACT_DIR
116-
sudo rm -rf test/integration/logs
117-
name: "Cleanupartifactdir"
118-
displayName: "Cleanup artifact dir"
119-
condition: always()

‎.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml

+4-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ steps:
3434
- ${{ if eq(parameters.os, 'linux') }}:
3535
- script: |
3636
echo "Start Integration Tests on Overlay Cluster"
37-
sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true
37+
kubectl get po -owide -A
38+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
3839
retryCountOnTaskFailure: 2
3940
name: "integrationTest"
4041
displayName: "Run CNS Integration Tests on AKS Overlay"
@@ -47,6 +48,7 @@ steps:
4748
addSpnToEnvironment: true
4849
inlineScript: |
4950
set -e
51+
kubectl get po -owide -A
5052
clusterName=${{ parameters.clusterName }}
5153
echo "Restarting nodes"
5254
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
@@ -101,6 +103,7 @@ steps:
101103
scriptType: "bash"
102104
addSpnToEnvironment: true
103105
inlineScript: |
106+
set -e
104107
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
105108
echo "Windows node are successfully added to v4 Overlay Cluster"
106109
kubectl cluster-info

‎.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml

+1-34
Original file line numberDiff line numberDiff line change
@@ -73,31 +73,11 @@ steps:
7373
else
7474
CNS=$(make cns-version) IPAM=$(make azure-ipam-version)
7575
fi
76-
sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true
76+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true
7777
retryCountOnTaskFailure: 3
7878
name: "aziliumTest"
7979
displayName: "Run Azilium E2E on AKS Overlay"
8080
81-
- script: |
82-
echo "Status of the nodes and pods after the test"
83-
kubectl get nodes -o wide
84-
kubectl get pods -A -o wide
85-
echo "Logs will be available as a build artifact"
86-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
87-
echo $ARTIFACT_DIR
88-
sudo rm -rf $ARTIFACT_DIR
89-
sudo mkdir $ARTIFACT_DIR
90-
sudo cp test/integration/logs/* $ARTIFACT_DIR
91-
name: "GetLogs"
92-
displayName: "Get logs"
93-
condition: always()
94-
95-
- task: PublishBuildArtifacts@1
96-
inputs:
97-
artifactName: test-output
98-
pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
99-
condition: always()
100-
10181
- script: |
10282
kubectl get pods -A
10383
echo "Waiting < 2 minutes for cilium to be ready"
@@ -176,19 +156,6 @@ steps:
176156
name: "CiliumIdentities"
177157
displayName: "Verify Cilium Identities Deletion"
178158
179-
- script: |
180-
echo "validate pod IP assignment before CNS restart"
181-
kubectl get pod -owide -A
182-
make test-validate-state
183-
echo "restart CNS"
184-
kubectl rollout restart ds azure-cns -n kube-system
185-
kubectl rollout status ds azure-cns -n kube-system
186-
kubectl get pod -owide -A
187-
echo "validate pods after CNS restart"
188-
make test-validate-state
189-
name: "restartCNS"
190-
displayName: "Restart CNS and validate pods"
191-
192159
- script: |
193160
echo "Run wireserver and metadata connectivity Tests"
194161
bash test/network/wireserver_metadata_test.sh

‎.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml

+3-31
Original file line numberDiff line numberDiff line change
@@ -81,33 +81,14 @@ steps:
8181
else
8282
CNS=$(make cns-version) IPAM=$(make azure-ipam-version)
8383
fi
84-
sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true
84+
kubectl get po -owide -A
85+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true
8586
retryCountOnTaskFailure: 3
8687
name: "aziliumTest"
8788
displayName: "Run Azilium E2E on AKS Overlay"
8889
8990
- script: |
90-
echo "Status of the nodes and pods after the test"
91-
kubectl get nodes -o wide
92-
kubectl get pods -A -o wide
93-
echo "Logs will be available as a build artifact"
94-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
95-
echo $ARTIFACT_DIR
96-
sudo rm -rf $ARTIFACT_DIR
97-
sudo mkdir $ARTIFACT_DIR
98-
sudo cp test/integration/logs/* $ARTIFACT_DIR
99-
name: "GetLogs"
100-
displayName: "Get logs"
101-
condition: always()
102-
103-
- task: PublishBuildArtifacts@1
104-
inputs:
105-
artifactName: test-output
106-
pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
107-
condition: always()
108-
109-
- script: |
110-
kubectl get pods -A
91+
kubectl get po -owide -A
11192
echo "Waiting < 2 minutes for cilium to be ready"
11293
# Ensure Cilium is ready Xm\Xs
11394
cilium status --wait --wait-duration 2m
@@ -203,12 +184,3 @@ steps:
203184
retryCountOnTaskFailure: 3
204185
name: "WireserverMetadataConnectivityTests"
205186
displayName: "Run Wireserver and Metadata Connectivity Tests"
206-
207-
- script: |
208-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
209-
echo $ARTIFACT_DIR
210-
sudo rm -rf $ARTIFACT_DIR
211-
sudo rm -rf test/integration/logs
212-
name: "Cleanupartifactdir"
213-
displayName: "Cleanup artifact dir"
214-
condition: always()

‎.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml

+2-31
Original file line numberDiff line numberDiff line change
@@ -72,33 +72,13 @@ steps:
7272
- script: |
7373
echo "Start Azilium E2E Tests"
7474
kubectl get po -owide -A
75-
sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZILIUM=true
75+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZILIUM=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true
7676
retryCountOnTaskFailure: 3
7777
name: "aziliumTest"
7878
displayName: "Run Azilium E2E"
7979
8080
- script: |
81-
echo "Status of the nodes and pods after the test"
82-
kubectl get nodes -o wide
83-
kubectl get pods -A -o wide
84-
echo "Logs will be available as a build artifact"
85-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
86-
echo $ARTIFACT_DIR
87-
sudo rm -rf $ARTIFACT_DIR
88-
sudo mkdir $ARTIFACT_DIR
89-
sudo cp test/integration/logs/* $ARTIFACT_DIR
90-
name: "GetLogs"
91-
displayName: "Get logs"
92-
condition: always()
93-
94-
- task: PublishBuildArtifacts@1
95-
inputs:
96-
artifactName: test-output
97-
pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
98-
condition: always()
99-
100-
- script: |
101-
kubectl get pods -A
81+
kubectl get po -owide -A
10282
echo "Waiting < 2 minutes for cilium to be ready"
10383
# Ensure Cilium is ready Xm\Xs
10484
cilium status --wait --wait-duration 2m
@@ -147,12 +127,3 @@ steps:
147127
retryCountOnTaskFailure: 3
148128
name: "WireserverMetadataConnectivityTests"
149129
displayName: "Run Wireserver and Metadata Connectivity Tests"
150-
151-
- script: |
152-
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
153-
echo $ARTIFACT_DIR
154-
sudo rm -rf $ARTIFACT_DIR
155-
sudo rm -rf test/integration/logs
156-
name: "Cleanupartifactdir"
157-
displayName: "Cleanup artifact dir"
158-
condition: always()

‎.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,15 @@ steps:
3636
- script: |
3737
kubectl cluster-info
3838
kubectl get node
39-
sudo -E env "PATH=$PATH" make test-integration CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true
4039
kubectl get po -owide -A
40+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true
4141
retryCountOnTaskFailure: 3
4242
name: "integrationTest"
4343
displayName: "Run CNS Integration Tests on AKS DualStack Overlay"
4444
4545
- script: |
4646
set -e
47+
kubectl get po -owide -A
4748
cd test/integration/datapath
4849
echo "Dualstack Overlay Linux datapath IPv6 test"
4950
go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
@@ -108,6 +109,7 @@ steps:
108109
scriptType: "bash"
109110
addSpnToEnvironment: true
110111
inlineScript: |
112+
set -e
111113
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
112114
echo "Windows nodes have been successfully added to DualStack Overlay Cluster"
113115
kubectl cluster-info

‎cns/api.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ type HTTPService interface {
5050
GetPodIPConfigState() map[string]IPConfigurationStatus
5151
MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error)
5252
AttachIPConfigsHandlerMiddleware(IPConfigsHandlerMiddleware)
53+
MarkNIPsPendingRelease(n int) (map[string]IPConfigurationStatus, error)
5354
}
5455

5556
// IPConfigsHandlerFunc
@@ -281,13 +282,13 @@ type NodeConfiguration struct {
281282
NodeSubnet Subnet
282283
}
283284

285+
// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct
284286
type IPAMPoolMonitor interface {
285287
Start(ctx context.Context) error
286288
Update(nnc *v1alpha.NodeNetworkConfig) error
287289
GetStateSnapshot() IpamPoolMonitorStateSnapshot
288290
}
289291

290-
// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct
291292
type IpamPoolMonitorStateSnapshot struct {
292293
MinimumFreeIps int64
293294
MaximumFreeIps int64

‎cns/configuration/configuration.go

+15-13
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,26 @@ const (
2626
)
2727

2828
type CNSConfig struct {
29+
AZRSettings AZRSettings
30+
AsyncPodDeletePath string
31+
CNIConflistFilepath string
32+
CNIConflistScenario string
2933
ChannelMode string
34+
EnableAsyncPodDelete bool
35+
EnableCNIConflistGeneration bool
36+
EnableIPAMv2 bool
3037
EnablePprof bool
3138
EnableSubnetScarcity bool
3239
EnableSwiftV2 bool
33-
SWIFTV2Mode SWIFTV2Mode
3440
InitializeFromCNI bool
41+
KeyVaultSettings KeyVaultSettings
42+
MSISettings MSISettings
43+
ManageEndpointState bool
3544
ManagedSettings ManagedSettings
45+
MellanoxMonitorIntervalSecs int
3646
MetricsBindAddress string
47+
ProgramSNATIPTables bool
48+
SWIFTV2Mode SWIFTV2Mode
3749
SyncHostNCTimeoutMs int
3850
SyncHostNCVersionIntervalMs int
3951
TLSCertificatePath string
@@ -42,19 +54,8 @@ type CNSConfig struct {
4254
TLSSubjectName string
4355
TelemetrySettings TelemetrySettings
4456
UseHTTPS bool
57+
WatchPods bool `json:"-"`
4558
WireserverIP string
46-
KeyVaultSettings KeyVaultSettings
47-
MSISettings MSISettings
48-
ProgramSNATIPTables bool
49-
ManageEndpointState bool
50-
CNIConflistScenario string
51-
EnableCNIConflistGeneration bool
52-
CNIConflistFilepath string
53-
MellanoxMonitorIntervalSecs int
54-
AZRSettings AZRSettings
55-
WatchPods bool
56-
EnableAsyncPodDelete bool
57-
AsyncPodDeletePath string
5859
}
5960

6061
type TelemetrySettings struct {
@@ -219,4 +220,5 @@ func SetCNSConfigDefaults(config *CNSConfig) {
219220
if config.AsyncPodDeletePath == "" {
220221
config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs"
221222
}
223+
config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2
222224
}

‎cns/fakes/cnsfake.go

+9
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,11 @@ func (ipm *IPStateManager) ReleaseIPConfig(ipconfigID string) (cns.IPConfigurati
115115
return ipm.AvailableIPConfigState[ipconfigID], nil
116116
}
117117

118+
func (ipm *IPStateManager) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) {
119+
// MarkIPASPendingRelease actually already errors if it is unable to release all N IPs
120+
return ipm.MarkIPAsPendingRelease(n)
121+
}
122+
118123
func (ipm *IPStateManager) MarkIPAsPendingRelease(numberOfIPsToMark int) (map[string]cns.IPConfigurationStatus, error) {
119124
ipm.Lock()
120125
defer ipm.Unlock()
@@ -256,6 +261,10 @@ func (fake *HTTPServiceFake) GetPodIPConfigState() map[string]cns.IPConfiguratio
256261
return ipconfigs
257262
}
258263

264+
func (fake *HTTPServiceFake) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) {
265+
return fake.IPStateManager.MarkIPAsPendingRelease(n)
266+
}
267+
259268
// TODO: Populate on scale down
260269
func (fake *HTTPServiceFake) MarkIPAsPendingRelease(numberToMark int) (map[string]cns.IPConfigurationStatus, error) {
261270
return fake.IPStateManager.MarkIPAsPendingRelease(numberToMark)

0 commit comments

Comments
 (0)