diff --git a/cns/configuration/cns_config.json b/cns/configuration/cns_config.json index 23ec3116ad..2efba1def7 100644 --- a/cns/configuration/cns_config.json +++ b/cns/configuration/cns_config.json @@ -29,5 +29,6 @@ "MSISettings": { "ResourceID": "" }, - "PopulateHomeAzCacheRetryIntervalSecs": 15 + "PopulateHomeAzCacheRetryIntervalSecs": 15, + "MellanoxMonitorIntervalSecs": 30 } diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 8366b6cd4a..2753b4f1b5 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -43,6 +43,7 @@ type CNSConfig struct { EnableCNIConflistGeneration bool CNIConflistFilepath string PopulateHomeAzCacheRetryIntervalSecs int + MellanoxMonitorIntervalSecs int } type TelemetrySettings struct { diff --git a/cns/service/main.go b/cns/service/main.go index 77e41b1348..56c4204fd5 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -713,6 +713,17 @@ func main() { return } + // We are only setting the PriorityVLANTag in 'cns.Direct' mode, because it neatly maps today, to 'isUsingMultitenancy' + // In the future, we would want to have a better CNS flag, to explicitly say, this CNS is using multitenancy + if config.ChannelMode == cns.Direct { + // Set Mellanox adapter's PriorityVLANTag value to 3 if adapter exists + // reg key value for PriorityVLANTag = 3 --> Packet priority and VLAN enabled + // for more details goto https://docs.nvidia.com/networking/display/winof2v230/Configuring+the+Driver+Registry+Keys#ConfiguringtheDriverRegistryKeys-GeneralRegistryKeysGeneralRegistryKeys + if platform.HasMellanoxAdapter() { + go platform.MonitorAndSetMellanoxRegKeyPriorityVLANTag(rootCtx, cnsconfig.MellanoxMonitorIntervalSecs) + } + } + // Initialze state in if CNS is running in CRD mode // State must be initialized before we start HTTPRestService if config.ChannelMode == cns.CRD { diff --git a/platform/Makefile b/platform/Makefile new file mode 100644 index 0000000000..7314b0b9c9 --- /dev/null +++ b/platform/Makefile @@ -0,0 +1,11 @@ +REPO_ROOT = $(shell git rev-parse --show-toplevel) +TOOLS_BIN_DIR = $(REPO_ROOT)/build/tools/bin +MOCKGEN = $(TOOLS_BIN_DIR)/mockgen + +.PHONY: generate + +generate: $(MOCKGEN) ## Generate mock clients + $(MOCKGEN) -source=$(REPO_ROOT)/platform/windows/adapter/network_adapter.go -package=mocks NetworkAdapter > windows/adapter/mocks/networkadapter_generated.go + +$(MOCKGEN): + @make -C $(REPO_ROOT) $(MOCKGEN) diff --git a/platform/os_linux.go b/platform/os_linux.go index 6aae528a79..60356d54ce 100644 --- a/platform/os_linux.go +++ b/platform/os_linux.go @@ -179,3 +179,11 @@ func PrintDependencyPackageDetails() { func ReplaceFile(source, destination string) error { return os.Rename(source, destination) } + +// Mellanox adapter not applicable for linux +func HasMellanoxAdapter() bool { + return false +} + +// Not needed for Linux +func MonitorAndSetMellanoxRegKeyPriorityVLANTag(_ context.Context, _ int) {} diff --git a/platform/os_windows.go b/platform/os_windows.go index 648c82bc0d..6c5b450a3c 100644 --- a/platform/os_windows.go +++ b/platform/os_windows.go @@ -5,6 +5,7 @@ package platform import ( "bytes" + "context" "fmt" "os" "os/exec" @@ -68,6 +69,14 @@ const ( // Command to restart HNS service RestartHnsServiceCommand = "Restart-Service -Name hns" + + // Interval between successive checks for mellanox adapter's PriorityVLANTag value + defaultMellanoxMonitorInterval = 30 * time.Second + + // Value for reg key: PriorityVLANTag for adapter + // reg key value for PriorityVLANTag = 3 --> Packet priority and VLAN enabled + // for more details goto https://learn.microsoft.com/en-us/windows-hardware/drivers/network/standardized-inf-keywords-for-ndis-qos + desiredVLANTagForMellanox = 3 ) // Flag to check if sdnRemoteArpMacAddress registry key is set @@ -191,6 +200,68 @@ func SetSdnRemoteArpMacAddress() error { return nil } +func HasMellanoxAdapter() bool { + m := &mellanox.Mellanox{} + return hasNetworkAdapter(m) +} + +func hasNetworkAdapter(na adapter.NetworkAdapter) bool { + adapterName, err := na.GetAdapterName() + if err != nil { + log.Errorf("Error while getting network adapter name: %v", err) + return false + } + log.Printf("Name of the network adapter : %v", adapterName) + return true +} + +// Regularly monitors the Mellanox PriorityVLANGTag registry value and sets it to desired value if needed +func MonitorAndSetMellanoxRegKeyPriorityVLANTag(ctx context.Context, intervalSecs int) { + m := &mellanox.Mellanox{} + interval := defaultMellanoxMonitorInterval + if intervalSecs > 0 { + interval = time.Duration(intervalSecs) * time.Second + } + err := updatePriorityVLANTagIfRequired(m, desiredVLANTagForMellanox) + if err != nil { + log.Errorf("Error while monitoring mellanox, continuing: %v", err) + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + log.Printf("context cancelled, stopping Mellanox Monitoring: %v", ctx.Err()) + return + case <-ticker.C: + err := updatePriorityVLANTagIfRequired(m, desiredVLANTagForMellanox) + if err != nil { + log.Errorf("Error while monitoring mellanox, continuing: %v", err) + } + } + } +} + +// Updates the priority VLAN Tag of mellanox adapter if not already set to the desired value +func updatePriorityVLANTagIfRequired(na adapter.NetworkAdapter, desiredValue int) error { + currentVal, err := na.GetPriorityVLANTag() + if err != nil { + return fmt.Errorf("error while getting Priority VLAN Tag value: %w", err) + } + + if currentVal == desiredValue { + log.Printf("Adapter's PriorityVLANTag is already set to %v, skipping reset", desiredValue) + return nil + } + + err = na.SetPriorityVLANTag(desiredValue) + if err != nil { + return fmt.Errorf("error while setting Priority VLAN Tag value: %w", err) + } + + return nil +} + func GetOSDetails() (map[string]string, error) { return nil, nil } diff --git a/platform/windows/adapter/mellanox/mellanox.go b/platform/windows/adapter/mellanox/mellanox.go new file mode 100644 index 0000000000..e7d37cc2e7 --- /dev/null +++ b/platform/windows/adapter/mellanox/mellanox.go @@ -0,0 +1,222 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package mellanox + +import ( + "bytes" + "fmt" + "os/exec" + "strconv" + "strings" + + "github.com/Azure/azure-container-networking/log" +) + +const ( + // Search string to find adapter having Mellanox in description + mellanoxSearchString = "*Mellanox*" + + // PriorityVlanTag reg key for adapter + priorityVLANTagIdentifier = "*PriorityVLANTag" + + // Registry key Path Prefix + registryKeyPrefix = "HKLM:\\System\\CurrentControlSet\\Control\\Class\\" +) + +var ( + errorMellanoxAdapterNotFound = fmt.Errorf("no network adapter found with %s in description", mellanoxSearchString) + errorMellanoxDeviceNotFound = fmt.Errorf("no network device found with %s in description", mellanoxSearchString) + errorPowershellNotFound = fmt.Errorf("failed to find powershell executable") +) + +type Mellanox struct{} + +// GetAdapter returns name of Mellanox adapter if found +// Returns errorMellanoxAdapterNotFound if adapter is not found or adapter name empty +func (m *Mellanox) GetAdapterName() (string, error) { + // get mellanox adapter name + cmd := fmt.Sprintf(`Get-NetAdapter | Where-Object { $_.InterfaceDescription -like '%s' } | Select-Object -ExpandProperty Name`, mellanoxSearchString) + adapterName, err := executePowershellCommand(cmd) + if err != nil { + return "", fmt.Errorf("error while executing powershell command to get net adapter list: %w", err) + } + if adapterName == "" { + return "", errorMellanoxAdapterNotFound + } + return adapterName, nil +} + +// Set Mellanox adapter's PriorityVLANTag value to desired value if adapter exists +// 5/16/23 : right now setting desired reg key value for PriorityVLANTag = 3 --> Packet priority and VLAN enabled +// for more details goto https://docs.nvidia.com/networking/display/winof2v230/Configuring+the+Driver+Registry+Keys#ConfiguringtheDriverRegistryKeys-GeneralRegistryKeysGeneralRegistryKeys +func (m *Mellanox) SetPriorityVLANTag(desiredVal int) error { + adapterName, err := m.GetAdapterName() + if err != nil { + return fmt.Errorf("failed to find mellanox adapter: %w", err) + } + + // Find if adapter has property PriorityVLANTag (version 4 or up) or not (version 3) + cmd := fmt.Sprintf(`Get-NetAdapterAdvancedProperty | Where-Object { $_.RegistryKeyword -like '%s' -and $_.Name -eq '%s' } | Select-Object -ExpandProperty Name`, + priorityVLANTagIdentifier, adapterName) + adapterNameWithVLANTag, err := executePowershellCommand(cmd) + if err != nil { + return fmt.Errorf("error while executing powershell command to get VLAN Tag advance property of %s: %w", adapterName, err) + } + + if adapterNameWithVLANTag != "" { + return m.setMellanoxPriorityVLANTagValueForV4(adapterNameWithVLANTag, desiredVal) + } + return m.setMellanoxPriorityVLANTagValueForV3(adapterName, desiredVal) +} + +// Get PriorityVLANTag returns PriorityVLANTag value for Mellanox Adapter (both version 3 and version 4) +func (m *Mellanox) GetPriorityVLANTag() (int, error) { + adapterName, err := m.GetAdapterName() + if err != nil { + return 0, fmt.Errorf("failed to find mellanox adapter: %w", err) + } + + // Find if adapter has property PriorityVLANTag (version 4 or up) or not (version 3) + cmd := fmt.Sprintf(`Get-NetAdapterAdvancedProperty | Where-Object { $_.RegistryKeyword -like '%s' -and $_.Name -eq '%s' } | Select-Object -ExpandProperty Name`, + priorityVLANTagIdentifier, adapterName) + adapterNameWithVLANTag, err := executePowershellCommand(cmd) + if err != nil { + return 0, fmt.Errorf("error while executing powershell command to get VLAN Tag advance property of %s: %w", adapterName, err) + } + + if adapterNameWithVLANTag != "" { + return m.getMellanoxPriorityVLANTagValueForV4(adapterNameWithVLANTag) + } + + return m.getMellanoxPriorityVLANTagValueForV3() +} + +// Checks if a Mellanox adapter's PriorityVLANTag value +// for version 4 and up is set to the given expected value +func (m *Mellanox) getMellanoxPriorityVLANTagValueForV4(adapterName string) (int, error) { + cmd := fmt.Sprintf( + `Get-NetAdapterAdvancedProperty | Where-Object { $_.RegistryKeyword -like '%s' -and $_.Name -eq '%s' } | Select-Object -ExpandProperty RegistryValue`, + priorityVLANTagIdentifier, adapterName) + + regvalue, err := executePowershellCommand(cmd) + if err != nil { + return 0, err + } + + intValue, err := strconv.Atoi(regvalue) + if err != nil { + return 0, fmt.Errorf("failed to convert PriorityVLANTag value to integer: %w", err) + } + + return intValue, nil +} + +// Checks if a Mellanox adapter's PriorityVLANTag value +// for version 3 and below is set to the given expected value +func (m *Mellanox) getMellanoxPriorityVLANTagValueForV3() (int, error) { + registryKeyFullPath, err := m.getRegistryFullPath() + if err != nil { + return 0, err + } + + cmd := fmt.Sprintf( + `Get-ItemProperty -Path '%s' -Name '%s' | Select-Object -ExpandProperty '%s'`, registryKeyFullPath, priorityVLANTagIdentifier, priorityVLANTagIdentifier) + regvalue, err := executePowershellCommand(cmd) + if err != nil { + return 0, err + } + + intValue, err := strconv.Atoi(regvalue) + if err != nil { + return 0, fmt.Errorf("failed to convert PriorityVLANTag value to integer: %w", err) + } + + return intValue, nil +} + +// adapter is version 4 and up since adapter's advance property consists of reg key : PriorityVLANTag +// set reg value for Priorityvlantag of adapter to 3 if not set already +func (m *Mellanox) setMellanoxPriorityVLANTagValueForV4(adapterName string, desiredVal int) error { + cmd := fmt.Sprintf( + `Set-NetAdapterAdvancedProperty -Name '%s' -RegistryKeyword '%s' -RegistryValue %d`, + adapterName, priorityVLANTagIdentifier, desiredVal) + _, err := executePowershellCommand(cmd) + if err != nil { + return fmt.Errorf("error while setting up registry value for PriorityVLANTag for adapter: %w", err) + } + + log.Printf("Successfully set Mellanox Network Adapter: %s with %s property value as %d", + adapterName, priorityVLANTagIdentifier, desiredVal) + return nil +} + +// Adapter is version 3 or less as PriorityVLANTag was not found in advanced properties of mellanox adapter +func (m *Mellanox) setMellanoxPriorityVLANTagValueForV3(adapterName string, desiredVal int) error { + registryKeyFullPath, err := m.getRegistryFullPath() + if err != nil { + return err + } + + cmd := fmt.Sprintf(`New-ItemProperty -Path '%s' -Name '%s' -Value %d -PropertyType String -Force`, + registryKeyFullPath, priorityVLANTagIdentifier, desiredVal) + _, err = executePowershellCommand(cmd) + if err != nil { + return fmt.Errorf("error while executing powershell command to set Item property for adapter %s: %w", adapterName, err) + } + + log.Printf("Restarting Mellanox network adapter for regkey change to take effect") + cmd = fmt.Sprintf(`Restart-NetAdapter -Name '%s'`, adapterName) + _, err = executePowershellCommand(cmd) + if err != nil { + return fmt.Errorf("error while executing powershell command to restart net adapter %s: %w", adapterName, err) + } + log.Printf("For Mellanox CX-3 adapters, the reg key set to %d", desiredVal) + return nil +} + +// Get registry full path for Mellanox Adapter +func (m *Mellanox) getRegistryFullPath() (string, error) { + log.Printf("Searching through CIM instances for Network devices with %s in the name", mellanoxSearchString) + cmd := fmt.Sprintf( + `Get-CimInstance -Namespace root/cimv2 -ClassName Win32_PNPEntity | Where-Object PNPClass -EQ "Net" | Where-Object { $_.Name -like '%s' } | Select-Object -ExpandProperty DeviceID`, + mellanoxSearchString) + deviceid, err := executePowershellCommand(cmd) + if err != nil { + return "", fmt.Errorf("error while executing powershell command to get device id for Mellanox: %w", err) + } + if deviceid == "" { + return "", errorMellanoxDeviceNotFound + } + + cmd = fmt.Sprintf(`Get-PnpDeviceProperty -InstanceId '%s' | Where-Object KeyName -EQ "DEVPKEY_Device_Driver" | Select-Object -ExpandProperty Data`, deviceid) + registryKeySuffix, err := executePowershellCommand(cmd) + if err != nil { + return "", fmt.Errorf("error while executing powershell command to get registry suffix of device id %s: %w", deviceid, err) + } + + return registryKeyPrefix + registryKeySuffix, nil +} + +// ExecutePowershellCommand executes powershell command +func executePowershellCommand(command string) (string, error) { + ps, err := exec.LookPath("powershell.exe") + if err != nil { + return "", errorPowershellNotFound + } + + log.Printf("[Azure-Utils] %s", command) + + cmd := exec.Command(ps, command) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("%s:%w", stderr.String(), err) + } + + return strings.TrimSpace(stdout.String()), nil +} diff --git a/platform/windows/adapter/mocks/networkadapter_generated.go b/platform/windows/adapter/mocks/networkadapter_generated.go new file mode 100644 index 0000000000..42651c221f --- /dev/null +++ b/platform/windows/adapter/mocks/networkadapter_generated.go @@ -0,0 +1,78 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: /mnt/d/Projects/azure-container-networking/platform/windows/adapter/network_adapter.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockNetworkAdapter is a mock of NetworkAdapter interface. +type MockNetworkAdapter struct { + ctrl *gomock.Controller + recorder *MockNetworkAdapterMockRecorder +} + +// MockNetworkAdapterMockRecorder is the mock recorder for MockNetworkAdapter. +type MockNetworkAdapterMockRecorder struct { + mock *MockNetworkAdapter +} + +// NewMockNetworkAdapter creates a new mock instance. +func NewMockNetworkAdapter(ctrl *gomock.Controller) *MockNetworkAdapter { + mock := &MockNetworkAdapter{ctrl: ctrl} + mock.recorder = &MockNetworkAdapterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockNetworkAdapter) EXPECT() *MockNetworkAdapterMockRecorder { + return m.recorder +} + +// GetAdapterName mocks base method. +func (m *MockNetworkAdapter) GetAdapterName() (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetAdapterName") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAdapterName indicates an expected call of GetAdapterName. +func (mr *MockNetworkAdapterMockRecorder) GetAdapterName() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdapterName", reflect.TypeOf((*MockNetworkAdapter)(nil).GetAdapterName)) +} + +// GetPriorityVLANTag mocks base method. +func (m *MockNetworkAdapter) GetPriorityVLANTag() (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPriorityVLANTag") + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPriorityVLANTag indicates an expected call of GetPriorityVLANTag. +func (mr *MockNetworkAdapterMockRecorder) GetPriorityVLANTag() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriorityVLANTag", reflect.TypeOf((*MockNetworkAdapter)(nil).GetPriorityVLANTag)) +} + +// SetPriorityVLANTag mocks base method. +func (m *MockNetworkAdapter) SetPriorityVLANTag(arg0 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetPriorityVLANTag", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetPriorityVLANTag indicates an expected call of SetPriorityVLANTag. +func (mr *MockNetworkAdapterMockRecorder) SetPriorityVLANTag(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriorityVLANTag", reflect.TypeOf((*MockNetworkAdapter)(nil).SetPriorityVLANTag), arg0) +} diff --git a/platform/windows/adapter/network_adapter.go b/platform/windows/adapter/network_adapter.go new file mode 100644 index 0000000000..80f82a6539 --- /dev/null +++ b/platform/windows/adapter/network_adapter.go @@ -0,0 +1,16 @@ +// Copyright 2017 Microsoft. All rights reserved. +// MIT License + +package adapter + +type NetworkAdapter interface { + // GetAdapter returns name of adapter if found + // Must return error if adapter is not found or adapter name empty + GetAdapterName() (string, error) + + // Get PriorityVLANTag returns PriorityVLANTag value for Adapter + GetPriorityVLANTag() (int, error) + + // Set adapter's PriorityVLANTag value to desired value if adapter exists + SetPriorityVLANTag(int) error +}