set mellanox reg key (#1768)

(cherry picked from commit fa2de6d)

cns/configuration/cns_config.json

@@ -29,5 +29,6 @@
     "MSISettings": {
         "ResourceID": ""
     },
-    "PopulateHomeAzCacheRetryIntervalSecs": 15
+    "PopulateHomeAzCacheRetryIntervalSecs": 15,
+    "MellanoxMonitorIntervalSecs": 30
 }

cns/configuration/configuration.go

@@ -43,6 +43,7 @@ type CNSConfig struct {
 	EnableCNIConflistGeneration          bool
 	CNIConflistFilepath                  string
 	PopulateHomeAzCacheRetryIntervalSecs int
+	MellanoxMonitorIntervalSecs          int
 }
 
 type TelemetrySettings struct {

cns/service/main.go

@@ -713,6 +713,17 @@ func main() {
 		return
 	}
 
+	// We are only setting the PriorityVLANTag in 'cns.Direct' mode, because it neatly maps today, to 'isUsingMultitenancy'
+	// In the future, we would want to have a better CNS flag, to explicitly say, this CNS is using multitenancy
+	if config.ChannelMode == cns.Direct {
+		// Set Mellanox adapter's PriorityVLANTag value to 3 if adapter exists
+		// reg key value for PriorityVLANTag = 3  --> Packet priority and VLAN enabled
+		// for more details goto https://docs.nvidia.com/networking/display/winof2v230/Configuring+the+Driver+Registry+Keys#ConfiguringtheDriverRegistryKeys-GeneralRegistryKeysGeneralRegistryKeys
+		if platform.HasMellanoxAdapter() {
+			go platform.MonitorAndSetMellanoxRegKeyPriorityVLANTag(rootCtx, cnsconfig.MellanoxMonitorIntervalSecs)
+		}
+	}
+
 	// Initialze state in if CNS is running in CRD mode
 	// State must be initialized before we start HTTPRestService
 	if config.ChannelMode == cns.CRD {

platform/Makefile

@@ -0,0 +1,11 @@
+REPO_ROOT = $(shell git rev-parse --show-toplevel)
+TOOLS_BIN_DIR = $(REPO_ROOT)/build/tools/bin
+MOCKGEN = $(TOOLS_BIN_DIR)/mockgen
+
+.PHONY: generate
+
+generate: $(MOCKGEN) ## Generate mock clients
+	$(MOCKGEN) -source=$(REPO_ROOT)/platform/windows/adapter/network_adapter.go -package=mocks NetworkAdapter > windows/adapter/mocks/networkadapter_generated.go
+
+$(MOCKGEN):
+	@make -C $(REPO_ROOT) $(MOCKGEN)

platform/os_linux.go

@@ -179,3 +179,11 @@ func PrintDependencyPackageDetails() {
 func ReplaceFile(source, destination string) error {
 	return os.Rename(source, destination)
 }
+
+// Mellanox adapter not applicable for linux
+func HasMellanoxAdapter() bool {
+	return false
+}
+
+// Not needed for Linux
+func MonitorAndSetMellanoxRegKeyPriorityVLANTag(_ context.Context, _ int) {}

platform/os_windows.go

@@ -5,6 +5,7 @@ package platform
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
@@ -68,6 +69,14 @@ const (
 
 	// Command to restart HNS service
 	RestartHnsServiceCommand = "Restart-Service -Name hns"
+
+	// Interval between successive checks for mellanox adapter's PriorityVLANTag value
+	defaultMellanoxMonitorInterval = 30 * time.Second
+
+	// Value for reg key: PriorityVLANTag for adapter
+	// reg key value for PriorityVLANTag = 3  --> Packet priority and VLAN enabled
+	// for more details goto https://learn.microsoft.com/en-us/windows-hardware/drivers/network/standardized-inf-keywords-for-ndis-qos
+	desiredVLANTagForMellanox = 3
 )
 
 // Flag to check if sdnRemoteArpMacAddress registry key is set
@@ -191,6 +200,68 @@ func SetSdnRemoteArpMacAddress() error {
 	return nil
 }
 
+func HasMellanoxAdapter() bool {
+	m := &mellanox.Mellanox{}
+	return hasNetworkAdapter(m)
+}
+
+func hasNetworkAdapter(na adapter.NetworkAdapter) bool {
+	adapterName, err := na.GetAdapterName()
+	if err != nil {
+		log.Errorf("Error while getting network adapter name: %v", err)
+		return false
+	}
+	log.Printf("Name of the network adapter : %v", adapterName)
+	return true
+}
+
+// Regularly monitors the Mellanox PriorityVLANGTag registry value and sets it to desired value if needed
+func MonitorAndSetMellanoxRegKeyPriorityVLANTag(ctx context.Context, intervalSecs int) {
+	m := &mellanox.Mellanox{}
+	interval := defaultMellanoxMonitorInterval
+	if intervalSecs > 0 {
+		interval = time.Duration(intervalSecs) * time.Second
+	}
+	err := updatePriorityVLANTagIfRequired(m, desiredVLANTagForMellanox)
+	if err != nil {
+		log.Errorf("Error while monitoring mellanox, continuing: %v", err)
+	}
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			log.Printf("context cancelled, stopping Mellanox Monitoring: %v", ctx.Err())
+			return
+		case <-ticker.C:
+			err := updatePriorityVLANTagIfRequired(m, desiredVLANTagForMellanox)
+			if err != nil {
+				log.Errorf("Error while monitoring mellanox, continuing: %v", err)
+			}
+		}
+	}
+}
+
+// Updates the priority VLAN Tag of mellanox adapter if not already set to the desired value
+func updatePriorityVLANTagIfRequired(na adapter.NetworkAdapter, desiredValue int) error {
+	currentVal, err := na.GetPriorityVLANTag()
+	if err != nil {
+		return fmt.Errorf("error while getting Priority VLAN Tag value: %w", err)
+	}
+
+	if currentVal == desiredValue {
+		log.Printf("Adapter's PriorityVLANTag is already set to %v, skipping reset", desiredValue)
+		return nil
+	}
+
+	err = na.SetPriorityVLANTag(desiredValue)
+	if err != nil {
+		return fmt.Errorf("error while setting Priority VLAN Tag value: %w", err)
+	}
+
+	return nil
+}
+
 func GetOSDetails() (map[string]string, error) {
 	return nil, nil
 }

platform/windows/adapter/mellanox/mellanox.go

@@ -0,0 +1,222 @@
+// Copyright 2017 Microsoft. All rights reserved.
+// MIT License
+
+package mellanox
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/Azure/azure-container-networking/log"
+)
+
+const (
+	// Search string to find adapter having Mellanox in description
+	mellanoxSearchString = "*Mellanox*"
+
+	// PriorityVlanTag reg key for adapter
+	priorityVLANTagIdentifier = "*PriorityVLANTag"
+
+	// Registry key Path Prefix
+	registryKeyPrefix = "HKLM:\\System\\CurrentControlSet\\Control\\Class\\"
+)
+
+var (
+	errorMellanoxAdapterNotFound = fmt.Errorf("no network adapter found with %s in description", mellanoxSearchString)
+	errorMellanoxDeviceNotFound  = fmt.Errorf("no network device found with %s in description", mellanoxSearchString)
+	errorPowershellNotFound      = fmt.Errorf("failed to find powershell executable")
+)
+
+type Mellanox struct{}
+
+// GetAdapter returns name of Mellanox adapter if found
+// Returns errorMellanoxAdapterNotFound if adapter is not found or adapter name empty
+func (m *Mellanox) GetAdapterName() (string, error) {
+	// get mellanox adapter name
+	cmd := fmt.Sprintf(`Get-NetAdapter | Where-Object { $_.InterfaceDescription -like '%s' } | Select-Object -ExpandProperty Name`, mellanoxSearchString)
+	adapterName, err := executePowershellCommand(cmd)
+	if err != nil {
+		return "", fmt.Errorf("error while executing powershell command to get net adapter list: %w", err)
+	}
+	if adapterName == "" {
+		return "", errorMellanoxAdapterNotFound
+	}
+	return adapterName, nil
+}
+
+// Set Mellanox adapter's PriorityVLANTag value to desired value if adapter exists
+// 5/16/23 : right now setting desired reg key value for PriorityVLANTag = 3  --> Packet priority and VLAN enabled
+// for more details goto https://docs.nvidia.com/networking/display/winof2v230/Configuring+the+Driver+Registry+Keys#ConfiguringtheDriverRegistryKeys-GeneralRegistryKeysGeneralRegistryKeys
+func (m *Mellanox) SetPriorityVLANTag(desiredVal int) error {
+	adapterName, err := m.GetAdapterName()
+	if err != nil {
+		return fmt.Errorf("failed to find mellanox adapter: %w", err)
+	}
+
+	// Find if adapter has property PriorityVLANTag (version 4 or up) or not (version 3)
+	cmd := fmt.Sprintf(`Get-NetAdapterAdvancedProperty | Where-Object { $_.RegistryKeyword -like '%s' -and $_.Name -eq '%s' } | Select-Object -ExpandProperty Name`,
+		priorityVLANTagIdentifier, adapterName)
+	adapterNameWithVLANTag, err := executePowershellCommand(cmd)
+	if err != nil {
+		return fmt.Errorf("error while executing powershell command to get VLAN Tag advance property of %s: %w", adapterName, err)
+	}
+
+	if adapterNameWithVLANTag != "" {
+		return m.setMellanoxPriorityVLANTagValueForV4(adapterNameWithVLANTag, desiredVal)
+	}
+	return m.setMellanoxPriorityVLANTagValueForV3(adapterName, desiredVal)
+}
+
+// Get PriorityVLANTag returns PriorityVLANTag value for Mellanox Adapter (both version 3 and version 4)
+func (m *Mellanox) GetPriorityVLANTag() (int, error) {
+	adapterName, err := m.GetAdapterName()
+	if err != nil {
+		return 0, fmt.Errorf("failed to find mellanox adapter: %w", err)
+	}
+
+	// Find if adapter has property PriorityVLANTag (version 4 or up) or not (version 3)
+	cmd := fmt.Sprintf(`Get-NetAdapterAdvancedProperty | Where-Object { $_.RegistryKeyword -like '%s' -and $_.Name -eq '%s' } | Select-Object -ExpandProperty Name`,
+		priorityVLANTagIdentifier, adapterName)
+	adapterNameWithVLANTag, err := executePowershellCommand(cmd)
+	if err != nil {
+		return 0, fmt.Errorf("error while executing powershell command to get VLAN Tag advance property of %s: %w", adapterName, err)
+	}
+
+	if adapterNameWithVLANTag != "" {
+		return m.getMellanoxPriorityVLANTagValueForV4(adapterNameWithVLANTag)
+	}
+
+	return m.getMellanoxPriorityVLANTagValueForV3()
+}
+
+// Checks if a Mellanox adapter's PriorityVLANTag value
+// for version 4 and up is set to the given expected value
+func (m *Mellanox) getMellanoxPriorityVLANTagValueForV4(adapterName string) (int, error) {
+	cmd := fmt.Sprintf(
+		`Get-NetAdapterAdvancedProperty | Where-Object { $_.RegistryKeyword -like '%s' -and $_.Name -eq '%s' } | Select-Object -ExpandProperty RegistryValue`,
+		priorityVLANTagIdentifier, adapterName)
+
+	regvalue, err := executePowershellCommand(cmd)
+	if err != nil {
+		return 0, err
+	}
+
+	intValue, err := strconv.Atoi(regvalue)
+	if err != nil {
+		return 0, fmt.Errorf("failed to convert PriorityVLANTag value to integer: %w", err)
+	}
+
+	return intValue, nil
+}
+
+// Checks if a Mellanox adapter's PriorityVLANTag value
+// for version 3 and below is set to the given expected value
+func (m *Mellanox) getMellanoxPriorityVLANTagValueForV3() (int, error) {
+	registryKeyFullPath, err := m.getRegistryFullPath()
+	if err != nil {
+		return 0, err
+	}
+
+	cmd := fmt.Sprintf(
+		`Get-ItemProperty -Path '%s' -Name '%s' | Select-Object -ExpandProperty '%s'`, registryKeyFullPath, priorityVLANTagIdentifier, priorityVLANTagIdentifier)
+	regvalue, err := executePowershellCommand(cmd)
+	if err != nil {
+		return 0, err
+	}
+
+	intValue, err := strconv.Atoi(regvalue)
+	if err != nil {
+		return 0, fmt.Errorf("failed to convert PriorityVLANTag value to integer: %w", err)
+	}
+
+	return intValue, nil
+}
+
+// adapter is version 4 and up since adapter's advance property consists of reg key : PriorityVLANTag
+// set reg value for Priorityvlantag of adapter to 3 if not set already
+func (m *Mellanox) setMellanoxPriorityVLANTagValueForV4(adapterName string, desiredVal int) error {
+	cmd := fmt.Sprintf(
+		`Set-NetAdapterAdvancedProperty -Name '%s' -RegistryKeyword '%s' -RegistryValue %d`,
+		adapterName, priorityVLANTagIdentifier, desiredVal)
+	_, err := executePowershellCommand(cmd)
+	if err != nil {
+		return fmt.Errorf("error while setting up registry value for PriorityVLANTag for adapter: %w", err)
+	}
+
+	log.Printf("Successfully set Mellanox Network Adapter: %s with %s property value as %d",
+		adapterName, priorityVLANTagIdentifier, desiredVal)
+	return nil
+}
+
+// Adapter is version 3 or less as PriorityVLANTag was not found in advanced properties of mellanox adapter
+func (m *Mellanox) setMellanoxPriorityVLANTagValueForV3(adapterName string, desiredVal int) error {
+	registryKeyFullPath, err := m.getRegistryFullPath()
+	if err != nil {
+		return err
+	}
+
+	cmd := fmt.Sprintf(`New-ItemProperty -Path '%s' -Name '%s' -Value %d -PropertyType String -Force`,
+		registryKeyFullPath, priorityVLANTagIdentifier, desiredVal)
+	_, err = executePowershellCommand(cmd)
+	if err != nil {
+		return fmt.Errorf("error while executing powershell command to set Item property for adapter  %s: %w", adapterName, err)
+	}
+
+	log.Printf("Restarting Mellanox network adapter for regkey change to take effect")
+	cmd = fmt.Sprintf(`Restart-NetAdapter -Name '%s'`, adapterName)
+	_, err = executePowershellCommand(cmd)
+	if err != nil {
+		return fmt.Errorf("error while executing powershell command to restart net adapter  %s: %w", adapterName, err)
+	}
+	log.Printf("For Mellanox CX-3 adapters, the reg key set to %d", desiredVal)
+	return nil
+}
+
+// Get registry full path for Mellanox Adapter
+func (m *Mellanox) getRegistryFullPath() (string, error) {
+	log.Printf("Searching through CIM instances for Network devices with %s in the name", mellanoxSearchString)
+	cmd := fmt.Sprintf(
+		`Get-CimInstance -Namespace root/cimv2 -ClassName Win32_PNPEntity | Where-Object PNPClass -EQ "Net" | Where-Object { $_.Name -like '%s' } | Select-Object -ExpandProperty DeviceID`,
+		mellanoxSearchString)
+	deviceid, err := executePowershellCommand(cmd)
+	if err != nil {
+		return "", fmt.Errorf("error while executing powershell command to get device id for Mellanox: %w", err)
+	}
+	if deviceid == "" {
+		return "", errorMellanoxDeviceNotFound
+	}
+
+	cmd = fmt.Sprintf(`Get-PnpDeviceProperty -InstanceId '%s' | Where-Object KeyName -EQ "DEVPKEY_Device_Driver" | Select-Object -ExpandProperty Data`, deviceid)
+	registryKeySuffix, err := executePowershellCommand(cmd)
+	if err != nil {
+		return "", fmt.Errorf("error while executing powershell command to get registry suffix of device id %s: %w", deviceid, err)
+	}
+
+	return registryKeyPrefix + registryKeySuffix, nil
+}
+
+// ExecutePowershellCommand executes powershell command
+func executePowershellCommand(command string) (string, error) {
+	ps, err := exec.LookPath("powershell.exe")
+	if err != nil {
+		return "", errorPowershellNotFound
+	}
+
+	log.Printf("[Azure-Utils] %s", command)
+
+	cmd := exec.Command(ps, command)
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	err = cmd.Run()
+	if err != nil {
+		return "", fmt.Errorf("%s:%w", stderr.String(), err)
+	}
+
+	return strings.TrimSpace(stdout.String()), nil
+}