Merge branch 'master' into Multi-Asic-gnmi-Apply-patch

.gitignore

@@ -23,4 +23,4 @@ translib/
 swsscommon/swsscommon.go
 swsscommon/swsscommon.i
 swsscommon/swsscommon_wrap.cxx
-swsscommon/swsscommon_wrap.h
+swsscommon/swsscommon_wrap.h

Makefile

@@ -6,10 +6,28 @@ export PATH := $(PATH):$(GOPATH)/bin
 INSTALL := /usr/bin/install
 DBDIR := /var/run/redis/sonic-db/
 GO ?= /usr/local/go/bin/go
-TOP_DIR := $(abspath ..)
-MGMT_COMMON_DIR := $(TOP_DIR)/sonic-mgmt-common
+TOPDIR := $(abspath .)
+MGMT_COMMON_DIR := $(TOPDIR)/../sonic-mgmt-common
+BUILD_BASE := build
 BUILD_DIR := build/bin
+BUILD_GNOI_YANG_DIR := $(BUILD_BASE)/gnoi_yang
+BUILD_GNOI_YANG_PROTO_DIR := $(BUILD_GNOI_YANG_DIR)/proto
+BUILD_GNOI_YANG_SERVER_DIR := $(BUILD_GNOI_YANG_DIR)/server
+BUILD_GNOI_YANG_CLIENT_DIR := $(BUILD_GNOI_YANG_DIR)/client
+GNOI_YANG := $(BUILD_GNOI_YANG_PROTO_DIR)/.gnoi_yang_done
+TOOLS_DIR        := $(TOPDIR)/tools
+PYANG_PLUGIN_DIR := $(TOOLS_DIR)/pyang_plugins
+PYANG  ?= pyang
+GOROOT ?= $(shell $(GO) env GOROOT)
+FORMAT_CHECK = $(BUILD_DIR)/.formatcheck
+FORMAT_LOG = $(BUILD_DIR)/go_format.log
+# Find all .go files excluding vendor, build, and patches files
+GO_FILES := $(shell find . -type f -name '*.go' ! -path './vendor/*' ! -path './build/*' ! -path './patches/*' ! -path './proto/*' ! -path './swsscommon/*')
 export CVL_SCHEMA_PATH := $(MGMT_COMMON_DIR)/build/cvl/schema
+
+API_YANGS=$(shell find $(MGMT_COMMON_DIR)/build/yang -name '*.yang' -not -path '*/sonic/*' -not -path '*/annotations/*')
+SONIC_YANGS=$(shell find $(MGMT_COMMON_DIR)/models/yang/sonic -name '*.yang')
+
 export GOBIN := $(abspath $(BUILD_DIR))
 export PATH := $(PATH):$(GOBIN):$(shell dirname $(GO))
 export CGO_LDFLAGS := -lswsscommon -lhiredis
@@ -47,7 +65,7 @@ all: sonic-gnmi
 go.mod:
 	$(GO) mod init github.com/sonic-net/sonic-gnmi
 
-$(GO_DEPS): go.mod $(PATCHES) swsscommon_wrap
+$(GO_DEPS): go.mod $(PATCHES) swsscommon_wrap $(GNOI_YANG)
 	$(GO) mod vendor
 	$(GO) mod download github.com/google/[email protected]
 	cp -r $(GOPATH)/pkg/mod/github.com/google/[email protected]/* vendor/github.com/google/gnxi/
@@ -68,7 +86,7 @@ go-deps: $(GO_DEPS)
 go-deps-clean:
 	$(RM) -r vendor
 
-sonic-gnmi: $(GO_DEPS)
+sonic-gnmi: $(GO_DEPS) $(FORMAT_CHECK)
 # advancetls 1.0.0 release need following patch to build by go-1.19
 	patch -d vendor -p0 < patches/0002-Fix-advance-tls-build-with-go-119.patch
 # build service first which depends on advancetls
@@ -86,6 +104,9 @@ ifneq ($(ENABLE_DIALOUT_VALUE),0)
 endif
 	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/gnoi_client
 	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/gnmi_dump
+	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/build/gnoi_yang/client/gnoi_openconfig_client
+	$(GO) install -mod=vendor github.com/sonic-net/sonic-gnmi/build/gnoi_yang/client/gnoi_sonic_client
+
 endif
 
 # download and apply patch for gnmi client, which will break advancetls
@@ -124,11 +145,46 @@ swsscommon_wrap:
 
 PROTOC_PATH := $(PATH):$(GOBIN)
 PROTOC_OPTS := -I$(CURDIR)/vendor -I/usr/local/include -I/usr/include
+PROTOC_OPTS_WITHOUT_VENDOR := -I/usr/local/include -I/usr/include
 
 # Generate following go & grpc bindings using teh legacy protoc-gen-go
 PROTO_GO_BINDINGS += proto/sonic_internal.pb.go
 PROTO_GO_BINDINGS += proto/gnoi/sonic_debug.pb.go
 
+$(BUILD_GNOI_YANG_PROTO_DIR)/.proto_api_done: $(API_YANGS)
+	@echo "+++++ Generating PROTOBUF files for API Yang modules; +++++"
+	$(PYANG) \
+		-f proto \
+		--proto-outdir $(BUILD_GNOI_YANG_PROTO_DIR) \
+		--plugindir $(PYANG_PLUGIN_DIR) \
+		--server-rpc-outdir $(BUILD_GNOI_YANG_SERVER_DIR) \
+		--client-rpc-outdir $(BUILD_GNOI_YANG_CLIENT_DIR) \
+		-p $(MGMT_COMMON_DIR)/build/yang/common:$(MGMT_COMMON_DIR)/build/yang/extensions \
+		$(MGMT_COMMON_DIR)/build/yang/*.yang $(MGMT_COMMON_DIR)/build/yang/extensions/*.yang
+	@echo "+++++ Generation of protobuf files for API Yang modules completed +++++"
+	touch $@
+
+$(BUILD_GNOI_YANG_PROTO_DIR)/.proto_sonic_done: $(SONIC_YANGS)
+	@echo "+++++ Generating PROTOBUF files for SONiC Yang modules; +++++"
+	$(PYANG) \
+		-f proto \
+		--proto-outdir $(BUILD_GNOI_YANG_PROTO_DIR) \
+		--plugindir $(PYANG_PLUGIN_DIR) \
+		--server-rpc-outdir $(BUILD_GNOI_YANG_SERVER_DIR) \
+		--client-rpc-outdir $(BUILD_GNOI_YANG_CLIENT_DIR) \
+		-p $(MGMT_COMMON_DIR)/build/yang/common:$(MGMT_COMMON_DIR)/build/yang/sonic/common \
+		$(MGMT_COMMON_DIR)/build/yang/sonic/*.yang
+	@echo "+++++ Generation of protobuf files for SONiC Yang modules completed +++++"
+	touch $@
+
+$(GNOI_YANG): $(BUILD_GNOI_YANG_PROTO_DIR)/.proto_api_done $(BUILD_GNOI_YANG_PROTO_DIR)/.proto_sonic_done
+	@echo "+++++ Compiling PROTOBUF files; +++++"
+	$(GO) install github.com/gogo/protobuf/protoc-gen-gofast
+	@mkdir -p $(@D)
+	$(foreach file, $(wildcard $(BUILD_GNOI_YANG_PROTO_DIR)/*/*.proto), PATH=$(PROTOC_PATH) protoc -I$(@D) $(PROTOC_OPTS_WITHOUT_VENDOR) --gofast_out=plugins=grpc,Mgoogle/protobuf/struct.proto=github.com/gogo/protobuf/types:$(BUILD_GNOI_YANG_PROTO_DIR) $(file);)
+	@echo "+++++ PROTOBUF completion completed; +++++"
+	touch $@
+
 $(PROTO_GO_BINDINGS): $$(patsubst %.pb.go,%.proto,$$@) | $(GOBIN)/protoc-gen-go
 	PATH=$(PROTOC_PATH) protoc -I$(@D) $(PROTOC_OPTS) --go_out=plugins=grpc:$(@D) $<
 
@@ -160,11 +216,12 @@ endif
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-data.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_data_client
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-dbus.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_service_client
 	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(TESTENV) $(GO) test -race -coverprofile=coverage-translutils.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/transl_utils
+	sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(TESTENV) $(GO) test -race -coverprofile=coverage-gnoi-client-system.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/gnoi_client/system
 	$(GO) install github.com/axw/gocov/[email protected]
 	$(GO) install github.com/AlekSi/gocov-xml@latest
 	$(GO) mod vendor
 	gocov convert coverage-*.txt | gocov-xml -source $(shell pwd) > coverage.xml
-	rm -rf coverage-*.txt 
+	rm -rf coverage-*.txt
 
 check_memleak: $(DBCONFG) $(ENVFILE)
 	sudo CGO_LDFLAGS="$(MEMCHECK_CGO_LDFLAGS)" CGO_CXXFLAGS="$(MEMCHECK_CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-telemetry.txt -covermode=atomic -mod=vendor $(MEMCHECK_FLAGS) -v github.com/sonic-net/sonic-gnmi/telemetry
@@ -177,6 +234,22 @@ clean:
 	$(RM) -r build
 	$(RM) -r vendor
 
+# File target that generates a diff file if formatting is incorrect
+$(FORMAT_CHECK): $(GO_FILES)
+	@echo "Checking Go file formatting..."
+	@echo $(GO_FILES)
+	mkdir -p $(@D)
+	@$(GOROOT)/bin/gofmt -l $(GO_FILES) > $(FORMAT_LOG)
+	@if [ -s $(FORMAT_LOG) ]; then \
+		cat $(FORMAT_LOG); \
+		echo "Formatting issues found. Please run 'gofmt -w <file>' on the above files and commit the changes."; \
+		exit 1; \
+	else \
+		echo "All files are properly formatted."; \
+		rm -f $(FORMAT_LOG); \
+	fi
+	touch $@
+
 install:
 	$(INSTALL) -D $(BUILD_DIR)/telemetry $(DESTDIR)/usr/sbin/telemetry
 ifneq ($(ENABLE_DIALOUT_VALUE),0)
@@ -186,6 +259,8 @@ endif
 	$(INSTALL) -D $(BUILD_DIR)/gnmi_set $(DESTDIR)/usr/sbin/gnmi_set
 	$(INSTALL) -D $(BUILD_DIR)/gnmi_cli $(DESTDIR)/usr/sbin/gnmi_cli
 	$(INSTALL) -D $(BUILD_DIR)/gnoi_client $(DESTDIR)/usr/sbin/gnoi_client
+	$(INSTALL) -D $(BUILD_DIR)/gnoi_openconfig_client $(DESTDIR)/usr/sbin/gnoi_openconfig_client
+	$(INSTALL) -D $(BUILD_DIR)/gnoi_sonic_client $(DESTDIR)/usr/sbin/gnoi_sonic_client
 	$(INSTALL) -D $(BUILD_DIR)/gnmi_dump $(DESTDIR)/usr/sbin/gnmi_dump
 
 
@@ -197,6 +272,8 @@ endif
 	rm $(DESTDIR)/usr/sbin/gnmi_get
 	rm $(DESTDIR)/usr/sbin/gnmi_set
 	rm $(DESTDIR)/usr/sbin/gnoi_client
+	rm $(DESTDIR)/usr/sbin/gnoi_openconfig_client
+	rm $(DESTDIR)/usr/sbin/gnoi_sonic_client
 	rm $(DESTDIR)/usr/sbin/gnmi_dump
 
 

azure-pipelines.yml

@@ -146,6 +146,11 @@ stages:
         sudo dpkg -i python3-swsscommon_1.0.0_amd64.deb
       workingDirectory: $(Pipeline.Workspace)/
       displayName: 'Install libswsscommon package'
+    
+    - script: |
+        sudo apt-get install -y protobuf-compiler
+        protoc --version
+      displayName: 'Install protoc'
 
     - script: |
         set -ex

common_utils/constants.go

@@ -1,4 +1,3 @@
-
-package common_utils
-
-const GNMI_WORK_PATH = "/tmp"
+package common_utils
+
+const GNMI_WORK_PATH = "/tmp"

common_utils/context.go

@@ -58,6 +58,7 @@ const (
 	DBUS_IMAGE_DOWNLOAD
 	DBUS_IMAGE_INSTALL
 	DBUS_IMAGE_LIST
+	DBUS_IMAGE_ACTIVATE
 	COUNTER_SIZE
 )
 
@@ -103,6 +104,8 @@ func (c CounterType) String() string {
 		return "DBUS image install"
 	case DBUS_IMAGE_LIST:
 		return "DBUS image list"
+	case DBUS_IMAGE_ACTIVATE:
+		return "DBUS image activate"
 	default:
 		return ""
 	}

common_utils/shareMem.go

@@ -9,11 +9,11 @@ import (
 // Use share memory to dump GNMI internal counters,
 // GNMI server and gnmi_dump should use memKey to access the share memory,
 // memSize is 1024 bytes, so we can support 128 counters
-// memMode is 0x380, this value is O_RDWR|IPC_CREAT, 
+// memMode is 0x380, this value is O_RDWR|IPC_CREAT,
 // O_RDWR means: Owner can write and read the file, everyone else can't.
 // IPC_CREAT means: Create a shared memory segment if a shared memory identifier does not exist for memKey.
 var (
-	memKey = 7749
+	memKey  = 7749
 	memSize = 1024
 	memMode = 0x380
 )
@@ -61,4 +61,3 @@ func GetMemCounters(counters *[int(COUNTER_SIZE)]uint64) error {
 	}
 	return nil
 }
-

dialout/dialout_client/dialout_client.go

@@ -5,14 +5,14 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	spb "github.com/sonic-net/sonic-gnmi/proto"
-	sdc "github.com/sonic-net/sonic-gnmi/sonic_data_client"
-	sdcfg "github.com/sonic-net/sonic-gnmi/sonic_db_config"
 	"github.com/Workiva/go-datastructures/queue"
 	"github.com/go-redis/redis"
 	log "github.com/golang/glog"
 	gpb "github.com/openconfig/gnmi/proto/gnmi"
 	"github.com/openconfig/ygot/ygot"
+	spb "github.com/sonic-net/sonic-gnmi/proto"
+	sdc "github.com/sonic-net/sonic-gnmi/sonic_data_client"
+	sdcfg "github.com/sonic-net/sonic-gnmi/sonic_db_config"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"

dialout/dialout_client/dialout_client_test.go

@@ -28,10 +28,10 @@ import (
 	"testing"
 	"time"
 
+	gclient "github.com/openconfig/gnmi/client/gnmi"
 	sds "github.com/sonic-net/sonic-gnmi/dialout/dialout_server"
 	sdc "github.com/sonic-net/sonic-gnmi/sonic_data_client"
 	sdcfg "github.com/sonic-net/sonic-gnmi/sonic_db_config"
-	gclient "github.com/openconfig/gnmi/client/gnmi"
 )
 
 var clientTypes = []string{gclient.Type}
@@ -326,7 +326,6 @@ func serverOp(t *testing.T, sop ServerOp) {
 	}
 }
 
-//
 func TestGNMIDialOutPublish(t *testing.T) {
 
 	fileName := "../../testdata/COUNTERS_PORT_NAME_MAP.txt"
@@ -416,50 +415,50 @@ func TestGNMIDialOutPublish(t *testing.T) {
 				},
 			},
 		},
-	//}, {
-	//	desc: "DialOut to second collector in stream mode upon failure of first collector",
-	//	cmds: []string{
-	//		"redis-cli -n 4 hset TELEMETRY_CLIENT|DestinationGroup_HS dst_addr 127.0.0.1:8080,127.0.0.1:8081",
-	//		"redis-cli -n 4 hmset TELEMETRY_CLIENT|Subscription_HS_RDMA path_target COUNTERS_DB dst_group HS report_type stream paths COUNTERS/Ethernet*/SAI_PORT_STAT_PFC_7_RX_PKTS",
-	//	},
-	//	collector: "s2",
-	//	sop:       S1Stop,
-	//	updates: []tablePathValue{{
-	//		dbName:    "COUNTERS_DB",
-	//		tableName: "COUNTERS",
-	//		tableKey:  "oid:0x1000000000039", // "Ethernet68": "oid:0x1000000000039",
-	//		delimitor: ":",
-	//		field:     "SAI_PORT_STAT_PFC_7_RX_PKTS",
-	//		value:     "3", // be changed to 3 from 2
-	//	}, {
-	//		dbName:    "COUNTERS_DB",
-	//		tableName: "COUNTERS",
-	//		tableKey:  "oid:0x1000000000039", // "Ethernet68": "oid:0x1000000000039",
-	//		delimitor: ":",
-	//		field:     "SAI_PORT_STAT_PFC_7_RX_PKTS",
-	//		value:     "2", // be changed to 2 from 3
-	//	}},
-	//	waitTime: clientCfg.RetryInterval + time.Second,
-	//	wantRespVal: []*pb.SubscribeResponse{
-	//		&pb.SubscribeResponse{
-	//			Response: &pb.SubscribeResponse_Update{
-	//				Update: &pb.Notification{
-	//					Update: []*pb.Update{
-	//						{Val: &pb.TypedValue{
-	//							Value: &pb.TypedValue_JsonIetfVal{
-	//								JsonIetfVal: countersEthernetWildcardPfcByte,
-	//							}},
-	//						},
-	//					},
-	//				},
-	//			},
-	//		},
-	//		&pb.SubscribeResponse{
-	//			Response: &pb.SubscribeResponse_SyncResponse{
-	//				SyncResponse: true,
-	//			},
-	//		},
-	//	},
+		//}, {
+		//	desc: "DialOut to second collector in stream mode upon failure of first collector",
+		//	cmds: []string{
+		//		"redis-cli -n 4 hset TELEMETRY_CLIENT|DestinationGroup_HS dst_addr 127.0.0.1:8080,127.0.0.1:8081",
+		//		"redis-cli -n 4 hmset TELEMETRY_CLIENT|Subscription_HS_RDMA path_target COUNTERS_DB dst_group HS report_type stream paths COUNTERS/Ethernet*/SAI_PORT_STAT_PFC_7_RX_PKTS",
+		//	},
+		//	collector: "s2",
+		//	sop:       S1Stop,
+		//	updates: []tablePathValue{{
+		//		dbName:    "COUNTERS_DB",
+		//		tableName: "COUNTERS",
+		//		tableKey:  "oid:0x1000000000039", // "Ethernet68": "oid:0x1000000000039",
+		//		delimitor: ":",
+		//		field:     "SAI_PORT_STAT_PFC_7_RX_PKTS",
+		//		value:     "3", // be changed to 3 from 2
+		//	}, {
+		//		dbName:    "COUNTERS_DB",
+		//		tableName: "COUNTERS",
+		//		tableKey:  "oid:0x1000000000039", // "Ethernet68": "oid:0x1000000000039",
+		//		delimitor: ":",
+		//		field:     "SAI_PORT_STAT_PFC_7_RX_PKTS",
+		//		value:     "2", // be changed to 2 from 3
+		//	}},
+		//	waitTime: clientCfg.RetryInterval + time.Second,
+		//	wantRespVal: []*pb.SubscribeResponse{
+		//		&pb.SubscribeResponse{
+		//			Response: &pb.SubscribeResponse_Update{
+		//				Update: &pb.Notification{
+		//					Update: []*pb.Update{
+		//						{Val: &pb.TypedValue{
+		//							Value: &pb.TypedValue_JsonIetfVal{
+		//								JsonIetfVal: countersEthernetWildcardPfcByte,
+		//							}},
+		//						},
+		//					},
+		//				},
+		//			},
+		//		},
+		//		&pb.SubscribeResponse{
+		//			Response: &pb.SubscribeResponse_SyncResponse{
+		//				SyncResponse: true,
+		//			},
+		//		},
+		//	},
 	}}
 
 	rclient := getRedisClient(t)

dialout/dialout_client_cli/dialout_client_cli.go

@@ -4,9 +4,9 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	dc "github.com/sonic-net/sonic-gnmi/dialout/dialout_client"
 	log "github.com/golang/glog"
 	gpb "github.com/openconfig/gnmi/proto/gnmi"
+	dc "github.com/sonic-net/sonic-gnmi/dialout/dialout_client"
 	"golang.org/x/net/context"
 	"os"
 	"os/signal"

dialout/dialout_server/dialout_server.go

@@ -3,10 +3,10 @@ package dialout_server
 import (
 	"errors"
 	"fmt"
-	spb "github.com/sonic-net/sonic-gnmi/proto"
 	log "github.com/golang/glog"
 	"github.com/google/gnxi/utils"
 	gpb "github.com/openconfig/gnmi/proto/gnmi"
+	spb "github.com/sonic-net/sonic-gnmi/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/peer"

gnmi_server/basicAuth.go

@@ -1,8 +1,8 @@
 package gnmi
 
 import (
-	"github.com/sonic-net/sonic-gnmi/common_utils"
 	"github.com/golang/glog"
+	"github.com/sonic-net/sonic-gnmi/common_utils"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"