From f5a16ab391c06ab895d1802dc1f591ce22c2ebda Mon Sep 17 00:00:00 2001 From: Malay Kumar Parida Date: Wed, 5 Feb 2025 04:51:02 +0530 Subject: [PATCH] Fix CephFS volumes failing to mount after upgrade to 4.18 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until 4.18 Provider mode was using v1(6789) port as default, so v1 ports were present in the rook-ceph-mon-endpoints CM. Rook doesn’t update this CM to v2 3300 port until the mons are failed over, even after requireMsgr2 is set to true. Provider sends the mon endpoints from the same rook-ceph-mon-endpoints CM to the client, so the client uses the v1 (6789) port address it received in it’s ceph-csi-config CM. But client receives the cephFS kernel mount option from provider as ‘prefer-crc’ as requireMsgr2 is true. When mounting new cephFS volume on client side it tries to use the v1 6789 port with the ‘prefer-crc’ kernel mount option. Which can't work,thus cephFS volumes fail to mount. As since 4.18 we are using v2 port always, so the provider should send the v2 port address to the client by modifying the mon IPs. Similar implementation can be seen in rook. Signed-off-by: Malay Kumar Parida --- services/provider/server/server.go | 21 +++++++++++ services/provider/server/server_test.go | 49 ++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/services/provider/server/server.go b/services/provider/server/server.go index d0c86c606b..7b3e9354f8 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -1077,9 +1077,30 @@ func extractMonitorIps(data string) ([]string, error) { } // sorting here removes any positional change which reduces spurious reconciles slices.Sort(ips) + + // Rook does not update the rook-ceph-mon-endpoint ConfigMap until mons failover + // Starting from 4.18, RequireMsgr2 is always enabled, and encryption in transit is allowed on existing clusters. + // So, we need to replace the msgr1 port with msgr2 port. + replaceMsgr1PortWithMsgr2(ips) + return ips, nil } +func replaceMsgr1PortWithMsgr2(ips []string) { + const ( + // msgr2port is the listening port of the messenger v2 protocol + msgr2port = "3300" + // msgr1port is the listening port of the messenger v1 protocol + msgr1port = "6789" + ) + + for i, ip := range ips { + if strings.HasSuffix(ip, msgr1port) { + ips[i] = strings.TrimSuffix(ip, msgr1port) + msgr2port + } + } +} + func (s *OCSProviderServer) PeerStorageCluster(ctx context.Context, req *pb.PeerStorageClusterRequest) (*pb.PeerStorageClusterResponse, error) { pubKey, err := s.getOnboardingValidationKey(ctx) diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index 5863190771..b13edc004f 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strconv" "testing" @@ -113,7 +114,7 @@ var mockExtR = map[string]*externalResource{ "monitor-endpoints": { Name: "monitor-endpoints", Kind: "CephConnection", - Data: &csiopv1a1.CephConnectionSpec{Monitors: []string{"10.99.45.27:6789"}}, + Data: &csiopv1a1.CephConnectionSpec{Monitors: []string{"10.99.45.27:3300"}}, }, } @@ -486,6 +487,52 @@ func createMonConfigMapAndSecret(server *OCSProviderServer) (*v1.ConfigMap, *v1. return monCm, monSc } +func TestReplaceMsgr1PortWithMsgr2(t *testing.T) { + tests := []struct { + name string + input []string + expected []string + }{ + { + name: "no msgr1 port", + input: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"}, + expected: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"}, + }, + { + name: "all msgr1 ports", + input: []string{"192.168.1.1:6789", "192.168.1.2:6789", "192.168.1.3:6789"}, + expected: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"}, + }, + { + name: "mixed ports", + input: []string{"192.168.1.1:6789", "192.168.1.2:3300", "192.168.1.3:6789"}, + expected: []string{"192.168.1.1:3300", "192.168.1.2:3300", "192.168.1.3:3300"}, + }, + { + name: "empty slice", + input: []string{}, + expected: []string{}, + }, + { + name: "no port in IP", + input: []string{"192.168.1.1", "192.168.1.2:6789", "192.168.1.2:6789"}, + expected: []string{"192.168.1.1", "192.168.1.2:3300", "192.168.1.2:3300"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Make a copy of the input slice to avoid modifying the original + inputCopy := make([]string, len(tt.input)) + copy(inputCopy, tt.input) + replaceMsgr1PortWithMsgr2(inputCopy) + if !reflect.DeepEqual(inputCopy, tt.expected) { + t.Errorf("replaceMsgr1PortWithMsgr2() = %v, expected %v", inputCopy, tt.expected) + } + }) + } +} + func createCephClientAndSecret(name string, server *OCSProviderServer) (*rookCephv1.CephClient, *v1.Secret) { cephClient := &rookCephv1.CephClient{ ObjectMeta: metav1.ObjectMeta{