From 1762cbd7a52e201f89d7aa2838ac6743760f0e79 Mon Sep 17 00:00:00 2001 From: sunwp <244372610@qq.com> Date: Wed, 8 Dec 2021 00:56:48 +0800 Subject: [PATCH 1/5] unit test Signed-off-by: sunwp <244372610@qq.com> --- pkg/source/hdfsprotocol/hdfs_source_client_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/source/hdfsprotocol/hdfs_source_client_test.go b/pkg/source/hdfsprotocol/hdfs_source_client_test.go index e0b6bed4b62..c63962a1b75 100644 --- a/pkg/source/hdfsprotocol/hdfs_source_client_test.go +++ b/pkg/source/hdfsprotocol/hdfs_source_client_test.go @@ -43,7 +43,7 @@ const ( hdfsExistFileURL = "hdfs://" + hdfsExistFileHost + hdfsExistFilePath hdfsExistFileContentLength int64 = 12 hdfsExistFileContent = "Hello World\n" - hdfsExistFileLastModifiedMillis int64 = 1625218150000 + hdfsExistFileLastModifiedMillis int64 = 1136214245000 hdfsExistFileLastModified = "Mon, 02 Jan 2006 15:04:05 GMT" hdfsExistFileRangeStart int64 = 3 hdfsExistFileRangeEnd int64 = 10 @@ -52,6 +52,7 @@ const ( const ( hdfsNotExistFileURL = "hdfs://127.0.0.1:9000/user/root/input/f3.txt" hdfsNotExistFileContentLength int64 = source.UnknownSourceFileLen + hdfsNotExistLastModified int64 = -1 ) var fakeHDFSClient = &hdfs.Client{} @@ -344,7 +345,7 @@ func TestGetLastModified_FileNotExist(t *testing.T) { lastModifiedMillis, err := sourceClient.GetLastModified(request) assert.EqualError(t, err, "stat /user/root/input/f3.txt: file does not exist") - assert.Equal(t, hdfsNotExistFileContentLength, lastModifiedMillis) + assert.Equal(t, hdfsNotExistLastModified, lastModifiedMillis) } func TestNewHDFSSourceClient(t *testing.T) { From 6b78e06b09c46688472e7c0467e2871b86c10042 Mon Sep 17 00:00:00 2001 From: sunwp <244372610@qq.com> Date: Wed, 8 Dec 2021 12:44:10 +0800 Subject: [PATCH 2/5] hdfs unit test Signed-off-by: sunwp <244372610@qq.com> --- pkg/source/hdfsprotocol/hdfs_source_client_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/source/hdfsprotocol/hdfs_source_client_test.go b/pkg/source/hdfsprotocol/hdfs_source_client_test.go index e0b6bed4b62..c63962a1b75 100644 --- a/pkg/source/hdfsprotocol/hdfs_source_client_test.go +++ b/pkg/source/hdfsprotocol/hdfs_source_client_test.go @@ -43,7 +43,7 @@ const ( hdfsExistFileURL = "hdfs://" + hdfsExistFileHost + hdfsExistFilePath hdfsExistFileContentLength int64 = 12 hdfsExistFileContent = "Hello World\n" - hdfsExistFileLastModifiedMillis int64 = 1625218150000 + hdfsExistFileLastModifiedMillis int64 = 1136214245000 hdfsExistFileLastModified = "Mon, 02 Jan 2006 15:04:05 GMT" hdfsExistFileRangeStart int64 = 3 hdfsExistFileRangeEnd int64 = 10 @@ -52,6 +52,7 @@ const ( const ( hdfsNotExistFileURL = "hdfs://127.0.0.1:9000/user/root/input/f3.txt" hdfsNotExistFileContentLength int64 = source.UnknownSourceFileLen + hdfsNotExistLastModified int64 = -1 ) var fakeHDFSClient = &hdfs.Client{} @@ -344,7 +345,7 @@ func TestGetLastModified_FileNotExist(t *testing.T) { lastModifiedMillis, err := sourceClient.GetLastModified(request) assert.EqualError(t, err, "stat /user/root/input/f3.txt: file does not exist") - assert.Equal(t, hdfsNotExistFileContentLength, lastModifiedMillis) + assert.Equal(t, hdfsNotExistLastModified, lastModifiedMillis) } func TestNewHDFSSourceClient(t *testing.T) { From 2d39315a7ec5ff436c512c577deab570cf7aa873 Mon Sep 17 00:00:00 2001 From: sunwp <244372610@qq.com> Date: Wed, 8 Dec 2021 12:45:30 +0800 Subject: [PATCH 3/5] extract the do register action Signed-off-by: sunwp <244372610@qq.com> --- pkg/source/source_client.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pkg/source/source_client.go b/pkg/source/source_client.go index d7f3062e50a..ee5c7ef38b7 100644 --- a/pkg/source/source_client.go +++ b/pkg/source/source_client.go @@ -138,28 +138,37 @@ func NewManager() ClientManager { } func (m *clientManager) Register(scheme string, resourceClient ResourceClient, adaptor requestAdapter, hooks ...Hook) error { + scheme = strings.ToLower(scheme) m.mu.Lock() defer m.mu.Unlock() - if client, ok := m.clients[strings.ToLower(scheme)]; ok { + if client, ok := m.clients[scheme]; ok { if client.(*clientWrapper).rc != resourceClient { return errors.Errorf("client with scheme %s already exist, current client: %#v", scheme, client) } + logger.Warnf("client with scheme %s already exist, no need register again", scheme) + return nil } - m.clients[strings.ToLower(scheme)] = &clientWrapper{ + m.doRegister(scheme, &clientWrapper{ adapter: adaptor, hooks: hooks, rc: resourceClient, - } + }) return nil } +func (m *clientManager) doRegister(scheme string, resourceClient ResourceClient) { + logger.Debugf("register new client %#v for scheme: %s", resourceClient, scheme) + m.clients[strings.ToLower(scheme)] = resourceClient +} + func (m *clientManager) UnRegister(scheme string) { m.mu.Lock() defer m.mu.Unlock() - if client, ok := m.clients[strings.ToLower(scheme)]; ok { + scheme = strings.ToLower(scheme) + if client, ok := m.clients[scheme]; ok { logger.Infof("remove client %#v for scheme %s", client, scheme) } - delete(m.clients, strings.ToLower(scheme)) + delete(m.clients, scheme) } func (m *clientManager) GetClient(scheme string) (ResourceClient, bool) { @@ -184,7 +193,7 @@ func (m *clientManager) GetClient(scheme string) (ResourceClient, bool) { m.mu.Unlock() return nil, false } - m.clients[scheme] = client + m.doRegister(scheme, client) m.mu.Unlock() return client, true } From bdc8a8246bd6e4fc82d7f5d714b1cd0c0b460245 Mon Sep 17 00:00:00 2001 From: sunwp <244372610@qq.com> Date: Wed, 8 Dec 2021 13:12:00 +0800 Subject: [PATCH 4/5] print raw dfget version Signed-off-by: sunwp <244372610@qq.com> --- test/e2e/e2e_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7146e98b283..3863ae2e20a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -67,6 +67,7 @@ var _ = BeforeSuite(func() { rawDfgetVersion, err := pod.Command("dfget", "version").CombinedOutput() Expect(err).NotTo(HaveOccurred()) dfgetGitCommit := strings.Fields(string(rawDfgetVersion))[7] + fmt.Printf("raw dfget version: %s\n", rawDfgetVersion) fmt.Printf("dfget merge commit: %s\n", dfgetGitCommit) if mode == dfdaemonCompatibilityTestMode { From fb20349891e960a49ebc9550cd72fa0c5d384b19 Mon Sep 17 00:00:00 2001 From: sunwp <244372610@qq.com> Date: Wed, 8 Dec 2021 14:41:34 +0800 Subject: [PATCH 5/5] remove logger & add later Signed-off-by: sunwp <244372610@qq.com> --- pkg/source/source_client.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/source/source_client.go b/pkg/source/source_client.go index ee5c7ef38b7..eb23f5a72a4 100644 --- a/pkg/source/source_client.go +++ b/pkg/source/source_client.go @@ -157,7 +157,6 @@ func (m *clientManager) Register(scheme string, resourceClient ResourceClient, a } func (m *clientManager) doRegister(scheme string, resourceClient ResourceClient) { - logger.Debugf("register new client %#v for scheme: %s", resourceClient, scheme) m.clients[strings.ToLower(scheme)] = resourceClient }