diff --git a/tests/apitests/python/library/base.py b/tests/apitests/python/library/base.py index 082d8861d38..b84efe09257 100644 --- a/tests/apitests/python/library/base.py +++ b/tests/apitests/python/library/base.py @@ -91,6 +91,20 @@ def _get_string_from_unicode(udata): result = result + tmp.strip('\n\r\t') return result +def run_command_with_popen(command): + print("Command: ", subprocess.list2cmdline(command)) + + try: + proc = subprocess.Popen(command, universal_newlines=True, shell=True, + stdout=subprocess.PIPE,stderr=subprocess.STDOUT) + output, errors = proc.communicate() + except Exception as e: + print("Error:", e) + else: + print(proc.returncode, errors, output) + finally: + proc.stdout.close() + def run_command(command, expected_error_message = None): print("Command: ", subprocess.list2cmdline(command)) try: diff --git a/tests/apitests/python/library/docker_api.py b/tests/apitests/python/library/docker_api.py index 738e6e8e860..724ee5aac66 100644 --- a/tests/apitests/python/library/docker_api.py +++ b/tests/apitests/python/library/docker_api.py @@ -3,7 +3,7 @@ import base import subprocess import json -from testutils import DOCKER_USER, DOCKER_PWD +from testutils import DOCKER_USER, DOCKER_PWD, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME try: import docker @@ -40,9 +40,20 @@ def docker_manifest_create(index, manifests): def docker_images_all_list(): command = ["sudo", "docker","images","-a"] - print( "Docker images Command: ", command) base.run_command(command) +def docker_load_image(image): + command = ["sudo", "docker","load","-i", image] + base.run_command(command) + +def docker_image_clean_all(): + docker_images_all_list() + command = ["sudo docker rmi -f $(docker images -a -q)"] + base.run_command_with_popen(command) + command = ["sudo", "docker","system", "prune", "-a", "-f"] + base.run_command(command) + docker_images_all_list() + def docker_manifest_push(index): command = ["sudo", "docker","manifest","push",index] print( "Docker Manifest Command: ", command) @@ -121,15 +132,7 @@ def docker_login(self, registry, username, password, expected_error_message = No if str(err_message).lower().find("error".lower()) >= 0: raise Exception(r" It's was not suppose to catch error when login registry {}, return message is [{}]".format (registry, err_message)) - def docker_image_remove(self, image, tag = "latest"): - docker_images_all_list() - try: - self.DCLIENT.remove_image(image+":"+tag, force=True, noprune=False) - except Exception as err: - print( "Docker image remove catch exception:", str(err)) - docker_images_all_list() - - def docker_image_pull(self, image, tag = None, expected_error_message = None, is_remove_image = True): + def docker_image_pull(self, image, tag = None, expected_error_message = None, is_clean_all_img = True): ret = "" err_message = "" if tag is not None: @@ -157,8 +160,8 @@ def docker_image_pull(self, image, tag = None, expected_error_message = None, is else: if str(err_message).lower().find("error".lower()) >= 0: raise Exception(r" It's was not suppose to catch error when pull image {}, return message is [{}]".format (image, err_message)) - if is_remove_image: - self.docker_image_remove(image, _tag) + if is_clean_all_img: + docker_image_clean_all() def docker_image_tag(self, image, harbor_registry, tag = None): _tag = base._random_name("tag") @@ -175,6 +178,7 @@ def docker_image_tag(self, image, harbor_registry, tag = None): def docker_image_push(self, harbor_registry, tag, expected_error_message = None): ret = "" err_message = "" + docker_images_all_list() if expected_error_message is "": expected_error_message = None try: @@ -196,18 +200,19 @@ def docker_image_push(self, harbor_registry, tag, expected_error_message = None) else: if str(err_message).lower().find("error".lower()) >= 0: raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, err_message)) + docker_images_all_list() def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_message = None): ret = "" err_message = "" docker_images_all_list() try: - baseimage='busybox:latest' - self.DCLIENT.login(username=DOCKER_USER, password=DOCKER_PWD) + baseimage = BASE_IMAGE['name'] + ":" + BASE_IMAGE['tag'] if not self.DCLIENT.images(name=baseimage): - print( "Docker pull is triggered when building {}".format(harbor_registry)) - self.DCLIENT.pull(baseimage) - c=self.DCLIENT.create_container(image='busybox:latest', + print( "Docker load is triggered when building {}".format(harbor_registry)) + docker_load_image(BASE_IMAGE_ABS_PATH_NAME) + docker_images_all_list() + c = self.DCLIENT.create_container(image=baseimage, command='dd if=/dev/urandom of=test bs=1M count={}'.format(size)) self.DCLIENT.start(c) self.DCLIENT.wait(c) @@ -224,10 +229,7 @@ def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_ ret = self.DCLIENT.push(repo) print("docker_image_push ret:", ret) print("build image {} with size {}".format(repo, size)) - self.DCLIENT.remove_image(repo, force=True, noprune=False) self.DCLIENT.remove_container(c) - #self.DCLIENT.pull(repo) - #image = self.DCLIENT2.images.get(repo) except Exception as err: print( "Docker image build catch exception:", str(err)) err_message = str(err) @@ -245,4 +247,4 @@ def docker_image_build(self, harbor_registry, tags=None, size=1, expected_error_ else: if str(err_message).lower().find("error".lower()) >= 0: raise Exception(r" It's was not suppose to catch error when build image {}, return message is [{}]".format (harbor_registry, err_message)) - docker_images_all_list() \ No newline at end of file + docker_image_clean_all() \ No newline at end of file diff --git a/tests/apitests/python/library/project.py b/tests/apitests/python/library/project.py index a86aa86951d..de97ef10b9e 100644 --- a/tests/apitests/python/library/project.py +++ b/tests/apitests/python/library/project.py @@ -97,6 +97,7 @@ def get_project(self, project_id, expect_status_code = 200, expect_response_body base._assert_status_code(expect_status_code, status_code) base._assert_status_code(200, status_code) + print("Project {} info: {}".format(project_id, data)) return data def update_project(self, project_id, expect_status_code=200, metadata=None, cve_allowlist=None, **kwargs): diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index 112564ecc48..817e4e88222 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -3,6 +3,7 @@ import time import base import swagger_client +import docker_api from docker_api import DockerAPI from swagger_client.rest import ApiException from testutils import DOCKER_USER, DOCKER_PWD @@ -19,7 +20,7 @@ def push_image_to_project(project_name, registry, username, password, image, tag print("Start to push image {}/{}/{}:{}".format(registry, project_name, image, tag) ) _docker_api = DockerAPI() _docker_api.docker_login("docker", DOCKER_USER, DOCKER_PWD) - _docker_api.docker_image_pull(image, tag = tag, is_remove_image = False) + _docker_api.docker_image_pull(image, tag = tag, is_clean_all_img = False) _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) time.sleep(2) if expected_login_error_message != None: @@ -35,8 +36,7 @@ def push_image_to_project(project_name, registry, username, password, image, tag new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(original_name, tag), target_image, tag = tag) time.sleep(2) _docker_api.docker_image_push(new_harbor_registry, new_tag, expected_error_message = expected_error_message) - _docker_api.docker_image_remove(target_image, tag=tag) - _docker_api.docker_image_remove(original_name, tag=tag) + docker_api.docker_image_clean_all() return r'{}/{}'.format(project_name, image), new_tag def push_self_build_image_to_project(project_name, registry, username, password, image, tag, size=2, expected_login_error_message = None, expected_error_message = None): diff --git a/tests/apitests/python/library/robot.py b/tests/apitests/python/library/robot.py index 59f87c1bee0..5a5fc280aaa 100644 --- a/tests/apitests/python/library/robot.py +++ b/tests/apitests/python/library/robot.py @@ -50,7 +50,10 @@ def create_access_list(self, right_map = [True] * 10): access_list.append(robotAccountAccess) return access_list - def create_project_robot(self, project_name, duration, robot_name = None, robot_desc = None, has_pull_right = True, has_push_right = True, has_chart_read_right = True, has_chart_create_right = True, expect_status_code = 201, **kwargs): + def create_project_robot(self, project_name, duration, robot_name = None, robot_desc = None, + has_pull_right = True, has_push_right = True, has_chart_read_right = True, + has_chart_create_right = True, expect_status_code = 201, expect_response_body = None, + **kwargs): if robot_name is None: robot_name = base._random_name("robot") if robot_desc is None: @@ -82,10 +85,16 @@ def create_project_robot(self, project_name, duration, robot_name = None, robot_ client = self._get_client(**kwargs) data = [] - data, status_code, header = client.create_robot_with_http_info(robotAccountCreate) - base._assert_status_code(expect_status_code, status_code) - base._assert_status_code(201, status_code) - return base._get_id_from_header(header), data + try: + data, status_code, header = client.create_robot_with_http_info(robotAccountCreate) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + else: + base._assert_status_code(expect_status_code, status_code) + base._assert_status_code(201, status_code) + return base._get_id_from_header(header), data def get_robot_account_by_id(self, robot_id, **kwargs): client = self._get_client(**kwargs) diff --git a/tests/apitests/python/library/sign.py b/tests/apitests/python/library/sign.py index 679439dbb60..788481073cc 100644 --- a/tests/apitests/python/library/sign.py +++ b/tests/apitests/python/library/sign.py @@ -1,11 +1,15 @@ # -*- coding: utf-8 -*- import subprocess -from testutils import notary_url +from testutils import notary_url, BASE_IMAGE_ABS_PATH_NAME +from docker_api import docker_load_image, docker_image_clean_all def sign_image(registry_ip, project_name, image, tag): + docker_load_image(BASE_IMAGE_ABS_PATH_NAME) try: ret = subprocess.check_output(["./tests/apitests/python/sign_image.sh", registry_ip, project_name, image, tag, notary_url], shell=False) print("sign_image return: ", ret) - except subprocess.CalledProcessError as exc: - raise Exception("Failed to sign image error is {} {}.".format(exc.returncode, exc.output)) + except subprocess.CalledProcessError as e: + raise Exception("Failed to sign image error is {} {}.".format(e.returncode, e.output)) + finally: + docker_image_clean_all() diff --git a/tests/apitests/python/test_add_sys_label_to_tag.py b/tests/apitests/python/test_add_sys_label_to_tag.py index 37c20ddbe84..36e903c4b07 100644 --- a/tests/apitests/python/test_add_sys_label_to_tag.py +++ b/tests/apitests/python/test_add_sys_label_to_tag.py @@ -72,7 +72,7 @@ def testAddSysLabelToRepo(self): expected_project_id = TestProjects.project_add_g_lbl_id, **TestProjects.USER_add_g_lbl_CLIENT) #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_add_g_lbl_name, harbor_server, user_add_g_lbl_name, user_001_password, "hello-world", "latest") + TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_add_g_lbl_name, harbor_server, user_add_g_lbl_name, user_001_password, "test_sys_label", "latest") #6. Create a new label(LA) in project(PA) by admin; TestProjects.label_id, _ = self.label.create_label(**ADMIN_CLIENT) diff --git a/tests/apitests/python/test_assign_role_to_ldap_group.py b/tests/apitests/python/test_assign_role_to_ldap_group.py index c87d52453b4..8c648dd8a10 100644 --- a/tests/apitests/python/test_assign_role_to_ldap_group.py +++ b/tests/apitests/python/test_assign_role_to_ldap_group.py @@ -44,7 +44,7 @@ def testAssignRoleToLdapGroup(self): 9. Delete project(PA); """ url = ADMIN_CLIENT["endpoint"] - USER_ADMIN=dict(endpoint = url, username = "admin_user", password = "zhu88jie", repo = "hello-world") + USER_ADMIN=dict(endpoint = url, username = "admin_user", password = "zhu88jie", repo = "haproxy") USER_DEV=dict(endpoint = url, username = "dev_user", password = "zhu88jie", repo = "alpine") USER_GUEST=dict(endpoint = url, username = "guest_user", password = "zhu88jie", repo = "busybox") USER_TEST=dict(endpoint = url, username = "test", password = "123456") diff --git a/tests/apitests/python/test_copy_artifact_outside_project.py b/tests/apitests/python/test_copy_artifact_outside_project.py index 37701aa0e8e..085943a508f 100644 --- a/tests/apitests/python/test_copy_artifact_outside_project.py +++ b/tests/apitests/python/test_copy_artifact_outside_project.py @@ -83,7 +83,7 @@ def testRetag(self): self.project.update_project_member_role(TestProjects.project_dst_repo_id, retag_member_id, 3, **ADMIN_CLIENT) #5. Create a new repository(RA) in project(PA) by user(UA); - TestProjects.src_repo_name, tag_name = push_self_build_image_to_project(TestProjects.project_src_repo_name, harbor_server, 'admin', 'Harbor12345', "hello-world", pull_tag_name) + TestProjects.src_repo_name, tag_name = push_self_build_image_to_project(TestProjects.project_src_repo_name, harbor_server, 'admin', 'Harbor12345', "test_copy", pull_tag_name) #6. Get repository in project(PA), there should be one repository which was created by user(UA); src_repo_data = self.repo.list_repositories(TestProjects.project_src_repo_name, **TestProjects.USER_RETAG_CLIENT) diff --git a/tests/apitests/python/test_del_repo.py b/tests/apitests/python/test_del_repo.py index a92826174fe..c39194d496c 100644 --- a/tests/apitests/python/test_del_repo.py +++ b/tests/apitests/python/test_del_repo.py @@ -54,7 +54,7 @@ def testDelRepo(self): TestProjects.project_del_repo_id, TestProjects.project_del_repo_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_del_repo_CLIENT) #3. Create a new repository(RA) in project(PA) by user(UA); - repo_name, _ = push_self_build_image_to_project(TestProjects.project_del_repo_name, harbor_server, 'admin', 'Harbor12345', "hello-world", "latest") + repo_name, _ = push_self_build_image_to_project(TestProjects.project_del_repo_name, harbor_server, 'admin', 'Harbor12345', "test_del_repo", "latest") #4. Get repository in project(PA), there should be one repository which was created by user(UA); repo_data = self.repo.list_repositories(TestProjects.project_del_repo_name, **TestProjects.USER_del_repo_CLIENT) diff --git a/tests/apitests/python/test_manage_project_member.py b/tests/apitests/python/test_manage_project_member.py index 98a4d923a76..dfc6c65f619 100644 --- a/tests/apitests/python/test_manage_project_member.py +++ b/tests/apitests/python/test_manage_project_member.py @@ -70,7 +70,7 @@ def testManageProjectMember(self): TestProjects.project_alice_id, TestProjects.project_alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) #2.2 Add a repository to project(PA) by Alice - TestProjects.repo_name, _ = push_self_build_image_to_project(TestProjects.project_alice_name, harbor_server, user_alice_name, user_alice_password, "hello-world", "latest") + TestProjects.repo_name, _ = push_self_build_image_to_project(TestProjects.project_alice_name, harbor_server, user_alice_name, user_alice_password, "test_member", "latest") #3. Bob is not a member of project(PA); self.project.check_project_member_not_exist(TestProjects.project_alice_id, user_bob_name, **USER_ALICE_CLIENT) diff --git a/tests/apitests/python/test_project_level_policy_content_trust.py b/tests/apitests/python/test_project_level_policy_content_trust.py index f971aceb298..349a9092f6b 100644 --- a/tests/apitests/python/test_project_level_policy_content_trust.py +++ b/tests/apitests/python/test_project_level_policy_content_trust.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import unittest +import time from testutils import ADMIN_CLIENT, suppress_urllib3_warning from testutils import harbor_server @@ -11,6 +12,7 @@ from library.repository import Repository from library.repository import push_self_build_image_to_project from library.repository import pull_harbor_image +from library.docker_api import docker_image_clean_all class TestProjects(unittest.TestCase): @suppress_urllib3_warning def setUp(self): @@ -19,7 +21,7 @@ def setUp(self): self.artifact= Artifact() self.repo= Repository() - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + @unittest.skipIf(TEARDOWN == True, "Test data won't be erased.") def tearDown(self): #1. Delete repository(RA) by user(UA); self.repo.delete_repository(TestProjects.project_content_trust_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_CONTENT_TRUST_CLIENT) @@ -48,9 +50,7 @@ def testProjectLevelPolicyContentTrust(self): 3. Delete user(UA); """ url = ADMIN_CLIENT["endpoint"] - image = "hello-world" - admin_name = ADMIN_CLIENT["username"] - admin_password = ADMIN_CLIENT["password"] + image = "test_content_trust" user_content_trust_password = "Aa123456" #1. Create a new user(UA); @@ -62,20 +62,24 @@ def testProjectLevelPolicyContentTrust(self): TestProjects.project_content_trust_id, TestProjects.project_content_trust_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CONTENT_TRUST_CLIENT) #3. Push a new image(IA) in project(PA) by admin; - TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_content_trust_name, harbor_server, admin_name, admin_password, image, "latest") + TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_content_trust_name, harbor_server, ADMIN_CLIENT["username"], ADMIN_CLIENT["password"], image, "latest") #4. Image(IA) should exist; artifact = self.artifact.get_reference_info(TestProjects.project_content_trust_name, image, tag, **TestProjects.USER_CONTENT_TRUST_CLIENT) self.assertEqual(artifact.tags[0].name, tag) + docker_image_clean_all() #5. Pull image(IA) successfully; - pull_harbor_image(harbor_server, admin_name, admin_password, TestProjects.repo_name, tag) + pull_harbor_image(harbor_server, ADMIN_CLIENT["username"], ADMIN_CLIENT["password"], TestProjects.repo_name, tag) + self.project.get_project(TestProjects.project_content_trust_id) #6. Enable content trust in project(PA) configuration; self.project.update_project(TestProjects.project_content_trust_id, metadata = {"enable_content_trust": "true"}, **TestProjects.USER_CONTENT_TRUST_CLIENT) + self.project.get_project(TestProjects.project_content_trust_id) #7. Pull image(IA) failed and the reason is "The image is not signed in Notary". - pull_harbor_image(harbor_server, admin_name, admin_password, TestProjects.repo_name, tag, expected_error_message = "The image is not signed in Notary") + time.sleep(30) + pull_harbor_image(harbor_server, ADMIN_CLIENT["username"], ADMIN_CLIENT["password"], TestProjects.repo_name, tag, expected_error_message = "The image is not signed in Notary") if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_project_quota.py b/tests/apitests/python/test_project_quota.py index 71f671f6b1a..5bc09b1d674 100644 --- a/tests/apitests/python/test_project_quota.py +++ b/tests/apitests/python/test_project_quota.py @@ -45,6 +45,7 @@ def testProjectQuota(self): with created_project(metadata={"public": "false"}, user_id=user_id) as (project_id, project_name): #4. Push an image to project(PA) by user(UA), then check the project quota usage; -- {"count": 1, "storage": 2791709} image, tag = "goharbor/alpine", "3.10" + image_alias_name = "_alias" push_image_to_project(project_name, harbor_server, user_name, user_001_password, image, tag) #5. Get project quota @@ -52,7 +53,7 @@ def testProjectQuota(self): self.assertEqual(quota[0].used["storage"], 2789002) #6. Push the image with another tag to project(PA) by user(UA), the check the project quota usage; -- {"count": 1, "storage": 2791709} - push_image_to_project(project_name, harbor_server, user_name, user_001_password, image, tag) + push_image_to_project(project_name, harbor_server, user_name, user_001_password, image, tag, new_image=image+image_alias_name) #7. Get project quota quota = self.system.get_project_quota("project", project_id, **ADMIN_CLIENT) @@ -60,6 +61,7 @@ def testProjectQuota(self): #8. Delete repository(RA) by user(UA); self.repo.delete_repository(project_name, "goharbor%2Falpine", **ADMIN_CLIENT) + self.repo.delete_repository(project_name, "goharbor%2Falpine"+image_alias_name, **ADMIN_CLIENT) #9. Quota should be 0 quota = self.system.get_project_quota("project", project_id, **ADMIN_CLIENT) diff --git a/tests/apitests/python/test_push_cnab_bundle.py b/tests/apitests/python/test_push_cnab_bundle.py index f859eac0369..fc4d3652edd 100644 --- a/tests/apitests/python/test_push_cnab_bundle.py +++ b/tests/apitests/python/test_push_cnab_bundle.py @@ -71,7 +71,7 @@ def test_01_PushBundleByCnab(self): #3. Push bundle to harbor as repository(RA); target = harbor_server + "/" + TestCNAB.project_name + "/" + TestCNAB.cnab_repo_name + ":" + TestCNAB.cnab_tag - TestCNAB.reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, TestCNAB.user_name, TestCNAB.user_push_cnab_password, "containrrr/watchtower:latest", "kong:latest", target) + TestCNAB.reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, TestCNAB.user_name, TestCNAB.user_push_cnab_password, "goharbor/harbor-log:v1.10.0", "kong:latest", target) #4. Get repository from Harbor successfully; TestCNAB.cnab_bundle_data = TestCNAB.repo.get_repository(TestCNAB.project_name, TestCNAB.cnab_repo_name, **TestCNAB.USER_CLIENT) diff --git a/tests/apitests/python/test_push_image_with_special_name.py b/tests/apitests/python/test_push_image_with_special_name.py index cde993d8060..f406869004f 100644 --- a/tests/apitests/python/test_push_image_with_special_name.py +++ b/tests/apitests/python/test_push_image_with_special_name.py @@ -65,7 +65,7 @@ def testPushImageWithSpecialName(self): self.project.projects_should_exist(dict(public=False), expected_count = 1, expected_project_id = TestProjects.project_sign_image_id, **TestProjects.USER_sign_image_CLIENT) - image = "hello-world" + image = "redis" src_tag = "latest" profix = "aaa/bbb" diff --git a/tests/apitests/python/test_registry_api.py b/tests/apitests/python/test_registry_api.py index cd31270b9a0..003afe54589 100644 --- a/tests/apitests/python/test_registry_api.py +++ b/tests/apitests/python/test_registry_api.py @@ -27,7 +27,6 @@ def setUp(self): self.user= User() self.artifact = Artifact() self.repo = Repository() - self.repo_name = "hello-world" @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): @@ -72,15 +71,15 @@ def testRegistryAPI(self): #3. Push 3 images to project_Alice and Add 3 tags to the 3rd image. src_tag = "latest" - image_a = "busybox" + image_a = "image_a" TestProjects.repo_a, tag_a = push_self_build_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) - image_b = "alpine" + image_b = "image_b" TestProjects.repo_b, tag_b = push_self_build_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_b, src_tag) - image_c = "hello-world" + image_c = "image_c" TestProjects.repo_c, tag_c = push_self_build_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_c, src_tag) create_tags = ["1.0","2.0","3.0"] for tag in create_tags: - self.artifact.create_tag(TestProjects.project_Alice_name, self.repo_name, tag_c, tag, **USER_ALICE_CLIENT) + self.artifact.create_tag(TestProjects.project_Alice_name, image_c, tag_c, tag, **USER_ALICE_CLIENT) #4. Call the image_list_tags API tags = list_image_tags(harbor_server,TestProjects.repo_c,user_Alice_name,user_common_password) for tag in create_tags: diff --git a/tests/apitests/python/test_robot_account.py b/tests/apitests/python/test_robot_account.py index b4007f97f62..e03472e0403 100644 --- a/tests/apitests/python/test_robot_account.py +++ b/tests/apitests/python/test_robot_account.py @@ -47,9 +47,11 @@ def do_01_tearDown(self): self.project.delete_project(self.project_ra_id_a, **self.USER_RA_CLIENT) self.project.delete_project(self.project_ra_id_b, **self.USER_RA_CLIENT) self.project.delete_project(self.project_ra_id_c, **self.USER_RA_CLIENT) + self.project.delete_project(self.project_ra_id_d, **self.USER_RA_CLIENT) #3. Delete user(UA). self.user.delete_user(self.user_ra_id, **ADMIN_CLIENT) + self.user.delete_user(self.user_ra_id_b, **ADMIN_CLIENT) def test_01_ProjectlevelRobotAccount(self): """ @@ -70,14 +72,15 @@ def test_01_ProjectlevelRobotAccount(self): 12. Update action property of robot account(RA); 13. Pull image(ImagePA) from project(PA) by robot account(RA), it must be not successful; 14. Push image(ImageRA) to project(PA) by robot account(RA), it must be not successful; - 15. Delete robot account(RA), it must be not successful. + 15. Delete robot account(RA). + 16. Create user(UB), Create public project(PD) by user(UB), user(UA) can't create robot account for project(PD). Tear down: 1. Delete repository(RA) by user(UA); 2. Delete project(PA); 3. Delete user(UA). """ image_project_a = "haproxy" - image_project_b = "hello-world" + image_project_b = "image_project_b" image_project_c = "httpd" image_robot_account = "alpine" tag = "latest" @@ -97,42 +100,50 @@ def test_01_ProjectlevelRobotAccount(self): self.repo_name_in_project_c, tag_c = push_self_build_image_to_project(self.project_ra_name_c, harbor_server, user_ra_name, TestRobotAccount.user_ra_password, image_project_c, tag) #4. Create a new robot account(RA) with pull and push privilege in project(PA) by user(UA); - robot_id, robot_account = self.robot.create_project_robot(self.project_ra_name_a, + robot_id_a, robot_account_a = self.robot.create_project_robot(self.project_ra_name_a, + 30 ,**self.USER_RA_CLIENT) + robot_id_b, robot_account_b = self.robot.create_project_robot(self.project_ra_name_b, 30 ,**self.USER_RA_CLIENT) #5. Check robot account info, it should has both pull and push privilege; - data = self.robot.get_robot_account_by_id(robot_id, **self.USER_RA_CLIENT) - _assert_status_code(robot_account.name, data.name) + data = self.robot.get_robot_account_by_id(robot_id_a, **self.USER_RA_CLIENT) + _assert_status_code(robot_account_a.name, data.name) #6. Pull image(ImagePA) from project(PA) by robot account(RA), it must be successful; - pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, self.repo_name_in_project_a, tag_a) + pull_harbor_image(harbor_server, robot_account_a.name, robot_account_a.secret, self.repo_name_in_project_a, tag_a) #7. Push image(ImageRA) to project(PA) by robot account(RA), it must be successful; - self.repo_name_pa, _ = push_self_build_image_to_project(self.project_ra_name_a, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag) + self.repo_name_pa, _ = push_self_build_image_to_project(self.project_ra_name_a, harbor_server, robot_account_a.name, robot_account_a.secret, image_robot_account, tag) #8. Push image(ImageRA) to project(PB) by robot account(RA), it must be not successful; - push_self_build_image_to_project(self.project_ra_name_b, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") + push_self_build_image_to_project(self.project_ra_name_b, harbor_server, robot_account_a.name, robot_account_a.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") #9. Pull image(ImagePB) from project(PB) by robot account(RA), it must be not successful; - pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, self.repo_name_in_project_b, tag_b, expected_error_message = "unauthorized to access repository") + pull_harbor_image(harbor_server, robot_account_a.name, robot_account_a.secret, self.repo_name_in_project_b, tag_b, expected_error_message = "unauthorized to access repository") #10. Pull image from project(PC), it must be successful; - pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, self.repo_name_in_project_c, tag_c) + pull_harbor_image(harbor_server, robot_account_a.name, robot_account_a.secret, self.repo_name_in_project_c, tag_c) #11. Push image(ImageRA) to project(PC) by robot account(RA), it must be not successful; - push_self_build_image_to_project(self.project_ra_name_c, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") + push_self_build_image_to_project(self.project_ra_name_c, harbor_server, robot_account_a.name, robot_account_a.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") #12. Update action property of robot account(RA);" - self.robot.disable_robot_account(robot_id, True, **self.USER_RA_CLIENT) + self.robot.disable_robot_account(robot_id_a, True, **self.USER_RA_CLIENT) #13. Pull image(ImagePA) from project(PA) by robot account(RA), it must be not successful; - pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, self.repo_name_in_project_a, tag_a, expected_login_error_message = "unauthorized: authentication required") + pull_harbor_image(harbor_server, robot_account_a.name, robot_account_a.secret, self.repo_name_in_project_a, tag_a, expected_login_error_message = "unauthorized: authentication required") #14. Push image(ImageRA) to project(PA) by robot account(RA), it must be not successful; - push_self_build_image_to_project(self.project_ra_name_a, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_login_error_message = "unauthorized: authentication required") + push_self_build_image_to_project(self.project_ra_name_a, harbor_server, robot_account_a.name, robot_account_a.secret, image_robot_account, tag, expected_login_error_message = "unauthorized: authentication required") + + #15. Delete robot account(RA). + self.robot.delete_robot_account(robot_id_a, **self.USER_RA_CLIENT) - #15. Delete robot account(RA), it must be not successful. - self.robot.delete_robot_account(robot_id, **self.USER_RA_CLIENT) + #16. Create user(UB), Create public project(PD) by user(UB), user(UA) can't create robot account for project(PD). + self.user_ra_id_b, user_ra_name_b = self.user.create_user(user_password = TestRobotAccount.user_ra_password, **ADMIN_CLIENT) + self.USER_RA_CLIENT_B=dict(endpoint = TestRobotAccount.url, username = user_ra_name_b, password = TestRobotAccount.user_ra_password) + self.project_ra_id_d, self.project_ra_name_d = self.project.create_project(metadata = {"public": "true"}, **self.USER_RA_CLIENT_B) + self.robot.create_project_robot(self.project_ra_name_d, 30 , expect_status_code = 403, **self.USER_RA_CLIENT) self.do_01_tearDown() @@ -186,7 +197,8 @@ def test_02_SystemlevelRobotAccount(self): for i in range(2): base.run_command( ["curl", r"-o", "./tests/apitests/python/{}-{}.tgz".format(CHART_FILE_LIST[i]["name"], CHART_FILE_LIST[i]["version"]), "https://storage.googleapis.com/harbor-builds/helm-chart-test-files/{}-{}.tgz".format(CHART_FILE_LIST[i]["name"], CHART_FILE_LIST[i]["version"])]) - #Make sure that whether 'True' or 'False' must be included in each line or row. + # In this priviledge check list, make sure that each of lines and rows must + # contains both True and False value. check_list = [ [True, True, True, True, True, True, False, True, False, True], [False, False, False, False, True, True, False, True, True, False], diff --git a/tests/apitests/python/test_scan_image_artifact.py b/tests/apitests/python/test_scan_image_artifact.py index a8ae85b0f8d..2faa87fd61c 100644 --- a/tests/apitests/python/test_scan_image_artifact.py +++ b/tests/apitests/python/test_scan_image_artifact.py @@ -4,7 +4,7 @@ from testutils import harbor_server, suppress_urllib3_warning from testutils import TEARDOWN -from testutils import ADMIN_CLIENT +from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME from library.project import Project from library.user import User from library.repository import Repository @@ -103,12 +103,11 @@ def testScanSignedImage(self): #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image right after repository creation. - #Note:busybox is pulled in setup phase, and setup is a essential phase. - image = "busybox" - tag = "latest" + #Note:busybox is pulled in setup phase, and setup is an essential phase before scripts execution. + image = BASE_IMAGE['name'] + tag = BASE_IMAGE['tag'] #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - #TestScan.repo_name_1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, tag) - + # Push base image in function sign_image. sign_image(harbor_server, self.project_name, image, tag) #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; diff --git a/tests/apitests/python/test_tag_crud.py b/tests/apitests/python/test_tag_crud.py index a625c8cbf92..d06c65caeba 100644 --- a/tests/apitests/python/test_tag_crud.py +++ b/tests/apitests/python/test_tag_crud.py @@ -25,7 +25,7 @@ def setUp(self): self.repo= Repository() self.url = ADMIN_CLIENT["endpoint"] self.user_password = "Aa123456" - self.repo_name = "hello-world" + self.repo_name = "test_tag_crud" @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): diff --git a/tests/apitests/python/testutils.py b/tests/apitests/python/testutils.py index 36979ddb5ea..8368e57288d 100644 --- a/tests/apitests/python/testutils.py +++ b/tests/apitests/python/testutils.py @@ -33,6 +33,8 @@ DOCKER_USER = os.environ.get('DOCKER_USER', '') DOCKER_PWD = os.environ.get('DOCKER_PWD', '') METRIC_URL = os.environ.get('METRIC_URL', 'http://'+harbor_server+':9090') +BASE_IMAGE = dict(name='busybox', tag='latest') +BASE_IMAGE_ABS_PATH_NAME = '/' + BASE_IMAGE['name'] + '.tar' def GetProductApi(username, password, harbor_server= os.environ.get("HARBOR_HOST", '')): diff --git a/tests/ci/api_run.sh b/tests/ci/api_run.sh index ce3fd5474c6..f1d04aaa7e2 100755 --- a/tests/ci/api_run.sh +++ b/tests/ci/api_run.sh @@ -18,7 +18,7 @@ harbor_logs_bucket="harbor-ci-logs" #echo "content_language = en" >> $botofile #echo "default_project_id = $GS_PROJECT_ID" >> $botofile DIR="$(cd "$(dirname "$0")" && pwd)" -E2E_IMAGE="goharbor/harbor-e2e-engine:2.6.1" +E2E_IMAGE="goharbor/harbor-e2e-engine:2.6.2" # GS util function uploader { diff --git a/tests/e2e-image/Dockerfile b/tests/e2e-image/Dockerfile index aaf85aed1d6..3e75760f1e0 100644 --- a/tests/e2e-image/Dockerfile +++ b/tests/e2e-image/Dockerfile @@ -4,7 +4,8 @@ ENV LANG C.UTF-8 # V 2.0.1: upgrade docker to version 19.03.12 # V 2.5 Add support for e2e py-test (especially containerd) # V 2.6 docker 19.03.12 -# V 2.6.1 upgrade helm2, helm3 and docker 20.10.1 +# V 2.6.1 upgrade containerd(ctr) to v1.4.3, docker 20.10.3 +# V 2.6.2 package busybox in RUN apt-get update && apt-get install -y --no-install-recommends wget curl gnupg2 RUN apt-get install libseccomp2 @@ -83,12 +84,13 @@ RUN apt-get update && apt install libnss3-tools && \ RUN pip3 install pyasn1 google-apitools==0.5.31 gsutil robotframework==3.2.1 robotframework-sshlibrary robotframework-httplibrary requests dbbot robotframework-seleniumlibrary==4.3.0 robotframework-pabot robotframework-JSONLibrary --upgrade -ENV CRI_CONTAINERD_VERSION 1.3.4 -RUN wget https://storage.googleapis.com/cri-containerd-release/cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz && \ - tar --no-overwrite-dir -C / -xzf cri-containerd-${CRI_CONTAINERD_VERSION}.linux-amd64.tar.gz +ENV CONTAINERD_VERSION 1.4.3 +RUN wget https://github.com/containerd/containerd/releases/download/v1.4.3/containerd-$CONTAINERD_VERSION-linux-amd64.tar.gz && \ + tar zxvf containerd-$CONTAINERD_VERSION-linux-amd64.tar.gz && \ + cd bin && cp -f containerd ctr /usr/bin/ && cp -f containerd ctr /usr/local/bin/ # Install docker, docker compose -ENV DOCKER_VERSION 20.10.1 +ENV DOCKER_VERSION 20.10.3 RUN wget https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VERSION.tgz && \ tar --strip-components=1 -xvzf docker-$DOCKER_VERSION.tgz -C /usr/bin && \ curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && \ @@ -167,6 +169,8 @@ RUN wget "https://raw.githubusercontent.com/docker/docker/${DIND_COMMIT}/hack/di COPY containerd_config.toml /etc/containerd/config.toml +COPY busybox.tar / + # This container needs to be run in privileged mode(run with --privileged option) to make it work COPY dockerd-entrypoint.sh /usr/local/bin/dockerd-entrypoint.sh diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot index 64736034b03..8c2f4e0033c 100644 --- a/tests/resources/Docker-Util.robot +++ b/tests/resources/Docker-Util.robot @@ -35,7 +35,7 @@ Pull image Should Contain ${output} Status: Should Not Contain ${output} No such image: #Remove image for docker 20 - Wait Unitl Command Success docker rmi -f ${ip}/${project}/${image_with_tag} + Clean All Local Images Push image # If no tag provided in $(image_with_or_without_tag}, latest will be the tag pulled from docker-hub or read from local @@ -54,10 +54,7 @@ Push image Wait Unitl Command Success docker push ${ip}/${project}/${image_in_use_with_tag} Wait Unitl Command Success docker logout ${ip} #Remove image for docker 20 - ${output}= Wait Unitl Command Success docker rmi -f ${ip}/${project}/${image_in_use_with_tag} - Log All Docker rmi: ${output} - ${output}= Run Keyword If ${need_pull_first}==${true} Wait Unitl Command Success docker rmi -f ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image_in_use} - Log All Docker rmi: ${output} + Clean All Local Images Sleep 1 Push Image With Tag @@ -70,8 +67,11 @@ Push Image With Tag Wait Unitl Command Success docker push ${ip}/${project}/${image}:${tag} Wait Unitl Command Success docker logout ${ip} #Remove image for docker 20 - Wait Unitl Command Success docker rmi -f ${ip}/${project}/${image}:${tag} - Wait Unitl Command Success docker rmi -f ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:${tag1} + Clean All Local Images + +Clean All Local Images + Wait Unitl Command Success docker rmi -f $(docker images -a -q) + Wait Unitl Command Success docker system prune -a -f Cannot Docker Login Harbor [Arguments] ${ip} ${user} ${pwd} @@ -148,6 +148,29 @@ Start Docker Daemon Locally Sleep 2s [Return] ${handle} +Restart Docker Daemon Locally + FOR ${IDX} IN RANGE 5 + ${pid}= Run pidof dockerd + Exit For Loop If '${pid}' == '${EMPTY}' + ${result}= Run Process kill ${pid} shell=True + Log To Console Kill docker process: ${result} + Sleep 2s + END + ${pid}= Run pidof dockerd + Should Be Equal As Strings '${pid}' '${EMPTY}' + OperatingSystem.File Should Exist /usr/local/bin/dockerd-entrypoint.sh + ${result}= Run Process rm -rf /var/lib/docker/* shell=True + Log To Console Clear /var/lib/docker: ${result} + ${handle}= Start Process /usr/local/bin/dockerd-entrypoint.sh dockerd>./daemon-local.log 2>&1 shell=True + Process Should Be Running ${handle} + FOR ${IDX} IN RANGE 5 + ${pid}= Run pidof dockerd + Exit For Loop If '${pid}' != '${EMPTY}' + Sleep 2s + END + Sleep 2s + [Return] ${handle} + Start Containerd Daemon Locally ${handle}= Start Process containerd > ./daemon-local.log 2>&1 & shell=True FOR ${IDX} IN RANGE 5 diff --git a/tests/resources/Harbor-Pages/Configuration.robot b/tests/resources/Harbor-Pages/Configuration.robot index fad16ad42c6..d88347c5734 100644 --- a/tests/resources/Harbor-Pages/Configuration.robot +++ b/tests/resources/Harbor-Pages/Configuration.robot @@ -353,12 +353,16 @@ Get Project Storage Quota Text From Project Quotas List [Return] ${storage_quota} Check Automatic Onboarding And Save + Switch To Configure Retry Element Click ${cfg_auth_automatic_onboarding_checkbox} Retry Element Click xpath=${config_auth_save_button_xpath} Set User Name Claim And Save [Arguments] ${type} - Retry Text Input ${cfg_auth_user_name_claim_input} ${type} + Switch To Configure + Retry Clear Element Text ${cfg_auth_user_name_claim_input} + Run Keyword If '${type}'=='${null}' Retry Text Input ${cfg_auth_user_name_claim_input} anytext + ... ELSE Retry Text Input ${cfg_auth_user_name_claim_input} ${type} Retry Element Click xpath=${config_auth_save_button_xpath} Select Distribution diff --git a/tests/resources/Harbor-Pages/OIDC_Auth.robot b/tests/resources/Harbor-Pages/OIDC_Auth.robot index cf748049b99..673e6047f44 100644 --- a/tests/resources/Harbor-Pages/OIDC_Auth.robot +++ b/tests/resources/Harbor-Pages/OIDC_Auth.robot @@ -21,14 +21,16 @@ Resource ../../resources/Util.robot *** Keywords *** Sign In Harbor With OIDC User - [Arguments] ${url} ${username}=${OIDC_USERNAME} ${is_onboard}=${false} ${username_claim}=${null} - ${full_name}= Set Variable ${username}@example.com + [Arguments] ${url} ${username}=${OIDC_USERNAME} ${password}=password ${is_onboard}=${false} ${username_claim}=${null} ${login_with_provider}=email + ${full_name}= Set Variable If '${login_with_provider}' == 'email' ${username}@example.com ${username} ${head_username}= Set Variable If '${username_claim}' == 'email' xpath=//harbor-app/harbor-shell/clr-main-container/navigator/clr-header//clr-dropdown//button[contains(.,'${full_name}')] xpath=//harbor-app/harbor-shell/clr-main-container/navigator/clr-header//clr-dropdown//button[contains(.,'${username}')] Init Chrome Driver Go To ${url} Retry Element Click ${log_oidc_provider_btn} + Run Keyword If '${login_with_provider}' == 'email' Retry Element Click ${login_with_email_btn} + Run Keyword If '${login_with_provider}' == 'ldap' Retry Element Click ${login_with_ldap_btn} Retry Text Input ${dex_login_btn} ${full_name} - Retry Text Input ${dex_pwd_btn} password + Retry Text Input ${dex_pwd_btn} ${password} Retry Element Click ${submit_login_btn} Retry Element Click ${grant_btn} @@ -39,6 +41,7 @@ Sign In Harbor With OIDC User Run Keyword If ${is_onboard} == ${true} Should Not Be True ${isVisible} Run Keyword If '${isVisible}' == 'True' Run Keywords Retry Text Input ${oidc_username_input} ${username} AND Retry Element Click ${save_btn} Retry Wait Element ${head_username} + Capture Page Screenshot ${name_display}= Get Text xpath=//harbor-app/harbor-shell/clr-main-container/navigator/clr-header//clr-dropdown[2]//button/span Run Keyword If '${username_claim}' == 'email' Should Be Equal As Strings ${name_display} ${full_name} ... ELSE Should Be Equal As Strings ${name_display} ${username} diff --git a/tests/resources/Harbor-Pages/OIDC_Auth_Elements.robot b/tests/resources/Harbor-Pages/OIDC_Auth_Elements.robot index 42389d0e602..ecb03f81a30 100644 --- a/tests/resources/Harbor-Pages/OIDC_Auth_Elements.robot +++ b/tests/resources/Harbor-Pages/OIDC_Auth_Elements.robot @@ -17,6 +17,8 @@ Documentation This resource provides any keywords related to the Harbor private *** Variables *** ${log_oidc_provider_btn} //*[@id='log_oidc'] +${login_with_email_btn} //span[contains(., 'Log in with Email')] +${login_with_ldap_btn} //span[contains(., 'Log in with LDAP')] ${dex_login_btn} //*[@id='login'] ${dex_pwd_btn} //*[@id='password'] ${submit_login_btn} //*[@id='submit-login'] @@ -25,4 +27,4 @@ ${oidc_username_input} //*[@id='oidcUsername'] ${save_btn} //*[@id='saveButton'] ${OIDC_USERNAME} test1 ${generate_secret_btn} //*[@id='generate-cli-btn'] -${more_btn} //*[@id='hidden-generate-cli'] \ No newline at end of file +${more_btn} //*[@id='hidden-generate-cli'] diff --git a/tests/resources/Harbor-Pages/Project-Config.robot b/tests/resources/Harbor-Pages/Project-Config.robot index 75b13059cb8..eef5007db45 100644 --- a/tests/resources/Harbor-Pages/Project-Config.robot +++ b/tests/resources/Harbor-Pages/Project-Config.robot @@ -8,7 +8,9 @@ Resource ../../resources/Util.robot Goto Project Config Sleep 3 - Retry Element Click //project-detail//ul/li[contains(.,'Configuration')] + Retry Element Click //project-detail//ul/li[contains(.,'Summary')] + Sleep 3 + Retry Double Keywords When Error Retry Element Click //project-detail//ul/li[contains(.,'Configuration')] Retry Wait Element //clr-checkbox-wrapper/label[contains(.,'Enable content trust')] Sleep 2 Click Project Public diff --git a/tests/resources/Harbor-Pages/Project-Members.robot b/tests/resources/Harbor-Pages/Project-Members.robot index bcf00b2b1b5..f0aa84cd8cf 100644 --- a/tests/resources/Harbor-Pages/Project-Members.robot +++ b/tests/resources/Harbor-Pages/Project-Members.robot @@ -39,6 +39,7 @@ Clear Search Input And Go Into Project ${out} Run Keyword If ${has_image}==${false} Run Keywords Retry Element Click xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a AND Wait Until Element Is Visible And Enabled xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any repositories!\")] ... ELSE Run Keywords Retry Element Click xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a AND Wait Until Element Is Visible And Enabled xpath=//project-detail//hbr-repository-gridview//clr-dg-cell[contains(.,'${project}/')] Sleep 1 + Capture Page Screenshot Add User To Project Admin [Arguments] ${project} ${user} diff --git a/tests/resources/Harbor-Pages/Project-P2P-Preheat-Elements.robot b/tests/resources/Harbor-Pages/Project-P2P-Preheat-Elements.robot index 84dbc4323c9..7524ad2042a 100644 --- a/tests/resources/Harbor-Pages/Project-P2P-Preheat-Elements.robot +++ b/tests/resources/Harbor-Pages/Project-P2P-Preheat-Elements.robot @@ -28,6 +28,7 @@ ${p2p_preheat_edit_save_btn_id} //*[@id='edit-policy-save'] ${p2p_preheat_action_btn_id} //*[@id='action-policy'] ${p2p_preheat_del_btn_id} //*[@id='delete-policy'] ${p2p_preheat_edit_btn_id} //*[@id='edit-policy'] +${p2p_execution_header} //clr-main-container//project-detail//ng-component//h4[contains(.,'Executions')] diff --git a/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot b/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot index 9bcb7007c21..dcca4780127 100644 --- a/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot +++ b/tests/resources/Harbor-Pages/Project-P2P-Preheat.robot @@ -54,7 +54,7 @@ Create An New P2P Preheat Policy Edit A P2P Preheat Policy [Arguments] ${name} ${repo} ${trigger_type}=${null} Switch To P2P Preheat - Retry Double Keywords When Error Select P2P Preheat Policy ${name} Wait Until Element Is Visible //clr-main-container//project-detail/p2p-provider/ng-component//h4[contains(.,'Executions')] + Retry Double Keywords When Error Select P2P Preheat Policy ${name} Wait Until Element Is Visible ${p2p_execution_header} Retry Double Keywords When Error Retry Element Click ${p2p_preheat_action_btn_id} Wait Until Element Is Visible And Enabled ${p2p_preheat_edit_btn_id} Retry Double Keywords When Error Retry Element Click ${p2p_preheat_edit_btn_id} Wait Until Element Is Visible And Enabled ${p2p_preheat_name_input_id} Retry Text Input ${p2p_preheat_repoinput_id} ${repo} @@ -64,7 +64,7 @@ Edit A P2P Preheat Policy Delete A P2P Preheat Policy [Arguments] ${name} Switch To P2P Preheat - Retry Double Keywords When Error Select P2P Preheat Policy ${name} Wait Until Element Is Visible //clr-main-container//project-detail/p2p-provider/ng-component//h4[contains(.,'Executions')] + Retry Double Keywords When Error Select P2P Preheat Policy ${name} Wait Until Element Is Visible ${p2p_execution_header} Retry Double Keywords When Error Retry Element Click ${p2p_preheat_action_btn_id} Wait Until Element Is Visible And Enabled ${p2p_preheat_del_btn_id} Retry Double Keywords When Error Retry Element Click ${p2p_preheat_del_btn_id} Wait Until Element Is Visible And Enabled ${delete_confirm_btn} Retry Double Keywords When Error Retry Element Click ${delete_confirm_btn} Retry Wait Until Page Not Contains Element ${delete_confirm_btn} diff --git a/tests/resources/Harbor-Pages/Project-Tag-Retention.robot b/tests/resources/Harbor-Pages/Project-Tag-Retention.robot index b274a4c086f..d8d9b3b1467 100644 --- a/tests/resources/Harbor-Pages/Project-Tag-Retention.robot +++ b/tests/resources/Harbor-Pages/Project-Tag-Retention.robot @@ -27,8 +27,14 @@ Add A Tag Retention Rule Retry Element Click xpath=${project_tag_retention_save_add_button_xpath} Retry Wait Until Page Contains Element xpath=${project_tag_retention_rule_name_xpath} +Retry Add A Tag Immutability Rule + [Arguments] @{param} + Retry Keyword N Times When Error 5 Add A Tag Immutability Rule @{param} + Add A Tag Immutability Rule [Arguments] ${scope} ${tag} + Reload Page + Sleep 3 Retry Double Keywords When Error Retry Element Click xpath=${project_tag_retention_add_rule_xpath} Retry Wait Until Page Contains Element xpath=${project_tag_immutability_save_add_button_xpath} Retry Clear Element Text ${project_tag_immutability_scope_input_xpath} Retry Text Input ${project_tag_immutability_scope_input_xpath} ${scope} @@ -80,31 +86,34 @@ Set Daily Schedule Retry Wait Until Page Contains Element xpath=${project_tag_retention_span_daily_xpath} Execute Result Should Be - [Arguments] ${result} + [Arguments] ${image} ${result} FOR ${idx} IN RANGE 0 20 - ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(., '${result}')] + ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains Element xpath=//div[contains(@role, 'grid')]//div[contains(@class, 'datagrid-row-master') and contains(@role, 'row')]//clr-datagrid//div[contains(@role, 'grid')]//div[contains(@class, 'datagrid-row-master') and contains(@role, 'row')]//div[contains(@class, 'datagrid-row-scrollable') and contains(., '${result}') and contains(., '${image}')] Exit For Loop If '${out[0]}'=='PASS' - Sleep 6 + Sleep 1 Retry Element Click ${project_tag_retention_refresh_xpath} + Sleep 5 Retry Wait Until Page Contains Element xpath=${project_tag_retention_record_yes_xpath} + Retry Element Click ${project_tag_retention_list_expand_icon_xpath} END Should Be Equal As Strings '${out[0]}' 'PASS' Execute Dry Run + [Arguments] ${image} ${result} Retry Element Click xpath=${project_tag_retention_dry_run_xpath} Retry Wait Until Page Contains Element xpath=${project_tag_retention_record_yes_xpath} Sleep 5 Retry Element Click xpath=${project_tag_retention_record_yes_xpath} # memcached:123 should be deleted and hello-world:latest should be retained - Execute Result Should Be 0/1 - + Execute Result Should Be ${image} ${result} Execute Run + [Arguments] ${image} ${result} Retry Element Click xpath=${project_tag_retention_run_now_xpath} Retry Element Click xpath=${project_tag_retention_execute_run_xpath} Retry Wait Until Page Contains Element xpath=${project_tag_retention_record_no_xpath} Sleep 5 Retry Element Click xpath=${project_tag_retention_record_no_xpath} # memcached:123 should be deleted and hello-world:latest should be retained - Execute Result Should Be 0/1 + Execute Result Should Be ${image} ${result} diff --git a/tests/resources/Harbor-Pages/Project-Tag-Retention_Elements.robot b/tests/resources/Harbor-Pages/Project-Tag-Retention_Elements.robot index 598a8af7f39..687576d30b2 100644 --- a/tests/resources/Harbor-Pages/Project-Tag-Retention_Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Tag-Retention_Elements.robot @@ -33,6 +33,7 @@ ${project_tag_retention_span_daily_xpath} //cron-selection//div//span[contains( ${project_tag_retention_dry_run_xpath} //*[@id='dry-run'] ${project_tag_retention_refresh_xpath} //clr-dg-action-bar/button[4] ${project_tag_retention_record_yes_xpath} //clr-datagrid[contains(.,'Yes')] +${project_tag_retention_list_expand_icon_xpath} //project-detail/app-tag-feature-integration/tag-retention//clr-datagrid//clr-dg-row//clr-dg-cell[1]/clr-icon[contains(@shape, 'angle')] ${project_tag_retention_run_now_xpath} //*[@id='run-now'] ${project_tag_retention_execute_run_xpath} //*[@id='execute-run'] ${project_tag_retention_record_no_xpath} //clr-datagrid[contains(.,'No')] diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index ad64b468ce0..3d4c0088da6 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -217,13 +217,13 @@ Do Log Advanced Search Retry Click Repo Name [Arguments] ${repo_name_element} - FOR ${n} IN RANGE 1 10 + FOR ${n} IN RANGE 1 2 ${out} Run Keyword And Ignore Error Retry Double Keywords When Error Retry Element Click ${repo_name_element} Retry Wait Element ${tag_table_column_vulnerabilities} Exit For Loop If '${out[0]}'=='PASS' END Should Be Equal As Strings '${out[0]}' 'PASS' - FOR ${n} IN RANGE 1 10 + FOR ${n} IN RANGE 1 2 ${out} Run Keyword And Ignore Error Retry Wait Until Page Not Contains Element ${repo_list_spinner} Exit For Loop If '${out[0]}'=='PASS' END @@ -234,8 +234,9 @@ Go Into Repo Sleep 2 Retry Wait Until Page Not Contains Element ${repo_list_spinner} ${repo_name_element}= Set Variable xpath=//clr-dg-cell[contains(.,'${repoName}')]/a - Retry Element Click ${repo_search_icon} - FOR ${n} IN RANGE 1 10 + FOR ${n} IN RANGE 1 3 + Reload Page + Retry Element Click ${repo_search_icon} Retry Clear Element Text ${repo_search_input} Retry Text Input ${repo_search_input} ${repoName} ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains Element ${repo_name_element} diff --git a/tests/resources/Harbor-Pages/Replication_Elements.robot b/tests/resources/Harbor-Pages/Replication_Elements.robot index f4f79abadb7..861a38a5c8b 100644 --- a/tests/resources/Harbor-Pages/Replication_Elements.robot +++ b/tests/resources/Harbor-Pages/Replication_Elements.robot @@ -28,7 +28,6 @@ ${destination_password_xpath} //*[@id='destination_password'] ${replication_save_xpath} //button[contains(.,'OK')] ${replication_xpath} //clr-vertical-nav-group-children/a[contains(.,'Replication')] ${destination_insecure_xpath} //label[@id='destination_insecure_checkbox'] - ${new_replication-rule_button} //button[contains(.,'New Replication Rule')] ${link_to_registries} //clr-modal//span[contains(.,'Endpoint')] ${new_endpoint_button} //hbr-endpoint//button[contains(.,'New Endpoint')] @@ -42,6 +41,7 @@ ${schedule_type_select} //select[@name='scheduleType'] ${schedule_day_select} //select[@name='scheduleDay'] ${shcedule_time} //input[@type='time'] ${destination_insecure_checkbox} //hbr-create-edit-endpoint/clr-modal//input[@id='destination_insecure'] +#${destination_insecure_checkbox} //clr-checkbox-wrapper/label[contains(@for, 'destination_insecure')] ${ping_test_button} //button[contains(.,'Test')] ${nav_to_registries} //clr-vertical-nav//span[contains(.,'Registries')] ${nav_to_replications} //clr-vertical-nav//span[contains(.,'Replications')] diff --git a/tests/resources/Nightly-Util.robot b/tests/resources/Nightly-Util.robot index f17607bf915..df10740350a 100644 --- a/tests/resources/Nightly-Util.robot +++ b/tests/resources/Nightly-Util.robot @@ -34,7 +34,6 @@ Nightly Test Setup Run wget ${prometheus_chart_file_url} #Prepare docker image for push special image keyword in replication test Run Keyword If '${DOCKER_USER}' != '${EMPTY}' Docker Login "" ${DOCKER_USER} ${DOCKER_PWD} - Docker Pull busybox:latest CA Setup [Arguments] ${ip} ${HARBOR_PASSWORD} ${cert}=/ca/ca.crt diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index 2209ba9060b..83d6e38cfef 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -219,7 +219,7 @@ Clear Field Of Characters END Wait Unitl Command Success - [Arguments] ${cmd} ${times}=2 + [Arguments] ${cmd} ${times}=5 FOR ${n} IN RANGE 1 ${times} Log Trying ${cmd}: ${n} ... console=True ${rc} ${output}= Run And Return Rc And Output ${cmd} diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index df05777a14c..5adf0afb081 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -21,6 +21,11 @@ Test Case - Garbage Collection [Tags] gc Harbor API Test ./tests/apitests/python/test_garbage_collection.py +Test Case - Project Level Policy Content Trust + [Tags] content_trust + Restart Docker Daemon Locally + Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py + Test Case - Add Private Project Member and Check User Can See It [Tags] private_member Harbor API Test ./tests/apitests/python/test_add_member_to_private_project.py @@ -45,10 +50,6 @@ Test Case - Manage Project Member [Tags] member Harbor API Test ./tests/apitests/python/test_manage_project_member.py -Test Case - Project Level Policy Content Trust - [Tags] content_trust - Harbor API Test ./tests/apitests/python/test_project_level_policy_content_trust.py - Test Case - User View Logs [Tags] view_logs Harbor API Test ./tests/apitests/python/test_user_view_logs.py diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 645df5652e1..e5a33cd6f21 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -113,6 +113,7 @@ Test Case - Staticsinfo Should be equal as integers ${publicrepocount2} ${publicrepocount} Should be equal as integers ${totalprojcount2} ${totalprojcount} Should be equal as integers ${totalrepocount2} ${totalrepocount} + Close Browser Test Case - Push Image Init Chrome Driver @@ -123,6 +124,7 @@ Test Case - Push Image Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} test${d} hello-world:latest Go Into Project test${d} Wait Until Page Contains test${d}/hello-world + Close Browser Test Case - Project Level Policy Public Init Chrome Driver @@ -241,6 +243,7 @@ Test Case - User View Logs Push image ${ip} ${user} ${pwd} project${d} ${img}:${tag} Pull image ${ip} ${user} ${pwd} project${d} ${replication_image}:${replication_tag} + Close Browser Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${user} ${pwd} @@ -484,14 +487,14 @@ Test Case - Project Storage Quotas Dispaly And Control ${image_a}= Set Variable one_layer ${image_b}= Set Variable redis ${image_a_size}= Set Variable 330.83MB - ${image_b_size}= Set Variable 34.15MB + ${image_b_size}= Set Variable 34.1\\dMB ${image_a_ver}= Set Variable 1.0 ${image_b_ver}= Set Variable donotremove5.0 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project And Go Into Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_b} tag=${image_b_ver} tag1=${image_b_ver} ${storage_quota_ret}= Get Project Storage Quota Text From Project Quotas List project${d} - Should Be Equal As Strings ${storage_quota_ret} ${image_b_size} of ${storage_quota}${storage_quota_unit} + Should Match Regexp ${storage_quota_ret} ${image_b_size} of ${storage_quota}${storage_quota_unit} Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_a}:${image_a_ver} err_msg=adding 330.1 MiB of storage resource, which when updated to current usage of err_msg_2=MiB will exceed the configured upper limit of ${storage_quota}.0 MiB Go Into Project project${d} Delete Repo project${d} ${image_b} @@ -568,17 +571,21 @@ Test Case - Tag Retention Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= Get Current Date result_format=%m%s + ${image_sample_1}= Set Variable hello-world + ${image_sample_2}= Set Variable memcached Create An New Project And Go Into Project project${d} Switch To Tag Retention Add A Tag Retention Rule Delete A Tag Retention Rule Add A Tag Retention Rule Edit A Tag Retention Rule ** latest - Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world latest - Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} memcached 123 + Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_sample_1} latest + Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_sample_2} 123 Set Daily Schedule - Execute Dry Run - Execute Run + Execute Dry Run ${image_sample_2} 0/1 + Execute Run ${image_sample_2} 0/1 + Execute Dry Run ${image_sample_1} 1/1 + Execute Run ${image_sample_1} 1/1 Close Browser Test Case - Tag Immutability @@ -587,9 +594,11 @@ Test Case - Tag Immutability ${d}= Get Current Date result_format=%m%s Create An New Project And Go Into Project project${d} Switch To Tag Immutability - Add A Tag Immutability Rule 1212 3434 + @{param} Create List 1212 3434 + Retry Add A Tag Immutability Rule @{param} Delete A Tag Immutability Rule - Add A Tag Immutability Rule 5566 7788 + @{param} Create List 5566 7788 + Retry Add A Tag Immutability Rule @{param} Edit A Tag Immutability Rule hello-world latest Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world latest Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox latest @@ -612,6 +621,7 @@ Test Case - Tag Immutability # Log ${token} # Push image ${ip} robot${d} ${token} project${d} hello-world:latest is_robot=${true} # Pull image ${ip} robot${d} ${token} project${d} hello-world:latest is_robot=${true} +# Close Browser Test Case - Push Docker Manifest Index and Display Init Chrome Driver diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot index 2434d3abce3..0b5ba8c51c1 100644 --- a/tests/robot-cases/Group1-Nightly/Notary.robot +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -25,6 +25,7 @@ ${HARBOR_ADMIN} admin *** Test Cases *** Test Case - Project Level Policy Content Trust + Restart Docker Daemon Locally Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} diff --git a/tests/robot-cases/Group1-Nightly/OIDC.robot b/tests/robot-cases/Group1-Nightly/OIDC.robot index be4ae7e93c4..cacd830396c 100644 --- a/tests/robot-cases/Group1-Nightly/OIDC.robot +++ b/tests/robot-cases/Group1-Nightly/OIDC.robot @@ -79,24 +79,47 @@ Test Case - Generate User CLI Secret Cannot Docker Login Harbor ${ip} ${OIDC_USERNAME} ${secret_old} Pull image ${ip} ${OIDC_USERNAME} ${secret_new} project${d} ${image} Push image ${ip} ${OIDC_USERNAME} ${secret_new} project${d} ${image} + Close Browser Test Case - Helm CLI Push Init Chrome Driver Sign In Harbor With OIDC User ${HARBOR_URL} ${secret}= Get Secrete By API ${HARBOR_URL} Helm CLI Push Without Sign In Harbor ${OIDC_USERNAME} ${secret} + Close Browser Test Case - Onboard OIDC User Sign In Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Configure Check Automatic Onboarding And Save Logout Harbor Sign In Harbor With OIDC User ${HARBOR_URL} test8 is_onboard=${true} Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Configure Set User Name Claim And Save email Logout Harbor Sign In Harbor With OIDC User ${HARBOR_URL} test9 is_onboard=${true} username_claim=email - Sleep 2 \ No newline at end of file + Logout Harbor + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Set User Name Claim And Save ${null} + Sleep 2 + Close Browser + +Test Case - OIDC Group User + Init Chrome Driver + ${d}= Get current Date result_format=%m%s + ${image}= Set Variable hello-world + ${admin_user}= Set Variable admin_user + ${admin_pwd}= Set Variable zhu88jie + ${user}= Set Variable mike + ${pwd}= Set Variable ${admin_pwd} + Sign In Harbor With OIDC User ${HARBOR_URL} username=${admin_user} password=${admin_pwd} login_with_provider=ldap + Switch To Registries + Create A New Endpoint harbor test_oidc_admin https://cicd.harbor.vmwarecna.net ${null} ${null} Y + ${secret}= Get Secrete By API ${HARBOR_URL} username=${admin_user} + Push image ${ip} ${admin_user} ${secret} library ${image} + Logout Harbor + Sign In Harbor With OIDC User ${HARBOR_URL} username=${user} password=${pwd} login_with_provider=ldap + ${output}= Run Keyword And Ignore Error Switch To Configure + Should Be Equal As Strings '${output[0]}' 'FAIL' + Close Browser \ No newline at end of file diff --git a/tests/robot-cases/Group1-Nightly/Schedule.robot b/tests/robot-cases/Group1-Nightly/Schedule.robot index 8e6ca1bcaae..a9f72ea008a 100644 --- a/tests/robot-cases/Group1-Nightly/Schedule.robot +++ b/tests/robot-cases/Group1-Nightly/Schedule.robot @@ -134,7 +134,7 @@ Test Case - Scan Schedule Job END # After scan custom schedule is set, image should stay in unscanned status. Log To Console Sleep for 300 seconds...... - Sleep 300 + Sleep 180 Go Into Project ${project_name} Go Into Repo ${project_name}/${image} Retry Wait Until Page Contains Element ${not_scanned_icon} diff --git a/tests/robot-cases/Group3-Upgrade/data.json b/tests/robot-cases/Group3-Upgrade/data.json index d14889d8490..91e22269e20 100644 --- a/tests/robot-cases/Group3-Upgrade/data.json +++ b/tests/robot-cases/Group3-Upgrade/data.json @@ -46,7 +46,7 @@ "is_src_registry":true, "dest_namespace":"rule2-namespace", "trigger_type":"scheduled", - "cron":"6 7 8 * * *", + "cron":"0 0 */8 * * *", "deletion":false, "enabled":true, "override":true, diff --git a/tests/robot-cases/Group3-Upgrade/verify.robot b/tests/robot-cases/Group3-Upgrade/verify.robot index 13dd8da8de7..c1c8f89266e 100644 --- a/tests/robot-cases/Group3-Upgrade/verify.robot +++ b/tests/robot-cases/Group3-Upgrade/verify.robot @@ -88,7 +88,7 @@ Test Case - Upgrade Verify Run Keyword Verify Image Tag ${data} Run Keyword Verify Trivy Is Default Scanner Run Keyword Verify Artifact Index ${data} - Run Keyword Verify Quotas Display ${data} + #Run Keyword Verify Quotas Display ${data} Test Case - Upgrade Verify [Tags] 2.1-latest @@ -116,4 +116,4 @@ Test Case - Upgrade Verify Run Keyword Verify Proxy Cache Image Existence ${data} Run Keyword Verify Distributions ${data} Run Keyword Verify P2P Preheat Policy ${data} - Run Keyword Verify Quotas Display ${data} + #Run Keyword Verify Quotas Display ${data}