From 583c078c2ce1e02829dbad122593ee8ad7b76a5a Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 11:00:07 +0300 Subject: [PATCH 01/11] testing release dg --- .circleci/config.yml | 117 ++++++++++++++++++++++--------------------- version.txt | 2 +- 2 files changed, 62 insertions(+), 57 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7de474b9..89b7dd39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,8 @@ parameters: orb_version: type: string description: Deci ai ORB version https://circleci.com/developer/orbs/orb/deci-ai/circleci-common-orb - default: "10.5.1" -# default: "dev:alpha" +# default: "10.5.1" + default: "dev:alpha" ad_hoc_container_build: type: boolean default: false @@ -147,6 +147,8 @@ jobs: environment: CIRCLE_COMPARE_URL: << pipeline.project.git_url >>/compare/<< pipeline.git.base_revision >>..<> parameters: + version: + type: string py_version: type: string dev_venv_name: @@ -165,27 +167,26 @@ jobs: - run: name: edit package version command: | - echo $CIRCLE_TAG > version.txt + echo << parameters.version >> > version.txt - - deci-common/pip_upload_package_codeartifact_all_accounts: + - deci-common/pip_upload_package_codeartifact_dev: codeartifact_repository: "deci-packages" - - deci-common/pip_test_package_installation_codeartifact_dev: + - deci-common/pip_test_package_installation_codeartifact_dev: # we are testing dev only. so we won't publish trash to prod package_name: "data-gradients" - version: $CIRCLE_TAG + version: << parameters.version >> venv_name: << parameters.dev_venv_name >> - run: name: verify that the output of __version__ is what we expect command: | - . << parameters.dev_venv_name >>-data-gradients-$CIRCLE_TAG/bin/activate - python3 tests/verify_version.py $CIRCLE_TAG + . << parameters.dev_venv_name >>/bin/activate + python3 tests/verify_version.py << parameters.version >> # - - deci-common/pip_test_package_installation_codeartifact_prod: - package_name: "data-gradients" - version: $CIRCLE_TAG + - deci-common/pip_upload_package_codeartifact_prod: + codeartifact_repository: "deci-packages" - deci-common/git_commit_and_tag: - version: $CIRCLE_TAG + version: << parameters.version >> delete_remote_tag_before_tagging: true - deci-common/tag_as: @@ -195,8 +196,8 @@ jobs: - deci-common/github_create_release: github_cli_token: $GITHUB_CLI_TOKEN directory_to_cd_into: "." - tag: $CIRCLE_TAG - notes: "This GitHub Release was done automatically by CircleCI" + tag: << parameters.version >> + generate_notes: true workflows: @@ -207,17 +208,20 @@ workflows: - << pipeline.parameters.ad_hoc_container_build_code_only >> jobs: - deci-common/persist_version_info: - version_override: $CIRCLE_TAG - <<: *release_tag_filter + version_override: 0.0.6 #todo: rtemove +# version_override: $CIRCLE_TAG +# <<: *release_tag_filter - build: name: "build3.8" py_version: "3.8" package_name: "data-gradients" requires: - deci-common/persist_version_info - <<: *release_tag_filter +# <<: *release_tag_filter - release_version: py_version: "3.8" + version: 0.0.6 #todo: rtemove +# version: $CIRCLE_TAG requires: - "build3.8" context: @@ -225,51 +229,52 @@ workflows: - aws-prod - aws-master - github-token - <<: *release_tag_filter +# <<: *release_tag_filter - deci-common/pip_upload_package_from_codeartifact_to_global_pypi: package_name: "data-gradients" name: "upload_data_gradients_to_pypi" - version: $CIRCLE_TAG + version: 0.0.6 #todo: rtemove +# version: $CIRCLE_TAG requires: - "release_version" context: - pypi-datagradients - aws-prod - aws-master - <<: *release_tag_filter - - build_and_deploy: - unless: - or: - - << pipeline.parameters.ad_hoc_container_build >> - - << pipeline.parameters.ad_hoc_container_build_code_only >> - jobs: - - deci-common/persist_version_info: - use_rc: true - use_beta: false - - - build: - name: "build3.8" - py_version: "3.8" - package_name: "data-gradients" - requires: - - deci-common/persist_version_info - - - deci-common/codeartifact_login: - repo_name: "deci-packages" - dev_account: true - context: - - aws-dev - - aws-master - <<: *release_candidate_filter - - - release_candidate: # happens on merge - py_version: "3.8" - requires: - - "build3.8" - - deci-common/codeartifact_login - context: - - aws-dev - - aws-master - <<: *release_candidate_filter - +# <<: *release_tag_filter +# +# build_and_deploy: +# unless: +# or: +# - << pipeline.parameters.ad_hoc_container_build >> +# - << pipeline.parameters.ad_hoc_container_build_code_only >> +# jobs: +# - deci-common/persist_version_info: +# use_rc: true +# use_beta: false +# +# - build: +# name: "build3.8" +# py_version: "3.8" +# package_name: "data-gradients" +# requires: +# - deci-common/persist_version_info +# +# - deci-common/codeartifact_login: +# repo_name: "deci-packages" +# dev_account: true +# context: +# - aws-dev +# - aws-master +# <<: *release_candidate_filter +# +# - release_candidate: # happens on merge +# py_version: "3.8" +# requires: +# - "build3.8" +# - deci-common/codeartifact_login +# context: +# - aws-dev +# - aws-master +# <<: *release_candidate_filter +# diff --git a/version.txt b/version.txt index ceab6e11..70ccf005 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.1 \ No newline at end of file +for ci use only \ No newline at end of file From b784083098155c0a35086d9df226dc90283fdd2e Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:25:12 +0300 Subject: [PATCH 02/11] testing release dg --- .circleci/config.yml | 5 +++++ src/data_gradients/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 89b7dd39..c293ca81 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -169,6 +169,11 @@ jobs: command: | echo << parameters.version >> > version.txt + - run: + name: edit __version__ value + command: | + sed -i 's/^__version__.*/__version__ = "<< parameters.version >>"/g' src/data_gradients/__init__.py + - deci-common/pip_upload_package_codeartifact_dev: codeartifact_repository: "deci-packages" diff --git a/src/data_gradients/__init__.py b/src/data_gradients/__init__.py index a4e2017f..30982294 100644 --- a/src/data_gradients/__init__.py +++ b/src/data_gradients/__init__.py @@ -1 +1 @@ -__version__ = "0.1" +__version__ = "CI changes this value automatically" From f2cc01921471cf3b2a3d5a91792ffb796e56d11d Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:25:49 +0300 Subject: [PATCH 03/11] testing release dg --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c293ca81..56dfdb7a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -216,19 +216,19 @@ workflows: version_override: 0.0.6 #todo: rtemove # version_override: $CIRCLE_TAG # <<: *release_tag_filter - - build: - name: "build3.8" - py_version: "3.8" - package_name: "data-gradients" - requires: - - deci-common/persist_version_info +# - build: +# name: "build3.8" +# py_version: "3.8" +# package_name: "data-gradients" +# requires: +# - deci-common/persist_version_info # <<: *release_tag_filter - release_version: py_version: "3.8" version: 0.0.6 #todo: rtemove # version: $CIRCLE_TAG - requires: - - "build3.8" +# requires: +# - "build3.8" context: - aws-dev - aws-prod From 9b7a66f3748c8bd93a05b37fcc20247ca01e8160 Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:27:48 +0300 Subject: [PATCH 04/11] testing release dg --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 56dfdb7a..e4dabf22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -213,7 +213,7 @@ workflows: - << pipeline.parameters.ad_hoc_container_build_code_only >> jobs: - deci-common/persist_version_info: - version_override: 0.0.6 #todo: rtemove + version_override: 0.0.7 #todo: rtemove # version_override: $CIRCLE_TAG # <<: *release_tag_filter # - build: @@ -225,7 +225,7 @@ workflows: # <<: *release_tag_filter - release_version: py_version: "3.8" - version: 0.0.6 #todo: rtemove + version: 0.0.7 #todo: rtemove # version: $CIRCLE_TAG # requires: # - "build3.8" @@ -238,7 +238,7 @@ workflows: - deci-common/pip_upload_package_from_codeartifact_to_global_pypi: package_name: "data-gradients" name: "upload_data_gradients_to_pypi" - version: 0.0.6 #todo: rtemove + version: 0.0.7 #todo: rtemove # version: $CIRCLE_TAG requires: - "release_version" From 80fa15316157df4147294df7e14ad3c4ee03f0b2 Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:34:08 +0300 Subject: [PATCH 05/11] testing release dg --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4dabf22..eb379b33 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -165,14 +165,15 @@ jobs: command: | cp requirements.txt src/data_gradients/requirements.txt - run: - name: edit package version + name: edit package version assisting files command: | echo << parameters.version >> > version.txt + sed -i 's/^__version__.*/__version__ = "<< parameters.version >>"/g' src/data_gradients/__init__.py - run: - name: edit __version__ value + name: commit package version assisting files command: | - sed -i 's/^__version__.*/__version__ = "<< parameters.version >>"/g' src/data_gradients/__init__.py + git commit version.txt src/data_gradients/__init__.py - deci-common/pip_upload_package_codeartifact_dev: codeartifact_repository: "deci-packages" From c9a73e0d7b933970402e16b8c26a98399dbf5640 Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:39:14 +0300 Subject: [PATCH 06/11] testing release dg --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb379b33..8c077f9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,12 +164,12 @@ jobs: name: add requirements.txt a to source code command: | cp requirements.txt src/data_gradients/requirements.txt + - deci-common/git_config_automation_user - run: name: edit package version assisting files command: | echo << parameters.version >> > version.txt sed -i 's/^__version__.*/__version__ = "<< parameters.version >>"/g' src/data_gradients/__init__.py - - run: name: commit package version assisting files command: | @@ -194,6 +194,7 @@ jobs: - deci-common/git_commit_and_tag: version: << parameters.version >> delete_remote_tag_before_tagging: true + config_automation_user: false #already configured earlier - deci-common/tag_as: tag_name: "stable" @@ -205,7 +206,6 @@ jobs: tag: << parameters.version >> generate_notes: true - workflows: release: unless: From 8bb7ca8892b23ccdca4a5e3cafcc1dd1cae11aad Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:42:24 +0300 Subject: [PATCH 07/11] testing release dg --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c077f9c..e123ae82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,6 +44,9 @@ release_candidate_tag_filter: &release_candidate_tag_filter only: /^\d+\.\d+\.\d+rc\d+/ +#commands: +# handle_version_ + jobs: build: environment: @@ -173,7 +176,7 @@ jobs: - run: name: commit package version assisting files command: | - git commit version.txt src/data_gradients/__init__.py + git commit -m "Edited by Circleci" version.txt src/data_gradients/__init__.py - deci-common/pip_upload_package_codeartifact_dev: codeartifact_repository: "deci-packages" From 39e92906acb3ae10fba098e0bfa7f73fddb75f1b Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 15:49:57 +0300 Subject: [PATCH 08/11] testing release dg --- .circleci/config.yml | 59 ++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e123ae82..a6beada5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,8 +11,8 @@ parameters: orb_version: type: string description: Deci ai ORB version https://circleci.com/developer/orbs/orb/deci-ai/circleci-common-orb -# default: "10.5.1" - default: "dev:alpha" + default: "11.0.0" +# default: "dev:alpha" ad_hoc_container_build: type: boolean default: false @@ -44,8 +44,22 @@ release_candidate_tag_filter: &release_candidate_tag_filter only: /^\d+\.\d+\.\d+rc\d+/ -#commands: -# handle_version_ +commands: + handle_version_files: + parameters: + version: + type: string + steps: + - deci-common/git_config_automation_user + - run: + name: edit package version assisting files + command: | + echo << parameters.version >> > version.txt + sed -i 's/^__version__.*/__version__ = "<< parameters.version >>"/g' src/data_gradients/__init__.py + - run: + name: commit package version assisting files + command: | + git commit -m "Deci Services - Changed version to << parameters.version >>" version.txt src/data_gradients/__init__.py jobs: build: @@ -129,22 +143,20 @@ jobs: skip_md_files: << pipeline.parameters.skip_md_files >> - deci-common/get_persisted_version_info - - run: - name: edit package version - command: | - echo $NEW_VERSION > version.txt + + - handle_version_files: + version: $NEW_VERSION + - deci-common/pip_upload_package_codeartifact_dev: codeartifact_repository: "deci-packages" + - deci-common/pip_test_package_installation_codeartifact_dev: package_name: "data-gradients" version: $NEW_VERSION - - deci-common/git_config_automation_user - - run: - name: "commit version file" - command: | - git commit version.txt -m "Deci Services - Changed version to $NEW_VERSION" + - deci-common/git_commit_and_tag: version: $NEW_VERSION + config_automation_user: false #already configured earlier release_version: environment: @@ -167,16 +179,9 @@ jobs: name: add requirements.txt a to source code command: | cp requirements.txt src/data_gradients/requirements.txt - - deci-common/git_config_automation_user - - run: - name: edit package version assisting files - command: | - echo << parameters.version >> > version.txt - sed -i 's/^__version__.*/__version__ = "<< parameters.version >>"/g' src/data_gradients/__init__.py - - run: - name: commit package version assisting files - command: | - git commit -m "Edited by Circleci" version.txt src/data_gradients/__init__.py + + - handle_version_files: + version: << parameters.version >> - deci-common/pip_upload_package_codeartifact_dev: codeartifact_repository: "deci-packages" @@ -190,7 +195,7 @@ jobs: command: | . << parameters.dev_venv_name >>/bin/activate python3 tests/verify_version.py << parameters.version >> -# + - deci-common/pip_upload_package_codeartifact_prod: codeartifact_repository: "deci-packages" @@ -217,7 +222,7 @@ workflows: - << pipeline.parameters.ad_hoc_container_build_code_only >> jobs: - deci-common/persist_version_info: - version_override: 0.0.7 #todo: rtemove + version_override: 0.0.8 #todo: rtemove # version_override: $CIRCLE_TAG # <<: *release_tag_filter # - build: @@ -229,7 +234,7 @@ workflows: # <<: *release_tag_filter - release_version: py_version: "3.8" - version: 0.0.7 #todo: rtemove + version: 0.0.8 #todo: rtemove # version: $CIRCLE_TAG # requires: # - "build3.8" @@ -242,7 +247,7 @@ workflows: - deci-common/pip_upload_package_from_codeartifact_to_global_pypi: package_name: "data-gradients" name: "upload_data_gradients_to_pypi" - version: 0.0.7 #todo: rtemove + version: 0.0.8 #todo: rtemove # version: $CIRCLE_TAG requires: - "release_version" From a40b75cd08751b9e9aafaee01ed5be5fb0297360 Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 17:26:16 +0300 Subject: [PATCH 09/11] new and improved CI --- .circleci/config.yml | 135 ++++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 60 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a6beada5..b7dd752a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,6 +45,20 @@ release_candidate_tag_filter: &release_candidate_tag_filter commands: + + verify_that_package_version_is_aligned_with_code: + parameters: + version: + type: string + venv_name: + type: string + steps: + - run: + name: verify that the output of __version__ is what we expect + command: | + . << parameters.venv_name >>/bin/activate + python3 tests/verify_version.py << parameters.version >> + handle_version_files: parameters: version: @@ -152,7 +166,12 @@ jobs: - deci-common/pip_test_package_installation_codeartifact_dev: package_name: "data-gradients" - version: $NEW_VERSION + version: << parameters.version >> + venv_name: << parameters.dev_venv_name >> + + - verify_that_package_version_is_aligned_with_code: + venv_name: << parameters.dev_venv_name >> + version: << parameters.version >> - deci-common/git_commit_and_tag: version: $NEW_VERSION @@ -190,11 +209,10 @@ jobs: package_name: "data-gradients" version: << parameters.version >> venv_name: << parameters.dev_venv_name >> - - run: - name: verify that the output of __version__ is what we expect - command: | - . << parameters.dev_venv_name >>/bin/activate - python3 tests/verify_version.py << parameters.version >> + + - verify_that_package_version_is_aligned_with_code: + venv_name: << parameters.dev_venv_name >> + version: << parameters.version >> - deci-common/pip_upload_package_codeartifact_prod: codeartifact_repository: "deci-packages" @@ -222,73 +240,70 @@ workflows: - << pipeline.parameters.ad_hoc_container_build_code_only >> jobs: - deci-common/persist_version_info: - version_override: 0.0.8 #todo: rtemove -# version_override: $CIRCLE_TAG -# <<: *release_tag_filter -# - build: -# name: "build3.8" -# py_version: "3.8" -# package_name: "data-gradients" -# requires: -# - deci-common/persist_version_info -# <<: *release_tag_filter + version_override: $CIRCLE_TAG + <<: *release_tag_filter + - build: + name: "build3.8" + py_version: "3.8" + package_name: "data-gradients" + requires: + - deci-common/persist_version_info + <<: *release_tag_filter - release_version: py_version: "3.8" - version: 0.0.8 #todo: rtemove -# version: $CIRCLE_TAG -# requires: -# - "build3.8" + version: $CIRCLE_TAG + requires: + - "build3.8" context: - aws-dev - aws-prod - aws-master - github-token -# <<: *release_tag_filter + <<: *release_tag_filter - deci-common/pip_upload_package_from_codeartifact_to_global_pypi: package_name: "data-gradients" name: "upload_data_gradients_to_pypi" - version: 0.0.8 #todo: rtemove -# version: $CIRCLE_TAG + version: $CIRCLE_TAG requires: - "release_version" context: - pypi-datagradients - aws-prod - aws-master -# <<: *release_tag_filter -# -# build_and_deploy: -# unless: -# or: -# - << pipeline.parameters.ad_hoc_container_build >> -# - << pipeline.parameters.ad_hoc_container_build_code_only >> -# jobs: -# - deci-common/persist_version_info: -# use_rc: true -# use_beta: false -# -# - build: -# name: "build3.8" -# py_version: "3.8" -# package_name: "data-gradients" -# requires: -# - deci-common/persist_version_info -# -# - deci-common/codeartifact_login: -# repo_name: "deci-packages" -# dev_account: true -# context: -# - aws-dev -# - aws-master -# <<: *release_candidate_filter -# -# - release_candidate: # happens on merge -# py_version: "3.8" -# requires: -# - "build3.8" -# - deci-common/codeartifact_login -# context: -# - aws-dev -# - aws-master -# <<: *release_candidate_filter -# + <<: *release_tag_filter + + build_and_deploy: + unless: + or: + - << pipeline.parameters.ad_hoc_container_build >> + - << pipeline.parameters.ad_hoc_container_build_code_only >> + jobs: + - deci-common/persist_version_info: + use_rc: true + use_beta: false + + - build: + name: "build3.8" + py_version: "3.8" + package_name: "data-gradients" + requires: + - deci-common/persist_version_info + + - deci-common/codeartifact_login: + repo_name: "deci-packages" + dev_account: true + context: + - aws-dev + - aws-master + <<: *release_candidate_filter + + - release_candidate: # happens on merge + py_version: "3.8" + requires: + - "build3.8" + - deci-common/codeartifact_login + context: + - aws-dev + - aws-master + <<: *release_candidate_filter + From f6d62c62e3c1692f7d554fb08a72c75277d381b5 Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 17:27:44 +0300 Subject: [PATCH 10/11] new and improved CI --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7dd752a..08e0c0de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -149,6 +149,9 @@ jobs: parameters: py_version: type: string + dev_venv_name: + type: string + default: "dev-data_gradients-${CIRCLE_BUILD_NUM}" docker: - image: cimg/python:<< parameters.py_version >> steps: From ec0d0576d2e54d38996819972a98bbc34095d452 Mon Sep 17 00:00:00 2001 From: ranrubin Date: Thu, 15 Jun 2023 17:28:19 +0300 Subject: [PATCH 11/11] new and improved CI --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08e0c0de..0335fb2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -169,12 +169,12 @@ jobs: - deci-common/pip_test_package_installation_codeartifact_dev: package_name: "data-gradients" - version: << parameters.version >> + version: $NEW_VERSION venv_name: << parameters.dev_venv_name >> - verify_that_package_version_is_aligned_with_code: + version: $NEW_VERSION venv_name: << parameters.dev_venv_name >> - version: << parameters.version >> - deci-common/git_commit_and_tag: version: $NEW_VERSION