diff --git a/.github/workflows/cookiecutter-e2e.yml b/.github/workflows/cookiecutter-e2e.yml index 4dad4de4e..bd1e59290 100644 --- a/.github/workflows/cookiecutter-e2e.yml +++ b/.github/workflows/cookiecutter-e2e.yml @@ -18,32 +18,32 @@ env: jobs: lint: - name: Lint ${{ matrix.cookiecutter }}, Replay ${{ matrix.replay }} on ${{ matrix.python-version }} ${{ matrix.python-version }} / ${{ matrix.os }} + name: Cookiecutter E2E ${{ matrix.python-version }} ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: true matrix: include: - - { cookiecutter: "tap-template", replay: "tap-graphql-jwt.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-api_key-github.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-basic_auth.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-bearer_token.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-custom.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-jwt.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-rest-oauth2.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-other-custom.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "tap-template", replay: "tap-sql-custom.json", python-version: "3.9", os: "ubuntu-latest" } - - { cookiecutter: "target-template", replay: "target-per_record.json", python-version: "3.9", os: "ubuntu-latest" } + - { python-version: "3.10", os: "ubuntu-latest" } steps: - name: Check out the repository - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.3.0 + - name: Install Poetry & Tox + env: + PIP_CONSTRAINT: .github/workflows/constraints.txt + run: | + pipx install poetry + poetry --version + pipx install tox - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v4.5.0 with: python-version: ${{ matrix.python-version }} architecture: x64 + cache: 'pip' + cache-dependency-path: 'poetry.lock' - name: Upgrade pip env: @@ -52,32 +52,25 @@ jobs: pip install pip pip --version - - name: Install Poetry, Tox & Cookiecutter + - name: Install Nox env: PIP_CONSTRAINT: .github/workflows/constraints.txt run: | - pipx install poetry - pipx install cookiecutter - pipx install tox + pipx install --pip-args=--constraint=.github/workflows/constraints.txt nox + pipx inject --pip-args=--constraint=.github/workflows/constraints.txt nox nox-poetry + nox --version - - name: Build cookiecutter project - env: - CC_TEMPLATE: cookiecutter/${{ matrix.cookiecutter }} - REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} + - name: Run Nox run: | - chmod +x ./e2e-tests/cookiecutters/test_cookiecutter.sh - ./e2e-tests/cookiecutters/test_cookiecutter.sh $CC_TEMPLATE $REPLAY_FILE 0 - - - uses: actions/upload-artifact@v3 + nox --python=${{ matrix.python-version }} --session=test_cookiecutter + - name: Upload build artifacts + if: always() + uses: actions/upload-artifact@v3 with: - name: ${{ matrix.replay }} path: | - ${{ env.CC_TEST_OUTPUT }}/ - !${{ env.CC_TEST_OUTPUT }}/.mypy_cache/ - - - name: Run lint - env: - REPLAY_FILE: e2e-tests/cookiecutters/${{ matrix.replay }} - run: | - cd $CC_TEST_OUTPUT - poetry run tox -e lint + /tmp/tap-* + /tmp/target-* + !/tmp/tap-*/.mypy_cache/ + !/tmp/target-*/.mypy_cache/ + !/tmp/tap-*/.tox/ + !/tmp/target-*/.tox/ diff --git a/e2e-tests/cookiecutters/test_cookiecutter.sh b/e2e-tests/cookiecutters/test_cookiecutter.sh deleted file mode 100755 index a8a33afde..000000000 --- a/e2e-tests/cookiecutters/test_cookiecutter.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -CC_BUILD_PATH=/tmp -TAP_TEMPLATE=$(realpath $1) -REPLAY_FILE=$(realpath $2) -SDK_DIR=$(realpath $1/../..) -CC_OUTPUT_DIR=$(basename $REPLAY_FILE .json) # name of replay file without .json -RUN_LINT=${3:-1} - -usage() { - echo "test_cookiecutter.sh [tap_template] [replay_file.json]" - echo - echo "Uses the tap template to build an empty cookiecutter, and runs the lint task on the created test project" - echo "" - if [[ $# -eq 1 ]]; then - echo "ERROR: $1" - fi -} - -if [[ ! -d $TAP_TEMPLATE ]]; then - usage "Tap template folder not found" - exit 1 -fi -if [[ ! -f $REPLAY_FILE ]]; then - usage "Replay file not found" - exit 1 -fi - -CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR -if [[ -d "$CC_TEST_OUTPUT" ]]; then - rm -fr "$CC_TEST_OUTPUT" -fi - -if [[ -f $GITHUB_ENV ]]; then - echo CC_TEST_OUTPUT=$CC_BUILD_PATH/$CC_OUTPUT_DIR >> $GITHUB_ENV -fi - -cookiecutter --replay-file $REPLAY_FILE $TAP_TEMPLATE -o $CC_BUILD_PATH && - cd $CC_TEST_OUTPUT && - pwd && - sed -i.bak "s|singer-sdk =.*|singer-sdk = \{ path = \"$SDK_DIR\", develop = true \}|" pyproject.toml && - poetry lock && - poetry install - -if [[ $? -ne 0 ]]; then - exit $? -fi - -# before linting, auto-fix what can be autofixed -LIBRARY_NAME=$(ls * -d | egrep "tap|target") -poetry run black $LIBRARY_NAME && - poetry run isort $LIBRARY_NAME && - poetry run flake8 $LIBRARY_NAME && - poetry run mypy $LIBRARY_NAME -## - -if [[ $RUN_LINT -eq 1 ]] && [[ $? -eq 0 ]]; then - poetry run tox -e lint -fi diff --git a/noxfile.py b/noxfile.py index 995402f81..45c0c5974 100644 --- a/noxfile.py +++ b/noxfile.py @@ -2,9 +2,11 @@ from __future__ import annotations +import glob import os import shutil import sys +import tempfile from pathlib import Path from textwrap import dedent @@ -27,6 +29,7 @@ "mypy", "tests", "doctest", + "test_cookiecutter", ) test_dependencies = [ "coverage[toml]", @@ -170,3 +173,73 @@ def docs_serve(session: Session) -> None: shutil.rmtree(build_dir) session.run("sphinx-autobuild", *args) + + +@nox.parametrize("replay_file_path", glob.glob("./e2e-tests/cookiecutters/*.json")) +@session(python=main_python_version) +def test_cookiecutter(session: Session, replay_file_path) -> None: + """Uses the tap template to build an empty cookiecutter. + + Runs the lint task on the created test project. + """ + args = session.posargs or ["1"] + + cc_build_path = tempfile.gettempdir() + folder_base_path = "./cookiecutter" + + target_folder = ( + "tap-template" + if Path(replay_file_path).name.startswith("tap") + else "target-template" + ) + tap_template = Path(folder_base_path + "/" + target_folder).resolve() + replay_file = Path(replay_file_path).resolve() + + if not Path(tap_template).exists(): + return + + if not Path(replay_file).is_file(): + return + + sdk_dir = Path(Path(tap_template).parent).parent + cc_output_dir = Path(replay_file_path).name.replace(".json", "") + cc_test_output = cc_build_path + "/" + cc_output_dir + + if Path(cc_test_output).exists(): + session.run("rm", "-fr", cc_test_output, external=True) + + session.install(".") + session.install("cookiecutter", "pythonsed") + + session.run( + "cookiecutter", + "--replay-file", + str(replay_file), + str(tap_template), + "-o", + cc_build_path, + ) + session.chdir(cc_test_output) + + session.run( + "pythonsed", + "-i.bak", + 's|singer-sdk =.*|singer-sdk = \\{ path = "' + + str(sdk_dir) + + '", develop = true \\}|', + "pyproject.toml", + ) + session.run("poetry", "lock", external=True) + session.run("poetry", "install", external=True) + + for path in glob.glob(f"{Path.cwd()}/*", recursive=True): + if Path(path).name.startswith("tap") or Path( + path, + ).name.startswith("target"): + library_name = Path(path).name + + for argument in ["black", "isort", "flake8", "mypy"]: + session.run("poetry", "run", argument, library_name, external=True) + + if int(args[0]) == 1: + session.run("poetry", "run", "tox", "-e", "lint", external=True)