Switch to Poetry for dependency and build management (#679)

* Switch to Poetry for dependency and build management * Update k8s/README.md Co-authored-by: Alexander Dejanovski <[email protected]> * Add notes on python dependency * Fix directory name in offline instalation doc --------- Co-authored-by: Alexander Dejanovski <[email protected]>
thelastpickle · Mar 6, 2024 · 36cc3fd · 36cc3fd
1 parent b0cc958
commit 36cc3fd
Show file tree

Hide file tree

Showing 20 changed files with 2,494 additions and 198 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -16,4 +16,4 @@ values =
 
 [bumpversion:file:VERSION]
 
-[bumpversion:file:setup.py]
+[bumpversion:file:pyproject.toml]
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -30,27 +30,23 @@ jobs:
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m venv venv
-        . venv/bin/activate
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install tox
+        poetry install
 
     - name: Lint with flake8
       run: |
-        pip install flake8
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
     - name: Run tox
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
-        . venv/bin/activate
-        tox -e ${{ matrix.tox-py }}
+        poetry run tox -e ${{ matrix.tox-py }}
 
     - uses: codecov/codecov-action@v1
       name: Report code coverage  
@@ -173,23 +169,13 @@ jobs:
       with:
         java-version: '8.0.252'
         architecture: x64
+    - name: Setup Poetry
+      uses: snok/install-poetry@v1
     - name: Install dependencies
       run: |
-        python -m venv venv
-        . venv/bin/activate
-        sudo apt-get remove azure-cli
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements-test.txt
+        poetry install
+        sudo apt-get remove azure-cli || true
         pip install ccm
-        case '${{ matrix.it-backend }}' in
-          'azure'|'azure-hierarchical')
-            echo "No extra requirements for now."
-            ;;
-          'ibm'|'minio'|'s3')
-            echo "No extra requirements for now."
-            ;;
-        esac
 
     - name: Check if integration tests can run
       env:
@@ -216,7 +202,6 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
       run: |
         set -e
-        . venv/bin/activate
         if [[ "${{ matrix.it-backend }}" == "ibm" ]];
         then
           # Prevent awscli from using AWS secrets in case we're running against IBM cloud
@@ -292,7 +277,8 @@ jobs:
 
         # Move and convert the coverage analysis file to XML
         mv tests/integration/.coverage .
-        coverage xml
+        poetry run coverage xml
+
     - uses: codecov/codecov-action@v1
       name: Report code coverage
 
@@ -329,6 +315,8 @@ jobs:
         with:
           username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
           password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
+      - name: Set up Poetry
+        uses: snok/install-poetry@v1
       - uses: actions/checkout@v3
         with:
           path: cassandra-medusa
@@ -354,7 +342,7 @@ jobs:
         working-directory: cassandra-medusa
         run: |
           echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-          python setup.py build
+          poetry install && poetry build
       - name: Build Medusa image
         id: docker_build
         uses: docker/build-push-action@v4
@@ -417,10 +405,12 @@ jobs:
       shell: bash
       run: |
         echo "sha_short=$(git rev-parse --short=8 ${{ github.sha }})" >> $GITHUB_OUTPUT
+    - name: Set up Poetry
+      uses: snok/install-poetry@v1
     - name: Build Medusa
       run: |
         echo "Publishing release $(git rev-parse --short HEAD) in Docker Hub"
-        python setup.py build
+        poetry install && poetry build
     - name: Build image and push
       id: docker_build
       uses: docker/build-push-action@v4
@@ -496,10 +486,12 @@ jobs:
       shell: bash
       run: |
         echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT
+    - name: Set up Poetry
+      uses: snok/install-poetry@v1
     - name: Build Medusa
       run: |
         echo "Publishing release ${{ steps.vars.outputs.version}} in Docker Hub"
-        python setup.py build
+        poetry install && poetry build
     - name: Build image and push
       id: docker_build
       uses: docker/build-push-action@v4
@@ -535,5 +527,5 @@ jobs:
       run: |
         . venv/bin/activate
         pip install setuptools wheel twine
-        python setup.py sdist bdist_wheel
+        poetry install && poetry build
         twine upload dist/*
diff --git a/README.md b/README.md
@@ -21,6 +21,8 @@
 
 [![codecov](https://codecov.io/gh/thelastpickle/cassandra-medusa/branch/master/graph/badge.svg?token=KTDCRD82NU)](https://codecov.io/gh/thelastpickle/cassandra-medusa)
 
+![Python Version from PEP 621 TOML](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Fthelastpickle%2Fcassandra-medusa%2Fmaster%2Fpyproject.toml)
+
 Medusa for Apache Cassandra&trade;
 ==================================
 
@@ -58,4 +60,6 @@ You can find the Docker images for Cassandra Medusa at [https://hub.docker.com/r
 Dependencies
 ------------
 
+Medusa requires Python 3.8 or newer.
+
 For information on the packaged dependencies of Medusa for Apache Cassandra&reg; and their licenses, check out our [open source report](https://app.fossa.com/reports/cac72e73-1214-4e6d-8476-76567e08db21).
diff --git a/debian/cassandra-medusa.links b/debian/cassandra-medusa.links
@@ -1,2 +1,2 @@
 usr/share/cassandra-medusa/bin/medusa usr/bin/medusa
-usr/share/cassandra-medusa/bin/medusa-wrapper usr/bin/medusa-wrapper
+usr/share/cassandra-medusa/bin/medusa-wrapper usr/bin/medusa-wrapper
diff --git a/debian/rules b/debian/rules
@@ -21,6 +21,7 @@ PACKAGEVERSION = $(VERSION)-0~$(DISTRIBUTION)0
 PY3VER = $(shell py3versions -d)
 SSH2_LIBS_SUFFIX = debian/cassandra-medusa/usr/share/cassandra-medusa/lib/$(PY3VER)/site-packages/ssh2_python.libs/
 SSH_LIBS_SUFFIX = debian/cassandra-medusa/usr/share/cassandra-medusa/lib/$(PY3VER)/site-packages/ssh_python.libs/
+POETRY_VIRTUALENVS_IN_PROJECT = true
 
 export DH_ALWAYS_EXCLUDE = .git
 export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
@@ -29,8 +30,8 @@ export DH_VIRTUALENV_INSTALL_ROOT = /usr/share
 
 override_dh_virtualenv:
 	dh_virtualenv \
-	  --extra-pip-arg "--no-cache-dir" \
-	  --python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv
+	--python /usr/bin/python3 --preinstall=setuptools==40.3.0 --preinstall=pip==21.3.1 --preinstall=wheel --builtin-venv \
+	--preinstall=poetry==1.6.1 --preinstall=dh-poetry==0.2.0 --pip-tool=dh-poetry
 
 override_dh_strip:
 	dh_strip --no-automatic-dbgsym -X libssh2 -X libssh -X libgssapi_krb5 -X libcrypto -X libkrb5 -X libk5crypto
@@ -40,3 +41,7 @@ override_dh_shlibdeps:
 
 override_dh_gencontrol:
 	dh_gencontrol -- -v$(PACKAGEVERSION)
+
+override_dh_auto_configure:
+	# overriding with a no-op to avoid dh to look for setup.py
+	true
diff --git a/docs/Installation.md b/docs/Installation.md
@@ -26,23 +26,27 @@ Running the installation using `sudo` is necessary to have the `/usr/local/bin/m
 
 If your Cassandra servers do not have internet access:  
 
-- on a machine with the same target os and python version, clone the cassandra-medusa repo and cd into the root directory
-- run `mkdir pip_dependencies && pip download -r requirements.txt -d medusa_dependencies` to download the dependencies into a sub directory
-- run `cp requirements.txt medusa_dependencies/`
-- run `tar -zcf medusa_dependencies.tar.gz medusa_dependencies` to compress the dependencies
-- Upload the archive to all Cassandra nodes and decompress it
-- run `pip install -r medusa_dependencies/requirements.txt --no-index --find-links` to install the dependencies on the nodes
-- install Medusa using `python setup.py install` from the cassandra-medusa source directory
+- on a machine with the same target OS and Python version, clone the cassandra-medusa repo and cd into the root directory
+- run `POETRY_VIRTUALENVS_IN_PROJECT=true poetry install` to install Medusa's dependencies
+- run `poetry run pip freeze | grep -v cassandra-medusa > requirements.txt` to identify the dependencies
+- run `mkdir -p medusa_dependencies && pip download -r requirements.txt -d medusa_dependencies` to download the dependencies into a sub directory
+- run `tar --exclude=".venv" -zcf medusa.tar.gz .` to compress Medusa together with its dependencies but without Poetry's venv
+- upload the archive to all Cassandra nodes and decompress it (`mkdir -p cassandra-medusa && tar xf medusa.tar.gz -C cassandra-medusa`)
+- install Medusa's dependencies using `pip install --no-index --find-links medusa_dependencies -r requirements.txt`
+- install medusa itself using a command similar to `poetry build && pip install dist/cassandra_medusa-<version>-py3-none-any.whl`
+- test if the installation was successful by running `medusa` 
 
 #### Example of Offline installation using pipenv on RHEL, centos 7
 
 - install RPM pre-requisites `sudo yum install -y python3-pip python3-devel`
 - install pipenv `sudo pip3 install pipenv`
-- unpack your archive built using the procedure from previous section `tar zxvf my-archive-of-cassandra-medusa.tar.gz`
-- create your python env in the directory previously created `cd cassandra-medusa-0.7.1 && pipenv --python 3`
+- unpack your archive built using the procedure from previous section
+- create your python env in the directory previously created `cd cassandra-medusa && pipenv --python 3`
 - install python dependencies of medusa `pipenv run pip3 install -r requirements.txt --no-index --find-links medusa_dependencies/`
 - prepare an installation directory with appropriate privileges `sudo mkdir /opt/cassandra-medusa ; sudo chmod go+rwX  /opt/cassandra-medusa`
-- install medusa as non root user `pipenv run python3 setup.py install --prefix=. --root=/opt/cassandra-medusa`
+- build Medusa using `pipenv run pip3 install poetry && poetry build`
+- install medusa as non-root user `pipenv run pip3 install dist/cassandra_medusa-0.20.0.dev0-py3-none-any.whl --no-index --find-links medusa_dependencies`
+- test if the installation was successful by running `pipenv run medusa` 
 
 ## Debian packages
 ### Using apt-get

diff --git a/k8s/Dockerfile b/k8s/Dockerfile
@@ -27,21 +27,18 @@ ENV PATH=/root/.local/bin:$PATH
 COPY . /build/
 
 # General requirements
-RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user \
-    -r /build/requirements.txt \
-    -r /build/requirements-grpc-runtime.txt
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+RUN python3 -m pip install -U pip && pip3 install --ignore-installed --user poetry
 
 # Build medusa itself so we can add the executables in the final image
-RUN pip3 install --ignore-installed --user /build
+RUN cd /build && poetry install
 
 # Could be python:slim, but we have a .sh entrypoint
 FROM ubuntu:22.04
 
 ## add user
 RUN groupadd -r cassandra --gid=999 && useradd -r -g cassandra --uid=999 --create-home cassandra
 
-COPY --from=base /install /usr/local
-
 # wget could happen in the build-phase
 RUN apt-get update && apt-get install -y python3 python3-setuptools wget \
     && rm -rf /var/lib/apt/lists/*
@@ -59,6 +56,8 @@ ENV PATH=/home/cassandra/.local/bin:/home/cassandra/google-cloud-sdk/bin:$PATH
 
 COPY --from=base --chown=cassandra:cassandra /root/.local /home/cassandra/.local
 
+COPY --from=base --chown=cassandra:cassandra /build/.venv /home/cassandra/.venv
+COPY --from=base --chown=cassandra:cassandra /build/pyproject.toml /home/cassandra/pyproject.toml
 COPY --chown=cassandra:cassandra medusa /home/cassandra/medusa
 COPY --chown=cassandra:cassandra k8s/docker-entrypoint.sh /home/cassandra
 

diff --git a/k8s/README.md b/k8s/README.md
@@ -37,7 +37,6 @@ $ cd medusa/service/grpc
 $ python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. medusa.proto
 ``` 
 
-These steps should be integrated into the build or setup.py at some point. I am just not sure where the appropriate integrations points are yet.
 
 # Kubernetes Configuration
 There is a Kubernetes section of the Medusa configuration that will need to be set for all this to work in a K8s environment. Currently, the Cassandra image you use will need to support one of two APIs in order for the Medusa image to be able to perform backup and restore functions: JMX (via Jolokia) or the [Management API](https://github.com/datastax/management-api-for-apache-cassandra).
@@ -86,7 +85,7 @@ If you made any changes to `medusa.proto`, then you first need to run the protob
 Run the following from the project root:
 
 ```
-$ python setup.py build
+$ poetry install && poetry build
 
 $ docker build -t <tag name> -f k8s/Dockerfile .
 ```
diff --git a/k8s/docker-entrypoint.sh b/k8s/docker-entrypoint.sh
@@ -41,14 +41,14 @@ restore() {
         echo "Skipping restore operation"    
     else
         echo "Restoring backup $BACKUP_NAME"
-        python3 -m medusa.service.grpc.restore -- "/etc/medusa/medusa.ini" $RESTORE_KEY
+        poetry run python -m medusa.service.grpc.restore -- "/etc/medusa/medusa.ini" $RESTORE_KEY
         echo $RESTORE_KEY > $last_restore_file
     fi
 }
 
 grpc() {
     echo "Starting Medusa gRPC service"
-    exec python3 -m medusa.service.grpc.server server.py
+    exec poetry run python -m medusa.service.grpc.server server.py
 }
 
 echo "sleeping for $DEBUG_SLEEP sec"

diff --git a/medusa/service/grpc/restore.py b/medusa/service/grpc/restore.py
@@ -104,9 +104,9 @@ def restore_backup(in_place, config):
 
 
 if __name__ == '__main__':
-    if len(sys.argv) > 3:
-        config_file_path = sys.argv[2]
-        restore_key = sys.argv[3]
+    if len(sys.argv) > 2:
+        config_file_path = sys.argv[1]
+        restore_key = sys.argv[2]
     else:
         logging.error("Usage: {} <config_file_path> <restore_key>".format(sys.argv[0]))
         sys.exit(1)

diff --git a/packaging/docker-build/Dockerfile b/packaging/docker-build/Dockerfile
@@ -33,8 +33,8 @@ RUN apt-get update && \
 # install dependencies
 RUN apt-get update \
     && apt-get install -y \
-        libffi-dev \   
-        libssl-dev \  
+        libffi-dev \
+        libssl-dev \
         debhelper \
         gdebi-core \
         gcc \
@@ -58,13 +58,14 @@ RUN apt-get update \
 RUN pip3 install --upgrade pip
 
 RUN cd /tmp && \
-    wget http://mirrors.kernel.org/ubuntu/pool/universe/d/dh-virtualenv/dh-virtualenv_1.0-1_all.deb && \
+    wget http://mirrors.kernel.org/ubuntu/pool/universe/d/dh-virtualenv/dh-virtualenv_1.2.2-1.4_all.deb && \
     gdebi -n dh-virtualenv*.deb && \
     rm dh-virtualenv_*.deb
 
 
 RUN pip3 install greenlet
 RUN pip3 install gevent
+RUN pip3 install poetry
 
 # Add entrypoint script
 COPY packaging/docker-build/docker-entrypoint.sh ${WORKDIR}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -16,4 +16,4 @@ values =

		[bumpversion:file:VERSION]

		[bumpversion:file:setup.py]
		[bumpversion:file:pyproject.toml]