Upgrade to python 3.12 (#449)

* Upgrade to python 3.12

* Upd pytest

* Del scrypt mock

* Upd dockerfilel and CI

* Disable poetry cache

* Add asyncio_default_fixture_loop_scope

* Revert "Disable poetry cache"

This reverts commit f7fb1bf.

* Set version v3.0.0

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.10.14
+          python-version: 3.12.8
 
       # Install poetry
       - name: Load cached Poetry installation
@@ -58,7 +58,7 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.10.14
+          python-version: 3.12.8
 
       # Install poetry
       - name: Load cached Poetry installation
@@ -96,7 +96,7 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.10.14
+          python-version: 3.12.8
 
       # Install poetry
       - name: Load cached Poetry installation

.github/workflows/release.yaml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.10.5
+          python-version: 3.12.8
 
       - name: Install Poetry
         uses: Gr1N/setup-poetry@v8
@@ -37,7 +37,7 @@ jobs:
       matrix:
         include:
           - OS: ubuntu-20.04
-            PYTHON_VERSION: 3.10.14
+            PYTHON_VERSION: 3.12.8
             BUILD_CMD: |
               export PYTHONHASHSEED=42
               export BUILD_FILE_NAME=operator-${RELEASE_VERSION}-linux-amd64;
@@ -52,7 +52,7 @@ jobs:
               sha256sum ${BUILD_FILE_NAME}.tar.gz | head -c 64 > /tmp/artifacts/${BUILD_FILE_NAME}.sha256;
 
           - OS: linux-arm-runner
-            PYTHON_VERSION: 3.10.14
+            PYTHON_VERSION: 3.12.8
             BUILD_CMD: |
               export PYTHONHASHSEED=42
               export BUILD_FILE_NAME=operator-${RELEASE_VERSION}-linux-arm64;
@@ -68,7 +68,7 @@ jobs:
               sha256sum ${BUILD_FILE_NAME}.tar.gz | head -c 64 > /tmp/artifacts/${BUILD_FILE_NAME}.sha256;
 
           - OS: macos-13
-            PYTHON_VERSION: 3.10.14
+            PYTHON_VERSION: 3.12.8
             BUILD_CMD: |
               export PYTHONHASHSEED=42
               export BUILD_FILE_NAME=operator-${RELEASE_VERSION}-darwin-amd64;
@@ -83,7 +83,7 @@ jobs:
               shasum -a 256 ${BUILD_FILE_NAME}.tar.gz | head -c 64 > /tmp/artifacts/${BUILD_FILE_NAME}.sha256
 
           - OS: windows-latest
-            PYTHON_VERSION: 3.10.11
+            PYTHON_VERSION: 3.12.8
             BUILD_CMD: |
               $RELEASE_VERSION = $env:GITHUB_REF.replace('refs/tags/', '')
               $BUILD_FILE_NAME = "operator-" + $RELEASE_VERSION + "-windows-amd64"

Dockerfile

@@ -1,5 +1,5 @@
 # `python-base` sets up all our shared environment variables
-FROM python:3.10.15-slim-bookworm as python-base
+FROM python:3.12.8-slim-bookworm as python-base
 
     # python
 ENV PYTHONUNBUFFERED=1 \

README.md

@@ -154,14 +154,14 @@ Head to [Usage](#usage) to launch your operator service.
 Pull the latest docker operator docker image:
 
 ```bash
-docker pull europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.2.1
+docker pull europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v3.0.0
 ```
 
 You can also build the docker image from source by cloning this repo and executing the following command from within
 the `v3-operator` folder:
 
 ```bash
-docker build --pull -t europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.2.1 .
+docker build --pull -t europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v3.0.0 .
 ```
 
 You will execute Operator Service commands using the format below (note the use of flags are optional):
@@ -170,7 +170,7 @@ You will execute Operator Service commands using the format below (note the use
 docker run --rm -ti \
 -u $(id -u):$(id -g) \
 -v ~/.stakewise/:/data \
-europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.2.1 \
+europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v3.0.0 \
 src/main.py COMMAND \
 --flagA=123 \
 --flagB=xyz
@@ -393,7 +393,7 @@ below:
 docker run --restart on-failure:10 \
 -u $(id -u):$(id -g) \
 -v ~/.stakewise/:/data \
-europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v2.2.1 \
+europe-west4-docker.pkg.dev/stakewiselabs/public/v3-operator:v3.0.0 \
 src/main.py start \
 --vault=0x3320ad928c20187602a2b2c04eeaa813fa899468 \
 --data-dir=/data \

pyproject.toml

@@ -1,39 +1,39 @@
 [tool.poetry]
 name = "v3-operator"
-version = "v2.2.1"
+version = "v3.0.0"
 description = "StakeWise operator service for registering vault validators"
 authors = ["StakeWise Labs <[email protected]>"]
 package-mode = false
 
 [tool.poetry.dependencies]
-python = ">=3.10,<3.11"
+python = ">=3.12,<3.13"
 python-decouple = "==3.8"
 sentry-sdk = "==1.32.0"
 py-ecc = "==6.0.0"
 gql = {extras = ["aiohttp"], version = "==3.5.0"}
 multiproof = { git = "https://github.com/stakewise/multiproof.git", rev = "v0.1.8" }
 sw-utils = {git = "https://github.com/stakewise/sw-utils.git", rev = "v0.7.4"}
-staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.4.0" }
+staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.8.0" }
 pycryptodomex = "3.19.1"
 click = "==8.1.7"
 tomli = "~2"
-eciespy = "==0.4.0"
+eciespy = "==0.4.3"
 prometheus-client = "==0.17.1"
 psycopg2 = "==2.9.9"
 pyyaml = "==6.0.1"
 python-json-logger = "==2.0.7"
-aiohttp = "==3.10.11"
+aiohttp = "==3.11.11"
 
 [tool.poetry.group.dev.dependencies]
 pylint = "==3.3.3"
 mypy = "==1.14.1"
 isort = "==5.12.0"
-pytest = "==7.4.2"
-pytest-asyncio = "==0.21.1"
+pytest = "==8.3.3"
+pytest-asyncio = "==0.25.2"
 pre-commit = "==3.5.0"
 Flake8-pyproject = "==1.2.3"
 bandit = { version = "==1.7.5", extras = ["toml"] }
-black = { version = "==23.10.0", extras = ["d"] }
+black = { version = "==24.10.0", extras = ["d"] }
 faker = "==19.11.0"
 flake8-print = "==5.0.0"
 flake8-datetimez = "==20.10.0"
@@ -75,13 +75,17 @@ good-names = ["db", "i", "w3"]
 ignored-modules=["milagro_bls_binding"]
 
 [tool.flake8]
-extend-ignore = ["E203", "E501"]  # line length will be checked by pylint
+extend-ignore = [
+    "E203",  # Whitespace before ':'
+    "E501",  # Line too long
+    "E701"   # Multiple statements on one line (colon)
+]
 exclude = ["conftest.py"]
 
 [tool.mypy]
 exclude = ["test"]
 ignore_missing_imports = true
-python_version = "3.10"
+python_version = "3.12"
 disallow_untyped_defs = true
 disallow_incomplete_defs = true
 warn_redundant_casts = true
@@ -120,6 +124,7 @@ fail_under = 66
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "session"
 
 [tool.vulture]
 exclude = ["*/test*", "conftest.py", "networks.py"]

scripts/install.sh

@@ -281,7 +281,7 @@ http_copy() {
 github_release() {
   owner_repo=$1
   version=$2
-  test -z "$version" && version="v2.2.1"
+  test -z "$version" && version="v3.0.0"
   giturl="https://github.com/${owner_repo}/releases/${version}"
   json=$(http_copy "$giturl" "Accept:application/json")
   test -z "$json" && return 1

src/commands/create_keys.py

@@ -153,12 +153,15 @@ def _export_keystores(
 
     if not per_keystore_password:
         password = get_or_create_password_file(password_file)
-    with click.progressbar(
-        credentials,
-        label='Exporting validator keystores\t\t',
-        show_percent=False,
-        show_pos=True,
-    ) as progress_bar, Pool(processes=pool_size) as pool:
+    with (
+        click.progressbar(
+            credentials,
+            label='Exporting validator keystores\t\t',
+            show_percent=False,
+            show_pos=True,
+        ) as progress_bar,
+        Pool(processes=pool_size) as pool,
+    ):
         results = [
             pool.apply_async(
                 cred.save_signing_keystore,

src/commands/tests/test_remote_signer_setup.py

@@ -12,7 +12,9 @@
 
 class TestOperatorRemoteSignerSetup:
     @pytest.mark.usefixtures(
-        '_init_vault', '_create_keys', 'mocked_remote_signer', 'mock_scrypt_keystore'
+        '_init_vault',
+        '_create_keys',
+        'mocked_remote_signer',
     )
     async def test_basic(
         self,
@@ -52,7 +54,7 @@ async def test_basic(
             pubkeys_remote_signer = {pubkey_dict.get('validating_pubkey') for pubkey_dict in data}
             assert len(pubkeys_remote_signer) == key_count
 
-    @pytest.mark.usefixtures('_init_vault', 'mocked_remote_signer', 'mock_scrypt_keystore')
+    @pytest.mark.usefixtures('_init_vault', 'mocked_remote_signer')
     def test_add_more_keys_later(
         self,
         vault_address: HexAddress,

src/commands/validators_exit.py

@@ -208,7 +208,7 @@ async def main(count: int | None) -> None:
 
     click.confirm(
         f'Are you sure you want to exit {len(validators_exits)} validators '
-        f'with indexes: {", ".join(str(x.index) for x in validators_exits)}?',
+        f'with indexes: {', '.join(str(x.index) for x in validators_exits)}?',
         abort=True,
     )
     exited_indexes = []
@@ -239,7 +239,7 @@ async def main(count: int | None) -> None:
 
     if exited_indexes:
         click.secho(
-            f'Validators {", ".join(str(index) for index in exited_indexes)} '
+            f'Validators {', '.join(str(index) for index in exited_indexes)} '
             f'({len(exited_indexes)} of {len(validators_exits)}) '
             f'exits successfully initiated',
             bold=True,

src/common/credentials.py

@@ -66,7 +66,7 @@ def save_signing_keystore(
         self, password: str, folder: str, per_keystore_password: bool = False
     ) -> str:
         keystore = self.encrypt_signing_keystore(password)
-        file_name = f'keystore-{keystore.path.replace("/", "_")}-{int(time.time())}'
+        file_name = f'keystore-{keystore.path.replace('/', '_')}-{int(time.time())}'
         file_path = path.join(folder, f'{file_name}.json')
 
         if per_keystore_password:

src/common/startup_check.py

@@ -279,7 +279,7 @@ async def _check_consensus_nodes_network() -> None:
         consensus_network = chain_id_to_network.get(consensus_chain_id)
         if settings.network_config.CHAIN_ID != consensus_chain_id:
             raise ValueError(
-                f'Consensus node network is {consensus_network or "unknown"}, '
+                f'Consensus node network is {consensus_network or 'unknown'}, '
                 f'while {settings.network} is passed in "--network" parameter'
             )
 
@@ -298,7 +298,7 @@ async def _check_execution_nodes_network() -> None:
         execution_network = chain_id_to_network.get(execution_chain_id)
         if settings.network_config.CHAIN_ID != execution_chain_id:
             raise ValueError(
-                f'Execution node network is {execution_network or "unknown"}, '
+                f'Execution node network is {execution_network or 'unknown'}, '
                 f'while {settings.network} is passed in "--network" parameter'
             )
 

src/common/utils.py

@@ -99,13 +99,11 @@ def process_oracles_approvals(
 
 
 @overload
-def chunkify(items: list[T], size: int) -> Iterator[list[T]]:
-    ...
+def chunkify(items: list[T], size: int) -> Iterator[list[T]]: ...
 
 
 @overload
-def chunkify(items: range, size: int) -> Iterator[range]:
-    ...
+def chunkify(items: range, size: int) -> Iterator[range]: ...
 
 
 def chunkify(items, size):  # type: ignore[no-untyped-def]

src/conftest.py

@@ -8,7 +8,6 @@
 import pytest
 from _pytest.fixtures import SubRequest
 from click.testing import CliRunner
-from Cryptodome.Protocol.KDF import scrypt as raw_scrypt
 from eth_typing import HexAddress, HexStr
 from sw_utils.tests import faker
 from sw_utils.tests.factories import get_mocked_protocol_config
@@ -17,7 +16,7 @@
 from src.commands.create_keys import create_keys
 from src.commands.create_wallet import create_wallet
 from src.commands.remote_signer_setup import remote_signer_setup
-from src.common.credentials import CredentialManager, ScryptKeystore
+from src.common.credentials import CredentialManager
 from src.common.vault_config import VaultConfig
 from src.config.networks import HOLESKY
 from src.config.settings import settings
@@ -77,38 +76,12 @@ def runner() -> CliRunner:
     return CliRunner()
 
 
-def _scrypt_without_validation(
-    *, password: str, salt: str, n: int, r: int, p: int, dklen: int
-) -> bytes:
-    """
-    Shortened version of `staking_deposit.utils.crypto.scrypt`.
-    All validations are deleted to allow small number of hash iterations (`n`).
-    The function is not secure. Use it in tests only.
-    """
-    res = raw_scrypt(password=password, salt=salt, key_len=dklen, N=n, r=r, p=p)
-    return res if isinstance(res, bytes) else res[0]  # PyCryptodome can return Tuple[bytes]
-
-
-@pytest.fixture
-def mock_scrypt_keystore():
-    """
-    Decreases number of iterations of password hashing. Original value is ~200k.
-    This improves speed of keystore encryption.
-    Not secure.
-    """
-    with mock.patch.dict(ScryptKeystore.crypto.kdf.params, {'n': 2}), mock.patch(
-        'staking_deposit.key_handling.keystore.scrypt', new=_scrypt_without_validation
-    ):
-        yield
-
-
 @pytest.fixture
 def _create_keys(
     test_mnemonic: str,
     vault_address: HexAddress,
     data_dir: Path,
     _test_keystore_password_file: Path,
-    mock_scrypt_keystore,
     runner: CliRunner,
 ) -> None:
     count = 3

src/remote_db/commands.py

@@ -100,7 +100,7 @@ def setup(ctx: Context) -> None:
     click.echo(
         f'Successfully configured remote database.\n'
         f'Encryption key: {greenify(encryption_key)}\n'
-        f'{click.style("NB! You must store your encryption in a secure cold storage!", bold=True)}'
+        f'{click.style('NB! You must store your encryption in a secure cold storage!', bold=True)}'
     )
 
 

src/remote_db/database.py

@@ -91,7 +91,7 @@ def remove_keypairs(self, in_public_keys: set[HexStr] | None = None) -> None:
         query = f'''
                 DELETE FROM {self.table}
         '''
-        query += f'WHERE {" AND ".join(where_list)}\n'
+        query += f'WHERE {' AND '.join(where_list)}\n'
 
         with self.db_connection.cursor() as cur:
             cur.execute(query, params)