diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 306f7c8001..4df90eb2b0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,31 +36,34 @@ repos: - id: lint name: Linter stages: [pre-commit, pre-merge-commit, manual] - entry: "scripts/lint-pre-commit.sh" + entry: just linter language: python types: [python] require_serial: true verbose: true + pass_filenames: false - repo: local hooks: - id: static-analysis name: Static analysis - entry: "scripts/static-pre-commit.sh" + entry: just static-analysis language: python types: [python] require_serial: true verbose: true + pass_filenames: false - repo: local hooks: - id: docs name: Build docs - entry: "scripts/build-docs-pre-commit.sh" + entry: just docs-build language: python files: ^docs require_serial: true verbose: true + pass_filenames: false - repo: https://github.com/Yelp/detect-secrets rev: v1.5.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b8fb8c0a3f..7a7dff8d44 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,69 +4,89 @@ After cloning the project, you'll need to set up the development environment. Here are the guidelines on how to do this. -## Virtual Environment with `venv` +## Install Justfile Utility -Create a virtual environment in a directory using Python's `venv` module: +Install justfile on your system: ```bash -python -m venv venv +brew install justfile ``` -That will create a `./venv/` directory with Python binaries, allowing you to install packages in an isolated environment. +View all available commands: -## Activate the Environment +```bash +just +``` -Activate the new environment with: +## Init development environment + +Build Python and create a virtual environment: ```bash -source ./venv/bin/activate +just init ``` -Ensure you have the latest pip version in your virtual environment: +By default, this builds Python 3.8. If you need another version, pass it as an argument to the just command: ```bash -python -m pip install --upgrade pip +just init 3.11.5 ``` -## Installing Dependencies +To check available Python versions, refer to the pyproject.toml file in the project root. -After activating the virtual environment as described above, run: +## Activate the Environment + +Activate the new environment with + +For Unix-based systems: ```bash -pip install -e ".[dev]" +source ./venv/bin/activate ``` -This will install all the dependencies and your local **FastStream** in your virtual environment. +For Windows (PowerShell): -### Using Your local **FastStream** +```bash +.\venv\Scripts\Activate.ps1 +``` -If you create a Python file that imports and uses **FastStream**, and run it with the Python from your local environment, it will use your local **FastStream** source code. +Install and configure pre-commit: -Whenever you update your local **FastStream** source code, it will automatically use the latest version when you run your Python file again. This is because it is installed with `-e`. +```bash +just pre-commit-install +``` -This way, you don't have to "install" your local version to be able to test every change. +## Run all Dependencies -To use your local **FastStream CLI**, type: +Start all dependencies as docker containers: ```bash -python -m faststream ... +just up +``` + +Once you are done with development and running tests, you can stop the dependencies' docker containers by running: + +```bash +just stop +# or +just down ``` ## Running Tests -### Pytest +To run tests, use: + +```bash +just test +``` -To run tests with your current **FastStream** application and Python environment, use: +To run tests with coverage: ```bash -pytest tests -# or -./scripts/test.sh -# with coverage output -./scripts/test-cov.sh +just coverage-test ``` -In your project, you'll find some *pytest marks*: +In your project, some tests are grouped under specific pytest marks: * **slow** * **rabbit** @@ -75,34 +95,40 @@ In your project, you'll find some *pytest marks*: * **redis** * **all** -By default, running *pytest* will execute "not slow" tests. - -To run all tests use: +By default, will execute "all" tests. You can specify marks to include or exclude tests: ```bash -pytest -m 'all' +just test kafka +# or +just test rabbit +# or +just test 'not confluent' +# or +just test 'not confluent and not nats' +# or +just coverage-test kafka ``` -If you don't have a local broker instance running, you can run tests without those dependencies: +## Linter + +Run all linters: ```bash -pytest -m 'not rabbit and not kafka and not nats and not redis and not confluent' +just linter ``` -To run tests based on RabbitMQ, Kafka, or other dependencies, the following dependencies are needed to be started as docker containers: +## Static analysis -```yaml -{! includes/docker-compose.yaml !} -``` - -You can start the dependencies easily using provided script by running: +Run static analysis tools: ```bash -./scripts/start_test_env.sh +just static-analysis ``` -Once you are done with development and running tests, you can stop the dependencies' docker containers by running: +## Pre-commit + +Run pre-commit checks: ```bash -./scripts/stop_test_env.sh +just pre-commit ``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..8828eb6dfe --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +ARG PYTHON_VERSION=3.8 + +FROM python:$PYTHON_VERSION + +ENV PYTHONUNBUFFERED=1 + +COPY . /src + +WORKDIR /src + +RUN pip install --upgrade pip && pip install -e ".[dev]" diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000000..7eed4d8ca7 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,52 @@ +services: + rabbitmq: + image: rabbitmq:alpine + ports: + - "5672:5672" + # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges + security_opt: + - no-new-privileges:true + + kafka: + image: bitnami/kafka:3.5.0 + ports: + - "9092:9092" + environment: + KAFKA_ENABLE_KRAFT: "true" + KAFKA_CFG_NODE_ID: "1" + KAFKA_CFG_PROCESS_ROLES: "broker,controller" + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: "CONTROLLER" + KAFKA_CFG_LISTENERS: "PLAINTEXT://:9092,CONTROLLER://:9093" + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT" + KAFKA_CFG_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:9092" + KAFKA_BROKER_ID: "1" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: "1@kafka:9093" + ALLOW_PLAINTEXT_LISTENER: "true" + # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges + security_opt: + - no-new-privileges:true + + nats: + image: nats + command: -js + ports: + - 4222:4222 + - 8222:8222 # management + # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges + security_opt: + - no-new-privileges:true + + redis: + image: redis:alpine + ports: + - 6379:6379 + # https://semgrep.dev/r?q=yaml.docker-compose.security.no-new-privileges.no-new-privileges + security_opt: + - no-new-privileges:true + + faststream: + build: . + volumes: + - ./:/src + network_mode: "host" + tty: true diff --git a/docs/docs/en/getting-started/contributing/CONTRIBUTING.md b/docs/docs/en/getting-started/contributing/CONTRIBUTING.md index 745f963830..8c832fda59 100644 --- a/docs/docs/en/getting-started/contributing/CONTRIBUTING.md +++ b/docs/docs/en/getting-started/contributing/CONTRIBUTING.md @@ -12,69 +12,89 @@ search: After cloning the project, you'll need to set up the development environment. Here are the guidelines on how to do this. -## Virtual Environment with `venv` +## Install Justfile Utility -Create a virtual environment in a directory using Python's `venv` module: +Install justfile on your system: ```bash -python -m venv venv +brew install justfile ``` -That will create a `./venv/` directory with Python binaries, allowing you to install packages in an isolated environment. +View all available commands: -## Activate the Environment +```bash +just +``` -Activate the new environment with: +## Init development environment + +Build Python and create a virtual environment: ```bash -source ./venv/bin/activate +just init ``` -Ensure you have the latest pip version in your virtual environment: +By default, this builds Python 3.8. If you need another version, pass it as an argument to the just command: ```bash -python -m pip install --upgrade pip +just init 3.11.5 ``` -## Installing Dependencies +To check available Python versions, refer to the pyproject.toml file in the project root. -After activating the virtual environment as described above, run: +## Activate the Environment + +Activate the new environment with + +For Unix-based systems: ```bash -pip install -e ".[dev]" +source ./venv/bin/activate ``` -This will install all the dependencies and your local **FastStream** in your virtual environment. +For Windows (PowerShell): -### Using Your local **FastStream** +```bash +.\venv\Scripts\Activate.ps1 +``` -If you create a Python file that imports and uses **FastStream**, and run it with the Python from your local environment, it will use your local **FastStream** source code. +Install and configure pre-commit: -Whenever you update your local **FastStream** source code, it will automatically use the latest version when you run your Python file again. This is because it is installed with `-e`. +```bash +just pre-commit-install +``` -This way, you don't have to "install" your local version to be able to test every change. +## Run all Dependencies -To use your local **FastStream CLI**, type: +Start all dependencies as docker containers: ```bash -python -m faststream ... +just up +``` + +Once you are done with development and running tests, you can stop the dependencies' docker containers by running: + +```bash +just stop +# or +just down ``` ## Running Tests -### Pytest +To run tests, use: + +```bash +just test +``` -To run tests with your current **FastStream** application and Python environment, use: +To run tests with coverage: ```bash -pytest tests -# or -./scripts/test.sh -# with coverage output -./scripts/test-cov.sh +just coverage-test ``` -In your project, you'll find some *pytest marks*: +In your project, some tests are grouped under specific pytest marks: * **slow** * **rabbit** @@ -83,34 +103,40 @@ In your project, you'll find some *pytest marks*: * **redis** * **all** -By default, running *pytest* will execute "not slow" tests. - -To run all tests use: +By default, will execute "all" tests. You can specify marks to include or exclude tests: ```bash -pytest -m 'all' +just test kafka +# or +just test rabbit +# or +just test 'not confluent' +# or +just test 'not confluent and not nats' +# or +just coverage-test kafka ``` -If you don't have a local broker instance running, you can run tests without those dependencies: +## Linter + +Run all linters: ```bash -pytest -m 'not rabbit and not kafka and not nats and not redis and not confluent' +just linter ``` -To run tests based on RabbitMQ, Kafka, or other dependencies, the following dependencies are needed to be started as docker containers: +## Static analysis -```yaml -{! includes/docker-compose.yaml !} -``` - -You can start the dependencies easily using provided script by running: +Run static analysis tools: ```bash -./scripts/start_test_env.sh +just static-analysis ``` -Once you are done with development and running tests, you can stop the dependencies' docker containers by running: +## Pre-commit + +Run pre-commit checks: ```bash -./scripts/stop_test_env.sh +just pre-commit ``` diff --git a/justfile b/justfile new file mode 100644 index 0000000000..6cc1b7ea56 --- /dev/null +++ b/justfile @@ -0,0 +1,188 @@ +[doc("All command information")] +default: + @just --list --unsorted --list-heading $'FastStream commands…\n' + + +# Infra +[doc("Init infra")] +[group("infra")] +init python="3.8": + docker build . --build-arg PYTHON_VERSION={{python}} + python -m venv venv + +[doc("Run all containers")] +[group("infra")] +up: + docker compose up -d + +[doc("Stop all containers")] +[group("infra")] +stop: + docker compose stop + +[doc("Down all containers")] +[group("infra")] +down: + docker compose down + + +[doc("Run tests")] +[group("tests")] +test target="all": + docker compose exec faststream pytest -m "{{target}}" + +[doc("Run all tests")] +[group("tests")] +coverage-test target="all": + -docker compose exec faststream sh -c "coverage run -m pytest -m '{{target}}' && coverage combine && coverage report --show-missing --skip-covered --sort=cover --precision=2 && rm .coverage*" + + +# Docs +[doc("Build docs")] +[group("docs")] +docs-build: + docker compose exec -T faststream sh -c "cd docs && python docs.py build" + +[doc("Serve docs")] +[group("docs")] +docs-serve: + docker compose exec faststream sh -c "cd docs && python docs.py live" + + +# Linter +[doc("Ruff check")] +[group("linter")] +ruff-check: + -ruff check --exit-non-zero-on-fix + +[doc("Ruff format")] +[group("linter")] +ruff-format: + -ruff format + +[doc("Codespell check")] +[group("linter")] +codespell: + -codespell + +[doc("Linter run")] +[group("linter")] +linter: ruff-check ruff-format codespell + + +# Static analysis +[doc("Mypy check")] +[group("static analysis")] +mypy: + -mypy + +[doc("Bandit check")] +[group("static analysis")] +bandit: + -bandit -c pyproject.toml -r faststream + +[doc("Semgrep check")] +[group("static analysis")] +semgrep: + -semgrep scan --config auto --error + +[doc("Static analysis check")] +[group("static analysis")] +static-analysis: mypy bandit semgrep + + +# Pre-commit +[doc("Pre-commit install")] +[group("pre-commit")] +pre-commit-install: + pip install pre-commit + pre-commit install + +[doc("Pre-commit run")] +[group("pre-commit")] +pre-commit: + -pre-commit run --all + + +# Kafka +[doc("Run kafka container")] +[group("kafka")] +kafka-up: + docker compose up -d kafka + +[doc("Stop kafka container")] +[group("kafka")] +kafka-stop: + docker compose stop kafka + +[doc("Show kafka logs")] +[group("kafka")] +kafka-logs: + docker compose logs -f kafka + +[doc("Run kafka tests")] +[group("kafka")] +kafka-tests: (test "kafka") + + +# RabbitMQ +[doc("Run rabbitmq container")] +[group("rabbitmq")] +rabbit-up: + docker compose up -d rabbitmq + +[doc("Stop rabbitmq container")] +[group("rabbitmq")] +rabbit-stop: + docker compose stop rabbitmq + +[doc("Show rabbitmq logs")] +[group("rabbitmq")] +rabbit-logs: + docker compose logs -f rabbitmq + +[doc("Run rabbitmq tests")] +[group("rabbitmq")] +rabbit-tests: (test "rabbit") + + +# Redis +[doc("Run redis container")] +[group("redis")] +redis-up: + docker compose up -d redis + +[doc("Stop redis container")] +[group("redis")] +redis-stop: + docker compose stop redis + +[doc("Show redis logs")] +[group("redis")] +redis-logs: + docker compose logs -f redis + +[doc("Run redis tests")] +[group("redis")] +redis-tests: (test "redis") + + +# Nats +[doc("Run nats container")] +[group("nats")] +nats-up: + docker compose up -d nats + +[doc("Stop nats container")] +[group("nats")] +nats-stop: + docker compose stop nats + +[doc("Show nats logs")] +[group("nats")] +nats-logs: + docker compose logs -f nats + +[doc("Run nats tests")] +[group("nats")] +nats-tests: (test "nats") diff --git a/pyproject.toml b/pyproject.toml index 4860b6e730..b26c0e4b4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,8 @@ devdocs = [ types = [ "faststream[optionals]", - "mypy==1.15.0", + "mypy==1.15.0; python_version >= '3.9'", + "mypy==1.14.1; python_version < '3.9'", # mypy extensions "types-Deprecated", "types-PyYAML", diff --git a/scripts/build-docs-pre-commit.sh b/scripts/build-docs-pre-commit.sh deleted file mode 100755 index fba2388ec4..0000000000 --- a/scripts/build-docs-pre-commit.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# from: https://jaredkhan.com/blog/mypy-pre-commit - -# A script for running mypy, -# with all its dependencies installed. - -set -o errexit - -# Change directory to the project root directory. -cd "$(dirname "$0")"/.. - -# Install the dependencies into the mypy env. -# Note that this can take seconds to run. -# In my case, I need to use a custom index URL. -# Avoid pip spending time quietly retrying since -# likely cause of failure is lack of VPN connection. -pip install --editable ".[dev]" \ - --retries 1 \ - --no-input \ - --quiet - -# Run on all files, -# ignoring the paths passed to this script, -# so as not to miss type errors. -# My repo makes use of namespace packages. -# Use the namespace-packages flag -# and specify the package to run on explicitly. -# Note that we do not use --ignore-missing-imports, -# as this can give us false confidence in our results. -# mypy faststream -./scripts/build-docs.sh diff --git a/scripts/build-docs.sh b/scripts/build-docs.sh deleted file mode 100755 index 254f4a0077..0000000000 --- a/scripts/build-docs.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -cd docs; python docs.py build diff --git a/scripts/lint-pre-commit.sh b/scripts/lint-pre-commit.sh deleted file mode 100755 index ff6c82ccfc..0000000000 --- a/scripts/lint-pre-commit.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# from: https://jaredkhan.com/blog/mypy-pre-commit - -# A script for running mypy, -# with all its dependencies installed. - -set -o errexit - -# Change directory to the project root directory. -cd "$(dirname "$0")"/.. - -# Install the dependencies into the mypy env. -# Note that this can take seconds to run. -# In my case, I need to use a custom index URL. -# Avoid pip spending time quietly retrying since -# likely cause of failure is lack of VPN connection. -pip install --editable ".[dev]" \ - --retries 1 \ - --no-input \ - --quiet - -# Run on all files, -# ignoring the paths passed to this script, -# so as not to miss type errors. -# My repo makes use of namespace packages. -# Use the namespace-packages flag -# and specify the package to run on explicitly. -# Note that we do not use --ignore-missing-imports, -# as this can give us false confidence in our results. -# mypy faststream -./scripts/lint.sh diff --git a/scripts/lint.sh b/scripts/lint.sh deleted file mode 100755 index e9d341a0a0..0000000000 --- a/scripts/lint.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e - -echo "Running ruff linter (isort, flake, pyupgrade, etc. replacement)..." -ruff check --exit-non-zero-on-fix - -echo "Running ruff formatter (black replacement)..." -ruff format - -echo "Running codespell to find typos..." -codespell diff --git a/scripts/serve-docs.sh b/scripts/serve-docs.sh deleted file mode 100755 index 5cd16f63d6..0000000000 --- a/scripts/serve-docs.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -x - -cd docs; python docs.py live "$@" diff --git a/scripts/start_test_env.sh b/scripts/start_test_env.sh deleted file mode 100755 index 906556db41..0000000000 --- a/scripts/start_test_env.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source ./scripts/set_variables.sh - -docker compose -p $DOCKER_COMPOSE_PROJECT -f docs/includes/docker-compose.yaml up -d --no-recreate diff --git a/scripts/static-analysis.sh b/scripts/static-analysis.sh deleted file mode 100755 index bdf2183f4b..0000000000 --- a/scripts/static-analysis.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -e - -echo "Running mypy..." -mypy - -echo "Running bandit..." -bandit -c pyproject.toml -r faststream - -echo "Running semgrep..." -semgrep scan --config auto --error diff --git a/scripts/static-pre-commit.sh b/scripts/static-pre-commit.sh deleted file mode 100755 index d984100832..0000000000 --- a/scripts/static-pre-commit.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -# taken from: https://jaredkhan.com/blog/mypy-pre-commit - -# A script for running mypy, -# with all its dependencies installed. - -set -o errexit - -# Change directory to the project root directory. -cd "$(dirname "$0")"/.. - -# Install the dependencies into the mypy env. -# Note that this can take seconds to run. -# In my case, I need to use a custom index URL. -# Avoid pip spending time quietly retrying since -# likely cause of failure is lack of VPN connection. -pip install --editable ".[dev]" \ - --retries 1 \ - --no-input \ - --quiet - -# Run on all files, -# ignoring the paths passed to this script, -# so as not to miss type errors. -# My repo makes use of namespace packages. -# Use the namespace-packages flag -# and specify the package to run on explicitly. -# Note that we do not use --ignore-missing-imports, -# as this can give us false confidence in our results. -# mypy faststream -./scripts/static-analysis.sh diff --git a/scripts/stop_test_env.sh b/scripts/stop_test_env.sh deleted file mode 100755 index 76ab4a3ee0..0000000000 --- a/scripts/stop_test_env.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -source ./scripts/set_variables.sh - -docker compose -p $DOCKER_COMPOSE_PROJECT -f docs/includes/docker-compose.yaml down diff --git a/scripts/test-cov.sh b/scripts/test-cov.sh deleted file mode 100755 index f8a4389eab..0000000000 --- a/scripts/test-cov.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -bash scripts/test.sh -m "all" "$@" - -coverage combine -coverage report --show-missing --skip-covered --sort=cover --precision=2 - -rm .coverage* diff --git a/scripts/test.sh b/scripts/test.sh deleted file mode 100755 index 09960abead..0000000000 --- a/scripts/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -coverage run -m pytest -x --ff "$@" || \ -coverage run -m pytest -x --ff "$@" || \ -coverage run -m pytest -x --ff "$@"