diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a3dcf12b9d..cdc7867fc8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,27 +115,16 @@ jobs: path: ${{ steps.pip-cache.outputs.dir }} restore-keys: | pip-ci-${{ runner.os }}-${{ matrix.pyver }}-{{ matrix.no-extensions }}- - - name: Install cython - if: ${{ matrix.no-extensions == '' }} - uses: py-actions/py-dependency-install@v2 - with: - path: requirements/cython.txt - name: Cythonize if: ${{ matrix.no-extensions == '' }} run: | make cythonize - - name: Install dependencies - uses: py-actions/py-dependency-install@v2 - with: - path: requirements/test.txt - env: - AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} - name: Run unittests env: COLOR: 'yes' AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }} run: | - python -m pytest tests -vv + make vvtest python -m coverage xml - name: Upload coverage uses: codecov/codecov-action@v1 @@ -168,10 +157,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Install cython - uses: py-actions/py-dependency-install@v2 - with: - path: requirements/cython.txt - name: Cythonize run: | make cythonize @@ -210,11 +195,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Install cython - if: ${{ matrix.no-extensions == '' }} - uses: py-actions/py-dependency-install@v2 - with: - path: requirements/cython.txt - name: Cythonize if: ${{ matrix.no-extensions == '' }} run: | @@ -255,11 +235,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.pyver }} - - name: Install cython - if: ${{ matrix.no-extensions == '' }} - uses: py-actions/py-dependency-install@v2 - with: - path: requirements/cython.txt - name: Cythonize if: ${{ matrix.no-extensions == '' }} run: | diff --git a/.gitignore b/.gitignore index 0b83f6e5228..69f52e10d87 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ .envrc .flake .gitconfig +.hash .idea .install-cython .install-deps diff --git a/Makefile b/Makefile index 6327d654fbc..13cd76487eb 100644 --- a/Makefile +++ b/Makefile @@ -1,47 +1,67 @@ # Some simple testing tasks (sorry, UNIX only). -to-md5 = $1 $(addsuffix .md5,$1) +to-hash-one = $(dir $1).hash/$(addsuffix .hash,$(notdir $1)) +to-hash = $(foreach fname,$1,$(call to-hash-one,$(fname))) -CYS = $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi) $(wildcard aiohttp/*.pxd) -PYXS = $(wildcard aiohttp/*.pyx) -CS = $(wildcard aiohttp/*.c) -PYS = $(wildcard aiohttp/*.py) -REQS = $(wildcard requirements/*.txt) -SRC = aiohttp examples tests setup.py +CYS := $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi) $(wildcard aiohttp/*.pxd) +PYXS := $(wildcard aiohttp/*.pyx) +CS := $(wildcard aiohttp/*.c) +PYS := $(wildcard aiohttp/*.py) +REQS := $(wildcard requirements/*.txt) +ALLS := $(sort $(CYS) $(CS) $(PYS) $(REQS)) .PHONY: all all: test -# Recipe from https://www.cmcrossroads.com/article/rebuilding-when-files-checksum-changes -%.md5: FORCE - @$(if $(filter-out $(shell cat $@ 2>/dev/null),$(shell md5sum $*)),md5sum $* > $@) +tst: + @echo $(call to-hash,requirements/cython.txt) + @echo $(call to-hash,aiohttp/%.pyx) + +# Recipe from https://www.cmcrossroads.com/article/rebuilding-when-files-checksum-changes FORCE: +# check_sum.py works perfectly fine but slow when called for every file from $(ALLS) +# (perhaps even several times for each file). +# That is why much less readable but faster solution exists +ifneq (, $(shell which sha256sum)) +%.hash: FORCE + $(eval $@_ABS := $(abspath $@)) + $(eval $@_NAME := $($@_ABS)) + $(eval $@_HASHDIR := $(dir $($@_ABS))) + $(eval $@_TMP := $($@_HASHDIR)../$(notdir $($@_ABS))) + $(eval $@_ORIG := $(subst /.hash/../,/,$(basename $($@_TMP)))) + @#echo ==== $($@_ABS) $($@_HASHDIR) $($@_NAME) $($@_TMP) $($@_ORIG) + @if ! (sha256sum --check $($@_ABS) 1>/dev/null 2>/dev/null); then \ + mkdir -p $($@_HASHDIR); \ + echo re-hash $($@_ORIG); \ + sha256sum $($@_ORIG) > $($@_ABS); \ + fi +else +%.hash: FORCE + @./tools/check_sum.py $@ # --debug +endif + # Enumerate intermediate files to don't remove them automatically. -# The target must exist, no need to execute it. -.PHONY: _keep-intermediate-files -_keep-intermediate-files: $(addsuffix .md5,$(CYS))\ - $(addsuffix .md5,$(CS))\ - $(addsuffix .md5,$(PYS))\ - $(addsuffix .md5,$(REQS)) - -.install-cython: $(call to-md5,requirements/cython.txt) +.SECONDARY: $(call to-hash,$(ALLS)) + + +.install-cython: $(call to-hash,requirements/cython.txt) pip install -r requirements/cython.txt @touch .install-cython -aiohttp/_find_header.c: $(call to-md5,aiohttp/hdrs.py) +aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py) ./tools/gen.py # _find_headers generator creates _headers.pyi as well -aiohttp/%.c: $(call to-md5,aiohttp/%.pyx) aiohttp/_find_header.c +aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c cython -3 -o $@ $< -I aiohttp .PHONY: cythonize cythonize: .install-cython $(PYXS:.pyx=.c) -.install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-md5,$(CYS) $(REQS)) +.install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS)) pip install -r requirements/dev.txt @touch .install-deps @@ -56,7 +76,7 @@ fmt format: mypy: mypy aiohttp -.develop: .install-deps $(call to-md5,$(PYS) $(CYS) $(CS)) +.develop: .install-deps $(call to-hash,$(PYS) $(CYS) $(CS)) pip install -e . @touch .develop @@ -68,9 +88,15 @@ test: .develop vtest: .develop @pytest -s -v +.PHONY: vvtest +vvtest: .develop + @pytest -vv + .PHONY: clean clean: @rm -rf `find . -name __pycache__` + @rm -rf `find . -name .hash` + @rm -rf `find . -name .md5` # old styling @rm -f `find . -type f -name '*.py[co]' ` @rm -f `find . -type f -name '*~' ` @rm -f `find . -type f -name '.*~' ` @@ -78,6 +104,7 @@ clean: @rm -f `find . -type f -name '#*#' ` @rm -f `find . -type f -name '*.orig' ` @rm -f `find . -type f -name '*.rej' ` + @rm -f `find . -type f -name '*.md5' ` # old styling @rm -f .coverage @rm -rf htmlcov @rm -rf build diff --git a/requirements/lint.txt b/requirements/lint.txt index 7ed1d685283..969b7909bad 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -1,4 +1,4 @@ -black==20.8b1; python_version >= "3.6" +black==20.8b1; implementation_name=="cpython" flake8==3.8.4 flake8-pyi==20.10.0 isort==5.6.4 diff --git a/tools/check_sum.py b/tools/check_sum.py new file mode 100755 index 00000000000..50dec4d2be5 --- /dev/null +++ b/tools/check_sum.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import argparse +import hashlib +import pathlib +import sys + +PARSER = argparse.ArgumentParser( + description="Helper for check file hashes in Makefile instead of bare timestamps" +) +PARSER.add_argument("dst", metavar="DST", type=pathlib.Path) +PARSER.add_argument("-d", "--debug", action="store_true", default=False) + + +def main(argv): + args = PARSER.parse_args(argv) + dst = args.dst + assert dst.suffix == ".hash" + dirname = dst.parent + if dirname.name != ".hash": + if args.debug: + print(f"Invalid name {dst} -> dirname {dirname}", file=sys.stderr) + return 0 + dirname.mkdir(exist_ok=True) + src_dir = dirname.parent + src_name = dst.stem # drop .hash + full_src = src_dir / src_name + hasher = hashlib.sha256() + try: + hasher.update(full_src.read_bytes()) + except OSError: + if args.debug: + print(f"Cannot open {full_src}", file=sys.stderr) + return 0 + src_hash = hasher.hexdigest() + if dst.exists(): + dst_hash = dst.read_text() + else: + dst_hash = "" + if src_hash != dst_hash: + dst.write_text(src_hash) + print(f"re-hash {src_hash}") + else: + if args.debug: + print(f"Skip {src_hash} checksum, up-to-date") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))