pre-commit, contribution guideline, script/test now creates one virtualenv using -m venv

Author: temoto

.github/workflows/publish.yaml

@@ -39,6 +39,8 @@ jobs:
         run: check-manifest
       - name: twine check
         run: twine check dist/*
+      - name: script/test install_check_version
+        run: stages=venv,check_version script/test
 
       - uses: actions/upload-artifact@v2
         with:

.github/workflows/style.yaml

@@ -28,10 +28,7 @@ jobs:
       - name: pip install
         run: pip install black isort -r requirements-test.txt
       - name: isort
-        run: isort --check python2/ python3/ tests/ || echo FIXME
+        run: isort --check python3/ tests/ || echo FIXME
       - name: black
-        run: black --check --diff python2/ python3/ tests/ || echo FIXME
-      - name: flake8 python2/
-        run: flake8 python2/
-      - name: flake8 python3/ tests/
-        run: flake8 python3/ tests/
+        run: black --check --diff python3/ tests/ || echo FIXME
+      - run: stages=venv,style script/test

.github/workflows/test.yaml

@@ -22,6 +22,8 @@ jobs:
           - { py: 3.7, ignore-error: false }
           - { py: 3.8, ignore-error: false }
           - { py: 3.9, ignore-error: false }
+          - { py: "3.10", ignore-error: false }
+          - { py: "3.11", ignore-error: false }
           - { py: 3.x, ignore-error: true }
           - { py: pypy2, ignore-error: false }
           - { py: pypy3, ignore-error: false }
@@ -46,7 +48,7 @@ jobs:
       - run: pip install tox-gh
         if: startsWith(matrix.py, '3.') && matrix.py >= '3.6'
       - run: pip install 'codecov>=2.0.15' -r requirements-test.txt
-      - run: script/test -sv
+      - run: stages=venv,test script/test -sv
       - run: codecov --flags=$(echo ${{ matrix.py }} |tr -d -- '-.')
 
   test_py37_openssl11:

.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-ast
+      - id: check-executables-have-shebangs
+      - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-toml
+      - id: check-yaml
+      # - id: end-of-file-fixer
+      # - id: trailing-whitespace
+
+  # TODO enable after py2/3 merge and initial black/isort run
+  # - repo: https://github.com/asottile/pyupgrade
+  #   rev: v3.1.0
+  #   hooks:
+  #     - id: pyupgrade
+
+  - repo: local
+    hooks:
+      - id: pre-commit-validate-config
+        name: validate pre-commit config
+        files: "^.pre-commit-config.yaml$"
+        types: [yaml]
+        entry: pre-commit validate-config
+        language: system
+        fail_fast: true
+
+      - id: test
+        name: run code style and unit tests
+        types_or: [text, python, shell, toml, yaml]
+        exclude: "^(build|dist|doc|python2|ref|venv[^/]*?)/"
+        entry: script/test -sv
+        language: system
+        pass_filenames: false
+        require_serial: true

README.md

@@ -4,6 +4,8 @@ Introduction
 httplib2 is a comprehensive HTTP client library, httplib2.py supports many
 features left out of other HTTP libraries.
 
+If you want to help this project by bug report or code change, [contribution guidelines](contributing.md) may contain useful information.
+
 ### HTTP and HTTPS
 
 HTTPS support is only available if the socket module was

contributing.md

@@ -0,0 +1,26 @@
+# Welcome
+
+You have come to the right place if you want to support httplib2 by doing. Thank you!
+
+- [Reporting security / sensitive issues](SECURITY.md)
+- [Regular issues](https://github.com/httplib2/httplib2/issues?q=) please try to find existing solution first, search without issue/open filters.
+- If you intend to propose code change, there is no need to open an issue first. Pull request is more than enough. `git diff` in message is fine too.
+- Tests use TLS certificates/keys generated by `script/generate-tls`
+
+
+## Submitting changes (pull request policy)
+
+- Run `pre-commit install` once to setup hooks that detect/fix common style issues and run unit tests. [pre-commit website](https://pre-commit.com/)
+- If you don't want pre-commit hooks, please run `script/test` before commit
+- Test is required, or at least coverage must not decrease
+- One commit is strongly preferred, except for very big changes
+- Feel free to append yourself into __contributors__
+- Commit message should follow the following formula:
+
+>subsystem: description of why the change is useful
+>
+>optional details
+>
+>links to related issues or websites
+
+The why part is very important. Diff already says what is changed. Explain why.

script/test

@@ -8,61 +8,62 @@ if [[ -z "${CI:-}" ]] && [[ -t 1 ]] ; then
 fi
 test_flags=(
 	$@
-	$flag_forked
+	"$flag_forked"
 	tests/
 )
+if [[ -n "${CI:-}" ]] ; then
+	test_flags+=(--fulltrace)
+fi
+: "${stages="venv,style,test,check_version"}"
 
 main() {
 	cd "$( dirname "${BASH_SOURCE[0]}" )/.."
-	if [[ -n "${CI:-}" ]] ; then
-		case "${test_group-}" in
-		package)
-			# TODO: sdist bdist_wheel
-			# but wheels don't roll well with our 2/3 split code base
-			python setup.py sdist
-			install_check_version "pip"
-			;;
-		*)
-			pip install -e .
-			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_303 $@ tests/ || true
-			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_head_301 $@ tests/ || true
-			pytest --fulltrace ${test_flags[@]}
-			;;
-		esac
-	else
-		if [[ ! -d ./venv-27 ]] ; then
-			virtualenv --python=python2.7 ./venv-27
-		fi
-		if [[ ! -d ./venv-36 ]] ; then
-			virtualenv --python=python3.6 ./venv-36
-		fi
+	if enabled venv && [[ -z "${CI:-}${VIRTUAL_ENV:-}" ]] ; then
+		echo "environment: neither CI nor existing virtualenv, using \`python -m venv ./venv\`" >&2
+		[[ -x venv/bin/pip ]] || python -m venv venv
+		source venv/bin/activate
+	fi
 
-		./venv-27/bin/pip install -e . -r requirements-test.txt
-		./venv-27/bin/pytest ${test_flags[@]}
-		./venv-36/bin/pip install -e . -r requirements-test.txt
-		./venv-36/bin/pytest ${test_flags[@]}
+	if enabled style ; then
+		pip install black isort
+		grep -E 'flake8' <requirements-test.txt |xargs pip install
+		# TODO enable after py2/3 merge and initial black/isort run
+        # isort python3/ tests/
+        # black python3/ tests/
+        flake8 python3/ tests/
+	fi
 
-		# FIXME: too many errors
-		# ./venv-27/bin/flake8 python2/
-		# ./venv-36/bin/flake8 python3/ tests/
+	if enabled test ; then
+		pip install -e . -r requirements-test.txt
+		if [[ -n "${CI:-}" ]] ; then
+			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_303 "$@" tests/ || true
+			httplib2_test_still_run_skipped=1 pytest --fulltrace -k test_head_301 "$@" tests/ || true
+		fi
+		pytest --fulltrace "${test_flags[@]}"
+	fi
 
+	if enabled check_version ; then
 		# TODO: sdist bdist_wheel
 		# but wheels don't roll well with our 2/3 split code base
-		./venv-36/bin/python setup.py sdist
-		install_check_version "./venv-27/bin/pip"
-		install_check_version "./venv-36/bin/pip"
+		python setup.py sdist
+		install_check_version
 	fi
+
 	rm -rf ./_httplib2_test_cache
 }
 
+enabled() {
+	[[ ",${stages}," = *,$1,* ]] || return 1
+}
+
 install_check_version() {
-	local pip="$1"
-	$pip install dist/httplib2*
+	# FIXME replace gz with whl after py2/3 merge
+	pip install dist/httplib2*gz
 	version_source=$(python setup.py --version)
-	version_installed=$($pip show httplib2 |fgrep Version: |cut -d' ' -f2)
+	version_installed=$(pip show httplib2 |grep -F Version: |cut -d' ' -f2)
 	if [[ "$version_source" != "$version_installed" ]] ; then
 		echo "error: installed package version=$version_installed does not match source=$version_source" >&2
-		exit 1
+		return 1
 	fi
 }