From 50525a2abf54abaca89437473c3659597b9e9a33 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 17 Oct 2023 05:00:37 +0000 Subject: [PATCH 01/10] Create linter --- .../adapters/requirements_txt_linter.py | 189 ++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 lintrunner_adapters/adapters/requirements_txt_linter.py diff --git a/lintrunner_adapters/adapters/requirements_txt_linter.py b/lintrunner_adapters/adapters/requirements_txt_linter.py new file mode 100644 index 0000000..088e2e3 --- /dev/null +++ b/lintrunner_adapters/adapters/requirements_txt_linter.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import argparse +import logging +import os +import re +import sys +from typing import IO +import concurrent.futures + +from lintrunner_adapters import ( + LintMessage, + LintSeverity, + add_default_options, +) + + +LINTER_CODE = "REQUIREMENTS-TXT" + + +class Requirement: + UNTIL_COMPARISON = re.compile(b"={2,3}|!=|~=|>=?|<=?") + UNTIL_SEP = re.compile(rb"[^;\s]+") + + def __init__(self) -> None: + self.value: bytes | None = None + self.comments: list[bytes] = [] + + @property + def name(self) -> bytes: + assert self.value is not None, self.value + name = self.value.lower() + for egg in (b"#egg=", b"&egg="): + if egg in self.value: + return name.partition(egg)[-1] + + m = self.UNTIL_SEP.match(name) + assert m is not None + + name = m.group() + m = self.UNTIL_COMPARISON.search(name) + if not m: + return name + + return name[: m.start()] + + def __lt__(self, requirement: Requirement) -> bool: + # \n means top of file comment, so always return True, + # otherwise just do a string comparison with value. + assert self.value is not None, self.value + if self.value == b"\n": + return True + elif requirement.value == b"\n": + return False + else: + return self.name < requirement.name + + def is_complete(self) -> bool: + return self.value is not None and not self.value.rstrip(b"\r\n").endswith(b"\\") + + def append_value(self, value: bytes) -> None: + if self.value is not None: + self.value += value + else: + self.value = value + + +def fix_requirements(f: IO[bytes]) -> bytes: + requirements: list[Requirement] = [] + before = list(f) + after: list[bytes] = [] + + before_string = b"".join(before) + + # adds new line in case one is missing + # AND a change to the requirements file is needed regardless: + if before and not before[-1].endswith(b"\n"): + before[-1] += b"\n" + + # If the file is empty (i.e. only whitespace/newlines) exit early + if before_string.strip() == b"": + return PASS + + for line in before: + # If the most recent requirement object has a value, then it's + # time to start building the next requirement object. + + if not len(requirements) or requirements[-1].is_complete(): + requirements.append(Requirement()) + + requirement = requirements[-1] + + # If we see a newline before any requirements, then this is a + # top of file comment. + if len(requirements) == 1 and line.strip() == b"": + if len(requirement.comments) and requirement.comments[0].startswith(b"#"): + requirement.value = b"\n" + else: + requirement.comments.append(line) + elif line.lstrip().startswith(b"#") or line.strip() == b"": + requirement.comments.append(line) + else: + requirement.append_value(line) + + # if a file ends in a comment, preserve it at the end + if requirements[-1].value is None: + rest = requirements.pop().comments + else: + rest = [] + + # find and remove pkg-resources==0.0.0 + # which is automatically added by broken pip package under Debian + requirements = [ + req for req in requirements if req.value != b"pkg-resources==0.0.0\n" + ] + + for requirement in sorted(requirements): + after.extend(requirement.comments) + assert requirement.value, requirement.value + after.append(requirement.value) + after.extend(rest) + + after_string = b"".join(after) + + return after_string + + +def check_file( + filename: str +) -> list[LintMessage]: + with open(filename, "rb") as f: + original = f.read() + with open(filename, "rb") as f: + replacement = fix_requirements(f) + + if original == replacement: + return [] + + return [ + LintMessage( + path=filename, + line=None, + char=None, + code=LINTER_CODE, + severity=LintSeverity.WARNING, + name="format", + original=original.decode("utf-8"), + replacement=replacement.decode("utf-8"), + description="Run `lintrunner -a` to apply this patch.", + ) + ] + +def main() -> None: + parser = argparse.ArgumentParser( + description=f"add-trailing-comma wrapper linter. Linter code: {LINTER_CODE}", + fromfile_prefix_chars="@", + ) + add_default_options(parser) + args = parser.parse_args() + + logging.basicConfig( + format="<%(threadName)s:%(levelname)s> %(message)s", + level=logging.NOTSET + if args.verbose + else logging.DEBUG + if len(args.filenames) < 1000 + else logging.INFO, + stream=sys.stderr, + ) + + with concurrent.futures.ThreadPoolExecutor( + max_workers=os.cpu_count(), + thread_name_prefix="Thread", + ) as executor: + futures = { + executor.submit(check_file, x, args.retries, args.timeout): x + for x in args.filenames + } + for future in concurrent.futures.as_completed(futures): + try: + for lint_message in future.result(): + lint_message.display() + except Exception: + logging.critical('Failed at "%s".', futures[future]) + raise + + +if __name__ == "__main__": + main() From f8789545f5548246df74c42021cd86555e8e5165 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 17 Oct 2023 05:04:34 +0000 Subject: [PATCH 02/10] Fix --- .lintrunner.toml | 12 ++++++++++++ .../adapters/requirements_txt_linter.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index f6027da..221e7ea 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -370,3 +370,15 @@ init_command = [ '--dry-run={{DRYRUN}}', 'toml-sort==0.23.1', ] + + +[[linter]] +code = 'REQUIREMENTS-TXT' +is_formatter = true +include_patterns = ['requirements*.txt'] +exclude_patterns = [] +command = [ + 'python', + 'lintrunner_adapters/adapters/requirements_txt_linter.py', + '@{{PATHSFILE}}', +] diff --git a/lintrunner_adapters/adapters/requirements_txt_linter.py b/lintrunner_adapters/adapters/requirements_txt_linter.py index 088e2e3..3e198b8 100644 --- a/lintrunner_adapters/adapters/requirements_txt_linter.py +++ b/lintrunner_adapters/adapters/requirements_txt_linter.py @@ -173,7 +173,7 @@ def main() -> None: thread_name_prefix="Thread", ) as executor: futures = { - executor.submit(check_file, x, args.retries, args.timeout): x + executor.submit(check_file, x): x for x in args.filenames } for future in concurrent.futures.as_completed(futures): From 2ee032b7b66a32d7412145b43bf2a2f2a2c617a6 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 17 Oct 2023 05:04:50 +0000 Subject: [PATCH 03/10] Test example --- requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-test.txt b/requirements-test.txt index 61a4040..39570be 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,5 @@ # Test dependencies to avoid doctest import errors pyupgrade==3.3.1 +ufmt==2.0.1 refurb==1.10.0;python_version>="3.10" ufmt==2.0.1 From 2bdef991dac00462a76b9e9cbf85bada26d8cba7 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 17 Oct 2023 05:08:03 +0000 Subject: [PATCH 04/10] Format --- .../adapters/requirements_txt_linter.py | 26 +++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/lintrunner_adapters/adapters/requirements_txt_linter.py b/lintrunner_adapters/adapters/requirements_txt_linter.py index 3e198b8..8ca1af7 100644 --- a/lintrunner_adapters/adapters/requirements_txt_linter.py +++ b/lintrunner_adapters/adapters/requirements_txt_linter.py @@ -1,19 +1,14 @@ from __future__ import annotations import argparse +import concurrent.futures import logging import os import re import sys from typing import IO -import concurrent.futures - -from lintrunner_adapters import ( - LintMessage, - LintSeverity, - add_default_options, -) +from lintrunner_adapters import LintMessage, LintSeverity, add_default_options LINTER_CODE = "REQUIREMENTS-TXT" @@ -79,7 +74,7 @@ def fix_requirements(f: IO[bytes]) -> bytes: # If the file is empty (i.e. only whitespace/newlines) exit early if before_string.strip() == b"": - return PASS + return before_string for line in before: # If the most recent requirement object has a value, then it's @@ -103,10 +98,7 @@ def fix_requirements(f: IO[bytes]) -> bytes: requirement.append_value(line) # if a file ends in a comment, preserve it at the end - if requirements[-1].value is None: - rest = requirements.pop().comments - else: - rest = [] + rest = requirements.pop().comments if requirements[-1].value is None else [] # find and remove pkg-resources==0.0.0 # which is automatically added by broken pip package under Debian @@ -125,9 +117,7 @@ def fix_requirements(f: IO[bytes]) -> bytes: return after_string -def check_file( - filename: str -) -> list[LintMessage]: +def check_file(filename: str) -> list[LintMessage]: with open(filename, "rb") as f: original = f.read() with open(filename, "rb") as f: @@ -150,6 +140,7 @@ def check_file( ) ] + def main() -> None: parser = argparse.ArgumentParser( description=f"add-trailing-comma wrapper linter. Linter code: {LINTER_CODE}", @@ -172,10 +163,7 @@ def main() -> None: max_workers=os.cpu_count(), thread_name_prefix="Thread", ) as executor: - futures = { - executor.submit(check_file, x): x - for x in args.filenames - } + futures = {executor.submit(check_file, x): x for x in args.filenames} for future in concurrent.futures.as_completed(futures): try: for lint_message in future.result(): From 10d4b7dc95b97264990f17b528f63fdc067f132e Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Tue, 17 Oct 2023 05:13:46 +0000 Subject: [PATCH 05/10] Linter --- lintrunner_adapters/adapters/requirements_txt_linter.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lintrunner_adapters/adapters/requirements_txt_linter.py b/lintrunner_adapters/adapters/requirements_txt_linter.py index 8ca1af7..0993379 100644 --- a/lintrunner_adapters/adapters/requirements_txt_linter.py +++ b/lintrunner_adapters/adapters/requirements_txt_linter.py @@ -45,10 +45,9 @@ def __lt__(self, requirement: Requirement) -> bool: assert self.value is not None, self.value if self.value == b"\n": return True - elif requirement.value == b"\n": + if requirement.value == b"\n": return False - else: - return self.name < requirement.name + return self.name < requirement.name def is_complete(self) -> bool: return self.value is not None and not self.value.rstrip(b"\r\n").endswith(b"\\") @@ -80,7 +79,7 @@ def fix_requirements(f: IO[bytes]) -> bytes: # If the most recent requirement object has a value, then it's # time to start building the next requirement object. - if not len(requirements) or requirements[-1].is_complete(): + if not requirements or requirements[-1].is_complete(): requirements.append(Requirement()) requirement = requirements[-1] From 67886897649a4f5ee8a3ea6c7ecb39ed8f3f0544 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Mon, 16 Oct 2023 22:15:47 -0700 Subject: [PATCH 06/10] Update requirements_txt_linter.py --- .../adapters/requirements_txt_linter.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lintrunner_adapters/adapters/requirements_txt_linter.py b/lintrunner_adapters/adapters/requirements_txt_linter.py index 0993379..981e3e9 100644 --- a/lintrunner_adapters/adapters/requirements_txt_linter.py +++ b/lintrunner_adapters/adapters/requirements_txt_linter.py @@ -1,3 +1,24 @@ +# https://github.com/pre-commit/pre-commit-hooks/blob/main/pre_commit_hooks/requirements_txt_fixer.py +# Copyright (c) 2014 pre-commit dev team: Anthony Sottile, Ken Struys + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + from __future__ import annotations import argparse From 9953c997c0c60de05bdc5d1550509353bfeac12d Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Mon, 16 Oct 2023 22:16:24 -0700 Subject: [PATCH 07/10] Update .lintrunner.toml --- .lintrunner.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index 221e7ea..99baaf3 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -371,7 +371,6 @@ init_command = [ 'toml-sort==0.23.1', ] - [[linter]] code = 'REQUIREMENTS-TXT' is_formatter = true From 86d0c7dd098d8827ac4b38089c495f38efe7b442 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Mon, 16 Oct 2023 22:17:27 -0700 Subject: [PATCH 08/10] Update requirements_txt_linter.py --- lintrunner_adapters/adapters/requirements_txt_linter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lintrunner_adapters/adapters/requirements_txt_linter.py b/lintrunner_adapters/adapters/requirements_txt_linter.py index 981e3e9..0791034 100644 --- a/lintrunner_adapters/adapters/requirements_txt_linter.py +++ b/lintrunner_adapters/adapters/requirements_txt_linter.py @@ -163,7 +163,7 @@ def check_file(filename: str) -> list[LintMessage]: def main() -> None: parser = argparse.ArgumentParser( - description=f"add-trailing-comma wrapper linter. Linter code: {LINTER_CODE}", + description=f"Format Python requirements.txt files. Linter code: {LINTER_CODE}", fromfile_prefix_chars="@", ) add_default_options(parser) From 9eb3930c4531f1e67fdd8e7545e020805e5cb510 Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Mon, 16 Oct 2023 22:37:21 -0700 Subject: [PATCH 09/10] Update .lintrunner.toml --- .lintrunner.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index 99baaf3..daa8e1d 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -282,6 +282,7 @@ command = [ 'run', 'pyupgrade_linter', '--py37-plus', + '--', '@{{PATHSFILE}}', ] init_command = [ @@ -335,6 +336,7 @@ command = [ 'ruff_linter', '--config=pyproject.toml', '--show-disable', + '--', '@{{PATHSFILE}}', ] init_command = [ @@ -359,6 +361,7 @@ command = [ 'lintrunner_adapters', 'run', 'toml_sort_linter', + '--', '@{{PATHSFILE}}', ] init_command = [ @@ -378,6 +381,10 @@ include_patterns = ['requirements*.txt'] exclude_patterns = [] command = [ 'python', - 'lintrunner_adapters/adapters/requirements_txt_linter.py', + '-m', + 'lintrunner_adapters', + 'run', + 'requirements_txt_linter', + '--', '@{{PATHSFILE}}', ] From 6bdd61ee350978a3a6e504ce46507413ea394c2e Mon Sep 17 00:00:00 2001 From: Justin Chu Date: Mon, 16 Oct 2023 22:39:48 -0700 Subject: [PATCH 10/10] Update requirements-test.txt --- requirements-test.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 39570be..61a4040 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,4 @@ # Test dependencies to avoid doctest import errors pyupgrade==3.3.1 -ufmt==2.0.1 refurb==1.10.0;python_version>="3.10" ufmt==2.0.1