Merge pull request #800 from atugushev/normalize-compile-command-header

Normalize «command to run» in pip-compile headers.

piptools/__init__.py

@@ -0,0 +1,11 @@
+import locale
+
+from piptools.click import secho
+
+# Needed for locale.getpreferredencoding(False) to work
+# in pip._internal.utils.encoding.auto_decode
+try:
+    locale.setlocale(locale.LC_ALL, "")
+except locale.Error as e:  # pragma: no cover
+    # setlocale can apparently crash if locale are uninitialized
+    secho("Ignoring error when setting locale: {}".format(e), fg="red")

piptools/scripts/compile.py

@@ -394,6 +394,7 @@ def cli(
     writer = OutputWriter(
         src_files,
         output_file,
+        click_ctx=ctx,
         dry_run=dry_run,
         emit_header=header,
         emit_index=index,

piptools/utils.py

@@ -5,10 +5,21 @@
 from collections import OrderedDict
 from itertools import chain, groupby
 
+import six
+from six.moves import shlex_quote
+
 from ._compat import install_req_from_line
 from .click import style
 
 UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"}
+COMPILE_EXCLUDE_OPTIONS = {
+    "--dry-run",
+    "--quiet",
+    "--rebuild",
+    "--upgrade",
+    "--upgrade-package",
+    "--verbose",
+}
 
 
 def key_from_ireq(ireq):
@@ -260,3 +271,93 @@ def get_hashes_from_ireq(ireq):
         for hash_ in hexdigests:
             result.append("{}:{}".format(algorithm, hash_))
     return result
+
+
+def force_text(s):
+    """
+    Return a string representing `s`.
+    """
+    if s is None:
+        return ""
+    if not isinstance(s, six.string_types):
+        return six.text_type(s)
+    return s
+
+
+def get_compile_command(click_ctx):
+    """
+    Returns a normalized compile command depending on cli context.
+
+    The command will be normalized by:
+        - expanding options short to long
+        - removing values that are already default
+        - sorting the arguments
+        - removing one-off arguments like '--upgrade'
+        - removing arguments that don't change build behaviour like '--verbose'
+    """
+    from piptools.scripts.compile import cli
+
+    # Map of the compile cli options (option name -> click.Option)
+    compile_options = {option.name: option for option in cli.params}
+
+    left_args = []
+    right_args = []
+
+    for option_name, value in click_ctx.params.items():
+        option = compile_options[option_name]
+
+        # Get the latest option name (usually it'll be a long name)
+        option_long_name = option.opts[-1]
+
+        # Collect variadic args separately, they will be added
+        # at the end of the command later
+        if option.nargs < 0:
+            right_args.extend([shlex_quote(force_text(val)) for val in value])
+            continue
+
+        # Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...)
+        # or options that don't change compile behaviour (--verbose/--dry-run/...)
+        if option_long_name in COMPILE_EXCLUDE_OPTIONS:
+            continue
+
+        # Skip options without a value
+        if option.default is None and not value:
+            continue
+
+        # Skip options with a default value
+        if option.default == value:
+            continue
+
+        # Use a file name for file-like objects
+        if (
+            hasattr(value, "write")
+            and hasattr(value, "read")
+            and hasattr(value, "name")
+        ):
+            value = value.name
+
+        # Convert value to the list
+        if not isinstance(value, (tuple, list)):
+            value = [value]
+
+        for val in value:
+            # Flags don't have a value, thus add to args true or false option long name
+            if option.is_flag:
+                # If there are false-options, choose an option name depending on a value
+                if option.secondary_opts:
+                    # Get the latest false-option
+                    secondary_option_long_name = option.secondary_opts[-1]
+                    arg = option_long_name if val else secondary_option_long_name
+                # There are no false-options, use true-option
+                else:
+                    arg = option_long_name
+                left_args.append(shlex_quote(arg))
+            # Append to args the option with a value
+            else:
+                left_args.append(
+                    "{option}={value}".format(
+                        option=option_long_name, value=shlex_quote(force_text(val))
+                    )
+                )
+
+    return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args))

piptools/writer.py

@@ -1,17 +1,26 @@
+from __future__ import unicode_literals
+
 import os
-import sys
 from itertools import chain
 
-from .click import get_os_args, unstyle
+from .click import unstyle
 from .logging import log
-from .utils import UNSAFE_PACKAGES, comment, dedup, format_requirement, key_from_req
+from .utils import (
+    UNSAFE_PACKAGES,
+    comment,
+    dedup,
+    format_requirement,
+    get_compile_command,
+    key_from_req,
+)
 
 
 class OutputWriter(object):
     def __init__(
         self,
         src_files,
         dst_file,
+        click_ctx,
         dry_run,
         emit_header,
         emit_index,
@@ -27,6 +36,7 @@ def __init__(
     ):
         self.src_files = src_files
         self.dst_file = dst_file
+        self.click_ctx = click_ctx
         self.dry_run = dry_run
         self.emit_header = emit_header
         self.emit_index = emit_index
@@ -49,13 +59,10 @@ def write_header(self):
             yield comment("# This file is autogenerated by pip-compile")
             yield comment("# To update, run:")
             yield comment("#")
-            custom_cmd = os.environ.get("CUSTOM_COMPILE_COMMAND")
-            if custom_cmd:
-                yield comment("#    {}".format(custom_cmd))
-            else:
-                prog = os.path.basename(sys.argv[0])
-                args = " ".join(get_os_args())
-                yield comment("#    {prog} {args}".format(prog=prog, args=args))
+            compile_command = os.environ.get(
+                "CUSTOM_COMPILE_COMMAND"
+            ) or get_compile_command(self.click_ctx)
+            yield comment("#    {}".format(compile_command))
             yield comment("#")
 
     def write_index_options(self):

tests/conftest.py

@@ -1,7 +1,6 @@
 import json
 from contextlib import contextmanager
 from functools import partial
-from tempfile import NamedTemporaryFile
 
 from click.testing import CliRunner
 from pip._vendor.packaging.version import Version
@@ -139,6 +138,6 @@ def runner():
 
 
 @fixture
-def tmp_file():
-    with NamedTemporaryFile("wt") as fp:
-        yield fp
+def tmpdir_cwd(tmpdir):
+    with tmpdir.as_cwd():
+        yield tmpdir

tests/test_utils.py

@@ -1,13 +1,22 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import os
+
 import six
 from pytest import mark, raises
+from six.moves import shlex_quote
 
+from piptools.scripts.compile import cli as compile_cli
 from piptools.utils import (
     as_tuple,
     dedup,
     flat_map,
+    force_text,
     format_requirement,
     format_specifier,
     fs_str,
+    get_compile_command,
     get_hashes_from_ireq,
     is_pinned_requirement,
     name_from_req,
@@ -164,12 +173,128 @@ def test_name_from_req_with_project_name(from_line):
 
 
 def test_fs_str():
-    assert fs_str(u"some path component/Something") == "some path component/Something"
+    assert fs_str("some path component/Something") == "some path component/Something"
     assert isinstance(fs_str("whatever"), str)
-    assert isinstance(fs_str(u"whatever"), str)
 
 
 @mark.skipif(six.PY2, reason="Not supported in py2")
 def test_fs_str_with_bytes():
     with raises(AssertionError):
         fs_str(b"whatever")
+
+
+@mark.parametrize(
+    "value, expected_text", [(None, ""), (42, "42"), ("foo", "foo"), ("bãr", "bãr")]
+)
+def test_force_text(value, expected_text):
+    assert force_text(value) == expected_text
+
+
+@mark.parametrize(
+    "cli_args, expected_command",
+    [
+        # Check empty args
+        ([], "pip-compile"),
+        # Check all options which will be excluded from command
+        (["-v"], "pip-compile"),
+        (["--verbose"], "pip-compile"),
+        (["-n"], "pip-compile"),
+        (["--dry-run"], "pip-compile"),
+        (["-q"], "pip-compile"),
+        (["--quiet"], "pip-compile"),
+        (["-r"], "pip-compile"),
+        (["--rebuild"], "pip-compile"),
+        (["-U"], "pip-compile"),
+        (["--upgrade"], "pip-compile"),
+        (["-P", "django"], "pip-compile"),
+        (["--upgrade-package", "django"], "pip-compile"),
+        # Check options
+        (["--max-rounds", "42"], "pip-compile --max-rounds=42"),
+        (["--index-url", "https://foo"], "pip-compile --index-url=https://foo"),
+        # Check that short options will be expanded to long options
+        (["-p"], "pip-compile --pre"),
+        (["-f", "links"], "pip-compile --find-links=links"),
+        (["-i", "https://foo"], "pip-compile --index-url=https://foo"),
+        # Check positive flags
+        (["--generate-hashes"], "pip-compile --generate-hashes"),
+        (["--pre"], "pip-compile --pre"),
+        (["--allow-unsafe"], "pip-compile --allow-unsafe"),
+        # Check negative flags
+        (["--no-index"], "pip-compile --no-index"),
+        (["--no-emit-trusted-host"], "pip-compile --no-emit-trusted-host"),
+        (["--no-annotate"], "pip-compile --no-annotate"),
+        # Check that default values will be removed from the command
+        (["--emit-trusted-host"], "pip-compile"),
+        (["--annotate"], "pip-compile"),
+        (["--index"], "pip-compile"),
+        (["--max-rounds=10"], "pip-compile"),
+        (["--no-build-isolation"], "pip-compile"),
+        # Check options with multiple values
+        (
+            ["--find-links", "links1", "--find-links", "links2"],
+            "pip-compile --find-links=links1 --find-links=links2",
+        ),
+        # Check that option values will be quoted
+        (["-f", "foo;bar"], "pip-compile --find-links='foo;bar'"),
+        (["-f", "συνδέσεις"], "pip-compile --find-links='συνδέσεις'"),
+        (["-o", "my file.txt"], "pip-compile --output-file='my file.txt'"),
+        (["-o", "απαιτήσεις.txt"], "pip-compile --output-file='απαιτήσεις.txt'"),
+    ],
+)
+def test_get_compile_command(tmpdir_cwd, cli_args, expected_command):
+    """
+    Test general scenarios for the get_compile_command function.
+    """
+    with compile_cli.make_context("pip-compile", cli_args) as ctx:
+        assert get_compile_command(ctx) == expected_command
+
+
+@mark.parametrize(
+    "filename", ["requirements.in", "my requirements.in", "απαιτήσεις.txt"]
+)
+def test_get_compile_command_with_files(tmpdir_cwd, filename):
+    """
+    Test that get_compile_command returns a command with correct
+    and sanitized file names.
+    """
+    os.mkdir("sub")
+
+    path = os.path.join("sub", filename)
+    with open(path, "w"):
+        pass
+
+    args = [path, "--output-file", "requirements.txt"]
+    with compile_cli.make_context("pip-compile", args) as ctx:
+        assert get_compile_command(
+            ctx
+        ) == "pip-compile --output-file=requirements.txt {src_file}".format(
+            src_file=shlex_quote(path)
+        )
+
+
+def test_get_compile_command_sort_args(tmpdir_cwd):
+    """
+    Test that get_compile_command correctly sorts arguments.
+
+    The order is "pip-compile {sorted options} {sorted src files}".
+    """
+    with open("setup.py", "w"), open("requirements.in", "w"):
+        pass
+
+    args = [
+        "--no-index",
+        "--no-emit-trusted-host",
+        "--no-annotate",
+        "setup.py",
+        "--find-links",
+        "foo",
+        "--find-links",
+        "bar",
+        "requirements.in",
+    ]
+    with compile_cli.make_context("pip-compile", args) as ctx:
+        assert get_compile_command(ctx) == (
+            "pip-compile --find-links=bar --find-links=foo "
+            "--no-annotate --no-emit-trusted-host --no-index "
+            "requirements.in setup.py"
+        )