fix: git thought some php files were binary and refused to diff (#928)

* fix: git thought some php files were binary and refused to diff

Pass the --binary flag to git diff so it can diff files no matter if they're binary or text.

Fixes #918

* fix: add the --binary flag to one more 'git diff' statement

About 1/3 of the APIs are still failing due to binary files.

* fix: don't attempt to parse the result of git diff as text

* fix: update regex code to work with binary

* fix: yet another binary-utf8 issue

autosynth/git.py

@@ -145,7 +145,7 @@ def patch_merge(
     """
     with open(patch_file_path, "wb+") as patch_file:
         executor.check_call(
-            ["git", "diff", "HEAD", branch_name], stdout=patch_file, cwd=git_repo_dir
+            ["git", "diff", "--binary", "HEAD", branch_name], stdout=patch_file, cwd=git_repo_dir
         )
     if os.stat(patch_file_path).st_size:
         executor.check_call(["git", "apply", patch_file_path], cwd=git_repo_dir)

autosynth/multi.py

@@ -124,7 +124,7 @@ def synthesize_library(
         logger.error(f"Synthesis failed for {library['name']}")
     return {
         "name": library["name"],
-        "output": output.decode("utf-8"),
+        "output": output.decode("utf-8", errors="ignore"),
         "error": error,
         "skipped": skipped,
     }

autosynth/synth_toolbox.py

@@ -337,9 +337,9 @@ def _compose_pr_title(
 
 def git_branches_differ(branch_a: str, branch_b: str, metadata_path: str) -> bool:
     # Check to see if any files besides synth.metadata were added, modified, deleted.
-    diff_cmd = ["git", "diff", f"{branch_a}..{branch_b}"]
+    diff_cmd = ["git", "diff", "--binary", f"{branch_a}..{branch_b}"]
     diff_cmd.extend(["--", ".", f":(exclude){metadata_path}"])
-    proc = executor.run(diff_cmd, stdout=subprocess.PIPE, universal_newlines=True)
+    proc = executor.run(diff_cmd, stdout=subprocess.PIPE)
     proc.check_returncode()
     if bool(proc.stdout):
         return True

synthtool/transforms.py

@@ -224,17 +224,28 @@ def move(
 
 
 def _replace_in_file(path, expr, replacement):
-    with path.open("r+") as fh:
-        content = fh.read()
-        content, count = expr.subn(replacement, content)
-
-        # Don't bother writing the file if we didn't change
-        # anything.
-        if count:
-            fh.seek(0)
-            fh.write(content)
-            fh.truncate()
-        return count
+    try:
+        with path.open("r+") as fh:
+            return _replace_in_file_handle(fh, expr, replacement)
+    except UnicodeDecodeError:
+        pass  # It's a binary file.  Try again with a binary regular expression.
+    flags = expr.flags & ~re.UNICODE
+    expr = re.compile(expr.pattern.encode(), flags)
+    with path.open("rb+") as fh:
+        return _replace_in_file_handle(fh, expr, replacement.encode())
+
+
+def _replace_in_file_handle(fh, expr, replacement):
+    content = fh.read()
+    content, count = expr.subn(replacement, content)
+
+    # Don't bother writing the file if we didn't change
+    # anything.
+    if count:
+        fh.seek(0)
+        fh.write(content)
+        fh.truncate()
+    return count
 
 
 def replace(