Skip to content

Commit

Permalink
chore: use protos from googleapis (#126)
Browse files Browse the repository at this point in the history
  • Loading branch information
@parthea
parthea authored May 26, 2022
1 parent adb59d0 commit bb7e31a
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 142 deletions.
15 changes: 0 additions & 15 deletions packages/google-cloud-access-context-manager/noxfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@
"system",
"lint",
"test",
"generate_protos",
"lint_setup_py",
"blacken",
"docs",
Expand Down Expand Up @@ -257,24 +256,10 @@ def generate_protos(session):
session.install("grpcio-tools")
protos = [str(p) for p in (pathlib.Path(".").glob("google/**/*.proto"))]

# Clone googleapis/api-common-protos
api_common_protos = "api-common-protos"
try:
session.run("git", "-C", api_common_protos, "pull", external=True)
except nox.command.CommandFailed:
session.run(
"git",
"clone",
"--single-branch",
f"https://github.com/googleapis/{api_common_protos}",
external=True,
)

session.run(
"python",
"-m",
"grpc_tools.protoc",
"--proto_path=api-common-protos",
"--proto_path=.",
"--python_out=.",
*protos,
Expand Down
160 changes: 33 additions & 127 deletions packages/google-cloud-access-context-manager/owlbot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,159 +11,65 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import shutil
import subprocess


import synthtool as s
import synthtool.gcp as gcp
from synthtool import tmp
from synthtool.languages import python
from synthtool.sources import git

GOOGLEAPIS_REPO = "googleapis/googleapis"

# Clean up googleapis
shutil.rmtree('googleapis', ignore_errors=True)

# Clone googleapis
googleapis_url = git.make_repo_clone_url(GOOGLEAPIS_REPO)
subprocess.run(["git", "clone", googleapis_url])

# This is required in order for s.copy() to work
s._tracked_paths.add("googleapis")

os.makedirs("google/api", exist_ok=True)
os.makedirs("google/type", exist_ok=True)

s.copy("googleapis/google/api/annotations.proto", "google/api")
s.copy("googleapis/google/api/http.proto", "google/api")
s.copy("googleapis/google/type/expr.proto", "google/type")

# Clean up googleapis
shutil.rmtree('googleapis')

# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
common = gcp.CommonTemplates()
templated_files = common.py_library(microgenerator=True)

s.move(
templated_files, excludes=[".coveragerc", ".gitignore", ".github/workflows"],
templated_files, excludes=[".coveragerc", ".gitignore", ".github/workflows", "noxfile.py"],
)

python.py_samples(skip_readmes=True)

# ----------------------------------------------------------------------------
# Customize noxfile.py
# ----------------------------------------------------------------------------

old_sessions = """
"unit",
"system",
"cover",
"lint",
"""

new_sessions = """
"system",
"lint",
"test",
"generate_protos",
"""

# Remove `unit` and `cover` sessions.
# Add `test` and `generate_protos` sessions.
s.replace("noxfile.py", old_sessions, new_sessions)

# Remove unit* sessions
s.replace("noxfile.py",
"""@nox.session\(python=UNIT_TEST_PYTHON_VERSIONS\)
def unit\(session\):""",
"""def unit(session):""",
)

# Remove cover session
s.replace("noxfile.py",
"""@nox.session\(python=DEFAULT_PYTHON_VERSION\)
def cover.*
session.run\("coverage", "erase"\)""",
"""""",
flags=re.MULTILINE | re.DOTALL,
)
# Generate _pb2.py files and format them
s.shell.run(["nox", "-s", "generate_protos"])

def place_before(path, text, *before_text, escape=None):
replacement = "\n".join(before_text) + "\n" + text
if escape:
for c in escape:
text = text.replace(c, '\\' + c)
s.replace([path], text, replacement)

custom_nox_sessions = """
@nox.session(python=["3.6", "3.7", "3.8", "3.9"])
@nox.parametrize(
"library", ["python-asset"], ids=["asset"],
)
def test(session, library):
\"\"\"Run tests from a downstream libraries.
To verify that any changes we make here will not break downstream libraries, clone
a few and run their unit and system tests.
NOTE: The unit and system test functions above are copied from the templates.
They will need to be updated when the templates change.
\"\"\"
try:
session.run("git", "-C", library, "pull", external=True)
except nox.command.CommandFailed:
session.run(
"git",
"clone",
"--single-branch",
f"https://github.com/googleapis/{library}",
external=True,
)
session.cd(library)
unit(session)
# system tests are run 3.7 only
if session.python == "3.7":
system(session)
@nox.session(python="3.8")
def generate_protos(session):
\"\"\"Generates the protos using protoc.
Some notes on the `google` directory:
1. The `_pb2.py` files are produced by protoc.
2. The .proto files are non-functional but are left in the repository
to make it easier to understand diffs.
3. The `google` directory also has `__init__.py` files to create proper modules.
If a new subdirectory is added, you will need to create more `__init__.py`
files.
\"\"\"
session.install("grpcio-tools")
protos = [str(p) for p in (pathlib.Path(".").glob("google/**/*.proto"))]
# Clone googleapis/api-common-protos
api_common_protos = "api-common-protos"
try:
session.run("git", "-C", api_common_protos, "pull", external=True)
except nox.command.CommandFailed:
session.run(
"git",
"clone",
"--single-branch",
f"https://github.com/googleapis/{api_common_protos}",
external=True,
)
session.run(
"python",
"-m",
"grpc_tools.protoc",
"--proto_path=api-common-protos",
"--proto_path=.",
"--python_out=.",
*protos,
)
"""

# `google-cloud-context-manager` contains only protos
# and has a customized `test` session to test
# downstream client `google-cloud-asset`
place_before(
"noxfile.py",
"@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)\n"
"def system(session):",
custom_nox_sessions,
escape="()"
)
# Clean up
shutil.rmtree('google/api')
shutil.rmtree('google/type')

s.shell.run(["nox", "-s", "generate_protos"])
s.shell.run(["nox", "-s", "blacken"], hide_output=False)

# Add license headers
python.fix_pb2_headers()

LICENSE = """
# Copyright 2020 Google LLC
# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down

0 comments on commit bb7e31a

Please sign in to comment.