Merge pull request #1925 from tseaver/bigtable-v2-generate_v2_protos_…

…redux

Regenerate from final Bigtable v2  v2 protos

.gitignore

@@ -57,3 +57,4 @@ scripts/pylintrc_reduced
 generated_python/
 cloud-bigtable-client/
 googleapis-pb/
+grpc_python_venv/

Makefile.bigtable_v1

@@ -1,11 +1,11 @@
+GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
 GENERATED_DIR=$(shell pwd)/generated_python
 GENERATED_SUBDIR=_generated
 BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
-GRPC_PLUGIN=grpc_python_plugin
-PROTOC_CMD=protoc
-BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
-BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
+PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
 GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
+BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
+BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-client-core-parent/bigtable-protos/src/main/proto
 
 help:
 	@echo 'Makefile for gcloud-python Bigtable protos                      '
@@ -15,19 +15,22 @@ help:
 	@echo '   make clean                    Clean generated files          '
 
 generate:
+	# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
+	[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
+	$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
 	# Retrieve git repos that have our *.proto files.
 	[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
 	cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
-	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
-	cd googleapis-pb && git pull origin master
+	[ -d $(GOOGLEAPIS_PROTOS_DIR) ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
+	cd $(GOOGLEAPIS_PROTOS_DIR) && git pull origin master
 	# Make the directory where our *_pb2.py files will go.
 	mkdir -p $(GENERATED_DIR)
 	# Generate all *_pb2.py files that require gRPC.
 	$(PROTOC_CMD) \
 	    --proto_path=$(BIGTABLE_PROTOS_DIR) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
 	    --python_out=$(GENERATED_DIR) \
-	    --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
-	    --grpc_out=$(GENERATED_DIR) \
+	    --grpc_python_out=$(GENERATED_DIR) \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \
 	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto
@@ -52,7 +55,7 @@ generate:
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(BIGTABLE_DIR)
 	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(BIGTABLE_DIR)
-	cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
 	# Rename all *.proto files in our library with an
 	# underscore and remove executable bit.
 	cd $(BIGTABLE_DIR) && \
@@ -63,8 +66,8 @@ generate:
 	# Separate the gRPC parts of the operations service from the
 	# non-gRPC parts so that the protos from `googleapis-common-protos`
 	# can be used without gRPC.
-	PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
-		GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+	GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
+	GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
 		python scripts/make_operations_grpc.py
 	# Rewrite the imports in the generated *_pb2.py files.
 	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
@@ -73,6 +76,6 @@ check_generate:
 	python scripts/check_generate.py
 
 clean:
-	rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
+	rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
 
 .PHONY: generate check_generate clean

Makefile.bigtable_v2

@@ -1,10 +1,8 @@
+GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
 GENERATED_DIR=$(shell pwd)/generated_python
 GENERATED_SUBDIR=_generated_v2
 BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
-GRPC_PLUGIN=grpc_python_plugin
-PROTOC_CMD=protoc
-BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
-BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
+PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
 GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
 
 help:
@@ -15,41 +13,39 @@ help:
 	@echo '   make clean                    Clean generated files          '
 
 generate:
+	# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
+	[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
+	$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
 	# Retrieve git repos that have our *.proto files.
-	[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
-	cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
 	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
 	cd googleapis-pb && git pull origin master
 	# Make the directory where our *_pb2.py files will go.
 	mkdir -p $(GENERATED_DIR)
 	# Generate all *_pb2.py files that require gRPC.
 	$(PROTOC_CMD) \
-	    --proto_path=$(BIGTABLE_PROTOS_DIR) \
+	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
 	    --python_out=$(GENERATED_DIR) \
-	    --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
-	    --grpc_out=$(GENERATED_DIR) \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
+	    --grpc_python_out=$(GENERATED_DIR) \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
 	# Generate all *_pb2.py files that do not require gRPC.
 	$(PROTOC_CMD) \
-	    --proto_path=$(BIGTABLE_PROTOS_DIR) \
 	    --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
 	    --python_out=$(GENERATED_DIR) \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/data.proto \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
-	    $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/data.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
+	    $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
 	# Move the newly generated *_pb2.py files into our library.
 	cp $(GENERATED_DIR)/google/bigtable/v2/* $(BIGTABLE_DIR)
 	cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
-	cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
 	# Remove all existing *.proto files before we replace
 	rm -f $(BIGTABLE_DIR)/*.proto
 	# Copy over the *.proto files into our library.
-	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
-	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
-	cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
+	cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
 	# Rename all *.proto files in our library with an
 	# underscore and remove executable bit.
 	cd $(BIGTABLE_DIR) && \
@@ -60,8 +56,8 @@ generate:
 	# Separate the gRPC parts of the operations service from the
 	# non-gRPC parts so that the protos from `googleapis-common-protos`
 	# can be used without gRPC.
-	PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
-		GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
+	GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
+	GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
 		python scripts/make_operations_grpc.py
 	# Rewrite the imports in the generated *_pb2.py files.
 	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
@@ -70,6 +66,6 @@ check_generate:
 	python scripts/check_generate.py
 
 clean:
-	rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
+	rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
 
 .PHONY: generate check_generate clean

Makefile.datastore

@@ -1,7 +1,7 @@
+GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
 GENERATED_DIR=$(shell pwd)/generated_python
 DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
-GRPC_PLUGIN=grpc_python_plugin
-PROTOC_CMD=protoc
+PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
 GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
 
 help:
@@ -12,6 +12,9 @@ help:
 	@echo '   make clean                    Clean generated files          '
 
 generate:
+	# Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
+	[ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
+	$(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
 	# Retrieve git repos that have our *.proto files.
 	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
 	cd googleapis-pb && git pull origin master
@@ -39,7 +42,8 @@ generate:
 	    done
 	# Separate the gRPC parts of the datastore service from the
 	# non-gRPC parts so that the protos can be used without gRPC.
-	PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
+	GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
+	GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
 		python scripts/make_datastore_grpc.py
 	# Rewrite the imports in the generated *_pb2.py files.
 	python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py