chore(tests): add conformance tests

.github/workflows/conformance.yaml

@@ -0,0 +1,54 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Github action job to test core java library features on
+# downstream client libraries before they are released.
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+name: Conformance
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py-version: [ 3.8 ]
+        client-type: [ "Legacy" ]
+    name: "Python ${{ matrix.py-version }}"
+    steps:
+    - uses: actions/checkout@v3
+      name: "Checkout python-bigtable"
+    - uses: actions/checkout@v3
+      name: "Checkout conformance tests"
+      with:
+        repository: googleapis/cloud-bigtable-clients-test
+        ref: main
+        path: cloud-bigtable-clients-test
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.py-version }}
+    - uses: actions/setup-go@v4
+      with:
+        go-version: '>=1.20.2'
+    - run: chmod +x .kokoro/conformance.sh
+    - run: pip install -e .
+      name: "Install python-bigtable from HEAD"
+    - run: go version
+    - run: .kokoro/conformance.sh
+      name: "Run tests"
+      env:
+        CLIENT_TYPE: ${{ matrix.client-type }}
+        PYTHONUNBUFFERED: 1
+

.kokoro/conformance.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+## cd to the parent directory, i.e. the root of the git repo
+cd $(dirname $0)/..
+
+PROXY_ARGS=""
+TEST_ARGS=""
+if [[ "${CLIENT_TYPE^^}" == "LEGACY" ]]; then
+  echo "Using legacy client"
+  PROXY_ARGS="--legacy-client"
+  # legacy client does not expose mutate_row. Disable those tests
+  TEST_ARGS="-skip TestMutateRow_"
+fi
+
+# Build and start the proxy in a separate process
+PROXY_PORT=9999
+pushd test_proxy
+nohup python test_proxy.py --port $PROXY_PORT $PROXY_ARGS &
+proxyPID=$!
+popd
+
+# Kill proxy on exit
+function cleanup() {
+    echo "Cleanup testbench";
+    kill $proxyPID
+}
+trap cleanup EXIT
+
+# Run the conformance test
+pushd cloud-bigtable-clients-test/tests
+eval "go test -v -proxy_addr=:$PROXY_PORT $TEST_ARGS"
+RETURN_CODE=$?
+popd
+
+echo "exiting with ${RETURN_CODE}"
+exit ${RETURN_CODE}

test_proxy/README.md

@@ -0,0 +1,42 @@
+# CBT Python Test Proxy
+
+The CBT test proxy is intended for running conformance tests for Cloud Bigtable Python Client.
+
+### Start test proxy
+
+You can use `test_proxy.py` to launch a new test proxy process directly
+
+```
+cd python-bigtable/test_proxy
+python test_proxy.py
+```
+
+The port can be set by passing in an extra positional argument
+
+```
+cd python-bigtable/test_proxy
+python test_proxy.py --port 8080
+```
+
+### Run the test cases
+
+Prerequisites:
+- If you have not already done so, [install golang](https://go.dev/doc/install).
+- Before running tests, [launch an instance of the test proxy](#start-test-proxy) 
+in a separate shell session, and make note of the port
+
+
+Clone and navigate to the go test library:
+
+```
+git clone https://github.com/googleapis/cloud-bigtable-clients-test.git
+cd cloud-bigtable-clients-test/tests
+```
+
+
+Launch the tests
+
+```
+go test -v -proxy_addr=:50055
+```
+

test_proxy/handlers/client_handler_data.py

@@ -0,0 +1,122 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This module contains the client handler process for proxy_server.py.
+"""
+import os
+
+from google.cloud.environment_vars import BIGTABLE_EMULATOR
+
+
+def error_safe(func):
+    """
+    Catch and pass errors back to the grpc_server_process
+    Also check if client is closed before processing requests
+    """
+    async def wrapper(self, *args, **kwargs):
+        try:
+            if self.closed:
+                raise RuntimeError("client is closed")
+            return await func(self, *args, **kwargs)
+        except (Exception, NotImplementedError) as e:
+            # exceptions should be raised in grpc_server_process
+            encoded = encode_exception(e)
+            return encoded
+
+    return wrapper
+
+
+def encode_exception(exc):
+    """
+    Encode an exception or chain of exceptions to pass back to grpc_handler
+    """
+    from google.api_core.exceptions import GoogleAPICallError
+    error_msg = f"{type(exc).__name__}: {exc}"
+    result = {"error": error_msg}
+    if exc.__cause__:
+        result["cause"] = encode_exception(exc.__cause__)
+    if hasattr(exc, "exceptions"):
+        result["subexceptions"] = [encode_exception(e) for e in exc.exceptions]
+    if hasattr(exc, "index"):
+        result["index"] = exc.index
+    if isinstance(exc, GoogleAPICallError):
+        if exc.grpc_status_code is not None:
+            result["code"] = exc.grpc_status_code.value[0]
+        elif exc.code is not None:
+            result["code"] = int(exc.code)
+        else:
+            result["code"] = -1
+    elif result.get("cause", {}).get("code", None):
+        # look for code code in cause
+        result["code"] = result["cause"]["code"]
+    elif result.get("subexceptions", None):
+        # look for code in subexceptions
+        for subexc in result["subexceptions"]:
+            if subexc.get("code", None):
+                result["code"] = subexc["code"]
+    return result
+
+
+class TestProxyClientHandler:
+    """
+    Implements the same methods as the grpc server, but handles the client
+    library side of the request.
+
+    Requests received in TestProxyGrpcServer are converted to a dictionary,
+    and supplied to the TestProxyClientHandler methods as kwargs.
+    The client response is then returned back to the TestProxyGrpcServer
+    """
+
+    def __init__(
+        self,
+        data_target=None,
+        project_id=None,
+        instance_id=None,
+        app_profile_id=None,
+        per_operation_timeout=None,
+        **kwargs,
+    ):
+        raise NotImplementedError
+
+
+    def close(self):
+        raise NotImplementedError
+
+    @error_safe
+    async def ReadRows(self, request, **kwargs):
+        raise NotImplementedError
+
+    @error_safe
+    async def ReadRow(self, row_key, **kwargs):
+        raise NotImplementedError
+
+    @error_safe
+    async def MutateRow(self, request, **kwargs):
+        raise NotImplementedError
+
+    @error_safe
+    async def BulkMutateRows(self, request, **kwargs):
+        raise NotImplementedError
+
+    @error_safe
+    async def CheckAndMutateRow(self, request, **kwargs):
+        raise NotImplementedError
+
+    @error_safe
+    async def ReadModifyWriteRow(self, request, **kwargs):
+        raise NotImplementedError
+
+    @error_safe
+    async def SampleRowKeys(self, request, **kwargs):
+        raise NotImplementedError