feat(tests): [Bigtable] add conformance tests and proxy (#7959)

* feat(tests): [Bigtable] add conformance tests and proxy

* exclude gencode in cs

* add proxy github action for testing

* add workflow dispatch

* install roadrunner in conformance actions

* fix rr version

* fix rr version (again)

* add sudo

* cleanup imports, use InseureCredentials

* add logic for CheckAndMutateRows

* fix timeout, fix numworkers

* fix  timeout error

* fix call to checkAndMutateRow

* use InsecureCredentialsWrapper

* set hasEmulator to true to use http instead of https

* add back insecure credentials wrapper

* enable grpc in github actions

* more work on the proxy

* add support for ReadModifyWriteRow

* add skip_tests

* remove index

* WIP

* fixes NonTransientErrors test, moves conformance.sh

* checkout to proxy v0.0.3, remove ping.php

* update protos, add ExecuteQuery rpc

.github/workflows/conformance-tests-bigtable-emulator.yaml

@@ -0,0 +1,58 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Github action job to test core java library features on
+# downstream client libraries before they are released.
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'Bigtable/**'
+      - '.github/workflows/conformance-tests-bigtable-emulator.yaml'
+  pull_request:
+    paths:
+      - 'Bigtable/**'
+      - '.github/workflows/conformance-tests-bigtable-emulator.yaml'
+  workflow_dispatch:
+name: Run Bigtable Conformance Tests With Emulator
+jobs:
+  conformance:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.1'
+          extensions: grpc
+
+      - name: Checkout Bigtable conformance tests
+        uses: actions/checkout@v4
+        with:
+          repository: googleapis/cloud-bigtable-clients-test
+          ref: main
+          path: cloud-bigtable-clients-test
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.20.2'
+
+      - name: Install Road Runner PHP
+        run: |
+          wget https://github.com/roadrunner-server/roadrunner/releases/download/v2024.3.1/roadrunner-2024.3.1-linux-amd64.deb
+          sudo dpkg -i roadrunner-2024.3.1-linux-amd64.deb
+
+      - run: bash Bigtable/tests/Conformance/proxy/conformance.sh

.github/workflows/conformance-tests-storage-emulator.yaml

@@ -4,11 +4,11 @@ on:
       - main
     paths:
       - 'Storage/**'
-      - '.github/workflows/emulator-conformance-tests-storage.yaml'
+      - '.github/workflows/conformance-tests-storage-emulator.yaml'
   pull_request:
     paths:
       - 'Storage/**'
-      - '.github/workflows/emulator-conformance-tests-storage.yaml'
+      - '.github/workflows/conformance-tests-storage-emulator.yaml'
 name: Run Storage Retry Conformance Tests With Emulator
 jobs:
   test:

Bigtable/src/ResumableStream.php

@@ -163,6 +163,7 @@ public function readAll()
                     $headers['bigtable-attempt'] = [(string) $totalAttempt];
                     ($this->delayFunction)($currentAttempt);
                 }
+
                 $stream = call_user_func_array(
                     [$this->gapicClient, $this->method],
                     [$this->request, ['headers' => $headers] + $this->callOptions]

Bigtable/src/Table.php

@@ -573,7 +573,8 @@ private function mutateRowsWithEntries(array $entries, array $options = [])
                         $rowMutationsFailedResponse[] = [
                             'rowKey' => $entries[$mutateRowsResponseEntry->getIndex()]->getRowKey(),
                             'statusCode' => $mutateRowsResponseEntry->getStatus()->getCode(),
-                            'message' => $mutateRowsResponseEntry->getStatus()->getMessage()
+                            'message' => $mutateRowsResponseEntry->getStatus()->getMessage(),
+                            'index' => $mutateRowsResponseEntry->getIndex(),
                         ];
                     } else {
                         unset($entries[$mutateRowsResponseEntry->getIndex()]);

Bigtable/tests/Conformance/proxy/.rr.yaml

@@ -0,0 +1,30 @@
+version: "3"
+rpc:
+  listen: "tcp://127.0.0.1:6001"
+
+server:
+  command: "php worker.php"
+  relay: "pipes"
+  relay_timeout: "20s"
+
+kv:
+  memory-cache:
+    driver: memory
+    config: { }
+
+grpc:
+  listen: "tcp://127.0.0.1:9999"
+  proto: ["test_proxy.proto"]
+  max_send_msg_size: 50
+  max_recv_msg_size: 50
+  max_connection_idle: 0s
+  max_connection_age: 0s
+  max_connection_age_grace: 0s
+  max_concurrent_streams: 10
+  ping_time: 1s
+  timeout: 200s
+  pool:
+    num_workers: 5
+    max_jobs: 0
+    allocate_timeout: 60s
+    destroy_timeout: 60s

Bigtable/tests/Conformance/proxy/composer.json

@@ -0,0 +1,24 @@
+{
+    "name": "google-testing/proxy-bigtable-server",
+    "description": "Proxy Bigtable Server",
+    "repositories": [
+        {
+            "type": "path",
+            "url": "../../.."
+        }
+    ],
+    "require": {
+        "google/cloud-bigtable": "*",
+        "grpc/grpc": "^1.36",
+        "spiral/roadrunner-cli": "^2.6",
+        "spiral/roadrunner-grpc": "^3.4",
+        "spiral/roadrunner-kv": "^4.3"
+    },
+    "autoload": {
+        "psr-4": {
+            "": "src"
+        }
+    },
+    "minimum-stability": "dev",
+    "prefer-stable": true
+}

Bigtable/tests/Conformance/proxy/conformance.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+## Get the directory of the build script
+proxyDir=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+projectDir=$(realpath $proxyDir/../../../..)
+
+# Install composer dependencies in the "proxy" directory
+composer update -d $proxyDir
+
+set +e
+
+# declare -a configs=("default" "enable_all")
+declare -a configs=("default") # PHP only supports "default" feature flags for now
+for config in "${configs[@]}"
+do
+  # Start the proxy in a separate process
+  rr serve -w $proxyDir &
+  proxyPID=$!
+
+  pushd .
+  cd $projectDir/cloud-bigtable-clients-test/tests
+
+  skipTests=$(sed -n 'H;${x;s/\n/\\|/g;s/^\\|//;p;};d' < $proxyDir/known_failures.txt)
+  eval "go test -v -proxy_addr=:9999 -skip ${skipTests}"
+  returnCode=$?
+  popd
+
+  # Stop the proxy
+  kill ${proxyPID}
+
+  if [[ ${returnCode} -gt 0 ]]
+  then
+    echo "Conformance test failed for config: ${config}"
+    RETURN_CODE=${returnCode}
+  else
+    echo "Conformance test passed for config: ${config}"
+  fi
+done
+
+exit ${RETURN_CODE}

Bigtable/tests/Conformance/proxy/known_failures.txt

@@ -0,0 +1,10 @@
+_Retry_WithRoutingCookie
+_Retry_WithRetryInfo
+TestMutateRows_Generic_DeadlineExceeded
+TestReadRow_Generic_DeadlineExceeded
+TestReadRows_NoRetry_OutOfOrderError
+TestReadRows_Retry_StreamReset
+TestReadRows_Retry_PausedScan
+TestReadRows_Retry_LastScannedRow
+TestReadRows_ReverseScans_FeatureFlag_Enabled
+TestReadRows_Generic_DeadlineExceeded