From db792e8c876f55f5779c7e292282e453dc850fdd Mon Sep 17 00:00:00 2001 From: Daniel Baah Date: Tue, 25 May 2021 23:53:26 -0700 Subject: [PATCH 1/8] build: add kokoro & ci changes --- .github/CODEOWNERS | 12 +++ .github/blunderbuss.yml | 7 ++ .github/generated-files-bot.yml | 12 +++ .github/readme/synth.metadata/synth.metadata | 18 ++++ .github/readme/synth.py | 19 +++++ .github/release-please.yml | 8 +- .github/snippet-bot.yml | 0 .github/sync-repo-settings.yaml | 43 ++++++++++ .github/workflows/approve-readme.yaml | 54 ++++++++++++ .github/workflows/auto-release.yaml | 88 ++++++++++++++++++++ .github/workflows/ci.yaml | 79 ++++++++++++++++++ .github/workflows/samples.yaml | 14 ++++ .kokoro/build.bat | 2 +- .kokoro/build.sh | 49 +++++++---- .kokoro/common.sh | 58 +++++++++++++ .kokoro/continuous/readme.cfg | 55 ++++++++++++ .kokoro/dependencies.sh | 67 ++++++++++++++- .kokoro/nightly/integration.cfg | 35 ++++++-- .kokoro/nightly/samples.cfg | 24 +++--- .kokoro/populate-secrets.sh | 43 ++++++++++ .kokoro/presubmit/integration.cfg | 28 ++++--- .kokoro/presubmit/samples.cfg | 24 +++--- .kokoro/readme.sh | 45 ++++++++++ .kokoro/release/publish_javadoc.cfg | 4 + .kokoro/release/publish_javadoc.sh | 10 +-- .kokoro/release/publish_javadoc11.cfg | 30 +++++++ .kokoro/release/publish_javadoc11.sh | 67 +++++++++++++++ .kokoro/release/snapshot.sh | 2 +- .kokoro/release/stage.cfg | 31 +------ .kokoro/release/stage.sh | 17 ++-- .kokoro/trampoline.sh | 2 + 31 files changed, 841 insertions(+), 106 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/blunderbuss.yml create mode 100644 .github/generated-files-bot.yml create mode 100644 .github/readme/synth.metadata/synth.metadata create mode 100644 .github/readme/synth.py create mode 100644 .github/snippet-bot.yml create mode 100644 .github/sync-repo-settings.yaml create mode 100644 .github/workflows/approve-readme.yaml create mode 100644 .github/workflows/auto-release.yaml create mode 100644 .github/workflows/ci.yaml create mode 100644 .github/workflows/samples.yaml create mode 100644 .kokoro/common.sh create mode 100644 .kokoro/continuous/readme.cfg create mode 100755 .kokoro/populate-secrets.sh create mode 100755 .kokoro/readme.sh create mode 100644 .kokoro/release/publish_javadoc11.cfg create mode 100755 .kokoro/release/publish_javadoc11.sh diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..af60dcee54 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,12 @@ +# Code owners file. +# This file controls who is tagged for review for any given pull request. + +# For syntax help see: +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax + +# The @googleapis/storage-dpe is the default owner for changes in this repo +* @googleapis/yoshi-java @googleapis/storage-dpe +**/*.java @googleapis/storage-dpe + +# The java-samples-reviewers team is the default owner for samples changes +samples/**/*.java @googleapis/java-samples-reviewers diff --git a/.github/blunderbuss.yml b/.github/blunderbuss.yml new file mode 100644 index 0000000000..1a23ea42b1 --- /dev/null +++ b/.github/blunderbuss.yml @@ -0,0 +1,7 @@ +# Configuration for the Blunderbuss GitHub app. For more info see +# https://github.com/googleapis/repo-automation-bots/tree/master/packages/blunderbuss +assign_prs_by: +- labels: + - samples + to: + - googleapis/java-samples-reviewers \ No newline at end of file diff --git a/.github/generated-files-bot.yml b/.github/generated-files-bot.yml new file mode 100644 index 0000000000..c644a24e11 --- /dev/null +++ b/.github/generated-files-bot.yml @@ -0,0 +1,12 @@ +externalManifests: +- type: json + file: 'synth.metadata' + jsonpath: '$.generatedFiles[*]' +- type: json + file: '.github/readme/synth.metadata/synth.metadata' + jsonpath: '$.generatedFiles[*]' +ignoreAuthors: +- 'renovate-bot' +- 'yoshi-automation' +- 'release-please[bot]' +- 'gcf-owl-bot[bot]' diff --git a/.github/readme/synth.metadata/synth.metadata b/.github/readme/synth.metadata/synth.metadata new file mode 100644 index 0000000000..1ef244f2d5 --- /dev/null +++ b/.github/readme/synth.metadata/synth.metadata @@ -0,0 +1,18 @@ +{ + "sources": [ + { + "git": { + "name": ".", + "remote": "https://github.com/googleapis/java-storage.git", + "sha": "c734edf01084419b25825cee1bc1cf9ff8c6369c" + } + }, + { + "git": { + "name": "synthtool", + "remote": "https://github.com/googleapis/synthtool.git", + "sha": "4f4b1b9b8d8b52f1e9e4a76165896debce5ab7f1" + } + } + ] +} \ No newline at end of file diff --git a/.github/readme/synth.py b/.github/readme/synth.py new file mode 100644 index 0000000000..7b48cc28d3 --- /dev/null +++ b/.github/readme/synth.py @@ -0,0 +1,19 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated the README for this library.""" + +from synthtool.languages import java + +java.custom_templates(["java_library/README.md"]) diff --git a/.github/release-please.yml b/.github/release-please.yml index dce2c84509..8b1f9130ff 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1,2 +1,8 @@ +bumpMinorPreMajor: true +handleGHRelease: true releaseType: java-yoshi -bumpMinorPreMajor: true \ No newline at end of file +branches: + - bumpMinorPreMajor: true + handleGHRelease: true + releaseType: java-lts + branch: 1.113.14-sp diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml new file mode 100644 index 0000000000..3a7beef33b --- /dev/null +++ b/.github/sync-repo-settings.yaml @@ -0,0 +1,43 @@ +rebaseMergeAllowed: false +squashMergeAllowed: true +mergeCommitAllowed: false +branchProtectionRules: + - pattern: master + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - dependencies (8) + - dependencies (11) + - linkage-monitor + - lint + - clirr + - units (7) + - units (8) + - units (11) + - 'Kokoro - Test: Integration' + - cla/google + - pattern: 1.113.14-sp + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - dependencies (8) + - dependencies (11) + - linkage-monitor + - lint + - clirr + - units (7) + - units (8) + - units (11) + - 'Kokoro - Test: Integration' + - cla/google +permissionRules: + - team: yoshi-admins + permission: admin + - team: yoshi-java-admins + permission: admin + - team: yoshi-java + permission: push diff --git a/.github/workflows/approve-readme.yaml b/.github/workflows/approve-readme.yaml new file mode 100644 index 0000000000..7513acaebc --- /dev/null +++ b/.github/workflows/approve-readme.yaml @@ -0,0 +1,54 @@ +on: + pull_request: +name: auto-merge-readme +jobs: + approve: + runs-on: ubuntu-latest + if: github.repository_owner == 'googleapis' && github.head_ref == 'autosynth-readme' + steps: + - uses: actions/github-script@v3 + with: + github-token: ${{secrets.YOSHI_APPROVER_TOKEN}} + script: | + // only approve PRs from yoshi-automation + if (context.payload.pull_request.user.login !== "yoshi-automation") { + return; + } + + // only approve PRs like "chore: release " + if (!context.payload.pull_request.title === "chore: regenerate README") { + return; + } + + // only approve PRs with README.md and synth.metadata changes + const files = new Set( + ( + await github.paginate( + github.pulls.listFiles.endpoint({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }) + ) + ).map(file => file.filename) + ); + if (files.size != 2 || !files.has("README.md") || !files.has(".github/readme/synth.metadata/synth.metadata")) { + return; + } + + // approve README regeneration PR + await github.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Rubber stamped PR!', + pull_number: context.payload.pull_request.number, + event: 'APPROVE' + }); + + // attach automerge label + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + labels: ['automerge'] + }); diff --git a/.github/workflows/auto-release.yaml b/.github/workflows/auto-release.yaml new file mode 100644 index 0000000000..9b4fd4d834 --- /dev/null +++ b/.github/workflows/auto-release.yaml @@ -0,0 +1,88 @@ +on: + pull_request: +name: auto-release +jobs: + approve: + runs-on: ubuntu-latest + if: contains(github.head_ref, 'release-please') + steps: + - uses: actions/github-script@v3 + with: + github-token: ${{secrets.YOSHI_APPROVER_TOKEN}} + debug: true + script: | + // only approve PRs from release-please[bot] + if (context.payload.pull_request.user.login !== "release-please[bot]") { + return; + } + + // only approve PRs like "chore: release " + if ( !context.payload.pull_request.title.startsWith("chore: release") ) { + return; + } + + // only approve PRs with pom.xml and versions.txt changes + const filesPromise = github.pulls.listFiles.endpoint({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + }); + const changed_files = await github.paginate(filesPromise) + + if ( changed_files.length < 1 ) { + console.log( "Not proceeding since PR is empty!" ) + return; + } + + if ( !changed_files.some(v => v.filename.includes("pom")) || !changed_files.some(v => v.filename.includes("versions.txt")) ) { + console.log( "PR file changes do not have pom.xml or versions.txt -- something is wrong. PTAL!" ) + return; + } + + // trigger auto-release when + // 1) it is a SNAPSHOT release (auto-generated post regular release) + // 2) there are dependency updates only + // 3) there are no open dependency update PRs in this repo (to avoid multiple releases) + if ( + context.payload.pull_request.body.includes("Fix") || + context.payload.pull_request.body.includes("Build") || + context.payload.pull_request.body.includes("Documentation") || + context.payload.pull_request.body.includes("BREAKING CHANGES") || + context.payload.pull_request.body.includes("Features") + ) { + console.log( "Not auto-releasing since it is not a dependency-update-only release." ); + return; + } + + const promise = github.pulls.list.endpoint({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open' + }); + const open_pulls = await github.paginate(promise) + + if ( open_pulls.length > 1 && !context.payload.pull_request.title.includes("SNAPSHOT") ) { + for ( const pull of open_pulls ) { + if ( pull.title.startsWith("deps: update dependency") ) { + console.log( "Not auto-releasing yet since there are dependency update PRs open in this repo." ); + return; + } + } + } + + // approve release PR + await github.pulls.createReview({ + owner: context.repo.owner, + repo: context.repo.repo, + body: 'Rubber stamped release!', + pull_number: context.payload.pull_request.number, + event: 'APPROVE' + }); + + // attach kokoro:force-run and automerge labels + await github.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + labels: ['kokoro:force-run', 'automerge'] + }); diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..a72fdb96f6 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,79 @@ +on: + push: + branches: + - 1.106.1 + pull_request: +name: ci +jobs: + units: + runs-on: ubuntu-latest + strategy: + matrix: + java: [7, 8, 11] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: ${{matrix.java}} + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: test + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.bat + env: + JOB_TYPE: test + dependencies: + runs-on: ubuntu-latest + strategy: + matrix: + java: [8, 11] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: ${{matrix.java}} + - run: java -version + - run: .kokoro/dependencies.sh + linkage-monitor: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - name: Install artifacts to local Maven repository + run: .kokoro/build.sh + shell: bash + - name: Validate any conflicts with regard to com.google.cloud:libraries-bom (latest release) + uses: GoogleCloudPlatform/cloud-opensource-java/linkage-monitor@v1-linkagemonitor + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: lint + clirr: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - run: java -version + - run: .kokoro/build.sh + env: + JOB_TYPE: clirr diff --git a/.github/workflows/samples.yaml b/.github/workflows/samples.yaml new file mode 100644 index 0000000000..c46230a78c --- /dev/null +++ b/.github/workflows/samples.yaml @@ -0,0 +1,14 @@ +on: + pull_request: +name: samples +jobs: + checkstyle: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 8 + - name: Run checkstyle + run: mvn -P lint --quiet --batch-mode checkstyle:check + working-directory: samples/snippets diff --git a/.kokoro/build.bat b/.kokoro/build.bat index 1a8a450299..05826ad93f 100644 --- a/.kokoro/build.bat +++ b/.kokoro/build.bat @@ -1,3 +1,3 @@ :: See documentation in type-shell-output.bat -"C:\Program Files\Git\bin\bash.exe" github/java-storage/.kokoro/build.sh +"C:\Program Files\Git\bin\bash.exe" %~dp0build.sh diff --git a/.kokoro/build.sh b/.kokoro/build.sh index f833a06211..9e0dd51f73 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -20,21 +20,26 @@ scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) ## cd to the parent directory, i.e. the root of the git repo cd ${scriptDir}/.. +# include common functions +source ${scriptDir}/common.sh + # Print out Java version java -version echo ${JOB_TYPE} -mvn install -B -V \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ - -Dgcloud.download.skip=true \ - -T 1C +# attempt to install 3 times with exponential backoff (starting with 10 seconds) +retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dgcloud.download.skip=true \ + -T 1C -# if GOOGLE_APPLICATION_CREDIENTIALS is specified as a relative path prepend Kokoro root directory onto it +# if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then - export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_ROOT}/src/${GOOGLE_APPLICATION_CREDENTIALS}) + export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}) fi RETURN_CODE=0 @@ -46,9 +51,7 @@ test) RETURN_CODE=$? ;; lint) - mvn \ - -Penable-samples \ - com.coveo:fmt-maven-plugin:check + mvn com.coveo:fmt-maven-plugin:check RETURN_CODE=$? ;; javadoc) @@ -57,6 +60,7 @@ javadoc) ;; integration) mvn -B ${INTEGRATION_TEST_ARGS} \ + -ntp \ -Penable-integration-tests \ -DtrimStackTrace=false \ -Dclirr.skip=true \ @@ -66,11 +70,24 @@ integration) RETURN_CODE=$? ;; samples) - if [[ -f samples/pom.xml ]] + SAMPLES_DIR=samples + # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. + if [[ ! -z ${KOKORO_GITHUB_PULL_REQUEST_NUMBER} ]] then - pushd samples + SAMPLES_DIR=samples/snapshot + fi + + if [[ -f ${SAMPLES_DIR}/pom.xml ]] + then + for FILE in ${KOKORO_GFILE_DIR}/secret_manager/*-samples-secrets; do + [[ -f "$FILE" ]] || continue + source "$FILE" + done + + pushd ${SAMPLES_DIR} mvn -B \ -Penable-samples \ + -ntp \ -DtrimStackTrace=false \ -Dclirr.skip=true \ -Denforcer.skip=true \ @@ -100,8 +117,8 @@ bash .kokoro/coerce_logs.sh if [[ "${ENABLE_BUILD_COP}" == "true" ]] then - chmod +x ${KOKORO_GFILE_DIR}/linux_amd64/buildcop - ${KOKORO_GFILE_DIR}/linux_amd64/buildcop -repo=googleapis/java-storage + chmod +x ${KOKORO_GFILE_DIR}/linux_amd64/flakybot + ${KOKORO_GFILE_DIR}/linux_amd64/flakybot -repo=googleapis/java-storage fi echo "exiting with ${RETURN_CODE}" diff --git a/.kokoro/common.sh b/.kokoro/common.sh new file mode 100644 index 0000000000..ace89f45a9 --- /dev/null +++ b/.kokoro/common.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function retry_with_backoff { + attempts_left=$1 + sleep_seconds=$2 + shift 2 + command=$@ + + + # store current flag state + flags=$- + + # allow a failures to continue + set +e + ${command} + exit_code=$? + + # restore "e" flag + if [[ ${flags} =~ e ]] + then set -e + else set +e + fi + + if [[ $exit_code == 0 ]] + then + return 0 + fi + + # failure + if [[ ${attempts_left} > 0 ]] + then + echo "failure (${exit_code}), sleeping ${sleep_seconds}..." + sleep ${sleep_seconds} + new_attempts=$((${attempts_left} - 1)) + new_sleep=$((${sleep_seconds} * 2)) + retry_with_backoff ${new_attempts} ${new_sleep} ${command} + fi + + return $exit_code +} + +## Helper functionss +function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; } +function msg() { println "$*" >&2; } +function println() { printf '%s\n' "$(now) $*"; } \ No newline at end of file diff --git a/.kokoro/continuous/readme.cfg b/.kokoro/continuous/readme.cfg new file mode 100644 index 0000000000..233f84ab15 --- /dev/null +++ b/.kokoro/continuous/readme.cfg @@ -0,0 +1,55 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/python-multi" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/java-storage/.kokoro/readme.sh" +} + +# Build logs will be here +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "**/*sponge_log.log" + } +} + +# The github token is stored here. +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "yoshi-automation-github-key" + # TODO(theacodes): remove this after secrets have globally propagated + backend_type: FASTCONFIGPUSH + } + } +} + +# Common env vars for all repositories and builds. +env_vars: { + key: "GITHUB_USER" + value: "yoshi-automation" +} +env_vars: { + key: "GITHUB_EMAIL" + value: "yoshi-automation@google.com" +} diff --git a/.kokoro/dependencies.sh b/.kokoro/dependencies.sh index 04a288b1ad..59d2aafc79 100755 --- a/.kokoro/dependencies.sh +++ b/.kokoro/dependencies.sh @@ -15,7 +15,13 @@ set -eo pipefail -cd github/java-storage/ +## Get the directory of the build script +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +## cd to the parent directory, i.e. the root of the git repo +cd ${scriptDir}/.. + +# include common functions +source ${scriptDir}/common.sh # Print out Java java -version @@ -24,8 +30,61 @@ echo $JOB_TYPE export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=128m" # this should run maven enforcer -mvn install -B -V \ - -DskipTests=true \ - -Dclirr.skip=true +retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ + -DskipTests=true \ + -Dmaven.javadoc.skip=true \ + -Dclirr.skip=true mvn -B dependency:analyze -DfailOnWarning=true + +echo "****************** DEPENDENCY LIST COMPLETENESS CHECK *******************" +## Run dependency list completeness check +function completenessCheck() { + # Output dep list with compile scope generated using the original pom + # Running mvn dependency:list on Java versions that support modules will also include the module of the dependency. + # This is stripped from the output as it is not present in the flattened pom. + # Only dependencies with 'compile' or 'runtime' scope are included from original dependency list. + msg "Generating dependency list using original pom..." + mvn dependency:list -f pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' | sed -e s/\\s--\\smodule.*// >.org-list.txt + + # Output dep list generated using the flattened pom (only 'compile' and 'runtime' scopes) + msg "Generating dependency list using flattened pom..." + mvn dependency:list -f .flattened-pom.xml -DincludeScope=runtime -Dsort=true | grep '\[INFO] .*:.*:.*:.*:.*' >.new-list.txt + + # Compare two dependency lists + msg "Comparing dependency lists..." + diff .org-list.txt .new-list.txt >.diff.txt + if [[ $? == 0 ]] + then + msg "Success. No diff!" + else + msg "Diff found. See below: " + msg "You can also check .diff.txt file located in $1." + cat .diff.txt + return 1 + fi +} + +# Allow failures to continue running the script +set +e + +error_count=0 +for path in $(find -name ".flattened-pom.xml") +do + # Check flattened pom in each dir that contains it for completeness + dir=$(dirname "$path") + pushd "$dir" + completenessCheck "$dir" + error_count=$(($error_count + $?)) + popd +done + +if [[ $error_count == 0 ]] +then + msg "All checks passed." + exit 0 +else + msg "Errors found. See log statements above." + exit 1 +fi diff --git a/.kokoro/nightly/integration.cfg b/.kokoro/nightly/integration.cfg index 8bf59c02ed..7fbde2eef7 100644 --- a/.kokoro/nightly/integration.cfg +++ b/.kokoro/nightly/integration.cfg @@ -6,16 +6,37 @@ env_vars: { value: "gcr.io/cloud-devrel-kokoro-resources/java8" } +env_vars: { + key: "JOB_TYPE" + value: "integration" +} +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "gcloud-devel" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" +} + env_vars: { key: "ENABLE_BUILD_COP" value: "true" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" +} + +env_vars: { + key: "IT_SERVICE_ACCOUNT_EMAIL" + value: "it-service-account@gcloud-devel.iam.gserviceaccount.com" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" } diff --git a/.kokoro/nightly/samples.cfg b/.kokoro/nightly/samples.cfg index b4b051cd01..f25429314f 100644 --- a/.kokoro/nightly/samples.cfg +++ b/.kokoro/nightly/samples.cfg @@ -11,26 +11,28 @@ env_vars: { value: "samples" } +# TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "java-docs-samples-testing" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "java-docs-samples-testing" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + value: "secret_manager/java-docs-samples-service-account" } env_vars: { - key: "ENABLE_BUILD_COP" - value: "true" + key: "SECRET_MANAGER_KEYS" + value: "java-docs-samples-service-account" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } +env_vars: { + key: "ENABLE_BUILD_COP" + value: "true" } diff --git a/.kokoro/populate-secrets.sh b/.kokoro/populate-secrets.sh new file mode 100755 index 0000000000..f52514257e --- /dev/null +++ b/.kokoro/populate-secrets.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright 2020 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function now { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n' ;} +function msg { println "$*" >&2 ;} +function println { printf '%s\n' "$(now) $*" ;} + + +# Populates requested secrets set in SECRET_MANAGER_KEYS from service account: +# kokoro-trampoline@cloud-devrel-kokoro-resources.iam.gserviceaccount.com +SECRET_LOCATION="${KOKORO_GFILE_DIR}/secret_manager" +msg "Creating folder on disk for secrets: ${SECRET_LOCATION}" +mkdir -p ${SECRET_LOCATION} +for key in $(echo ${SECRET_MANAGER_KEYS} | sed "s/,/ /g") +do + msg "Retrieving secret ${key}" + docker run --entrypoint=gcloud \ + --volume=${KOKORO_GFILE_DIR}:${KOKORO_GFILE_DIR} \ + gcr.io/google.com/cloudsdktool/cloud-sdk \ + secrets versions access latest \ + --project cloud-devrel-kokoro-resources \ + --secret ${key} > \ + "${SECRET_LOCATION}/${key}" + if [[ $? == 0 ]]; then + msg "Secret written to ${SECRET_LOCATION}/${key}" + else + msg "Error retrieving secret ${key}" + fi +done diff --git a/.kokoro/presubmit/integration.cfg b/.kokoro/presubmit/integration.cfg index 4afaa4bf25..4989fa2695 100644 --- a/.kokoro/presubmit/integration.cfg +++ b/.kokoro/presubmit/integration.cfg @@ -11,26 +11,28 @@ env_vars: { value: "integration" } +# TODO: remove this after we've migrated all tests and scripts env_vars: { - key: "GCLOUD_PROJECT" - value: "gcloud-devel" + key: "GCLOUD_PROJECT" + value: "gcloud-devel" } env_vars: { - key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + key: "GOOGLE_CLOUD_PROJECT" + value: "gcloud-devel" } env_vars: { - key: "IT_SERVICE_ACCOUNT_EMAIL" - value: "it-service-account@gcloud-devel.iam.gserviceaccount.com" + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-it-service-account" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } +env_vars: { + key: "IT_SERVICE_ACCOUNT_EMAIL" + value: "it-service-account@gcloud-devel.iam.gserviceaccount.com" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-it-service-account" } diff --git a/.kokoro/presubmit/samples.cfg b/.kokoro/presubmit/samples.cfg index fa7b493d0b..01e0960047 100644 --- a/.kokoro/presubmit/samples.cfg +++ b/.kokoro/presubmit/samples.cfg @@ -11,21 +11,23 @@ env_vars: { value: "samples" } +# TODO: remove this after we've migrated all tests and scripts env_vars: { - key: "GCLOUD_PROJECT" - value: "gcloud-devel" + key: "GCLOUD_PROJECT" + value: "java-docs-samples-testing" } env_vars: { - key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "keystore/73713_java_it_service_account" + key: "GOOGLE_CLOUD_PROJECT" + value: "java-docs-samples-testing" } -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "java_it_service_account" - } - } +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/java-docs-samples-service-account" } + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "java-docs-samples-service-account" +} \ No newline at end of file diff --git a/.kokoro/readme.sh b/.kokoro/readme.sh new file mode 100755 index 0000000000..013ce452f6 --- /dev/null +++ b/.kokoro/readme.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +cd ${KOKORO_ARTIFACTS_DIR}/github/java-storage + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Kokoro exposes this as a file, but the scripts expect just a plain variable. +export GITHUB_TOKEN=$(cat ${KOKORO_KEYSTORE_DIR}/73713_yoshi-automation-github-key) + +# Setup git credentials +echo "https://${GITHUB_TOKEN}:@github.com" >> ~/.git-credentials +git config --global credential.helper 'store --file ~/.git-credentials' + +python3.6 -m pip install git+https://github.com/googleapis/synthtool.git#egg=gcp-synthtool + +set +e +python3.6 -m autosynth.synth \ + --repository=googleapis/java-storage \ + --synth-file-name=.github/readme/synth.py \ + --metadata-path=.github/readme/synth.metadata \ + --pr-title="chore: regenerate README" \ + --branch-suffix="readme" + +# autosynth returns 28 to signal there are no changes +RETURN_CODE=$? +if [[ ${RETURN_CODE} -ne 0 && ${RETURN_CODE} -ne 28 ]] +then + exit ${RETURN_CODE} +fi diff --git a/.kokoro/release/publish_javadoc.cfg b/.kokoro/release/publish_javadoc.cfg index 82745e5824..ab50da64c9 100644 --- a/.kokoro/release/publish_javadoc.cfg +++ b/.kokoro/release/publish_javadoc.cfg @@ -1,4 +1,7 @@ # Format: //devtools/kokoro/config/proto/build.proto + +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/doc-templates/" + env_vars: { key: "STAGING_BUCKET" value: "docs-staging" @@ -9,6 +12,7 @@ env_vars: { value: "github/java-storage/.kokoro/release/publish_javadoc.sh" } + before_action { fetch_keystore { keystore_resource { diff --git a/.kokoro/release/publish_javadoc.sh b/.kokoro/release/publish_javadoc.sh index 219d37c602..b70d3dbf93 100755 --- a/.kokoro/release/publish_javadoc.sh +++ b/.kokoro/release/publish_javadoc.sh @@ -31,13 +31,13 @@ pushd $(dirname "$0")/../../ python3 -m pip install gcp-docuploader # compile all packages -mvn clean install -B -DskipTests=true +mvn clean install -B -q -DskipTests=true -NAME=google-cloud-storage -VERSION=$(grep ${NAME}: versions.txt | cut -d: -f3) +export NAME=google-cloud-storage +export VERSION=$(grep ${NAME}: versions.txt | cut -d: -f3) # build the docs -mvn site -B +mvn site -B -q pushd target/site/apidocs @@ -51,5 +51,3 @@ python3 -m docuploader create-metadata \ python3 -m docuploader upload . \ --credentials ${CREDENTIALS} \ --staging-bucket ${STAGING_BUCKET} - -popd diff --git a/.kokoro/release/publish_javadoc11.cfg b/.kokoro/release/publish_javadoc11.cfg new file mode 100644 index 0000000000..6aa8afa690 --- /dev/null +++ b/.kokoro/release/publish_javadoc11.cfg @@ -0,0 +1,30 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# cloud-rad production +env_vars: { + key: "STAGING_BUCKET_V2" + value: "docs-staging-v2" +} + +# Configure the docker image for kokoro-trampoline +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/java-storage/.kokoro/release/publish_javadoc11.sh" +} + +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "docuploader_service_account" + } + } +} + +# Downloads docfx doclet resource. This will be in ${KOKORO_GFILE_DIR}/ +gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/docfx" diff --git a/.kokoro/release/publish_javadoc11.sh b/.kokoro/release/publish_javadoc11.sh new file mode 100755 index 0000000000..af8238dc6c --- /dev/null +++ b/.kokoro/release/publish_javadoc11.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright 2021 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +if [[ -z "${CREDENTIALS}" ]]; then + CREDENTIALS=${KOKORO_KEYSTORE_DIR}/73713_docuploader_service_account +fi + +if [[ -z "${STAGING_BUCKET_V2}" ]]; then + echo "Need to set STAGING_BUCKET_V2 environment variable" + exit 1 +fi + +# work from the git root directory +pushd $(dirname "$0")/../../ + +# install docuploader package +python3 -m pip install gcp-docuploader + +# compile all packages +mvn clean install -B -q -DskipTests=true + +export NAME=google-cloud-storage +export VERSION=$(grep ${NAME}: versions.txt | cut -d: -f3) + +# V3 generates docfx yml from javadoc +# generate yml +mvn clean site -B -q -P docFX + +# copy README to docfx-yml dir and rename index.md +cp README.md target/docfx-yml/index.md +# copy CHANGELOG to docfx-yml dir and rename history.md +cp CHANGELOG.md target/docfx-yml/history.md + +pushd target/docfx-yml + +# create metadata +python3 -m docuploader create-metadata \ + --name ${NAME} \ + --version ${VERSION} \ + --xrefs devsite://java/gax \ + --xrefs devsite://java/google-cloud-core \ + --xrefs devsite://java/api-common \ + --xrefs devsite://java/proto-google-common-protos \ + --xrefs devsite://java/google-api-client \ + --xrefs devsite://java/google-http-client \ + --xrefs devsite://java/protobuf \ + --language java + +# upload yml to production bucket +python3 -m docuploader upload . \ + --credentials ${CREDENTIALS} \ + --staging-bucket ${STAGING_BUCKET_V2} \ + --destination-prefix docfx diff --git a/.kokoro/release/snapshot.sh b/.kokoro/release/snapshot.sh index 098168a737..1f55b77024 100755 --- a/.kokoro/release/snapshot.sh +++ b/.kokoro/release/snapshot.sh @@ -25,7 +25,7 @@ grep SNAPSHOT versions.txt setup_environment_secrets create_settings_xml_file "settings.xml" -mvn clean install deploy -B \ +mvn clean deploy -B \ --settings ${MAVEN_SETTINGS_FILE} \ -DperformRelease=true \ -Dgpg.executable=gpg \ diff --git a/.kokoro/release/stage.cfg b/.kokoro/release/stage.cfg index f82c3ffb99..8dc93be45b 100644 --- a/.kokoro/release/stage.cfg +++ b/.kokoro/release/stage.cfg @@ -13,32 +13,7 @@ action { } } -# Fetch the token needed for reporting release status to GitHub -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "yoshi-automation-github-key" - } - } -} - -# Fetch magictoken to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "releasetool-magictoken" - } - } -} - -# Fetch api key to use with Magic Github Proxy -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "magic-github-proxy-api-key" - } - } +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } diff --git a/.kokoro/release/stage.sh b/.kokoro/release/stage.sh index 3c482cbc55..8a1033843c 100755 --- a/.kokoro/release/stage.sh +++ b/.kokoro/release/stage.sh @@ -20,19 +20,22 @@ python3 -m pip install gcp-releasetool python3 -m releasetool publish-reporter-script > /tmp/publisher-script; source /tmp/publisher-script source $(dirname "$0")/common.sh +source $(dirname "$0")/../common.sh MAVEN_SETTINGS_FILE=$(realpath $(dirname "$0")/../../)/settings.xml pushd $(dirname "$0")/../../ setup_environment_secrets create_settings_xml_file "settings.xml" -mvn clean install deploy -B \ - --settings ${MAVEN_SETTINGS_FILE} \ - -DskipTests=true \ - -DperformRelease=true \ - -Dgpg.executable=gpg \ - -Dgpg.passphrase=${GPG_PASSPHRASE} \ - -Dgpg.homedir=${GPG_HOMEDIR} +# attempt to stage 3 times with exponential backoff (starting with 10 seconds) +retry_with_backoff 3 10 \ + mvn clean deploy -B \ + --settings ${MAVEN_SETTINGS_FILE} \ + -DskipTests=true \ + -DperformRelease=true \ + -Dgpg.executable=gpg \ + -Dgpg.passphrase=${GPG_PASSPHRASE} \ + -Dgpg.homedir=${GPG_HOMEDIR} if [[ -n "${AUTORELEASE_PR}" ]] then diff --git a/.kokoro/trampoline.sh b/.kokoro/trampoline.sh index ba17ce0146..9da0f83987 100644 --- a/.kokoro/trampoline.sh +++ b/.kokoro/trampoline.sh @@ -21,4 +21,6 @@ function cleanup() { echo "cleanup"; } trap cleanup EXIT + +$(dirname $0)/populate-secrets.sh # Secret Manager secrets. python3 "${KOKORO_GFILE_DIR}/trampoline_v1.py" From 932f643eb3d8206c4fc229079b96cbe0040e9d32 Mon Sep 17 00:00:00 2001 From: Daniel Baah Date: Wed, 26 May 2021 12:40:59 -0700 Subject: [PATCH 2/8] add samples folder to pass ci --- samples/install-without-bom/pom.xml | 84 +++++++++++++++++++ samples/pom.xml | 56 +++++++++++++ samples/snapshot/pom.xml | 83 ++++++++++++++++++ samples/snippets/pom.xml | 60 +++++++++++++ .../com/example/storage/QuickstartSample.java | 40 +++++++++ .../example/storage/QuickstartSampleIT.java | 68 +++++++++++++++ 6 files changed, 391 insertions(+) create mode 100644 samples/install-without-bom/pom.xml create mode 100644 samples/pom.xml create mode 100644 samples/snapshot/pom.xml create mode 100644 samples/snippets/pom.xml create mode 100644 samples/snippets/src/main/java/com/example/storage/QuickstartSample.java create mode 100644 samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml new file mode 100644 index 0000000000..262e2167b1 --- /dev/null +++ b/samples/install-without-bom/pom.xml @@ -0,0 +1,84 @@ + + + 4.0.0 + com.google.cloud + storage-install-without-bom + jar + Google Google Cloud Storage Install Without Bom + https://github.com/googleapis/java-storage + + + + com.google.cloud.samples + shared-configuration + 1.0.22 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + com.google.cloud + google-cloud-storage + 1.114.0 + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.0.1 + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.2.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + diff --git a/samples/pom.xml b/samples/pom.xml new file mode 100644 index 0000000000..026e8e6285 --- /dev/null +++ b/samples/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + com.google.cloud + google-cloud-storage-samples + 0.0.1-SNAPSHOT + pom + Google Google Cloud Storage Samples Parent + https://github.com/googleapis/java-storage + + Java idiomatic client for Google Cloud Platform services. + + + + + com.google.cloud.samples + shared-configuration + 1.0.22 + + + + 1.8 + 1.8 + UTF-8 + + + + install-without-bom + snapshot + snippets + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + true + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + + true + + + + + diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml new file mode 100644 index 0000000000..751e2fecae --- /dev/null +++ b/samples/snapshot/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + com.google.cloud + storage-snapshot + jar + Google Google Cloud Storage Snapshot Samples + https://github.com/googleapis/java-storage + + + + com.google.cloud.samples + shared-configuration + 1.0.22 + + + + 1.8 + 1.8 + UTF-8 + + + + + + com.google.cloud + google-cloud-storage + 1.114.0 + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.0.1 + test + + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.2.0 + + + add-snippets-source + + add-source + + + + ../snippets/src/main/java + + + + + add-snippets-tests + + add-test-source + + + + ../snippets/src/test/java + + + + + + + + \ No newline at end of file diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml new file mode 100644 index 0000000000..f5ce98e31c --- /dev/null +++ b/samples/snippets/pom.xml @@ -0,0 +1,60 @@ + + + 4.0.0 + com.google.cloud + storage-snippets + jar + Google Google Cloud Storage Snippets + https://github.com/googleapis/java-storage + + + + com.google.cloud.samples + shared-configuration + 1.0.22 + + + + 1.8 + 1.8 + UTF-8 + + + + + + + + com.google.cloud + libraries-bom + 20.4.0 + pom + import + + + + + + + com.google.cloud + google-cloud-storage + + + + + junit + junit + 4.13.2 + test + + + com.google.truth + truth + 1.0.1 + test + + + diff --git a/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java b/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java new file mode 100644 index 0000000000..3307e18345 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.storage; + +// [START storage_quickstart] +// Imports the Google Cloud client library +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +public class QuickstartSample { + public static void main(String... args) throws Exception { + // Instantiates a client + Storage storage = StorageOptions.getDefaultInstance().getService(); + + // The name for the new bucket + String bucketName = args[0]; // "my-new-bucket"; + + // Creates the new bucket + Bucket bucket = storage.create(BucketInfo.of(bucketName)); + + System.out.printf("Bucket %s created.%n", bucket.getName()); + } +} +// [END storage_quickstart] diff --git a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java new file mode 100644 index 0000000000..0e84f1291d --- /dev/null +++ b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java @@ -0,0 +1,68 @@ +/* + * Copyright 2015 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.storage; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.UUID; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for quickstart sample. + */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class QuickstartSampleIT { + private String bucketName; + private ByteArrayOutputStream bout; + private PrintStream out; + + private static final void deleteBucket(String bucketName) { + Storage storage = StorageOptions.getDefaultInstance().getService(); + storage.delete(bucketName); + } + + @Before + public void setUp() { + bucketName = "my-new-bucket-" + UUID.randomUUID().toString(); + + bout = new ByteArrayOutputStream(); + out = new PrintStream(bout); + System.setOut(out); + } + + @After + public void tearDown() { + System.setOut(null); + deleteBucket(bucketName); + } + + @Test + public void testQuickstart() throws Exception { + QuickstartSample.main(bucketName); + String got = bout.toString(); + assertThat(got).contains(String.format("Bucket %s created.", bucketName)); + } +} From 64597019d7e3d69688e3309405221c97fa30806c Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Wed, 26 May 2021 19:02:31 -0400 Subject: [PATCH 3/8] build: pin fmt-maven-plugin version to 2.9 fmt-maven-plugin v2.10 is compiled for java 11, v2.9 is compiled for java 8 port of fix from https://github.com/googleapis/java-shared-config/pull/90 --- pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pom.xml b/pom.xml index d1d69a416b..b240265956 100644 --- a/pom.xml +++ b/pom.xml @@ -233,6 +233,17 @@ + + + com.coveo + fmt-maven-plugin + 2.9 + + + true + + + From 22f628acc802b701f723cecbdc1024839a89a965 Mon Sep 17 00:00:00 2001 From: Daniel Baah Date: Thu, 3 Jun 2021 17:48:01 -0700 Subject: [PATCH 4/8] remove samples directory & ci --- .github/workflows/samples.yaml | 14 ---- samples/install-without-bom/pom.xml | 84 ------------------- samples/pom.xml | 56 ------------- samples/snapshot/pom.xml | 83 ------------------ samples/snippets/pom.xml | 60 ------------- .../com/example/storage/QuickstartSample.java | 40 --------- .../example/storage/QuickstartSampleIT.java | 68 --------------- 7 files changed, 405 deletions(-) delete mode 100644 .github/workflows/samples.yaml delete mode 100644 samples/install-without-bom/pom.xml delete mode 100644 samples/pom.xml delete mode 100644 samples/snapshot/pom.xml delete mode 100644 samples/snippets/pom.xml delete mode 100644 samples/snippets/src/main/java/com/example/storage/QuickstartSample.java delete mode 100644 samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java diff --git a/.github/workflows/samples.yaml b/.github/workflows/samples.yaml deleted file mode 100644 index c46230a78c..0000000000 --- a/.github/workflows/samples.yaml +++ /dev/null @@ -1,14 +0,0 @@ -on: - pull_request: -name: samples -jobs: - checkstyle: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-java@v1 - with: - java-version: 8 - - name: Run checkstyle - run: mvn -P lint --quiet --batch-mode checkstyle:check - working-directory: samples/snippets diff --git a/samples/install-without-bom/pom.xml b/samples/install-without-bom/pom.xml deleted file mode 100644 index 262e2167b1..0000000000 --- a/samples/install-without-bom/pom.xml +++ /dev/null @@ -1,84 +0,0 @@ - - - 4.0.0 - com.google.cloud - storage-install-without-bom - jar - Google Google Cloud Storage Install Without Bom - https://github.com/googleapis/java-storage - - - - com.google.cloud.samples - shared-configuration - 1.0.22 - - - - 1.8 - 1.8 - UTF-8 - - - - - - - com.google.cloud - google-cloud-storage - 1.114.0 - - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.0.1 - test - - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.2.0 - - - add-snippets-source - - add-source - - - - ../snippets/src/main/java - - - - - add-snippets-tests - - add-test-source - - - - ../snippets/src/test/java - - - - - - - - diff --git a/samples/pom.xml b/samples/pom.xml deleted file mode 100644 index 026e8e6285..0000000000 --- a/samples/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-storage-samples - 0.0.1-SNAPSHOT - pom - Google Google Cloud Storage Samples Parent - https://github.com/googleapis/java-storage - - Java idiomatic client for Google Cloud Platform services. - - - - - com.google.cloud.samples - shared-configuration - 1.0.22 - - - - 1.8 - 1.8 - UTF-8 - - - - install-without-bom - snapshot - snippets - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - true - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - - true - - - - - diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml deleted file mode 100644 index 751e2fecae..0000000000 --- a/samples/snapshot/pom.xml +++ /dev/null @@ -1,83 +0,0 @@ - - - 4.0.0 - com.google.cloud - storage-snapshot - jar - Google Google Cloud Storage Snapshot Samples - https://github.com/googleapis/java-storage - - - - com.google.cloud.samples - shared-configuration - 1.0.22 - - - - 1.8 - 1.8 - UTF-8 - - - - - - com.google.cloud - google-cloud-storage - 1.114.0 - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.0.1 - test - - - - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.2.0 - - - add-snippets-source - - add-source - - - - ../snippets/src/main/java - - - - - add-snippets-tests - - add-test-source - - - - ../snippets/src/test/java - - - - - - - - \ No newline at end of file diff --git a/samples/snippets/pom.xml b/samples/snippets/pom.xml deleted file mode 100644 index f5ce98e31c..0000000000 --- a/samples/snippets/pom.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - 4.0.0 - com.google.cloud - storage-snippets - jar - Google Google Cloud Storage Snippets - https://github.com/googleapis/java-storage - - - - com.google.cloud.samples - shared-configuration - 1.0.22 - - - - 1.8 - 1.8 - UTF-8 - - - - - - - - com.google.cloud - libraries-bom - 20.4.0 - pom - import - - - - - - - com.google.cloud - google-cloud-storage - - - - - junit - junit - 4.13.2 - test - - - com.google.truth - truth - 1.0.1 - test - - - diff --git a/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java b/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java deleted file mode 100644 index 3307e18345..0000000000 --- a/samples/snippets/src/main/java/com/example/storage/QuickstartSample.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.storage; - -// [START storage_quickstart] -// Imports the Google Cloud client library -import com.google.cloud.storage.Bucket; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; - -public class QuickstartSample { - public static void main(String... args) throws Exception { - // Instantiates a client - Storage storage = StorageOptions.getDefaultInstance().getService(); - - // The name for the new bucket - String bucketName = args[0]; // "my-new-bucket"; - - // Creates the new bucket - Bucket bucket = storage.create(BucketInfo.of(bucketName)); - - System.out.printf("Bucket %s created.%n", bucket.getName()); - } -} -// [END storage_quickstart] diff --git a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java b/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java deleted file mode 100644 index 0e84f1291d..0000000000 --- a/samples/snippets/src/test/java/com/example/storage/QuickstartSampleIT.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2015 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.example.storage; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.StorageOptions; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.UUID; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** - * Tests for quickstart sample. - */ -@RunWith(JUnit4.class) -@SuppressWarnings("checkstyle:abbreviationaswordinname") -public class QuickstartSampleIT { - private String bucketName; - private ByteArrayOutputStream bout; - private PrintStream out; - - private static final void deleteBucket(String bucketName) { - Storage storage = StorageOptions.getDefaultInstance().getService(); - storage.delete(bucketName); - } - - @Before - public void setUp() { - bucketName = "my-new-bucket-" + UUID.randomUUID().toString(); - - bout = new ByteArrayOutputStream(); - out = new PrintStream(bout); - System.setOut(out); - } - - @After - public void tearDown() { - System.setOut(null); - deleteBucket(bucketName); - } - - @Test - public void testQuickstart() throws Exception { - QuickstartSample.main(bucketName); - String got = bout.toString(); - assertThat(got).contains(String.format("Bucket %s created.", bucketName)); - } -} From dd68f7b972bcdd41389c6b4729fc48d950b36898 Mon Sep 17 00:00:00 2001 From: Daniel Baah Date: Mon, 7 Jun 2021 16:31:11 -0700 Subject: [PATCH 5/8] add ci block --- .github/sync-repo-settings.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 3a7beef33b..c5a910acf0 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -34,6 +34,21 @@ branchProtectionRules: - units (11) - 'Kokoro - Test: Integration' - cla/google + - pattern: 1.106.1 + isAdminEnforced: true + requiredApprovingReviewCount: 1 + requiresCodeOwnerReviews: true + requiresStrictStatusChecks: false + requiredStatusCheckContexts: + - dependencies (8) + - dependencies (11) + - lint + - clirr + - units (7) + - units (8) + - units (11) + - 'Kokoro - Test: Integration' + - cla/google permissionRules: - team: yoshi-admins permission: admin From 8f8152c3da156516541b9847cd9221844f16c26c Mon Sep 17 00:00:00 2001 From: Daniel Baah Date: Mon, 7 Jun 2021 21:13:52 -0700 Subject: [PATCH 6/8] remove snippet bot config --- .github/snippet-bot.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/snippet-bot.yml diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml deleted file mode 100644 index e69de29bb2..0000000000 From dc12641776c5cdd4cdefd4fa38a347c7f1022f8f Mon Sep 17 00:00:00 2001 From: Daniel B Date: Wed, 9 Jun 2021 11:32:50 -0700 Subject: [PATCH 7/8] sync ci changes --- .github/release-please.yml | 4 ++++ .github/sync-repo-settings.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/release-please.yml b/.github/release-please.yml index 8b1f9130ff..6c61aeddb5 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -6,3 +6,7 @@ branches: handleGHRelease: true releaseType: java-lts branch: 1.113.14-sp + - bumpMinorPreMajor: true + handleGHRelease: true + releaseType: java-yoshi + branch: 1.106.1-patch diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index c5a910acf0..9eca33798a 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -34,7 +34,7 @@ branchProtectionRules: - units (11) - 'Kokoro - Test: Integration' - cla/google - - pattern: 1.106.1 + - pattern: 1.106.1-patch isAdminEnforced: true requiredApprovingReviewCount: 1 requiresCodeOwnerReviews: true From 85d640012455e58c44910ad926ee22bbaae56483 Mon Sep 17 00:00:00 2001 From: BenWhitehead Date: Tue, 16 Mar 2021 11:15:07 -0400 Subject: [PATCH 8/8] fix(test): update blob paths used in storage.it.ITStorageTest#testDownloadPublicBlobWithoutAuthentication (#759) Port of https://github.com/googleapis/google-cloud-go/pull/3806 Fixes https://github.com/googleapis/java-storage/issues/755 --- .../java/com/google/cloud/storage/it/ITStorageTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index 6abc233fb8..276c176112 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -2023,11 +2023,11 @@ public void testDownloadPublicBlobWithoutAuthentication() { // try to download blobs from a public bucket String landsatBucket = "gcp-public-data-landsat"; - String landsatPrefix = "LC08/PRE/044/034/LC80440342016259LGN00/"; - String landsatBlob = landsatPrefix + "LC80440342016259LGN00_MTL.txt"; + String landsatPrefix = "LC08/01/001/002/LC08_L1GT_001002_20160817_20170322_01_T2/"; + String landsatBlob = landsatPrefix + "LC08_L1GT_001002_20160817_20170322_01_T2_ANG.txt"; byte[] bytes = unauthorizedStorage.readAllBytes(landsatBucket, landsatBlob); - assertThat(bytes.length).isEqualTo(7903); + assertThat(bytes.length).isEqualTo(117255); int numBlobs = 0; Iterator blobIterator = unauthorizedStorage @@ -2038,7 +2038,7 @@ public void testDownloadPublicBlobWithoutAuthentication() { numBlobs++; blobIterator.next(); } - assertThat(numBlobs).isEqualTo(13); + assertThat(numBlobs).isEqualTo(14); // try to download blobs from a bucket that requires authentication // authenticated client will succeed