diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java index 5f242517bb..a5dd1375e0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java @@ -101,12 +101,7 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { long totalRead = 0; do { if (sbc == null) { - try { - sbc = Retrying.run(options, resultRetryAlgorithm, this::open, Function.identity()); - } catch (StorageException e) { - result.setException(e); - throw e; - } + sbc = Retrying.run(options, resultRetryAlgorithm, this::open, Function.identity()); } long totalRemaining = Buffers.totalRemaining(dsts, offset, length); @@ -129,17 +124,13 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { sbc = null; } else if (t instanceof IOException) { IOException ioE = (IOException) t; - StorageException translate = StorageException.translate(ioE); - if (resultRetryAlgorithm.shouldRetry(translate, null)) { + if (resultRetryAlgorithm.shouldRetry(StorageException.translate(ioE), null)) { sbc = null; } else { - result.setException(translate); throw ioE; } } else { - BaseServiceException coalesce = StorageException.coalesce(t); - result.setException(coalesce); - throw new IOException(coalesce); + throw new IOException(StorageException.coalesce(t)); } } finally { long totalRemainingAfter = Buffers.totalRemaining(dsts, offset, length); @@ -216,17 +207,20 @@ private ScatteringByteChannel open() { if (xGoogGeneration != null) { int statusCode = e.getStatusCode(); if (statusCode == 404) { - StorageException storageException = - new StorageException(404, "Failure while trying to resume download", e); - result.setException(storageException); - throw storageException; + throw new StorageException(404, "Failure while trying to resume download", e); } } - throw StorageException.translate(e); + StorageException translate = StorageException.translate(e); + result.setException(translate); + throw translate; } catch (IOException e) { - throw StorageException.translate(e); + StorageException translate = StorageException.translate(e); + result.setException(translate); + throw translate; } catch (Throwable t) { - throw StorageException.coalesce(t); + BaseServiceException coalesce = StorageException.coalesce(t); + result.setException(coalesce); + throw coalesce; } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java index 06d26a13d1..f5f076327c 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BaseStorageWriteChannel.java @@ -101,10 +101,12 @@ public final synchronized int write(ByteBuffer src) throws IOException { } int write = tmp.write(src); return write; + } catch (StorageException e) { + throw new IOException(e); } catch (IOException e) { throw e; } catch (Exception e) { - throw StorageException.coalesce(e); + throw new IOException(StorageException.coalesce(e)); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java index 64bccd4acd..e814387171 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/BlobReadChannelV2.java @@ -44,7 +44,10 @@ final class BlobReadChannelV2 extends BaseStorageReadChannel { this.opts = opts; this.blobReadChannelContext = blobReadChannelContext; this.autoGzipDecompression = - Utils.isAutoGzipDecompression(opts, /*defaultWhenUndefined=*/ false); + // RETURN_RAW_INPUT_STREAM means do not add GZIPInputStream to the pipeline. Meaning, if + // RETURN_RAW_INPUT_STREAM is false, automatically attempt to decompress if Content-Encoding + // gzip. + Boolean.FALSE.equals(opts.get(StorageRpc.Option.RETURN_RAW_INPUT_STREAM)); } @Override diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java index a5613ff6ea..94ed6b33b0 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java @@ -256,34 +256,11 @@ public Blob create( @Override public Blob create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) { - requireNonNull(blobInfo, "blobInfo must be non null"); - - Opts opts = Opts.unwrap(options).resolveFrom(blobInfo).prepend(defaultOpts); - GrpcCallContext grpcCallContext = - opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault()); - WriteObjectRequest req = getWriteObjectRequest(blobInfo, opts); - - UnbufferedWritableByteChannelSession session = - ResumableMedia.gapic() - .write() - .byteChannel( - storageClient.writeObjectCallable().withDefaultCallContext(grpcCallContext)) - .setHasher(Hasher.enabled()) - .setByteStringStrategy(ByteStringStrategy.noCopy()) - .direct() - .unbuffered() - .setRequest(req) - .build(); - - // Specifically not in the try-with, so we don't close the provided stream - ReadableByteChannel src = - Channels.newChannel(firstNonNull(content, new ByteArrayInputStream(ZERO_BYTES))); - try (UnbufferedWritableByteChannel dst = session.open()) { - ByteStreams.copy(src, dst); - } catch (Exception e) { + try { + return createFrom(blobInfo, content, options); + } catch (IOException e) { throw StorageException.coalesce(e); } - return getBlob(session.getResult()); } @Override @@ -338,7 +315,7 @@ public Blob internalCreateFrom(Path path, BlobInfo info, Opts o } return codecs.blobInfo().decode(object).asBlob(this); } catch (InterruptedException | ExecutionException e) { - throw StorageException.coalesce(e.getCause()); + throw StorageException.coalesce(e); } } @@ -388,14 +365,7 @@ public Blob createFrom( @Override public Bucket get(String bucket, BucketGetOption... options) { Opts unwrap = Opts.unwrap(options); - try { - return internalBucketGet(bucket, unwrap); - } catch (StorageException e) { - if (e.getCode() == 404) { - return null; - } - throw e; - } + return internalBucketGet(bucket, unwrap); } @Override @@ -689,8 +659,7 @@ public byte[] readAllBytes(String bucket, String blob, BlobSourceOption... optio @Override public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) { - UnbufferedReadableByteChannelSession session = - unbufferedDefaultAutoGzipDecompressingReadSession(blob, options); + UnbufferedReadableByteChannelSession session = unbufferedReadSession(blob, options); ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (UnbufferedReadableByteChannel r = session.open(); @@ -718,19 +687,16 @@ public GrpcBlobReadChannel reader(BlobId blob, BlobSourceOption... options) { ReadObjectRequest request = getReadObjectRequest(blob, opts); Set codes = resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(request)); GrpcCallContext grpcCallContext = Retrying.newCallContext().withRetryableCodes(codes); - boolean autoGzipDecompression = - Utils.isAutoGzipDecompression(opts, /*defaultWhenUndefined=*/ false); return new GrpcBlobReadChannel( storageClient.readObjectCallable().withDefaultCallContext(grpcCallContext), request, - autoGzipDecompression); + !opts.autoGzipDecompression()); } @Override public void downloadTo(BlobId blob, Path path, BlobSourceOption... options) { - UnbufferedReadableByteChannelSession session = - unbufferedDefaultAutoGzipDecompressingReadSession(blob, options); + UnbufferedReadableByteChannelSession session = unbufferedReadSession(blob, options); try (UnbufferedReadableByteChannel r = session.open(); WritableByteChannel w = Files.newByteChannel(path, WRITE_OPS)) { @@ -743,8 +709,7 @@ public void downloadTo(BlobId blob, Path path, BlobSourceOption... options) { @Override public void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption... options) { - UnbufferedReadableByteChannelSession session = - unbufferedDefaultAutoGzipDecompressingReadSession(blob, options); + UnbufferedReadableByteChannelSession session = unbufferedReadSession(blob, options); try (UnbufferedReadableByteChannel r = session.open(); WritableByteChannel w = Channels.newChannel(outputStream)) { @@ -1842,20 +1807,18 @@ WriteObjectRequest getWriteObjectRequest(BlobInfo info, Opts op return opts.writeObjectRequest().apply(requestBuilder).build(); } - private UnbufferedReadableByteChannelSession - unbufferedDefaultAutoGzipDecompressingReadSession(BlobId blob, BlobSourceOption[] options) { + private UnbufferedReadableByteChannelSession unbufferedReadSession( + BlobId blob, BlobSourceOption[] options) { Opts opts = Opts.unwrap(options).resolveFrom(blob).prepend(defaultOpts); ReadObjectRequest readObjectRequest = getReadObjectRequest(blob, opts); Set codes = resultRetryAlgorithmToCodes(retryAlgorithmManager.getFor(readObjectRequest)); GrpcCallContext grpcCallContext = opts.grpcMetadataMapper().apply(Retrying.newCallContext().withRetryableCodes(codes)); - boolean autoGzipDecompression = - Utils.isAutoGzipDecompression(opts, /*defaultWhenUndefined=*/ true); return ResumableMedia.gapic() .read() .byteChannel(storageClient.readObjectCallable().withDefaultCallContext(grpcCallContext)) - .setAutoGzipDecompression(autoGzipDecompression) + .setAutoGzipDecompression(!opts.autoGzipDecompression()) .unbuffered() .setReadObjectRequest(readObjectRequest) .build(); diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GzipReadableByteChannel.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GzipReadableByteChannel.java index 19ab980fc7..967577ed0b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/GzipReadableByteChannel.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/GzipReadableByteChannel.java @@ -17,7 +17,6 @@ package com.google.cloud.storage; import com.google.api.core.ApiFuture; -import com.google.api.gax.rpc.ApiExceptions; import com.google.cloud.storage.UnbufferedReadableByteChannelSession.UnbufferedReadableByteChannel; import java.io.ByteArrayInputStream; import java.io.FilterInputStream; @@ -28,6 +27,7 @@ import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.channels.ScatteringByteChannel; +import java.util.concurrent.ExecutionException; import java.util.zip.GZIPInputStream; final class GzipReadableByteChannel implements UnbufferedReadableByteChannel { @@ -60,7 +60,7 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { source.read(wrap); try { // Step 2: wait for the object metadata, this is populated in the first message from GCS - String contentEncoding = ApiExceptions.callAndTranslateApiException(this.contentEncoding); + String contentEncoding = this.contentEncoding.get(); // if the Content-Encoding is gzip, Step 3: wire gzip decompression into the byte path // this will have a copy impact as we are no longer controlling all the buffers if ("gzip".equals(contentEncoding) || "x-gzip".equals(contentEncoding)) { @@ -86,9 +86,7 @@ public long read(ByteBuffer[] dsts, int offset, int length) throws IOException { bytesRead += Buffers.copy(wrap, dsts, offset, length); delegate = source; } - } catch (StorageException se) { - throw se; - } catch (Exception e) { + } catch (InterruptedException | ExecutionException e) { throw new IOException(e); } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java index 1540f452d8..2cf693e83f 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.Executors.callable; import com.google.api.core.ApiFuture; import com.google.api.gax.paging.Page; @@ -36,14 +37,12 @@ import com.google.cloud.Policy; import com.google.cloud.WriteChannel; import com.google.cloud.storage.Acl.Entity; -import com.google.cloud.storage.ApiaryUnbufferedReadableByteChannel.ApiaryReadRequest; import com.google.cloud.storage.BlobReadChannelV2.BlobReadChannelContext; import com.google.cloud.storage.HmacKey.HmacKeyMetadata; import com.google.cloud.storage.PostPolicyV4.ConditionV4Type; import com.google.cloud.storage.PostPolicyV4.PostConditionsV4; import com.google.cloud.storage.PostPolicyV4.PostFieldsV4; import com.google.cloud.storage.PostPolicyV4.PostPolicyV4Document; -import com.google.cloud.storage.UnbufferedReadableByteChannelSession.UnbufferedReadableByteChannel; import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt; import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt; import com.google.cloud.storage.UnifiedOpts.Opts; @@ -60,10 +59,9 @@ import com.google.common.collect.Maps; import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; -import com.google.common.io.ByteStreams; +import com.google.common.io.CountingOutputStream; import com.google.common.primitives.Ints; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -75,7 +73,6 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; import java.nio.file.Files; import java.nio.file.Path; import java.text.SimpleDateFormat; @@ -617,25 +614,9 @@ public byte[] readAllBytes(BlobId blob, BlobSourceOption... options) { Opts unwrap = Opts.unwrap(options); Opts resolve = unwrap.resolveFrom(blob); ImmutableMap optionsMap = resolve.getRpcOptions(); - boolean autoGzipDecompression = - Utils.isAutoGzipDecompression(resolve, /*defaultWhenUndefined=*/ true); - UnbufferedReadableByteChannelSession session = - ResumableMedia.http() - .read() - .byteChannel(BlobReadChannelContext.from(this)) - .setAutoGzipDecompression(autoGzipDecompression) - .unbuffered() - .setApiaryReadRequest( - new ApiaryReadRequest(storageObject, optionsMap, ByteRangeSpec.nullRange())) - .build(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (UnbufferedReadableByteChannel r = session.open(); - WritableByteChannel w = Channels.newChannel(baos)) { - ByteStreams.copy(r, w); - } catch (IOException e) { - throw StorageException.translate(e); - } - return baos.toByteArray(); + ResultRetryAlgorithm algorithm = + retryAlgorithmManager.getForObjectsGet(storageObject, optionsMap); + return run(algorithm, () -> storageRpc.load(storageObject, optionsMap), Function.identity()); } @Override @@ -667,26 +648,19 @@ public void downloadTo(BlobId blob, Path path, BlobSourceOption... options) { @Override public void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption... options) { + final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream); final StorageObject pb = codecs.blobId().encode(blob); - Opts resolve = Opts.unwrap(options).resolveFrom(blob); - ImmutableMap optionsMap = resolve.getRpcOptions(); - boolean autoGzipDecompression = - Utils.isAutoGzipDecompression(resolve, /*defaultWhenUndefined=*/ true); - UnbufferedReadableByteChannelSession session = - ResumableMedia.http() - .read() - .byteChannel(BlobReadChannelContext.from(this)) - .setAutoGzipDecompression(autoGzipDecompression) - .unbuffered() - .setApiaryReadRequest(new ApiaryReadRequest(pb, optionsMap, ByteRangeSpec.nullRange())) - .build(); - // don't close the provided stream - WritableByteChannel w = Channels.newChannel(outputStream); - try (UnbufferedReadableByteChannel r = session.open()) { - ByteStreams.copy(r, w); - } catch (IOException e) { - throw StorageException.translate(e); - } + ImmutableMap optionsMap = + Opts.unwrap(options).resolveFrom(blob).getRpcOptions(); + ResultRetryAlgorithm algorithm = retryAlgorithmManager.getForObjectsGet(pb, optionsMap); + run( + algorithm, + callable( + () -> { + storageRpc.read( + pb, optionsMap, countingOutputStream.getCount(), countingOutputStream); + }), + Function.identity()); } @Override diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java index 7765596b33..18f79be572 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java @@ -2339,6 +2339,18 @@ Mapper blobInfoMapper() { return fuseMappers(ObjectTargetOpt.class, ObjectTargetOpt::blobInfo); } + /** + * Here for compatibility. This should NOT be an "Opt" instead an attribute of the channel + * builder. When {@link ReturnRawInputStream} is removed, this method should be removed as well. + * + * @see + * GapicDownloadSessionBuilder.ReadableByteChannelSessionBuilder#setAutoGzipDecompression(boolean) + */ + @Deprecated + boolean autoGzipDecompression() { + return filterTo(ReturnRawInputStream.class).findFirst().map(r -> r.val).orElse(true); + } + Decoder clearBlobFields() { return filterTo(Fields.class) .findFirst() diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java index b51d5a393b..c24a68d4d6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/Utils.java @@ -24,8 +24,6 @@ import com.google.api.gax.rpc.ApiCallContext; import com.google.cloud.storage.Conversions.Codec; import com.google.cloud.storage.UnifiedOpts.NamedField; -import com.google.cloud.storage.UnifiedOpts.Opts; -import com.google.cloud.storage.spi.v1.StorageRpc; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; @@ -312,28 +310,4 @@ private static String crc32cEncode(int from) { static GrpcCallContext merge(@NonNull GrpcCallContext l, @NonNull GrpcCallContext r) { return (GrpcCallContext) l.merge(r); } - - /** - * RETURN_RAW_INPUT_STREAM means do not add GZIPInputStream to the pipeline. Meaning, if - * RETURN_RAW_INPUT_STREAM is false, automatically attempt to decompress if Content-Encoding gzip. - */ - static boolean isAutoGzipDecompression(Opts opts, boolean defaultWhenUndefined) { - return isAutoGzipDecompression(opts.getRpcOptions(), defaultWhenUndefined); - } - - /** - * RETURN_RAW_INPUT_STREAM means do not add GZIPInputStream to the pipeline. Meaning, if - * RETURN_RAW_INPUT_STREAM is false, automatically attempt to decompress if Content-Encoding gzip. - */ - static boolean isAutoGzipDecompression( - Map opts, boolean defaultWhenUndefined) { - // Option.getBoolean is package private, and we don't want to open it. - // if non-null explicitly compare to a boolean value to coerce it to a boolean result - Object returnRawInputStream = opts.get(StorageRpc.Option.RETURN_RAW_INPUT_STREAM); - if (returnRawInputStream == null) { - return defaultWhenUndefined; - } else { - return Boolean.FALSE.equals(returnRawInputStream); - } - } } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java index 04847ff0bd..2f1c3e2101 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java @@ -754,12 +754,6 @@ public byte[] load(StorageObject from, Map options) { .setIfGenerationNotMatch(Option.IF_GENERATION_NOT_MATCH.getLong(options)) .setUserProject(Option.USER_PROJECT.getString(options)); setEncryptionHeaders(getRequest.getRequestHeaders(), ENCRYPTION_KEY_PREFIX, options); - Boolean shouldReturnRawInputStream = Option.RETURN_RAW_INPUT_STREAM.getBoolean(options); - if (shouldReturnRawInputStream != null) { - getRequest.setReturnRawInputStream(shouldReturnRawInputStream); - } else { - getRequest.setReturnRawInputStream(false); - } ByteArrayOutputStream out = new ByteArrayOutputStream(); getRequest.executeMedia().download(out); return out.toByteArray(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java index 2538b10f60..12f1bd6b9a 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGrpcStorageImplUploadRetryTest.java @@ -18,7 +18,6 @@ import static com.google.cloud.storage.ByteSizeConstants._2MiB; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; import com.google.api.core.ApiFuture; import com.google.api.gax.grpc.GrpcCallContext; @@ -82,19 +81,16 @@ public void create_bytes() throws Exception { @Test public void create_inputStream() throws Exception { - Direct.FakeService service = Direct.FakeService.create(); + Resumable.FakeService service = Resumable.FakeService.create(); try (TmpFile tmpFile = DataGenerator.base64Characters().tempFile(baseDir, objectContentSize); FakeServer server = FakeServer.of(service); Storage s = server.getGrpcStorageOptions().getService(); InputStream in = Channels.newInputStream(tmpFile.reader())) { BlobInfo info = BlobInfo.newBuilder("buck", "obj").build(); - // create uses a direct upload, once the stream is consumed there is no means for us to retry - // if an error happens it should be surfaced - StorageException se = - assertThrows( - StorageException.class, () -> s.create(info, in, BlobWriteOption.doesNotExist())); - assertThat(se.getCode()).isEqualTo(500); + s.create(info, in, BlobWriteOption.doesNotExist()); } + + assertThat(service.returnError.get()).isFalse(); } @Test diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGzipReadableByteChannelTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGzipReadableByteChannelTest.java index e5ef2d47ba..2e8efa0589 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGzipReadableByteChannelTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/ITGzipReadableByteChannelTest.java @@ -245,10 +245,10 @@ public void autoGzipDecompress_default_disabled() throws IOException { } @Test - public void storage_readAllBytes_defaultUncompressed() { + public void storage_readAllBytes_defaultCompressed() { Storage s = storageFixture.getInstance(); byte[] actual = s.readAllBytes(BlobId.of("buck", "obj-compressed")); - assertThat(actual).isEqualTo(dataUncompressed); + assertThat(actual).isEqualTo(dataCompressed); } @Test diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java index 2fd5501d50..bda9d6a8bd 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java @@ -19,6 +19,12 @@ import static com.google.cloud.storage.SignedUrlEncodingHelper.Rfc3986UriEncode; import static com.google.cloud.storage.testing.ApiPolicyMatcher.eqApiPolicy; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.getCurrentArguments; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -26,10 +32,12 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.util.DateTime; import com.google.api.core.ApiClock; +import com.google.api.gax.retrying.RetrySettings; import com.google.api.services.storage.model.Policy.Bindings; import com.google.api.services.storage.model.StorageObject; import com.google.api.services.storage.model.TestIamPermissionsResponse; @@ -51,9 +59,12 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.BaseEncoding; +import java.io.File; +import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; +import java.nio.file.Files; import java.security.InvalidKeyException; import java.security.Key; import java.security.KeyFactory; @@ -75,6 +86,7 @@ import javax.crypto.spec.SecretKeySpec; import org.easymock.Capture; import org.easymock.EasyMock; +import org.easymock.IAnswer; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -680,6 +692,85 @@ public void testCopyMultipleRequests() { assertEquals(42L, writer.getBlobSize()); } + @Test + public void testReadAllBytes() { + EasyMock.expect( + storageRpcMock.load( + Conversions.json().blobId().encode(BlobId.of(BUCKET_NAME1, BLOB_NAME1)), + EMPTY_RPC_OPTIONS)) + .andReturn(BLOB_CONTENT); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + + @Test + public void testReadAllBytesWithOptions() { + EasyMock.expect( + storageRpcMock.load( + Conversions.json().blobId().encode(BlobId.of(BUCKET_NAME1, BLOB_NAME1)), + BLOB_SOURCE_OPTIONS)) + .andReturn(BLOB_CONTENT); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = + storage.readAllBytes( + BUCKET_NAME1, BLOB_NAME1, BLOB_SOURCE_GENERATION, BLOB_SOURCE_METAGENERATION); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + + @Test + public void testReadAllBytesWithDecriptionKey() { + EasyMock.expect( + storageRpcMock.load( + Conversions.json().blobId().encode(BlobId.of(BUCKET_NAME1, BLOB_NAME1)), + ENCRYPTION_KEY_OPTIONS)) + .andReturn(BLOB_CONTENT) + .times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = + storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + readBytes = + storage.readAllBytes(BUCKET_NAME1, BLOB_NAME1, BlobSourceOption.decryptionKey(BASE64_KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + + @Test + public void testReadAllBytesFromBlobIdWithOptions() { + EasyMock.expect( + storageRpcMock.load( + Conversions.json().blobId().encode(BLOB_INFO1.getBlobId()), BLOB_SOURCE_OPTIONS)) + .andReturn(BLOB_CONTENT); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = + storage.readAllBytes( + BLOB_INFO1.getBlobId(), + BLOB_SOURCE_GENERATION_FROM_BLOB_ID, + BLOB_SOURCE_METAGENERATION); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + + @Test + public void testReadAllBytesFromBlobIdWithDecriptionKey() { + EasyMock.expect( + storageRpcMock.load( + Conversions.json().blobId().encode(BLOB_INFO1.getBlobId()), ENCRYPTION_KEY_OPTIONS)) + .andReturn(BLOB_CONTENT) + .times(2); + EasyMock.replay(storageRpcMock); + initializeService(); + byte[] readBytes = + storage.readAllBytes(BLOB_INFO1.getBlobId(), BlobSourceOption.decryptionKey(KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + readBytes = + storage.readAllBytes(BLOB_INFO1.getBlobId(), BlobSourceOption.decryptionKey(BASE64_KEY)); + assertArrayEquals(BLOB_CONTENT, readBytes); + } + @Test public void testBatch() { RpcBatch batchMock = EasyMock.mock(RpcBatch.class); @@ -2157,4 +2248,97 @@ public void testBucketLifecycleRules() { assertEquals(30, lifecycleRule.getCondition().getDaysSinceCustomTime().intValue()); assertNotNull(lifecycleRule.getCondition().getCustomTimeBefore()); } + + @Test + public void testDownloadTo() throws Exception { + BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); + storage = options.toBuilder().build().getService(); + final byte[] expected = {1, 2}; + EasyMock.expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) + .andAnswer( + new IAnswer() { + @Override + public Long answer() throws Throwable { + ((OutputStream) getCurrentArguments()[3]).write(expected); + return 2l; + } + }); + EasyMock.replay(storageRpcMock); + File file = File.createTempFile("blob", ".tmp"); + storage.downloadTo(blob, file.toPath()); + byte actual[] = Files.readAllBytes(file.toPath()); + assertArrayEquals(expected, actual); + } + + @Test + public void testDownloadToWithRetries() throws Exception { + BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); + storage = + options + .toBuilder() + .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(2).build()) + .build() + .getService(); + final byte[] expected = {1, 2}; + expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) + .andAnswer( + new IAnswer() { + @Override + public Long answer() throws Throwable { + ((OutputStream) getCurrentArguments()[3]).write(expected[0]); + throw new StorageException(504, "error"); + } + }); + expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(1l), + anyObject(OutputStream.class))) + .andAnswer( + new IAnswer() { + @Override + public Long answer() throws Throwable { + ((OutputStream) getCurrentArguments()[3]).write(expected[1]); + return 1l; + } + }); + replay(storageRpcMock); + File file = File.createTempFile("blob", ".tmp"); + storage.downloadTo(blob, file.toPath()); + byte actual[] = Files.readAllBytes(file.toPath()); + assertArrayEquals(expected, actual); + } + + @Test + public void testDownloadToWithException() throws Exception { + BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); + storage = options.toBuilder().build().getService(); + Exception exception = new IllegalStateException("test"); + expect( + storageRpcMock.read( + anyObject(StorageObject.class), + anyObject(Map.class), + eq(0l), + anyObject(OutputStream.class))) + .andThrow(exception); + replay(storageRpcMock); + File file = File.createTempFile("blob", ".tmp"); + try { + storage.downloadTo(blob, file.toPath()); + fail(); + } catch (StorageException e) { + assertSame(exception, e.getCause()); + } + } } diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/UtilsTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/UtilsTest.java deleted file mode 100644 index 12a091bd3e..0000000000 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/UtilsTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.storage; - -import static com.google.cloud.storage.TestUtils.assertAll; -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.storage.UnifiedOpts.Opt; -import com.google.cloud.storage.UnifiedOpts.Opts; -import org.junit.Test; - -public final class UtilsTest { - private static final Opts autoGzipDecompress_undefined = Opts.empty(); - private static final Opts autoGzipDecompress_no = - Opts.from(UnifiedOpts.returnRawInputStream(true)); - private static final Opts autoGzipDecompress_yes = - Opts.from(UnifiedOpts.returnRawInputStream(false)); - - @Test - public void isAutoGzipDecompression() throws Exception { - assertAll( - () -> - assertThat( - Utils.isAutoGzipDecompression( - autoGzipDecompress_undefined, /*defaultWhenUndefined=*/ false)) - .isFalse(), - () -> - assertThat( - Utils.isAutoGzipDecompression( - autoGzipDecompress_undefined, /*defaultWhenUndefined=*/ true)) - .isTrue(), - () -> - assertThat( - Utils.isAutoGzipDecompression( - autoGzipDecompress_no, /*defaultWhenUndefined=*/ false)) - .isFalse(), - () -> - assertThat( - Utils.isAutoGzipDecompression( - autoGzipDecompress_no, /*defaultWhenUndefined=*/ true)) - .isFalse(), - () -> - assertThat( - Utils.isAutoGzipDecompression( - autoGzipDecompress_yes, /*defaultWhenUndefined=*/ false)) - .isTrue(), - () -> - assertThat( - Utils.isAutoGzipDecompression( - autoGzipDecompress_yes, /*defaultWhenUndefined=*/ true)) - .isTrue()); - } -} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java index 6c93e26882..8957473484 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/ITRetryConformanceTest.java @@ -21,7 +21,6 @@ import static com.google.cloud.storage.conformance.retry.Ctx.ctx; import static com.google.cloud.storage.conformance.retry.State.empty; import static com.google.common.truth.Truth.assertThat; -import static com.google.common.truth.Truth.assertWithMessage; import static java.util.Objects.requireNonNull; import static org.junit.Assert.assertNotNull; @@ -81,6 +80,7 @@ * RpcMethodMappings}. */ @RunWith(StorageITRunner.class) +// @CrossRun(transports = Transport.HTTP, backends = Backend.TEST_BENCH) @SingleBackend(Backend.TEST_BENCH) @Parameterized(RetryConformanceParameterProvider.class) @ParallelFriendly @@ -117,10 +117,9 @@ public void setUp() throws Throwable { public void tearDown() throws Throwable { LOGGER.fine("Running teardown..."); if (ctx != null) { - Ctx tearDownCtx = ctx.leftMap(s -> nonTestStorage); getReplaceStorageInObjectsFromCtx() .andThen(mapping.getTearDown()) - .apply(tearDownCtx, testRetryConformance); + .apply(ctx, testRetryConformance); } retryTestFixture.finished(null); LOGGER.fine("Running teardown complete"); @@ -178,9 +177,7 @@ public ImmutableList parameters() { } catch (IOException e) { throw new RuntimeException(e); } - assertWithMessage("Filter too strict. Resolved 0 retry test cases") - .that(retryTestCases) - .isNotEmpty(); + assertThat(retryTestCases).isNotEmpty(); return retryTestCases.stream() .map( rtc -> diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMapping.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMapping.java index 4b2c8ccf8b..1c3947adbf 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMapping.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMapping.java @@ -21,7 +21,6 @@ import static org.junit.Assert.fail; import com.google.cloud.storage.StorageException; -import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup; import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceTeardown; import com.google.cloud.storage.conformance.retry.Functions.CtxFunction; @@ -105,15 +104,8 @@ public CtxFunction getTest() { if (instructions.contains("return-401") && code == 401) { matchExpectedCode = true; } - if (instructions.contains("return-reset-connection")) { - if (c.getTransport() == Transport.HTTP && code == 0) { - matchExpectedCode = true; - } - // in grpc a broken connection is converted to an UNAVAILABLE - // our mapping from UNAVAILABLE to status code is 503 - if (c.getTransport() == Transport.GRPC && code == 503) { - matchExpectedCode = true; - } + if (instructions.contains("return-reset-connection") && code == 0) { + matchExpectedCode = true; } if (matchExpectedCode) { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java index a76173a129..9dc246373c 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/RpcMethodMappings.java @@ -53,7 +53,6 @@ import com.google.cloud.storage.Storage.SignUrlOption; import com.google.cloud.storage.Storage.UriScheme; import com.google.cloud.storage.StorageRoles; -import com.google.cloud.storage.TransportCompatibility.Transport; import com.google.cloud.storage.conformance.retry.CtxFunctions.Local; import com.google.cloud.storage.conformance.retry.CtxFunctions.ResourceSetup; import com.google.cloud.storage.conformance.retry.CtxFunctions.Rpc; @@ -1608,9 +1607,7 @@ private static void insert(ArrayList a) { .build()); a.add( RpcMethodMapping.newBuilder(54, objects.insert) - .withApplicable( - not(TestRetryConformance::isPreconditionsProvided) - .and(TestRetryConformance.transportIs(Transport.HTTP))) + .withApplicable(not(TestRetryConformance::isPreconditionsProvided)) .withSetup(defaultSetup.andThen(Local.blobInfoWithoutGeneration)) .withTest( (ctx, c) -> diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestRetryConformance.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestRetryConformance.java index 7644052fa3..65c0734807 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestRetryConformance.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/conformance/retry/TestRetryConformance.java @@ -42,7 +42,6 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -240,10 +239,6 @@ public String toString() { return getTestName(); } - public static Predicate transportIs(Transport t) { - return trc -> trc.transport == t; - } - private static Supplier resolvePathForResource(String objectName, Method method) { return () -> { try { diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAutomaticGzipDecompressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAutomaticGzipDecompressionTest.java deleted file mode 100644 index 912faa9d3b..0000000000 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITAutomaticGzipDecompressionTest.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.storage.it; - -import static com.google.cloud.storage.TestUtils.xxd; -import static com.google.common.truth.Truth.assertThat; - -import com.google.cloud.ReadChannel; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.BucketInfo; -import com.google.cloud.storage.Storage; -import com.google.cloud.storage.Storage.BlobSourceOption; -import com.google.cloud.storage.Storage.BlobTargetOption; -import com.google.cloud.storage.TestUtils; -import com.google.cloud.storage.TransportCompatibility.Transport; -import com.google.cloud.storage.it.runner.StorageITRunner; -import com.google.cloud.storage.it.runner.annotations.Backend; -import com.google.cloud.storage.it.runner.annotations.CrossRun; -import com.google.cloud.storage.it.runner.annotations.Inject; -import com.google.cloud.storage.it.runner.registry.Generator; -import com.google.common.io.ByteStreams; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -@RunWith(StorageITRunner.class) -@CrossRun( - backends = {Backend.PROD}, - transports = {Transport.HTTP, Transport.GRPC}) -public final class ITAutomaticGzipDecompressionTest { - - private static final byte[] helloWorldTextBytes = "hello world".getBytes(); - private static final byte[] helloWorldGzipBytes = TestUtils.gzipBytes(helloWorldTextBytes); - - @Inject public Storage storage; - @Inject public BucketInfo bucket; - @Inject public Generator generator; - - private BlobInfo info; - private BlobId blobId; - - @Before - public void setUp() throws Exception { - BlobInfo tmp = - BlobInfo.newBuilder(bucket, generator.randomObjectName()) - // define an object with explicit content type and encoding. - // JSON and gRPC have differing default behavior returning these values if they are - // either undefined, or match HTTP defaults. - .setContentType("text/plain") - .setContentEncoding("gzip") - .build(); - - Blob gen1 = storage.create(tmp, helloWorldGzipBytes, BlobTargetOption.doesNotExist()); - info = gen1.asBlobInfo(); - blobId = info.getBlobId(); - } - - @Test - public void readAllBytes_default_uncompressed() { - byte[] bytes = storage.readAllBytes(blobId); - assertThat(xxd(bytes)).isEqualTo(xxd(helloWorldTextBytes)); - } - - @Test - public void readAllBytes_returnRawInputStream_yes() { - byte[] bytes = storage.readAllBytes(blobId, BlobSourceOption.shouldReturnRawInputStream(true)); - assertThat(xxd(bytes)).isEqualTo(xxd(helloWorldGzipBytes)); - } - - @Test - public void readAllBytes_returnRawInputStream_no() { - byte[] bytes = storage.readAllBytes(blobId, BlobSourceOption.shouldReturnRawInputStream(false)); - assertThat(xxd(bytes)).isEqualTo(xxd(helloWorldTextBytes)); - } - - @Test - public void reader_default_compressed() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ReadChannel r = storage.reader(blobId)) { - WritableByteChannel w = Channels.newChannel(baos); - ByteStreams.copy(r, w); - } - - assertThat(xxd(baos.toByteArray())).isEqualTo(xxd(helloWorldGzipBytes)); - } - - @Test - public void reader_returnRawInputStream_yes() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ReadChannel r = - storage.reader(blobId, BlobSourceOption.shouldReturnRawInputStream(true))) { - WritableByteChannel w = Channels.newChannel(baos); - ByteStreams.copy(r, w); - } - - assertThat(xxd(baos.toByteArray())).isEqualTo(xxd(helloWorldGzipBytes)); - } - - @Test - public void reader_returnRawInputStream_no() throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ReadChannel r = - storage.reader(blobId, BlobSourceOption.shouldReturnRawInputStream(false))) { - WritableByteChannel w = Channels.newChannel(baos); - ByteStreams.copy(r, w); - } - - assertThat(xxd(baos.toByteArray())).isEqualTo(xxd(helloWorldTextBytes)); - } - - @Test - public void downloadTo_path_default_uncompressed() throws IOException { - Path helloWorldTxtGz = File.createTempFile(blobId.getName(), ".txt.gz").toPath(); - storage.downloadTo(blobId, helloWorldTxtGz); - - byte[] actualTxtGzBytes = Files.readAllBytes(helloWorldTxtGz); - assertThat(xxd(actualTxtGzBytes)).isEqualTo(xxd(helloWorldTextBytes)); - } - - @Test - public void downloadTo_path_returnRawInputStream_yes() throws IOException { - Path helloWorldTxtGz = File.createTempFile(blobId.getName(), ".txt.gz").toPath(); - storage.downloadTo(blobId, helloWorldTxtGz, BlobSourceOption.shouldReturnRawInputStream(true)); - - byte[] actualTxtGzBytes = Files.readAllBytes(helloWorldTxtGz); - assertThat(xxd(actualTxtGzBytes)).isEqualTo(xxd(helloWorldGzipBytes)); - } - - @Test - public void downloadTo_path_returnRawInputStream_no() throws IOException { - Path helloWorldTxt = File.createTempFile(blobId.getName(), ".txt").toPath(); - storage.downloadTo(blobId, helloWorldTxt, BlobSourceOption.shouldReturnRawInputStream(false)); - byte[] actualTxtBytes = Files.readAllBytes(helloWorldTxt); - assertThat(xxd(actualTxtBytes)).isEqualTo(xxd(helloWorldTextBytes)); - } - - @Test - public void downloadTo_outputStream_default_uncompressed() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - storage.downloadTo(blobId, baos); - byte[] actual = baos.toByteArray(); - assertThat(xxd(actual)).isEqualTo(xxd(helloWorldTextBytes)); - } - - @Test - public void downloadTo_outputStream_returnRawInputStream_yes() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - storage.downloadTo(blobId, baos, BlobSourceOption.shouldReturnRawInputStream(true)); - byte[] actual = baos.toByteArray(); - assertThat(xxd(actual)).isEqualTo(xxd(helloWorldGzipBytes)); - } - - @Test - public void downloadTo_outputStream_returnRawInputStream_no() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - storage.downloadTo(blobId, baos, BlobSourceOption.shouldReturnRawInputStream(false)); - byte[] actual = baos.toByteArray(); - assertThat(xxd(actual)).isEqualTo(xxd(helloWorldTextBytes)); - } -} diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITDownloadToTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITDownloadToTest.java new file mode 100644 index 0000000000..097609a57b --- /dev/null +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITDownloadToTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.it; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.fail; + +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.TestUtils; +import com.google.cloud.storage.TransportCompatibility.Transport; +import com.google.cloud.storage.it.runner.StorageITRunner; +import com.google.cloud.storage.it.runner.annotations.Backend; +import com.google.cloud.storage.it.runner.annotations.CrossRun; +import com.google.cloud.storage.it.runner.annotations.Inject; +import com.google.cloud.storage.it.runner.registry.Generator; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(StorageITRunner.class) +@CrossRun( + transports = {Transport.HTTP, Transport.GRPC}, + backends = {Backend.PROD}) +public final class ITDownloadToTest { + + private static final byte[] helloWorldTextBytes = "hello world".getBytes(); + private static final byte[] helloWorldGzipBytes = TestUtils.gzipBytes(helloWorldTextBytes); + + @Inject public Storage storage; + @Inject public BucketInfo bucket; + @Inject public Generator generator; + + private BlobId blobId; + + @Before + public void before() { + String objectString = generator.randomObjectName(); + blobId = BlobId.of(bucket.getName(), objectString); + BlobInfo blobInfo = + BlobInfo.newBuilder(blobId).setContentEncoding("gzip").setContentType("text/plain").build(); + storage.create(blobInfo, helloWorldGzipBytes); + } + + @Test + public void downloadTo_returnRawInputStream_yes() throws IOException { + Path helloWorldTxtGz = File.createTempFile(blobId.getName(), ".txt.gz").toPath(); + storage.downloadTo( + blobId, helloWorldTxtGz, Storage.BlobSourceOption.shouldReturnRawInputStream(true)); + + byte[] actualTxtGzBytes = Files.readAllBytes(helloWorldTxtGz); + if (Arrays.equals(actualTxtGzBytes, helloWorldTextBytes)) { + fail("expected gzipped bytes, but got un-gzipped bytes"); + } + assertThat(actualTxtGzBytes).isEqualTo(helloWorldGzipBytes); + } + + @Test + public void downloadTo_returnRawInputStream_no() throws IOException { + Path helloWorldTxt = File.createTempFile(blobId.getName(), ".txt").toPath(); + storage.downloadTo( + blobId, helloWorldTxt, Storage.BlobSourceOption.shouldReturnRawInputStream(false)); + byte[] actualTxtBytes = Files.readAllBytes(helloWorldTxt); + assertThat(actualTxtBytes).isEqualTo(helloWorldTextBytes); + } +}