-
Notifications
You must be signed in to change notification settings - Fork 61
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix: abstract batch resource and add method to determine if batch should be flushed #1790
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
/* | ||
* Copyright 2023 Google LLC | ||
* | ||
* Redistribution and use in source and binary forms, with or without | ||
* modification, are permitted provided that the following conditions are | ||
* met: | ||
* | ||
* * Redistributions of source code must retain the above copyright | ||
* notice, this list of conditions and the following disclaimer. | ||
* * Redistributions in binary form must reproduce the above | ||
* copyright notice, this list of conditions and the following disclaimer | ||
* in the documentation and/or other materials provided with the | ||
* distribution. | ||
* * Neither the name of Google LLC nor the names of its | ||
* contributors may be used to endorse or promote products derived from | ||
* this software without specific prior written permission. | ||
* | ||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | ||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | ||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | ||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | ||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | ||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | ||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | ||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | ||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | ||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
*/ | ||
package com.google.api.gax.batching; | ||
|
||
import com.google.api.core.InternalApi; | ||
|
||
/** | ||
* Represent the resource used by a batch including element and byte. It can also be extended to | ||
* other things to determine if adding a new element needs to be flow controlled or if the current | ||
* batch needs to be flushed. | ||
*/ | ||
@InternalApi("For google-cloud-java client use only.") | ||
public interface BatchResource { | ||
|
||
/** Adds the additional resource. */ | ||
BatchResource add(BatchResource resource); | ||
|
||
/** Returns the element count of this resource. */ | ||
long getElementCount(); | ||
|
||
/** Returns the byte count of this resource. */ | ||
long getByteCount(); | ||
|
||
/** | ||
* Checks if the current {@link BatchResource} should be flushed based on the maxElementThreshold | ||
* and maxBytesThreshold. | ||
*/ | ||
boolean shouldFlush(long maxElementThreshold, long maxBytesThreshold); | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -96,4 +96,17 @@ public interface BatchingDescriptor<ElementT, ElementResultT, RequestT, Response | |
|
||
/** Returns the size of the passed element object in bytes. */ | ||
long countBytes(ElementT element); | ||
|
||
/** Creates a new {@link BatchResource} with ElementT. */ | ||
default BatchResource createResource(ElementT element) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need this helper method in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, in bigtable client we want to override the BatchResource implementation, so we don't want to create BatchResource directly in the BatcherImpl code. |
||
return DefaultBatchResource.builder() | ||
.setElementCount(1) | ||
.setByteCount(countBytes(element)) | ||
.build(); | ||
} | ||
|
||
/** Create an empty {@link BatchResource}. */ | ||
default BatchResource createEmptyResource() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same comment for this method. |
||
return DefaultBatchResource.builder().setElementCount(0).setByteCount(0).build(); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
/* | ||
* Copyright 2023 Google LLC | ||
* | ||
* Redistribution and use in source and binary forms, with or without | ||
* modification, are permitted provided that the following conditions are | ||
* met: | ||
* | ||
* * Redistributions of source code must retain the above copyright | ||
* notice, this list of conditions and the following disclaimer. | ||
* * Redistributions in binary form must reproduce the above | ||
* copyright notice, this list of conditions and the following disclaimer | ||
* in the documentation and/or other materials provided with the | ||
* distribution. | ||
* * Neither the name of Google LLC nor the names of its | ||
* contributors may be used to endorse or promote products derived from | ||
* this software without specific prior written permission. | ||
* | ||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | ||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | ||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | ||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | ||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | ||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | ||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | ||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | ||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | ||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | ||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||
*/ | ||
package com.google.api.gax.batching; | ||
|
||
import com.google.auto.value.AutoValue; | ||
import com.google.common.base.Preconditions; | ||
|
||
/** | ||
* The default implementation of {@link BatchResource} which tracks the elementCount and byteCount. | ||
igorbernstein2 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
*/ | ||
@AutoValue | ||
abstract class DefaultBatchResource implements BatchResource { | ||
|
||
static DefaultBatchResource.Builder builder() { | ||
return new AutoValue_DefaultBatchResource.Builder(); | ||
} | ||
|
||
@Override | ||
public BatchResource add(BatchResource resource) { | ||
Preconditions.checkArgument( | ||
resource instanceof DefaultBatchResource, | ||
"Expect an instance of DefaultBatchResource, got " + resource.getClass()); | ||
DefaultBatchResource defaultResource = (DefaultBatchResource) resource; | ||
return new AutoValue_DefaultBatchResource.Builder() | ||
.setElementCount(getElementCount() + defaultResource.getElementCount()) | ||
.setByteCount(getByteCount() + defaultResource.getByteCount()) | ||
.build(); | ||
} | ||
|
||
@Override | ||
public abstract long getElementCount(); | ||
|
||
@Override | ||
public abstract long getByteCount(); | ||
|
||
@Override | ||
public boolean shouldFlush(long maxElementThreshold, long maxBytesThreshold) { | ||
return getElementCount() > maxElementThreshold || getByteCount() > maxBytesThreshold; | ||
} | ||
|
||
@AutoValue.Builder | ||
abstract static class Builder { | ||
abstract Builder setElementCount(long elementCount); | ||
|
||
abstract Builder setByteCount(long byteCount); | ||
|
||
abstract DefaultBatchResource build(); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Do we have to create an interface for it? The default implementation looks like a POJO to me, I would prefer not to have a separate interface if we don't plan to have multiple implementations.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
bigtable client will need to override it and have a different implementation :(