Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for templateSuffix to BigQuery InsertAllRequest #514

Merged
merged 1 commit into from
Dec 29, 2015
Merged

Add support for templateSuffix to BigQuery InsertAllRequest #514

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/BigQueryImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,7 @@ public InsertAllResponse insertAll(InsertAllRequest request) throws BigQueryExce
final TableDataInsertAllRequest requestPb = new TableDataInsertAllRequest();
requestPb.setIgnoreUnknownValues(request.ignoreUnknownValues());
requestPb.setSkipInvalidRows(request.skipInvalidRows());
requestPb.setTemplateSuffix(request.templateSuffix());
List<Rows> rowsPb = Lists.transform(request.rows(), new Function<RowToInsert, Rows>() {
@Override
public Rows apply(RowToInsert rowToInsert) {
Expand Down
36 changes: 34 additions & 2 deletions gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/InsertAllRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ public class InsertAllRequest implements Serializable {
private final List<RowToInsert> rows;
private final Boolean skipInvalidRows;
private final Boolean ignoreUnknownValues;
private final String templateSuffix;

/**
* A Google Big Query row to be inserted into a table. Each {@code RowToInsert} has an associated
Expand Down Expand Up @@ -140,6 +141,7 @@ public static final class Builder {
private List<RowToInsert> rows;
private Boolean skipInvalidRows;
private Boolean ignoreUnknownValues;
private String templateSuffix;

private Builder() {}

Expand Down Expand Up @@ -231,6 +233,20 @@ public Builder ignoreUnknownValues(boolean ignoreUnknownValues) {
return this;
}

/**
* If specified, the destination table is treated as a base template. Rows are inserted into an
* instance table named "{destination}{templateSuffix}". BigQuery will manage the creation of
* the instance table, using the schema of the base template table.
*
* @see <a
* href="https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables">
* Template Tables</a>
*/
public Builder templateSuffix(String templateSuffix) {
this.templateSuffix = templateSuffix;
return this;
}

public InsertAllRequest build() {
return new InsertAllRequest(this);
}
Expand All @@ -241,6 +257,7 @@ private InsertAllRequest(Builder builder) {
this.rows = ImmutableList.copyOf(checkNotNull(builder.rows));
this.ignoreUnknownValues = builder.ignoreUnknownValues;
this.skipInvalidRows = builder.skipInvalidRows;
this.templateSuffix = builder.templateSuffix;
}

/**
Expand Down Expand Up @@ -273,6 +290,19 @@ public Boolean skipInvalidRows() {
return skipInvalidRows;
}

/**
* If specified, the destination table is treated as a base template. Rows are inserted into an
* instance table named "{destination}{templateSuffix}". BigQuery will manage the creation of the
* instance table, using the schema of the base template table.
*
* @see <a
* href="https://cloud.google.com/bigquery/streaming-data-into-bigquery#template-tables">
* Template Tables</a>
*/
public String templateSuffix() {
return templateSuffix;
}

/**
* Returns a builder for an {@code InsertAllRequest} object given the destination table.
*/
Expand Down Expand Up @@ -384,12 +414,13 @@ public String toString() {
.add("rows", rows)
.add("ignoreUnknownValues", ignoreUnknownValues)
.add("skipInvalidRows", skipInvalidRows)
.add("templateSuffix", templateSuffix)
.toString();
}

@Override
public int hashCode() {
return Objects.hash(table, rows, ignoreUnknownValues, skipInvalidRows);
return Objects.hash(table, rows, ignoreUnknownValues, skipInvalidRows, templateSuffix);
}

@Override
Expand All @@ -401,6 +432,7 @@ public boolean equals(Object obj) {
return Objects.equals(table, other.table)
&& Objects.equals(rows, other.rows)
&& Objects.equals(ignoreUnknownValues, other.ignoreUnknownValues)
&& Objects.equals(skipInvalidRows, other.skipInvalidRows);
&& Objects.equals(skipInvalidRows, other.skipInvalidRows)
&& Objects.equals(templateSuffix, other.templateSuffix);
}
}
3 changes: 2 additions & 1 deletion gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/BigQueryImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,7 @@ public void testInsertAll() {
.rows(rows)
.skipInvalidRows(false)
.ignoreUnknownValues(true)
.templateSuffix("suffix")
.build();
TableDataInsertAllRequest requestPb = new TableDataInsertAllRequest().setRows(
Lists.transform(rows, new Function<RowToInsert, TableDataInsertAllRequest.Rows>() {
Expand All @@ -623,7 +624,7 @@ public TableDataInsertAllRequest.Rows apply(RowToInsert rowToInsert) {
.setJson(rowToInsert.content());
}
})
).setSkipInvalidRows(false).setIgnoreUnknownValues(true);
).setSkipInvalidRows(false).setIgnoreUnknownValues(true).setTemplateSuffix("suffix");
TableDataInsertAllResponse responsePb = new TableDataInsertAllResponse().setInsertErrors(
ImmutableList.of(new TableDataInsertAllResponse.InsertErrors().setIndex(0L).setErrors(
ImmutableList.of(new ErrorProto().setMessage("ErrorMessage")))));
Expand Down
35 changes: 35 additions & 0 deletions gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/ITBigQueryTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -475,6 +475,41 @@ public void testInsertAll() {
assertTrue(bigquery.delete(TableId.of(DATASET, tableName)));
}

@Test
public void testInsertAllWithSuffix() {
String tableName = "test_insert_all_with_suffix_table";
BaseTableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), TABLE_SCHEMA);
assertNotNull(bigquery.create(tableInfo));
InsertAllRequest request = InsertAllRequest.builder(tableInfo.tableId())
.addRow(ImmutableMap.<String, Object>of(
"TimestampField", "2014-08-19 07:41:35.220 -05:00",
"StringField", "stringValue",
"IntegerField", ImmutableList.of(0, 1),
"BooleanField", false,
"RecordField", ImmutableMap.of(
"TimestampField", "1969-07-20 20:18:04 UTC",
"IntegerField", ImmutableList.of(1, 0),
"BooleanField", true)))
.addRow(ImmutableMap.<String, Object>of(
"TimestampField", "2014-08-19 07:41:35.220 -05:00",
"StringField", "stringValue",
"IntegerField", ImmutableList.of(0, 1),
"BooleanField", false,
"RecordField", ImmutableMap.of(
"TimestampField", "1969-07-20 20:18:04 UTC",
"IntegerField", ImmutableList.of(1, 0),
"BooleanField", true)))
.templateSuffix("_suffix")
.build();
InsertAllResponse response = bigquery.insertAll(request);
assertFalse(response.hasErrors());
assertEquals(0, response.insertErrors().size());
String newTableName = tableName + "_suffix";
assertNotNull(bigquery.getTable(DATASET, newTableName, TableOption.fields()));
assertTrue(bigquery.delete(TableId.of(DATASET, tableName)));
assertTrue(bigquery.delete(TableId.of(DATASET, newTableName)));
}

@Test
public void testInsertAllWithErrors() {
String tableName = "test_insert_all_with_errors_table";
Expand Down
53 changes: 39 additions & 14 deletions gcloud-java-bigquery/src/test/java/com/google/gcloud/bigquery/InsertAllRequestTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;

import com.google.common.collect.ImmutableList;
Expand Down Expand Up @@ -45,6 +46,7 @@ public class InsertAllRequestTest {
private static final BaseTableInfo TABLE_INFO = TableInfo.of(TABLE_ID, TABLE_SCHEMA);
private static final boolean SKIP_INVALID_ROWS = true;
private static final boolean IGNORE_UNKNOWN_VALUES = false;
private static final String TEMPLATE_SUFFIX = "templateSuffix";
private static final InsertAllRequest INSERT_ALL_REQUEST1 = InsertAllRequest.builder(TABLE_ID)
.addRow(CONTENT1)
.addRow(CONTENT2)
Expand Down Expand Up @@ -90,20 +92,25 @@ public class InsertAllRequestTest {
.ignoreUnknownValues(IGNORE_UNKNOWN_VALUES)
.skipInvalidRows(SKIP_INVALID_ROWS)
.build();
private static final InsertAllRequest INSERT_ALL_REQUEST9 =
InsertAllRequest.builder(TABLE_INFO)
.addRow("id1", CONTENT1)
.addRow("id2", CONTENT2)
.ignoreUnknownValues(IGNORE_UNKNOWN_VALUES)
.skipInvalidRows(SKIP_INVALID_ROWS)
.build();
private static final InsertAllRequest INSERT_ALL_REQUEST10 =
InsertAllRequest.builder(TABLE_INFO)
.addRow("id1", CONTENT1)
.addRow("id2", CONTENT2)
.ignoreUnknownValues(true)
.skipInvalidRows(false)
.build();
private static final InsertAllRequest INSERT_ALL_REQUEST9 = InsertAllRequest.builder(TABLE_INFO)
.addRow("id1", CONTENT1)
.addRow("id2", CONTENT2)
.ignoreUnknownValues(IGNORE_UNKNOWN_VALUES)
.skipInvalidRows(SKIP_INVALID_ROWS)
.build();
private static final InsertAllRequest INSERT_ALL_REQUEST10 = InsertAllRequest.builder(TABLE_INFO)
.addRow("id1", CONTENT1)
.addRow("id2", CONTENT2)
.ignoreUnknownValues(true)
.skipInvalidRows(false)
.build();
private static final InsertAllRequest INSERT_ALL_REQUEST11 = InsertAllRequest.builder(TABLE_INFO)
.addRow("id1", CONTENT1)
.addRow("id2", CONTENT2)
.ignoreUnknownValues(true)
.skipInvalidRows(false)
.templateSuffix(TEMPLATE_SUFFIX)
.build();

@Test
public void testBuilder() {
Expand All @@ -117,6 +124,7 @@ public void testBuilder() {
assertEquals(TABLE_ID, INSERT_ALL_REQUEST8.table());
assertEquals(TABLE_ID, INSERT_ALL_REQUEST9.table());
assertEquals(TABLE_ID, INSERT_ALL_REQUEST10.table());
assertEquals(TABLE_ID, INSERT_ALL_REQUEST11.table());
assertEquals(ROWS, INSERT_ALL_REQUEST1.rows());
assertEquals(ROWS, INSERT_ALL_REQUEST2.rows());
assertEquals(ROWS, INSERT_ALL_REQUEST4.rows());
Expand All @@ -127,6 +135,7 @@ public void testBuilder() {
assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST8.rows());
assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST9.rows());
assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST10.rows());
assertEquals(ROWS_WITH_ID, INSERT_ALL_REQUEST11.rows());
assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST1.skipInvalidRows());
assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST2.skipInvalidRows());
assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST3.skipInvalidRows());
Expand All @@ -137,6 +146,7 @@ public void testBuilder() {
assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST8.skipInvalidRows());
assertEquals(SKIP_INVALID_ROWS, INSERT_ALL_REQUEST9.skipInvalidRows());
assertFalse(INSERT_ALL_REQUEST10.skipInvalidRows());
assertFalse(INSERT_ALL_REQUEST11.skipInvalidRows());
assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST1.ignoreUnknownValues());
assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST2.ignoreUnknownValues());
assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST3.ignoreUnknownValues());
Expand All @@ -147,6 +157,18 @@ public void testBuilder() {
assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST8.ignoreUnknownValues());
assertEquals(IGNORE_UNKNOWN_VALUES, INSERT_ALL_REQUEST9.ignoreUnknownValues());
assertTrue(INSERT_ALL_REQUEST10.ignoreUnknownValues());
assertTrue(INSERT_ALL_REQUEST11.ignoreUnknownValues());
assertNull(INSERT_ALL_REQUEST1.templateSuffix());
assertNull(INSERT_ALL_REQUEST2.templateSuffix());
assertNull(INSERT_ALL_REQUEST3.templateSuffix());
assertNull(INSERT_ALL_REQUEST4.templateSuffix());
assertNull(INSERT_ALL_REQUEST5.templateSuffix());
assertNull(INSERT_ALL_REQUEST6.templateSuffix());
assertNull(INSERT_ALL_REQUEST7.templateSuffix());
assertNull(INSERT_ALL_REQUEST8.templateSuffix());
assertNull(INSERT_ALL_REQUEST9.templateSuffix());
assertNull(INSERT_ALL_REQUEST10.templateSuffix());
assertEquals(TEMPLATE_SUFFIX, INSERT_ALL_REQUEST11.templateSuffix());
}

@Test
Expand Down Expand Up @@ -183,6 +205,8 @@ public void testEquals() {
compareInsertAllRequest(INSERT_ALL_REQUEST5, INSERT_ALL_REQUEST7);
compareInsertAllRequest(INSERT_ALL_REQUEST7, INSERT_ALL_REQUEST8);
compareInsertAllRequest(INSERT_ALL_REQUEST8, INSERT_ALL_REQUEST9);
compareInsertAllRequest(INSERT_ALL_REQUEST10, INSERT_ALL_REQUEST10);
compareInsertAllRequest(INSERT_ALL_REQUEST11, INSERT_ALL_REQUEST11);
}

private void compareInsertAllRequest(InsertAllRequest expected, InsertAllRequest value) {
Expand All @@ -193,5 +217,6 @@ private void compareInsertAllRequest(InsertAllRequest expected, InsertAllRequest
assertEquals(expected.rows(), value.rows());
assertEquals(expected.ignoreUnknownValues(), value.ignoreUnknownValues());
assertEquals(expected.skipInvalidRows(), value.skipInvalidRows());
assertEquals(expected.templateSuffix(), value.templateSuffix());
}
}