Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Spark: Don't allow branch_ usage with TIMESTAMP AS OF #10059

Merged
merged 1 commit into from
Mar 31, 2024
Merged

Spark: Don't allow branch_ usage with TIMESTAMP AS OF #10059

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ public Table loadTable(Identifier ident, long timestamp) throws NoSuchTableExcep
SparkTable sparkTable = (SparkTable) table;

Preconditions.checkArgument(
sparkTable.snapshotId() == null,
sparkTable.snapshotId() == null && sparkTable.branch() == null,
"Cannot do time-travel based on both table identifier and AS OF");

// convert the timestamp to milliseconds as Spark passes microseconds
Expand Down
6 changes: 6 additions & 0 deletions spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,12 @@ public void testInvalidTimeTravelAgainstBranchIdentifierWithAsOf() {
() -> sql("SELECT * FROM %s.branch_b1 VERSION AS OF %s", tableName, snapshotId))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot do time-travel based on both table identifier and AS OF");

// using branch_b1 in the table identifier and TIMESTAMP AS OF
Assertions.assertThatThrownBy(
() -> sql("SELECT * FROM %s.branch_b1 TIMESTAMP AS OF now()", tableName))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot do time-travel based on both table identifier and AS OF");
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ public Table loadTable(Identifier ident, long timestamp) throws NoSuchTableExcep
SparkTable sparkTable = (SparkTable) table;

Preconditions.checkArgument(
sparkTable.snapshotId() == null,
sparkTable.snapshotId() == null && sparkTable.branch() == null,
"Cannot do time-travel based on both table identifier and AS OF");

// convert the timestamp to milliseconds as Spark passes microseconds
Expand Down
6 changes: 6 additions & 0 deletions spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,12 @@ public void testInvalidTimeTravelAgainstBranchIdentifierWithAsOf() {
() -> sql("SELECT * FROM %s.branch_b1 VERSION AS OF %s", tableName, snapshotId))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot do time-travel based on both table identifier and AS OF");

// using branch_b1 in the table identifier and TIMESTAMP AS OF
Assertions.assertThatThrownBy(
() -> sql("SELECT * FROM %s.branch_b1 TIMESTAMP AS OF now()", tableName))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot do time-travel based on both table identifier and AS OF");
}

@Test
Expand Down
2 changes: 1 addition & 1 deletion spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ public Table loadTable(Identifier ident, long timestamp) throws NoSuchTableExcep
SparkTable sparkTable = (SparkTable) table;

Preconditions.checkArgument(
sparkTable.snapshotId() == null,
sparkTable.snapshotId() == null && sparkTable.branch() == null,
"Cannot do time-travel based on both table identifier and AS OF");

// convert the timestamp to milliseconds as Spark passes microseconds
Expand Down
5 changes: 5 additions & 0 deletions spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,11 @@ public void testInvalidTimeTravelAgainstBranchIdentifierWithAsOf() {
() -> sql("SELECT * FROM %s.branch_b1 VERSION AS OF %s", tableName, snapshotId))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot do time-travel based on both table identifier and AS OF");

// using branch_b1 in the table identifier and TIMESTAMP AS OF
assertThatThrownBy(() -> sql("SELECT * FROM %s.branch_b1 TIMESTAMP AS OF now()", tableName))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot do time-travel based on both table identifier and AS OF");
}

@TestTemplate
Expand Down