Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: read_modify_write and check_and_mutate_row #780

Merged
merged 62 commits into from
Jun 16, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
1d02154
added initial implementation of mutate_rows
daniel-sanche Apr 24, 2023
ab63cba
implemented mutation models
daniel-sanche Apr 24, 2023
cf9daa5
added retries to mutate_row
daniel-sanche Apr 24, 2023
1247da4
return exception group if possible
daniel-sanche Apr 24, 2023
3b3ed8c
check for idempotence
daniel-sanche Apr 24, 2023
5d20037
initial implementation for bulk_mutations
daniel-sanche Apr 24, 2023
3d322a1
include successes in bulk mutation error message
daniel-sanche Apr 24, 2023
a31232b
fixed style checks
daniel-sanche Apr 24, 2023
8da2d65
added basic system tests
daniel-sanche Apr 24, 2023
2b89d9c
added unit tests for mutate_row
daniel-sanche Apr 25, 2023
47c5985
ran blacken
daniel-sanche Apr 25, 2023
38fdcd7
improved exceptions
daniel-sanche Apr 25, 2023
504d2d8
added bulk_mutate_rows unit tests
daniel-sanche Apr 25, 2023
b16067f
ran blacken
daniel-sanche Apr 25, 2023
3ab1405
support __new___ for exceptions for python3.11+
daniel-sanche Apr 25, 2023
0a6c0c6
added exception unit tests
daniel-sanche Apr 25, 2023
ec043cf
makde exceptions tuple
daniel-sanche Apr 26, 2023
518530e
got exceptions to print consistently across versions
daniel-sanche Apr 26, 2023
9624729
added test for 311 rich traceback
daniel-sanche Apr 27, 2023
3087081
moved retryable row mutations to new file
daniel-sanche Apr 27, 2023
9df588f
use index map
daniel-sanche Apr 27, 2023
7ed8be3
added docstring
daniel-sanche Apr 27, 2023
2536cc4
added predicate check to failed mutations
daniel-sanche Apr 27, 2023
1f6875c
added _mutate_rows tests
daniel-sanche Apr 27, 2023
1ea24e6
improved client tests
daniel-sanche Apr 27, 2023
25ca2d2
refactored to loop by raising exception
daniel-sanche Apr 28, 2023
c0787db
refactored retry deadline logic into shared wrapper
daniel-sanche Apr 28, 2023
3ed5c3d
ran black
daniel-sanche Apr 28, 2023
a91fbcb
pulled in table default timeouts
daniel-sanche Apr 28, 2023
df8a058
added tests for shared deadline parsing function
daniel-sanche Apr 28, 2023
b866b57
added tests for mutation models
daniel-sanche Apr 28, 2023
54a4d43
fixed linter errors
daniel-sanche Apr 28, 2023
bd51dc4
added tests for BulkMutationsEntry
daniel-sanche Apr 28, 2023
921b05a
improved mutations documentation
daniel-sanche Apr 28, 2023
82ea61f
refactored mutate_rows logic into helper function
daniel-sanche May 2, 2023
fa42b86
implemented callbacks for mutate_rows
daniel-sanche May 2, 2023
01a16f3
made exceptions into a tuple
daniel-sanche May 5, 2023
e6df77e
improved and tested read_modify_write_rules models
daniel-sanche May 18, 2023
2d8ee3f
implemented read_modify_write
daniel-sanche May 18, 2023
af77dc3
added unit tests
daniel-sanche May 18, 2023
ebe2f94
added system test
daniel-sanche May 18, 2023
8af5c71
added test for large values
daniel-sanche May 18, 2023
1242836
allow string for append value rule
daniel-sanche May 18, 2023
afe839c
added append value system test
daniel-sanche May 18, 2023
d0781d0
added chained value system test
daniel-sanche May 18, 2023
ef30977
support creating SetValueMutation with int
daniel-sanche May 19, 2023
6140acb
remove aborted from retryable errors
daniel-sanche May 22, 2023
36ba2b6
improved SetCell mutation
daniel-sanche May 22, 2023
b3c9017
fixed mutations tests
daniel-sanche May 22, 2023
cac9e2d
SetCell timestamps use millisecond precision
daniel-sanche May 22, 2023
34b051f
renamed BulkMutationsEntry to RowMutationEntry
daniel-sanche May 22, 2023
baf3378
implemented check_and_mutate
daniel-sanche May 22, 2023
bad11e5
added system tests
daniel-sanche May 22, 2023
1d79202
fixed test issues
daniel-sanche May 23, 2023
63ac35c
Merge branch 'v3' into mutate_rows
daniel-sanche May 24, 2023
4138c89
Merge branch 'mutate_rows' into mutate_rows_other_rpcs
daniel-sanche May 24, 2023
3c27fb7
Merge branch 'v3' into mutate_rows_other_rpcs
daniel-sanche Jun 7, 2023
b9b9dac
adjusted tests; require kwargs for check_and_mutate
daniel-sanche Jun 7, 2023
234ea6c
added metadata
daniel-sanche Jun 7, 2023
fb818d4
clean up
daniel-sanche Jun 8, 2023
c9cebc2
changed timeout values
daniel-sanche Jun 14, 2023
ef8879e
Merge branch 'v3' into mutate_rows_other_rpcs
daniel-sanche Jun 16, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
added system tests
  • Loading branch information
daniel-sanche committed May 22, 2023
commit bad11e5e7368a15dcc96322191bfbf94e4039ce3
7 changes: 5 additions & 2 deletions google/cloud/bigtable/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -772,9 +772,11 @@ async def check_and_mutate_row(
true_case_mutations = [true_case_mutations]
if isinstance(false_case_mutations, Mutation):
false_case_mutations = [false_case_mutations]
true_case_mutations = [m._to_dict() for m in true_case_mutations or []]
false_case_mutations = [m._to_dict() for m in false_case_mutations or []]
if predicate is not None and not isinstance(predicate, dict):
predicate = predicate._to_dict()
return await self.client._gapic_client.check_and_mutate_row(
predicate = predicate.to_dict()
result = await self.client._gapic_client.check_and_mutate_row(
table_name=self.table_name,
row_key=row_key,
predicate_filter=predicate,
Expand All @@ -783,6 +785,7 @@ async def check_and_mutate_row(
app_profile_id=self.app_profile_id,
timeout=operation_timeout,
)
return result.predicate_matched

async def read_modify_write_row(
self,
Expand Down
32 changes: 32 additions & 0 deletions tests/system/test_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -310,3 +310,35 @@ async def test_read_modify_write_row_chained(client, table, temp_rows):
assert result[0].column_qualifier == qualifier
# result should be a bytes number string for the IncrementRules, followed by the AppendValueRule values
assert result[0].value == (start_amount+increment_amount).to_bytes(8, "big", signed=True) + b"helloworld!"

@pytest.mark.parametrize("start_val,predicate_range,expected_result", [
(1, (0,2), True),
(-1, (0,2), False),
])
@pytest.mark.asyncio
async def test_check_and_mutate(client, table, temp_rows, start_val, predicate_range, expected_result):
"""
test that check_and_mutate_row works applies the right mutations, and returns the right result
"""
from google.cloud.bigtable.mutations import SetCell
from google.cloud.bigtable.row_filters import ValueRangeFilter
row_key = b"test-row-key"
family = TEST_FAMILY
qualifier = b"test-qualifier"

await temp_rows.add_row(row_key, value=start_val, family=family, qualifier=qualifier)

false_mutation_value = -99
false_mutation_result = SetCell(
family=TEST_FAMILY, qualifier=qualifier, new_value=false_mutation_value
)
true_mutation_value = 55
true_mutation_result = SetCell(
family=TEST_FAMILY, qualifier=qualifier, new_value=true_mutation_value
)
predicate = ValueRangeFilter(predicate_range[0], predicate_range[1])
result = await table.check_and_mutate_row(
row_key, predicate, [false_mutation_result], [true_mutation_result]
)
assert result == expected_result
#TODO: add read_row assertions to verify the mutation was applied
49 changes: 39 additions & 10 deletions tests/unit/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1372,6 +1372,7 @@ def _make_client(self, *args, **kwargs):
@pytest.mark.parametrize("gapic_result", [True, False])
@pytest.mark.asyncio
async def test_check_and_mutate(self, gapic_result):
from google.cloud.bigtable_v2.types import CheckAndMutateRowResponse
app_profile = "app_profile_id"
async with self._make_client() as client:
async with client.get_table(
Expand All @@ -1380,11 +1381,11 @@ async def test_check_and_mutate(self, gapic_result):
with mock.patch.object(
client._gapic_client, "check_and_mutate_row"
) as mock_gapic:
mock_gapic.return_value = gapic_result
mock_gapic.return_value = CheckAndMutateRowResponse(predicate_matched=gapic_result)
row_key = b"row_key"
predicate = None
true_mutations = [object()]
false_mutations = [object(), object()]
true_mutations = [{}]
false_mutations = [{}, {}]
operation_timeout = 0.2
found = await table.check_and_mutate_row(
row_key,
Expand Down Expand Up @@ -1422,13 +1423,14 @@ async def test_check_and_mutate_bad_timeout(self):
async def test_check_and_mutate_single_mutations(self):
"""if single mutations are passed, they should be internally wrapped in a list"""
from google.cloud.bigtable.mutations import SetCell
from google.cloud.bigtable_v2.types import CheckAndMutateRowResponse

async with self._make_client() as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(
client._gapic_client, "check_and_mutate_row"
) as mock_gapic:
mock_gapic.return_value = True
mock_gapic.return_value = CheckAndMutateRowResponse(predicate_matched=True)
true_mutation = SetCell("family", b"qualifier", b"value")
false_mutation = SetCell("family", b"qualifier", b"value")
await table.check_and_mutate_row(
Expand All @@ -1438,30 +1440,57 @@ async def test_check_and_mutate_single_mutations(self):
false_mutation,
)
kwargs = mock_gapic.call_args[1]
assert kwargs["true_mutations"] == [true_mutation]
assert kwargs["false_mutations"] == [false_mutation]
assert kwargs["true_mutations"] == [true_mutation._to_dict()]
assert kwargs["false_mutations"] == [false_mutation._to_dict()]

@pytest.mark.asyncio
async def test_check_and_mutate_predicate_object(self):
"""predicate object should be converted to dict"""
from google.cloud.bigtable_v2.types import CheckAndMutateRowResponse
mock_predicate = mock.Mock()
fake_dict = {"fake": "dict"}
mock_predicate._to_dict.return_value = fake_dict
mock_predicate.to_dict.return_value = fake_dict
async with self._make_client() as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(
client._gapic_client, "check_and_mutate_row"
) as mock_gapic:
mock_gapic.return_value = True
mock_gapic.return_value = CheckAndMutateRowResponse(predicate_matched=True)
await table.check_and_mutate_row(
b"row_key",
mock_predicate,
None,
[object(), object()],
[{}, {}],
)
kwargs = mock_gapic.call_args[1]
assert kwargs["predicate_filter"] == fake_dict
assert mock_predicate._to_dict.call_count == 1
assert mock_predicate.to_dict.call_count == 1

@pytest.mark.asyncio
async def test_check_and_mutate_mutations_parsing(self):
"""mutations objects should be converted to dicts"""
from google.cloud.bigtable_v2.types import CheckAndMutateRowResponse
from google.cloud.bigtable.mutations import DeleteAllFromRow
mutations = [mock.Mock() for _ in range(5)]
for idx, mutation in enumerate(mutations):
mutation._to_dict.return_value = {"fake": idx}
mutations.append(DeleteAllFromRow())
async with self._make_client() as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(
client._gapic_client, "check_and_mutate_row"
) as mock_gapic:
mock_gapic.return_value = CheckAndMutateRowResponse(predicate_matched=True)
await table.check_and_mutate_row(
b"row_key",
None,
mutations[0:2],
mutations[2:],
)
kwargs = mock_gapic.call_args[1]
assert kwargs["true_mutations"] == [{"fake": 0}, {"fake": 1}]
assert kwargs["false_mutations"] == [{"fake": 2}, {"fake": 3}, {"fake": 4}, {"delete_from_row": {}}]
assert all(mutation._to_dict.call_count == 1 for mutation in mutations[:5])


class TestReadModifyWriteRow:
Expand Down