Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

vdk-core: Adopt 'method' argument in pre-process plugins #3074

Merged
merged 5 commits into from
Feb 19, 2024
Merged

vdk-core: Adopt 'method' argument in pre-process plugins #3074

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0570e6a
vdk-core: Adopt 'method' argument in pre-process plugins
doks5 Feb 2, 2024
b771aad
Merge branch 'main' into person/andonova/ingester-base
Feb 14, 2024
fc21ef6
Merge branch 'main' into person/andonova/ingester-base
Feb 15, 2024
83c4f73
Merge branch 'main' into person/andonova/ingester-base
duyguHsnHsn Feb 19, 2024
addd4a8
Merge branch 'main' into person/andonova/ingester-base
Feb 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion projects/vdk-core/src/vdk/internal/builtin_plugins/ingestion/ingester_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,9 +437,16 @@ def _payload_poster_thread(self):
destination_table: Optional[str] = None
target: Optional[str] = None
collection_id: Optional[str] = None
method: Optional[str] = None
try:
payload = self._payloads_queue.get()
payload_obj, destination_table, _, target, collection_id = payload
(
payload_obj,
destination_table,
method,
target,
collection_id,
) = payload

# If there are any pre-processors set, pass the payload object
# through them.
Expand All @@ -450,6 +457,7 @@ def _payload_poster_thread(self):
target=target,
collection_id=collection_id,
metadata=ingestion_metadata,
method=method,
)

# Verify payload after pre-processing it, since this preprocessing might be responsible for
Expand Down Expand Up @@ -512,6 +520,7 @@ def _payload_poster_thread(self):
collection_id=collection_id,
metadata=ingestion_metadata,
exception=exception,
method=method,
)
except Exception as e:
resolvable_by = errors.get_exception_resolvable_by(e)
Expand Down Expand Up @@ -575,6 +584,7 @@ def _pre_process_payload(
target: Optional[str],
collection_id: Optional[str],
metadata: Optional[IIngesterPlugin.IngestionMetadata],
method: Optional[str],
) -> Tuple[List[dict], Optional[IIngesterPlugin.IngestionMetadata]]:
for plugin in self._pre_processors:
try:
Expand All @@ -584,6 +594,7 @@ def _pre_process_payload(
target=target,
collection_id=collection_id,
metadata=metadata,
method=method,
)
except Exception as e:
raise PreProcessPayloadIngestionException(
Expand Down Expand Up @@ -611,6 +622,7 @@ def _execute_post_process_operations(
collection_id: Optional[str],
metadata: Optional[IIngesterPlugin.IngestionMetadata],
exception: Optional[Exception],
method: Optional[str],
):
for plugin in self._post_processors:
try:
Expand All @@ -621,6 +633,7 @@ def _execute_post_process_operations(
collection_id=collection_id,
metadata=metadata,
exception=exception,
method=method,
)
except Exception as e:
raise PostProcessPayloadIngestionException(
Expand Down
4 changes: 4 additions & 0 deletions projects/vdk-core/tests/vdk/internal/builtin_plugins/ingestion/test_ingester_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ def test_pre_ingestion_operation():
target=shared_test_values.get("target"),
collection_id=shared_test_values.get("collection_id"),
metadata=None,
method=shared_test_values.get("method"),
)
ingester_base._ingester.ingest_payload.assert_called_once()

Expand Down Expand Up @@ -276,6 +277,7 @@ def test_pre_ingestion_updated_dynamic_params():
target=shared_test_values.get("target"),
collection_id=shared_test_values.get("collection_id"),
metadata=None,
method=shared_test_values.get("method"),
)
ingester_base._ingester.ingest_payload.assert_called_with(
collection_id=shared_test_values.get("collection_id"),
Expand Down Expand Up @@ -335,6 +337,7 @@ def test_ingest_payload_and_post_ingestion_operation():
collection_id=shared_test_values.get("collection_id"),
metadata=test_ingestion_metadata,
exception=None,
method=shared_test_values.get("method"),
)


Expand All @@ -361,4 +364,5 @@ def test_post_ingestion_operation_with_exceptions():
collection_id=shared_test_values.get("collection_id"),
metadata=None,
exception=test_exception,
method=shared_test_values.get("method"),
)
Loading