Skip to content

Commit

Permalink
Ensure uniqueness of load table names (#1009)
Browse files Browse the repository at this point in the history
* Ensure uniqueness when load table name is cut off due to long provider name

* Shorten load table prefix
  • Loading branch information
stacimc authored Feb 24, 2023
1 parent 9d758e9 commit de078ce
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 3 deletions.
2 changes: 1 addition & 1 deletion openverse_catalog/dags/common/loader/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

logger = logging.getLogger(__name__)

LOAD_TABLE_NAME_STUB = "provider_data_"
LOAD_TABLE_NAME_STUB = "load_"
TABLE_NAMES = {AUDIO: AUDIO, IMAGE: IMAGE}
DB_USER_NAME = "deploy"
NOW = "NOW()"
Expand Down
6 changes: 5 additions & 1 deletion openverse_catalog/dags/providers/provider_dag_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,11 @@ def append_day_shift(id_str):
with TaskGroup(group_id=append_day_shift("ingest_data")) as ingest_data:
media_type_name = "mixed" if len(conf.media_types) > 1 else conf.media_types[0]
provider_name = conf.dag_id.replace("_workflow", "")
identifier = f"{provider_name}_{{{{ ts_nodash }}}}_{day_shift}"

# Unique identifier used to generate the load_table name
identifier = f"{{{{ ts_nodash }}}}_{provider_name}"
if is_reingestion:
identifier = f"{day_shift}_{identifier}"

ingestion_kwargs = {
"ingester_class": conf.ingester_class,
Expand Down
2 changes: 1 addition & 1 deletion tests/dags/common/loader/test_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def create_query_values(
@pytest.fixture
def load_table(identifier):
# Parallelized tests need to use distinct database tables
return f"provider_data_image_{identifier}"
return f"load_image_{identifier}"


@pytest.fixture
Expand Down

0 comments on commit de078ce

Please sign in to comment.