From 70ffcf88c7c47c1fc4d1268f0f557f415e626502 Mon Sep 17 00:00:00 2001 From: ayan <100955969+ayan1229@users.noreply.github.com> Date: Mon, 27 Jan 2025 11:46:46 -0500 Subject: [PATCH 1/8] SFR-2481/save_test_data --- processes/local_development/seed_test_data.py | 11 +++++++++++ test_data_ids.json | 1 + tests/integration/api/test_get_edition.py | 12 +++++++++++- 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test_data_ids.json diff --git a/processes/local_development/seed_test_data.py b/processes/local_development/seed_test_data.py index 4ec5ba0c855..afb477837b3 100644 --- a/processes/local_development/seed_test_data.py +++ b/processes/local_development/seed_test_data.py @@ -45,6 +45,8 @@ def runProcess(self): cluster_process = ClusterProcess('complete', None, None, self.test_data['uuid'], None) cluster_process.runProcess() + self.save_test_data_ids_to_file() + except Exception as e: logger.exception(f'Failed to seed test data') raise e @@ -68,3 +70,12 @@ def save_test_record(self): self.db_manager.session.add(test_record) self.db_manager.session.commit() + + def save_test_data_ids_to_file(self): + record = self.db_manager.session.query(Record).filter_by(source_id=self.test_data['source_id']).first() + if record: + with open('test_data_ids.json', 'w') as f: + json.dump({ + 'edition_id': str(record.edition_id), + 'uuid': str(record.uuid) + }, f) \ No newline at end of file diff --git a/test_data_ids.json b/test_data_ids.json new file mode 100644 index 00000000000..f64656ae6a8 --- /dev/null +++ b/test_data_ids.json @@ -0,0 +1 @@ +{"uuid": "3264ccb1-a870-43d7-9496-2153ca99acc6"} \ No newline at end of file diff --git a/tests/integration/api/test_get_edition.py b/tests/integration/api/test_get_edition.py index 65d10637ca7..c053de50916 100644 --- a/tests/integration/api/test_get_edition.py +++ b/tests/integration/api/test_get_edition.py @@ -1,10 +1,20 @@ import pytest import requests +import json from .constants import API_URL from .utils import assert_response_status +@pytest.fixture(scope="module", autouse=True) +def seed_test_data(): + process = SeedTestDataProcess() + process.runProcess() + +with open('test_data_ids.json', 'r') as f: + test_data_ids = json.load(f) + SEEDED_EDITION_ID = test_data_ids['edition_id'] + @pytest.mark.parametrize("endpoint, expected_status", [ - ("/editions/1982731", 200), + ("/editions/{SEEDED_EDITION_ID}", 200), ("/editions/00000000-0000-0000-0000-000000000000", 400), ("/editions/invalid_id_format", 400), ("/editions/", 404), From 79c57ce5af732572f852255cfe7cc5fd42abf496 Mon Sep 17 00:00:00 2001 From: ayan <100955969+ayan1229@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:20:17 -0500 Subject: [PATCH 2/8] removed seed_test_data and moved to conftest.py --- processes/__init__.py | 1 - processes/local_development/seed_test_data.py | 81 ------------------- test_data_ids.json | 2 +- tests/conftest.py | 72 ++++++++++++++++- tests/integration/api/test_get_edition.py | 15 +--- 5 files changed, 75 insertions(+), 96 deletions(-) delete mode 100644 processes/local_development/seed_test_data.py diff --git a/processes/__init__.py b/processes/__init__.py index 767afe27a06..0275efe4807 100644 --- a/processes/__init__.py +++ b/processes/__init__.py @@ -8,7 +8,6 @@ from .cluster import ClusterProcess from .local_development.local_development_setup import LocalDevelopmentSetupProcess from .local_development.seed_local_data import SeedLocalDataProcess -from .local_development.seed_test_data import SeedTestDataProcess from .file.s3_files import S3Process from .api import APIProcess from .ingest.muse import MUSEProcess diff --git a/processes/local_development/seed_test_data.py b/processes/local_development/seed_test_data.py deleted file mode 100644 index afb477837b3..00000000000 --- a/processes/local_development/seed_test_data.py +++ /dev/null @@ -1,81 +0,0 @@ -from datetime import datetime, timezone -import json -from uuid import uuid4 -from processes import ClusterProcess -from model import Record -from logger import create_log -from managers import DBManager - -logger = create_log(__name__) - -TEST_SOURCE = 'test_source' - -class SeedTestDataProcess(): - def __init__(self, *args): - self.db_manager = DBManager() - - flags = { 'catalog': False, 'download': False, 'reader': False, 'embed': True } - self.test_data = { - 'title': 'test data 1', - 'uuid' : uuid4(), - 'frbr_status': 'complete', - 'cluster_status': False, - "source": TEST_SOURCE, - 'authors': ['Ayan||true'], - 'languages': ['Serbian'], - 'dates': ['1907-|publication_date'], - 'publisher': ['Project Gutenberg Literary Archive Foundation||'], - 'identifiers': [], - 'source_id': '4064148285|test', - 'contributors': ['Metropolitan Museum of Art (New York, N.Y.)|||contributor','Metropolitan Museum of Art (New York, N.Y.)|||repository','Thomas J. Watson Library|||provider'], - 'extent': ('11, 164 p. ;'), - 'is_part_of': ['Tauchnitz edition|Vol. 4560|volume'], - 'abstract': ['test abstract 1', 'test abstract 2'], - 'subjects': ['test subjects 1||'], - 'rights': ('hathitrust|public_domain|expiration of copyright term for non-US work with corporate author|Public Domain|2021-10-02 05:25:13'), - 'has_part': [f'1|example.com/1.pdf|{TEST_SOURCE}|text/html|{json.dumps(flags)}'] - } - - def runProcess(self): - try: - self.db_manager.createSession() - - self.save_test_record() - - cluster_process = ClusterProcess('complete', None, None, self.test_data['uuid'], None) - cluster_process.runProcess() - - self.save_test_data_ids_to_file() - - except Exception as e: - logger.exception(f'Failed to seed test data') - raise e - finally: - self.db_manager.close_connection() - - def save_test_record(self): - existing_record = self.db_manager.session.query(Record).filter_by(source_id=self.test_data['source_id']).first() - - if existing_record: - for key, value in self.test_data.items(): - if key != 'uuid' and hasattr(existing_record, key): - setattr(existing_record, key, value) - - existing_record.date_modified = datetime.now(timezone.utc).replace(tzinfo=None) - - self.test_data['uuid'] = existing_record.uuid - - else: - test_record = Record(**self.test_data) - self.db_manager.session.add(test_record) - - self.db_manager.session.commit() - - def save_test_data_ids_to_file(self): - record = self.db_manager.session.query(Record).filter_by(source_id=self.test_data['source_id']).first() - if record: - with open('test_data_ids.json', 'w') as f: - json.dump({ - 'edition_id': str(record.edition_id), - 'uuid': str(record.uuid) - }, f) \ No newline at end of file diff --git a/test_data_ids.json b/test_data_ids.json index f64656ae6a8..79af2ef1b9a 100644 --- a/test_data_ids.json +++ b/test_data_ids.json @@ -1 +1 @@ -{"uuid": "3264ccb1-a870-43d7-9496-2153ca99acc6"} \ No newline at end of file +{"edition_id": "19855", "uuid": "3264ccb1-a870-43d7-9496-2153ca99acc6"} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index db73358ad6f..ebc791ff927 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,17 @@ import os import pytest - +from datetime import datetime, timezone +import json +from uuid import uuid4 +from processes import ClusterProcess +from model import Record, Item +from logger import create_log +from managers import DBManager from load_env import load_env_file +logger = create_log(__name__) + +TEST_SOURCE = 'test_source' def pytest_addoption(parser): parser.addoption('--env', action='store', default='local', help='Environment to use for tests') @@ -16,3 +25,64 @@ def setup_env(pytestconfig, request): if not running_unit_tests and environment in ['local', 'qa']: load_env_file(environment, file_string=f'config/{environment}.yaml') + +@pytest.fixture(scope='module') +def db_manager(): + db_manager = DBManager() + db_manager.createSession() + yield db_manager + db_manager.close_connection() + +@pytest.fixture(scope='module') +def seed_test_data(db_manager): + flags = { 'catalog': False, 'download': False, 'reader': False, 'embed': True } + test_data = { + 'title': 'test data 1', + 'uuid': uuid4(), + 'frbr_status': 'complete', + 'cluster_status': False, + "source": TEST_SOURCE, + 'authors': ['Ayan||true'], + 'languages': ['Serbian'], + 'dates': ['1907-|publication_date'], + 'publisher': ['Project Gutenberg Literary Archive Foundation||'], + 'identifiers': [], + 'source_id': '4064148285|test', + 'contributors': ['Metropolitan Museum of Art (New York, N.Y.)|||contributor','Metropolitan Museum of Art (New York, N.Y.)|||repository','Thomas J. Watson Library|||provider'], + 'extent': ('11, 164 p. ;'), + 'is_part_of': ['Tauchnitz edition|Vol. 4560|volume'], + 'abstract': ['test abstract 1', 'test abstract 2'], + 'subjects': ['test subjects 1||'], + 'rights': ('hathitrust|public_domain|expiration of copyright term for non-US work with corporate author|Public Domain|2021-10-02 05:25:13'), + 'has_part': [f'1|example.com/1.pdf|{TEST_SOURCE}|text/html|{json.dumps(flags)}'] + } + + existing_record = db_manager.session.query(Record).filter_by(source_id=test_data['source_id']).first() + + if existing_record: + for key, value in test_data.items(): + if key != 'uuid' and hasattr(existing_record, key): + setattr(existing_record, key, value) + existing_record.date_modified = datetime.now(timezone.utc).replace(tzinfo=None) + test_data['uuid'] = existing_record.uuid + test_record = existing_record + else: + test_record = Record(**test_data) + db_manager.session.add(test_record) + + db_manager.session.commit() + + item = db_manager.session.query(Item).filter_by(record_id=test_record.id).first() + edition_id = str(item.edition_id) if item else None + + with open('test_data_ids.json', 'w') as f: + json.dump({ + 'edition_id': edition_id, + 'uuid': str(test_data['uuid']) + }, f) + + return edition_id + +@pytest.fixture(scope='module') +def seeded_edition_id(seed_test_data): + return seed_test_data \ No newline at end of file diff --git a/tests/integration/api/test_get_edition.py b/tests/integration/api/test_get_edition.py index c053de50916..3ef4135aa02 100644 --- a/tests/integration/api/test_get_edition.py +++ b/tests/integration/api/test_get_edition.py @@ -4,24 +4,15 @@ from .constants import API_URL from .utils import assert_response_status -@pytest.fixture(scope="module", autouse=True) -def seed_test_data(): - process = SeedTestDataProcess() - process.runProcess() - -with open('test_data_ids.json', 'r') as f: - test_data_ids = json.load(f) - SEEDED_EDITION_ID = test_data_ids['edition_id'] - @pytest.mark.parametrize("endpoint, expected_status", [ - ("/editions/{SEEDED_EDITION_ID}", 200), + ("/editions/{seeded_edition_id}", 200), ("/editions/00000000-0000-0000-0000-000000000000", 400), ("/editions/invalid_id_format", 400), ("/editions/", 404), ("/editions/%$@!*", 400) ]) -def test_get_edition(endpoint, expected_status): - url = API_URL + endpoint +def test_get_edition(endpoint, expected_status, seeded_edition_id): + url = API_URL + endpoint.format(seeded_edition_id=seeded_edition_id) response = requests.get(url) assert response.status_code is not None From 18c34f0dec000127276e715c9786741287274d71 Mon Sep 17 00:00:00 2001 From: ayan <100955969+ayan1229@users.noreply.github.com> Date: Mon, 27 Jan 2025 16:45:40 -0500 Subject: [PATCH 3/8] remove test data file and use fixture for using data --- test_data_ids.json | 1 - tests/conftest.py | 21 +++++++++++++-------- tests/integration/api/test_get_edition.py | 1 + 3 files changed, 14 insertions(+), 9 deletions(-) delete mode 100644 test_data_ids.json diff --git a/test_data_ids.json b/test_data_ids.json deleted file mode 100644 index 79af2ef1b9a..00000000000 --- a/test_data_ids.json +++ /dev/null @@ -1 +0,0 @@ -{"edition_id": "19855", "uuid": "3264ccb1-a870-43d7-9496-2153ca99acc6"} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index ebc791ff927..65414e75844 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,14 +75,19 @@ def seed_test_data(db_manager): item = db_manager.session.query(Item).filter_by(record_id=test_record.id).first() edition_id = str(item.edition_id) if item else None - with open('test_data_ids.json', 'w') as f: - json.dump({ - 'edition_id': edition_id, - 'uuid': str(test_data['uuid']) - }, f) + return { + 'edition_id': edition_id, + 'uuid': str(test_data['uuid']) + } - return edition_id +@pytest.fixture(scope='module') +def seeded_edition_id(request, seed_test_data): + if 'functional' in request.keywords or 'integration' in request.keywords: + return seed_test_data['edition_id'] + return None @pytest.fixture(scope='module') -def seeded_edition_id(seed_test_data): - return seed_test_data \ No newline at end of file +def seeded_uuid(request, seed_test_data): + if 'functional' in request.keywords or 'integration' in request.keywords: + return seed_test_data['uuid'] + return None diff --git a/tests/integration/api/test_get_edition.py b/tests/integration/api/test_get_edition.py index 3ef4135aa02..e5e766dc977 100644 --- a/tests/integration/api/test_get_edition.py +++ b/tests/integration/api/test_get_edition.py @@ -4,6 +4,7 @@ from .constants import API_URL from .utils import assert_response_status +@pytest.mark.integration @pytest.mark.parametrize("endpoint, expected_status", [ ("/editions/{seeded_edition_id}", 200), ("/editions/00000000-0000-0000-0000-000000000000", 400), From 353864aef32f55b3a1589027e23df62d4959156c Mon Sep 17 00:00:00 2001 From: ayan <100955969+ayan1229@users.noreply.github.com> Date: Tue, 28 Jan 2025 13:39:04 -0500 Subject: [PATCH 4/8] for kyle --- config/qa.yaml | 19 ++++++++++--------- tests/conftest.py | 5 +++++ tests/integration/api/test_get_edition.py | 6 ++++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/config/qa.yaml b/config/qa.yaml index 02c8bdefcb6..117b6c012d1 100644 --- a/config/qa.yaml +++ b/config/qa.yaml @@ -4,23 +4,23 @@ ENVIRONMENT: qa LOG_LEVEL: info # POSTGRES CONNECTION DETAILS -# POSTGRES_USER, POSTGRES_PSWD, POSTGRES_ADMIN_USER and POSTGRES_ADMIN_PSWD must be configured in secrets file +# POSTGRES_USER, POSTGRES_PSWD, POSTGRES_ADMIN_USER and POSTGRES_ADMIN_PSWD must be configured in secrets file POSTGRES_HOST: sfr-new-metadata-production-cluster.cluster-cvy7z512hcjg.us-east-1.rds.amazonaws.com POSTGRES_NAME: dcdw_qa -POSTGRES_PORT: '5432' +POSTGRES_PORT: "5432" # REDIS CONFIGURATION # REDIS_HOST configured as part of ECS deployment -REDIS_PORT: '6379' +REDIS_PORT: "6379" # ELASTICSEARCH CONFIGURATION # ELASTICSEARCH_INDEX, ELASTICSEARCH_HOST, ELASTICSEARCH_SCHEME, ELASTICSEARCH_PORT must be configured in secrets file -ELASTICSEARCH_TIMEOUT: '10' +ELASTICSEARCH_TIMEOUT: "10" # RABBITMQ CONFIGURATION # RABBIT_USER and RABBIT_PSWD must be configured in secrets file RABBIT_HOST: qa.rmq.aws.nypl.org -RABBIT_PORT: '5672' +RABBIT_PORT: "5672" RABBIT_VIRTUAL_HOST: /sfr RABBIT_EXCHANGE: sfrIngestExchange OCLC_QUEUE: sfrOCLCCatalog @@ -30,7 +30,7 @@ FILE_ROUTING_KEY: sfrS3Files # OCLC CONFIGURATION # OCLC API keys must be configured in secrets file -OCLC_QUERY_LIMIT: '390000' +OCLC_QUERY_LIMIT: "390000" # AWS CONFIGURATION # AWS_ACCESS and AWS_SECRET must be configured in secrets file @@ -41,7 +41,7 @@ FILE_BUCKET: drb-files-qa # NYPL_BIB_USER and NYPL_BIB_PSWD must be configured in secrets file NYPL_BIB_HOST: bib-service-production-rep.cvy7z512hcjg.us-east-1.rds.amazonaws.com NYPL_BIB_NAME: bib_service_production -NYPL_BIB_PORT: '5432' +NYPL_BIB_PORT: "5432" # NYPL Location Code Lookup NYPL_LOCATIONS_BY_CODE: https://nypl-core-objects-mapping-qa.s3.amazonaws.com/by_sierra_location.json @@ -51,8 +51,9 @@ NYPL_LOCATIONS_BY_CODE: https://nypl-core-objects-mapping-qa.s3.amazonaws.com/by NYPL_API_CLIENT_TOKEN_URL: https://isso.nypl.org/oauth/token # DRB API Credentials -DRB_API_HOST: 'drb-api-qa.nypl.org' -DRB_API_PORT: '80' +DRB_API_HOST: "drb-api-qa.nypl.org" +DRB_API_PORT: "80" +DRB_API_URL: https://drb-api-qa.nypl.org # Bardo CCE API URL BARDO_CCE_API: http://sfr-c-ecsal-14v3injrieoy5-258691445.us-east-1.elb.amazonaws.com/search/ diff --git a/tests/conftest.py b/tests/conftest.py index 65414e75844..0e4b694fe29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,6 +69,11 @@ def seed_test_data(db_manager): else: test_record = Record(**test_data) db_manager.session.add(test_record) + + process_args = ('complete', None, None, str(test_data['uuid']), None) + + cluster_process = ClusterProcess(*process_args) + cluster_process.runProcess() db_manager.session.commit() diff --git a/tests/integration/api/test_get_edition.py b/tests/integration/api/test_get_edition.py index e5e766dc977..ad3afcdda2a 100644 --- a/tests/integration/api/test_get_edition.py +++ b/tests/integration/api/test_get_edition.py @@ -1,9 +1,11 @@ import pytest +import os import requests import json -from .constants import API_URL from .utils import assert_response_status +API_URL = os.getenv("DRB_API_URL") + @pytest.mark.integration @pytest.mark.parametrize("endpoint, expected_status", [ ("/editions/{seeded_edition_id}", 200), @@ -12,7 +14,7 @@ ("/editions/", 404), ("/editions/%$@!*", 400) ]) -def test_get_edition(endpoint, expected_status, seeded_edition_id): +def test_get_edition(endpoint, expected_status, seeded_edition_id): url = API_URL + endpoint.format(seeded_edition_id=seeded_edition_id) response = requests.get(url) From ad2413f5f038b83a82fa2dfa23a2bd4f3587e15f Mon Sep 17 00:00:00 2001 From: kyle Date: Tue, 28 Jan 2025 13:51:10 -0500 Subject: [PATCH 5/8] reverting quote changes --- config/qa.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/config/qa.yaml b/config/qa.yaml index 117b6c012d1..280e6caa091 100644 --- a/config/qa.yaml +++ b/config/qa.yaml @@ -7,20 +7,20 @@ LOG_LEVEL: info # POSTGRES_USER, POSTGRES_PSWD, POSTGRES_ADMIN_USER and POSTGRES_ADMIN_PSWD must be configured in secrets file POSTGRES_HOST: sfr-new-metadata-production-cluster.cluster-cvy7z512hcjg.us-east-1.rds.amazonaws.com POSTGRES_NAME: dcdw_qa -POSTGRES_PORT: "5432" +POSTGRES_PORT: '5432' # REDIS CONFIGURATION # REDIS_HOST configured as part of ECS deployment -REDIS_PORT: "6379" +REDIS_PORT: '6379' # ELASTICSEARCH CONFIGURATION # ELASTICSEARCH_INDEX, ELASTICSEARCH_HOST, ELASTICSEARCH_SCHEME, ELASTICSEARCH_PORT must be configured in secrets file -ELASTICSEARCH_TIMEOUT: "10" +ELASTICSEARCH_TIMEOUT: '10' # RABBITMQ CONFIGURATION # RABBIT_USER and RABBIT_PSWD must be configured in secrets file RABBIT_HOST: qa.rmq.aws.nypl.org -RABBIT_PORT: "5672" +RABBIT_PORT: '5672' RABBIT_VIRTUAL_HOST: /sfr RABBIT_EXCHANGE: sfrIngestExchange OCLC_QUEUE: sfrOCLCCatalog @@ -30,7 +30,7 @@ FILE_ROUTING_KEY: sfrS3Files # OCLC CONFIGURATION # OCLC API keys must be configured in secrets file -OCLC_QUERY_LIMIT: "390000" +OCLC_QUERY_LIMIT: '390000' # AWS CONFIGURATION # AWS_ACCESS and AWS_SECRET must be configured in secrets file @@ -41,7 +41,7 @@ FILE_BUCKET: drb-files-qa # NYPL_BIB_USER and NYPL_BIB_PSWD must be configured in secrets file NYPL_BIB_HOST: bib-service-production-rep.cvy7z512hcjg.us-east-1.rds.amazonaws.com NYPL_BIB_NAME: bib_service_production -NYPL_BIB_PORT: "5432" +NYPL_BIB_PORT: '5432' # NYPL Location Code Lookup NYPL_LOCATIONS_BY_CODE: https://nypl-core-objects-mapping-qa.s3.amazonaws.com/by_sierra_location.json @@ -51,8 +51,8 @@ NYPL_LOCATIONS_BY_CODE: https://nypl-core-objects-mapping-qa.s3.amazonaws.com/by NYPL_API_CLIENT_TOKEN_URL: https://isso.nypl.org/oauth/token # DRB API Credentials -DRB_API_HOST: "drb-api-qa.nypl.org" -DRB_API_PORT: "80" +DRB_API_HOST: 'drb-api-qa.nypl.org' +DRB_API_PORT: '80' DRB_API_URL: https://drb-api-qa.nypl.org # Bardo CCE API URL From c1a51de2b38c2eedcd6293d50898652986f79d80 Mon Sep 17 00:00:00 2001 From: kyle Date: Tue, 28 Jan 2025 14:05:06 -0500 Subject: [PATCH 6/8] fixing test --- tests/conftest.py | 8 +++----- tests/integration/api/test_get_edition.py | 4 +--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0e4b694fe29..173642a3524 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -69,14 +69,12 @@ def seed_test_data(db_manager): else: test_record = Record(**test_data) db_manager.session.add(test_record) - - process_args = ('complete', None, None, str(test_data['uuid']), None) - - cluster_process = ClusterProcess(*process_args) - cluster_process.runProcess() db_manager.session.commit() + cluster_process = ClusterProcess('complete', None, None, str(test_data['uuid']), None) + cluster_process.runProcess() + item = db_manager.session.query(Item).filter_by(record_id=test_record.id).first() edition_id = str(item.edition_id) if item else None diff --git a/tests/integration/api/test_get_edition.py b/tests/integration/api/test_get_edition.py index ad3afcdda2a..392476aa67b 100644 --- a/tests/integration/api/test_get_edition.py +++ b/tests/integration/api/test_get_edition.py @@ -1,10 +1,8 @@ import pytest import os import requests -import json from .utils import assert_response_status -API_URL = os.getenv("DRB_API_URL") @pytest.mark.integration @pytest.mark.parametrize("endpoint, expected_status", [ @@ -15,7 +13,7 @@ ("/editions/%$@!*", 400) ]) def test_get_edition(endpoint, expected_status, seeded_edition_id): - url = API_URL + endpoint.format(seeded_edition_id=seeded_edition_id) + url = os.getenv("DRB_API_URL") + endpoint.format(seeded_edition_id=seeded_edition_id) response = requests.get(url) assert response.status_code is not None From 214d5c311dcc62c8ac4ba5d3bc1775ee145f3f4d Mon Sep 17 00:00:00 2001 From: kyle Date: Tue, 28 Jan 2025 14:18:31 -0500 Subject: [PATCH 7/8] fixing source priority --- api/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/utils.py b/api/utils.py index 356b8e1a99a..03c302418ea 100644 --- a/api/utils.py +++ b/api/utils.py @@ -50,6 +50,8 @@ class APIUtils(): 'nypl': 11 } + DEFAULT_PRIORITY = 100 + @staticmethod def normalizeQueryParams(params): paramDict = params.to_dict(flat=False) @@ -374,13 +376,13 @@ def formatEdition( for itemDict in editionDict['items']: if itemDict['links'] == []: editionDict['items']\ - .sort(key=lambda x: (cls.SOURCE_PRIORITY[x['source']], x['links'] == [])) + .sort(key=lambda x: (cls.SOURCE_PRIORITY.get(x['source'], cls.DEFAULT_PRIORITY), x['links'] == [])) emptyListFlag = True break if emptyListFlag == False: editionDict['items']\ - .sort(key=lambda x: (cls.SOURCE_PRIORITY[x['source']], cls.sortByMediaType(x['links'][0]))) + .sort(key=lambda x: (cls.SOURCE_PRIORITY.get(x['source'], cls.DEFAULT_PRIORITY), cls.sortByMediaType(x['links'][0]))) if records is not None: itemsByLink = {} From f19d997043fe36fe64c216091ba5432c93744c87 Mon Sep 17 00:00:00 2001 From: kyle Date: Tue, 28 Jan 2025 15:32:44 -0500 Subject: [PATCH 8/8] Set env variables in CI --- .github/workflows/ci.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0e6d00b1fc0..b1e35973b6c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -18,6 +18,11 @@ jobs: AWS_ACCESS: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET: ${{ secrets.AWS_SECRET_ACCESS_KEY }} ENVIRONMENT: qa + ELASTICSEARCH_HOST: ${{ secrets.ELASTICSEARCH_HOST }} + ELASTICSEARCH_INDEX: ${{ secrets.ELASTICSEARCH_INDEX }} + ELASTICSEARCH_PORT: ${{ secrets.ELASTICSEARCH_PORT }} + ELASTICSEARCH_SCHEME: ${{ secrets.ELASTICSEARCH_SCHEME }} + REDIS_HOST: ${{ secrets.REDIS_HOST }} steps: - uses: actions/checkout@v4 - name: Set up Python 3.9 @@ -39,4 +44,4 @@ jobs: pip install -r requirements.txt - name: Run API tests run: | - pytest tests/integration/api + pytest tests/integration/api --env=qa