From 2f72220deedf4cf4eeb6fe46dbcd66e276597183 Mon Sep 17 00:00:00 2001 From: Kyle Villegas <86266231+kylevillegas93@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:04:41 -0400 Subject: [PATCH] SFR-2276: Deprecating ingest report (#407) --- config/development.yaml | 4 -- config/production.yaml | 4 -- config/qa.yaml | 4 -- managers/__init__.py | 1 - managers/smartsheet.py | 54 -------------- processes/__init__.py | 1 - processes/ingestReport.py | 71 ------------------- requirements.txt | 1 - tests/helper.py | 2 - tests/unit/test_ingestReport_process.py | 71 ------------------- tests/unit/test_smartsheet_manager.py | 93 ------------------------- 11 files changed, 306 deletions(-) delete mode 100644 managers/smartsheet.py delete mode 100644 processes/ingestReport.py delete mode 100644 tests/unit/test_ingestReport_process.py delete mode 100644 tests/unit/test_smartsheet_manager.py diff --git a/config/development.yaml b/config/development.yaml index 775d0f5ecf8..54e10ad37bc 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -74,10 +74,6 @@ DOAB_OAI_URL: https://directory.doabooks.org/oai/request? # ContentCafe2 API # CONTENT_CAFE_USER and CONTENT_CAFE_PSWD must be configured as secrets -# SmartSheet API -# SMARTSHEET_API_TOKEN must be configured as a secret -SMARTSHEET_SHEET_ID: '3683038090553220' - # Default Cover Image for OPDS2 Feed DEFAULT_COVER_URL: https://drb-files-qa.s3.amazonaws.com/covers/default/defaultCover.png diff --git a/config/production.yaml b/config/production.yaml index 6a630ac965b..68937b4e632 100644 --- a/config/production.yaml +++ b/config/production.yaml @@ -81,10 +81,6 @@ DOAB_OAI_URL: https://directory.doabooks.org/oai/request? # ContentCafe2 API # CONTENT_CAFE_USER and CONTENT_CAFE_PSWD must be configured as secrets -# SmartSheet API -# SMARTSHEET_API_TOKEN must be configured as a secret -SMARTSHEET_SHEET_ID: '3683038090553220' - # Default Cover Image for OPDS2 Feed DEFAULT_COVER_URL: https://drb-files-qa.s3.amazonaws.com/covers/default/defaultCover.png diff --git a/config/qa.yaml b/config/qa.yaml index 474a0111276..464b48ba543 100644 --- a/config/qa.yaml +++ b/config/qa.yaml @@ -81,10 +81,6 @@ DOAB_OAI_URL: https://directory.doabooks.org/oai/request? # ContentCafe2 API # CONTENT_CAFE_USER and CONTENT_CAFE_PSWD must be configured as secrets -# SmartSheet API -# SMARTSHEET_API_TOKEN must be configured as a secret -SMARTSHEET_SHEET_ID: '3683038090553220' - # Default Cover Image for OPDS2 Feed DEFAULT_COVER_URL: https://drb-files-qa.s3.amazonaws.com/covers/default/defaultCover.png diff --git a/managers/__init__.py b/managers/__init__.py index e827a37afab..1f6b0934522 100644 --- a/managers/__init__.py +++ b/managers/__init__.py @@ -13,5 +13,4 @@ from .elasticsearch import ElasticsearchManager from .sfrElasticRecord import SFRElasticRecordManager from .s3 import S3Manager -from .smartsheet import SmartSheetManager from .muse import MUSEError, MUSEManager \ No newline at end of file diff --git a/managers/smartsheet.py b/managers/smartsheet.py deleted file mode 100644 index 9a127bd6c60..00000000000 --- a/managers/smartsheet.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import smartsheet - -class SmartSheetManager: - def __init__(self, apiToken=None, sheetID=None): - super(SmartSheetManager, self).__init__() - self.apiToken = apiToken or os.environ['SMARTSHEET_API_TOKEN'] - self.sheetID = sheetID or int(os.environ['SMARTSHEET_SHEET_ID']) - self.client = None - - def createClient(self): - self.client = smartsheet.Smartsheet(self.apiToken) - self.client.errors_as_exceptions(True) - - def getColumns(self): - colResponse = self.client.Sheets.get_columns( - self.sheetID, include_all=True - ) - - return {c.title: c.id for c in colResponse.data} - - def insertRow(self, rowData): - newRow = self.createNewRow() - - columns = self.getColumns() - - rows = self.getRows(columnIDs=[columns['Col ID']], modifiedSince=rowData['Date']['value']) - lastPCNum = int(rows[-1].cells[0].value) if len(rows) > 0 else 0 - rowData['Col ID'] = {'value': lastPCNum + 1} - - for name, value in rowData.items(): - colID = columns[name] - - cellValue = {'column_id': colID, 'value': value['value']} - cellValue['format'] = ',,,,,,,,,28,,,,,,,' if value.get('anomaly', False) else ',,,,,,,,,3,,,,,,,' - - newRow.cells.append(cellValue) - - self.client.Sheets.add_rows(self.sheetID, [newRow]) - - def getRows(self, columnIDs=None, modifiedSince=None): - getSheetArgs = {} - - if columnIDs: getSheetArgs['column_ids'] = columnIDs - - if modifiedSince: getSheetArgs['rows_modified_since'] = modifiedSince - - sheetResp = self.client.Sheets.get_sheet(self.sheetID, **getSheetArgs) - - return sheetResp.rows - - @staticmethod - def createNewRow(): - return smartsheet.models.Row() \ No newline at end of file diff --git a/processes/__init__.py b/processes/__init__.py index 8f26b6cb197..13b218ea7d2 100644 --- a/processes/__init__.py +++ b/processes/__init__.py @@ -11,7 +11,6 @@ from .s3Files import S3Process from .api import APIProcess from .muse import MUSEProcess -from .ingestReport import IngestReportProcess from .met import METProcess from .db_maintenance import DatabaseMaintenanceProcess from .db_migration import MigrationProcess diff --git a/processes/ingestReport.py b/processes/ingestReport.py deleted file mode 100644 index 2f4212ef104..00000000000 --- a/processes/ingestReport.py +++ /dev/null @@ -1,71 +0,0 @@ -from datetime import datetime, timedelta -from sqlalchemy import func - -from .core import CoreProcess -from model import Work, Edition, Item, Record -from managers import SmartSheetManager -from logger import createLog - -logger = createLog(__name__) - - -class IngestReportProcess(CoreProcess): - def __init__(self, *args): - super(IngestReportProcess, self).__init__(*args[:4]) - - # PostgreSQL Connection - self.generateEngine() - self.createSession() - - # Smartsheet SDK - self.smartsheet = SmartSheetManager() - self.smartsheet.createClient() - - def runProcess(self): - self.generateReport() - - def generateReport(self): - ingestDate = datetime.today() - ingestDateStart = ingestDate.replace(hour=0, minute=0, second=0) - ingestDateEnd = ingestDateStart + timedelta(days=1) - - dailyRow = {'Date': {'value': ingestDate.strftime('%Y-%m-%d')}} - for table in [Work, Edition, Item, Record]: - totals = self.getTableCounts(table, ingestDateStart, ingestDateEnd) - dailyRow = {**dailyRow, **totals} - - self.smartsheet.insertRow(dailyRow) - - def getTableCounts(self, table, ingestStart, ingestEnd): - logger.info('Getting Totals for {}'.format(table.__name__)) - - totalCount = self.session.query(func.count(table.id)).scalar() - - modifiedCount = self.session.query(table.id)\ - .filter(table.date_modified > ingestStart)\ - .filter(table.date_modified < ingestEnd)\ - .filter(table.date_created < ingestStart)\ - .count() - - createdCount = self.session.query(table.id)\ - .filter(table.date_modified > ingestStart)\ - .filter(table.date_modified < ingestEnd)\ - .count() - - totals = self.setAnomalyStatus(totalCount, modifiedCount, createdCount) - - return { - 'Total {}s'.format(table.__name__): totals['total'], - '{}s Modified'.format(table.__name__): totals['modified'], - '{}s Created'.format(table.__name__): totals['created'] - } - - @staticmethod - def setAnomalyStatus(total, modified, created): - anomalyThreshold = total / 10 - - return { - 'total': {'value': total, 'anomaly': False}, - 'modified': {'value': modified, 'anomaly': modified > anomalyThreshold}, - 'created': {'value': created, 'anomaly': created > anomalyThreshold} - } diff --git a/requirements.txt b/requirements.txt index e2e55e3ba83..6966dc6f499 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,6 @@ pyyaml==6.0 redis==4.5.4 requests==2.28.2 requests_oauthlib==1.3.1 -smartsheet-python-sdk==3.0.0 scikit-learn==1.2.2 sqlalchemy==2.0.20 waitress==2.1.2 diff --git a/tests/helper.py b/tests/helper.py index 3499c7385e1..9d4b9502518 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -47,8 +47,6 @@ class TestHelpers: 'MUSE_MARC_URL': 'test_muse_url', 'MUSE_CSV_URL': 'test_muse_csv', 'DOAB_OAI_URL': 'test_doab_url', - 'SMARTSHEET_API_TOKEN': 'test_smartsheet_token', - 'SMARTSHEET_SHEET_ID': '1000', 'WEBPUB_CONVERSION_URL': 'test_conversion_url', 'WEBPUB_PDF_PROFILE': 'test_profile_uri' } diff --git a/tests/unit/test_ingestReport_process.py b/tests/unit/test_ingestReport_process.py deleted file mode 100644 index ed1b81b03b9..00000000000 --- a/tests/unit/test_ingestReport_process.py +++ /dev/null @@ -1,71 +0,0 @@ -from datetime import datetime, timedelta -import pytest - -from processes import IngestReportProcess -from model import Work, Edition, Item, Record - - -class TestIngestReportProcess: - @pytest.fixture - def testInstance(self, mocker): - class TestIngestReportProcess(IngestReportProcess): - def __init__(self, *args): - self.smartsheet = mocker.MagicMock() - self.session = mocker.MagicMock() - - return TestIngestReportProcess('TestProcess', 'testFile', 'testDate') - - def test_runProcess(self, testInstance, mocker): - mockGenerate = mocker.patch.object(IngestReportProcess, 'generateReport') - - testInstance.runProcess() - - mockGenerate.assert_called_once() - - def test_generateReport(self, testInstance, mocker): - now = datetime.today() - mockDate = mocker.patch('processes.ingestReport.datetime') - mockDate.today.return_value = now - startStr = now.strftime('%Y-%m-%d') - startTime = now.replace(hour=0, minute=0, second=0) - endTime = startTime + timedelta(days=1) - - mockGetTable = mocker.patch.object(IngestReportProcess, 'getTableCounts') - mockGetTable.side_effect = [ - {'Works': 'test'}, {'Editions': 'test'}, {'Items': 'test'}, {'Records': 'test'} - ] - - testInstance.generateReport() - - mockGetTable.assert_has_calls([ - mocker.call(Work, startTime, endTime), - mocker.call(Edition, startTime, endTime), - mocker.call(Item, startTime, endTime), - mocker.call(Record, startTime, endTime) - ]) - testInstance.smartsheet.insertRow.assert_called_once_with( - {'Date': {'value': startStr}, 'Works': 'test', 'Editions': 'test', 'Items': 'test', 'Records': 'test'} - ) - - def test_getTableCounts(self, testInstance, mocker): - testInstance.session.query().scalar.return_value = 10 # total - testInstance.session.query().filter().filter().filter().count.return_value = 3 # modified - testInstance.session.query().filter().filter().count.return_value = 2 # created - - mockAnomaly = mocker.patch.object(IngestReportProcess, 'setAnomalyStatus') - mockAnomaly.return_value = {'total': 10, 'modified': 3, 'created': 2} - - mockTable = mocker.MagicMock(__name__='Test', id=1, date_modified=1, date_created=1) - testCounts = testInstance.getTableCounts(mockTable, 0, 1) - - assert testCounts['Total Tests'] == 10 - assert testCounts['Tests Modified'] == 3 - assert testCounts['Tests Created'] == 2 - mockAnomaly.assert_called_once_with(10, 3, 2) - - def test_setAnomalyStatus(self): - testAnomalyObject = IngestReportProcess.setAnomalyStatus(100, 25, 5) - - assert testAnomalyObject['total'] == {'value': 100, 'anomaly': False} - assert testAnomalyObject['modified'] == {'value': 25, 'anomaly': True} - assert testAnomalyObject['created'] == {'value': 5, 'anomaly': False} diff --git a/tests/unit/test_smartsheet_manager.py b/tests/unit/test_smartsheet_manager.py deleted file mode 100644 index 10be27c89db..00000000000 --- a/tests/unit/test_smartsheet_manager.py +++ /dev/null @@ -1,93 +0,0 @@ -import pytest - -from managers import SmartSheetManager -from tests.helper import TestHelpers - - -class TestNYPLApiManager: - @classmethod - def setup_class(cls): - TestHelpers.setEnvVars() - - @classmethod - def teardown_class(cls): - TestHelpers.clearEnvVars() - - @pytest.fixture - def testInstance(self): - return SmartSheetManager() - - def test_initializer(self, testInstance): - assert testInstance.apiToken == 'test_smartsheet_token' - assert testInstance.sheetID == 1000 - assert testInstance.client == None - - def test_createClient(self, testInstance, mocker): - mockClient = mocker.MagicMock() - mockSheet = mocker.patch('managers.smartsheet.smartsheet') - mockSheet.Smartsheet.return_value = mockClient - - testInstance.createClient() - - assert testInstance.client == mockClient - mockSheet.Smartsheet.assert_called_once_with('test_smartsheet_token') - testInstance.client.errors_as_exceptions.assert_called_once_with(True) - - def test_getColumns(self, testInstance, mocker): - mockColumn = mocker.MagicMock(title='Test Column', id=1) - mockResp = mocker.MagicMock(data=[mockColumn]) - testInstance.client = mocker.MagicMock() - testInstance.client.Sheets.get_columns.return_value = mockResp - - testColumns = testInstance.getColumns() - - assert testColumns == {'Test Column': 1} - testInstance.client.Sheets.get_columns.assert_called_once_with( - 1000, include_all=True - ) - - def test_insertRow(self, testInstance, mocker): - testInstance.client = mocker.MagicMock() - sheetMocks = mocker.patch.multiple( - SmartSheetManager, - createNewRow=mocker.DEFAULT, - getColumns=mocker.DEFAULT, - getRows=mocker.DEFAULT - ) - mockRow = mocker.MagicMock() - sheetMocks['createNewRow'].return_value = mockRow - sheetMocks['getColumns'].return_value = {'Date': 1, 'Col ID': 2, 'Test': 3, 'Test2': 4} - sheetMocks['getRows'].return_value = [] - - testInstance.insertRow( - {'Date': {'value': 'testDate'}, 'Test': {'value': 10}, 'Test2': {'value': 25, 'anomaly': True}} - ) - - sheetMocks['createNewRow'].assert_called_once() - sheetMocks['getColumns'].assert_called_once() - sheetMocks['getRows'].assert_called_once_with(columnIDs=[2], modifiedSince='testDate') - mockRow.cells.append.assert_has_calls([ - mocker.call({'column_id': 1, 'value': 'testDate', 'format': ',,,,,,,,,3,,,,,,,'}), - mocker.call({'column_id': 3, 'value': 10, 'format': ',,,,,,,,,3,,,,,,,'}), - mocker.call({'column_id': 4, 'value': 25, 'format': ',,,,,,,,,28,,,,,,,'}), - mocker.call({'column_id': 2, 'value': 1, 'format': ',,,,,,,,,3,,,,,,,'}) - ]) - testInstance.client.Sheets.add_rows.assert_called_once_with(1000, [mockRow]) - - def test_getRows(self, testInstance, mocker): - mockClient = mocker.MagicMock() - testInstance.client = mockClient - mockClient.Sheets.get_sheet.return_value = mocker.MagicMock(rows=[1, 2]) - - testRows = testInstance.getRows(columnIDs=[1], modifiedSince='testDate') - - assert testRows == [1, 2] - mockClient.Sheets.get_sheet.assert_called_once_with( - 1000, column_ids=[1], rows_modified_since='testDate' - ) - - def test_createNewRow(self, mocker): - mockSheet = mocker.patch('managers.smartsheet.smartsheet') - mockSheet.models.Row.return_value = 'testRow' - - assert SmartSheetManager.createNewRow() == 'testRow' \ No newline at end of file