From 2fd7ca5d8458cc457b871197afb5ce17d3ba60f4 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 31 Jul 2024 09:53:58 +0100 Subject: [PATCH 01/26] Add function to get quotas from projects portal --- nlds/utils/get_quotas.py | 84 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 nlds/utils/get_quotas.py diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py new file mode 100644 index 00000000..b282ab03 --- /dev/null +++ b/nlds/utils/get_quotas.py @@ -0,0 +1,84 @@ +from ..server_config import load_config +from construct_url import construct_url +from retry import retry +import requests +import json + + + + +class Quotas(): + + _timeout = 10.0 + + def __init__(self): + self.config = load_config() + self.name = "jasmin_authenticator" + self.auth_name = "authentication" + + @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) + def get_projects_services(self, oauth_token: str, service_name): + """Make a call to the JASMIN Projects Portal to get the service information.""" + config = self.config[self.auth_name][self.name] + token_headers = { + "Content-Type": "application/x-ww-form-urlencoded", + "cache-control": "no-cache", + "Authorization": f"Bearer {oauth_token}", + } + # Contact the user_services_url to get the information about the services + url = construct_url([config["user_services_url"]], {"name":{service_name}}) + try: + response = requests.get( + url, + headers=token_headers, + timeout=Quotas._timeout, + ) + except requests.exceptions.ConnectionError: + raise RuntimeError(f"User services url {url} could not be reached.") + except KeyError: + raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") + if response.status_code == requests.codes.ok: # status code 200 + try: + response_json = json.loads(response.text) + return response_json + except json.JSONDecodeError: + raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") + else: + raise RuntimeError(f"Error getting data for {service_name}.") + + + def extract_tape_quota(self, oauth_token: str, service_name): + """Get the service information then process it to extract the quota for the service.""" + # Try to get the service information and throw an exception if an error is encountered + try: + result = self.get_projects_services(self, oauth_token, service_name) + except(RuntimeError, ValueError) as e: + raise type(e)(f"Error getting information for {service_name}: {e}") + + # Process the result to get the requirements + for attr in result: + # Check that the category is Group Workspace + if attr["category"] == 1: + # If there are no requirements, throw an error + if attr["requirements"]: + requirements = attr["requirements"] + else: + raise ValueError(f"Cannot find any requirements for {service_name}.") + else: + raise ValueError(f"Cannot find a Group Workspace with the name {service_name}. Check the category.") + + # Go through the requirements to find the tape resource requirement + for requirement in requirements: + # Only return provisioned requirements + if requirement["status"] == 50: + # Find the tape resource and get its quota + if requirement["resource"]["short_name"] == "tape": + if requirement["amount"]: + tape_quota = requirement["amount"] + return tape_quota + else: + raise ValueError(f"Issue getting tape quota for {service_name}. Either quota is zero or couldn't be found.") + else: + raise ValueError(f"No tape resources could be found for {service_name}") + else: + raise ValueError(f"No provisioned requirements found for {service_name}. Check the status of your requested resources.") \ No newline at end of file From fe2d7e1b3510439834f05fa4121d14359f2c915c Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 31 Jul 2024 13:57:33 +0100 Subject: [PATCH 02/26] Add initial tests for get_quotas --- nlds/utils/get_quotas.py | 2 +- tests/nlds/utils/test_get_quotas.py | 115 ++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 tests/nlds/utils/test_get_quotas.py diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py index b282ab03..49dd694a 100644 --- a/nlds/utils/get_quotas.py +++ b/nlds/utils/get_quotas.py @@ -1,5 +1,5 @@ from ..server_config import load_config -from construct_url import construct_url +from .construct_url import construct_url from retry import retry import requests import json diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py new file mode 100644 index 00000000..ea9db99f --- /dev/null +++ b/tests/nlds/utils/test_get_quotas.py @@ -0,0 +1,115 @@ +import pytest +import requests +import re +from nlds.utils.get_quotas import Quotas + + +@pytest.fixture +def quotas(): + return Quotas() + +user_services_url= "https://example.com/services" +url = f"{user_services_url}?name=test_service" + +def test_get_projects_services_success(monkeypatch, quotas): + def mock_load_config(): + return { + 'authentication': { + 'authenticator':{ + 'user_services_url': user_services_url + } + } + } + monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) + + def mock_construct_url(*args, **kwargs): + return f'https://example.com/services?name=test_service' + monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + class MockResponse: + status_code = 200 + text = '{"key": "value"}' + + def json(self): + return {"key": "value"} + + def mock_get(*args, **kwargs): + return MockResponse() + monkeypatch.setattr(requests, 'get', mock_get) + + result = quotas.get_projects_services('dummy_oauth_token', 'test_service') + + assert result == {"key": "value"} + + +def test_get_projects_services_connection_error(monkeypatch, quotas): + def mock_load_config(): + return { + 'authentication': { + 'authenticator': { + 'user_services_url': user_services_url + } + } + } + monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) + + def mock_construct_url(*args, **kwargs): + return f'https://example.com/services?name=test_service' + monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + def mock_get(*args, **kwargs): + raise requests.exceptions.ConnectionError + monkeypatch.setattr(requests, 'get', mock_get) + + with pytest.raises(RuntimeError, match=re.escape(f"User services url {url} could not be reached.")): + quotas.get_projects_services('dummy_oauth_token', 'test_service') + + +def test_extract_tape_quota_success(monkeypatch, quotas): + def mock_get_projects_services(*args, **kwargs): + return[{ + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + "amount": 100 + } + ] + }] + monkeypatch.setattr('nlds.utils.get_quotas.Quotas.get_projects_services', mock_get_projects_services) + + result = quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + + assert result == 100 + + +def test_extract_tape_quota_no_requirements(monkeypatch, quotas): + def mock_get_projects_services(*args, **kwargs): + return[{ + "category": 1, + "requirements": [] + }] + monkeypatch.setattr('nlds.utils.get_quotas.Quotas.get_projects_services', mock_get_projects_services) + + service_name = 'test_service' + + with pytest.raises(ValueError, match=f"Cannot find any requirements for {service_name}"): + quotas.extract_tape_quota('dummy_oauth_token', service_name) + +def test_extract_tape_quota_no_tape_resource(monkeypatch, quotas): + def mock_get_projects_services(*args, **kwargs): + return [{ + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "other"}, + "amount": 100 + } + ] + }] + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + with pytest.raises(ValueError): + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') \ No newline at end of file From 16513cd1dca12c70873cdf282871e387b2349fdf Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 31 Jul 2024 14:55:54 +0100 Subject: [PATCH 03/26] Add comments --- tests/nlds/utils/test_get_quotas.py | 41 +++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index ea9db99f..7bdd1c4a 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -4,15 +4,22 @@ from nlds.utils.get_quotas import Quotas +# Create an instance of Quotas @pytest.fixture def quotas(): return Quotas() + +# Consts needed in the tests user_services_url= "https://example.com/services" url = f"{user_services_url}?name=test_service" + def test_get_projects_services_success(monkeypatch, quotas): + """Test a successful instance of get_projects_services.""" + def mock_load_config(): + """Mock the load_config function to make it return the test config.""" return { 'authentication': { 'authenticator':{ @@ -23,10 +30,12 @@ def mock_load_config(): monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) def mock_construct_url(*args, **kwargs): + """Mock the construct_url function to make it return the test url.""" return f'https://example.com/services?name=test_service' monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) class MockResponse: + """Mock the response to return a 200 status code and the test text.""" status_code = 200 text = '{"key": "value"}' @@ -34,16 +43,22 @@ def json(self): return {"key": "value"} def mock_get(*args, **kwargs): + """Mock the get function to give the MockResponse.""" return MockResponse() monkeypatch.setattr(requests, 'get', mock_get) + # Call the get_projects_services function with the mocked functions result = quotas.get_projects_services('dummy_oauth_token', 'test_service') + # It should succeed and give the {"key":"value"} dict. assert result == {"key": "value"} def test_get_projects_services_connection_error(monkeypatch, quotas): + """Test an unsuccessful instance of get_projects_services""" + def mock_load_config(): + """Mock the load_config function to make it return the test config.""" return { 'authentication': { 'authenticator': { @@ -54,19 +69,26 @@ def mock_load_config(): monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) def mock_construct_url(*args, **kwargs): + """Mock the construct_url function to make it return the test url.""" return f'https://example.com/services?name=test_service' monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) def mock_get(*args, **kwargs): + """Mock the get function to give the MockResponse.""" raise requests.exceptions.ConnectionError monkeypatch.setattr(requests, 'get', mock_get) + # Check that the ConnectionError in the get triggers a RuntimeError with the right text. with pytest.raises(RuntimeError, match=re.escape(f"User services url {url} could not be reached.")): quotas.get_projects_services('dummy_oauth_token', 'test_service') def test_extract_tape_quota_success(monkeypatch, quotas): + """Test a succesful instance of extract_tape_quota""" + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with a provisioned tape requirement.""" return[{ "category": 1, "requirements": [ @@ -79,26 +101,34 @@ def mock_get_projects_services(*args, **kwargs): }] monkeypatch.setattr('nlds.utils.get_quotas.Quotas.get_projects_services', mock_get_projects_services) + # extract_tape_quota should return the quota value of 100 result = quotas.extract_tape_quota('dummy_oauth_token', 'test_service') - assert result == 100 def test_extract_tape_quota_no_requirements(monkeypatch, quotas): + """Test an unsuccesful instance of extract_tape_quota due to no requirements.""" + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with no requirements.""" return[{ "category": 1, "requirements": [] }] monkeypatch.setattr('nlds.utils.get_quotas.Quotas.get_projects_services', mock_get_projects_services) - service_name = 'test_service' + # A ValueError should be raised saying there's no requirements found. + with pytest.raises(ValueError, match="Cannot find any requirements for test_service"): + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') - with pytest.raises(ValueError, match=f"Cannot find any requirements for {service_name}"): - quotas.extract_tape_quota('dummy_oauth_token', service_name) def test_extract_tape_quota_no_tape_resource(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to no tape resources.""" + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with a requirement that isn't tape.""" return [{ "category": 1, "requirements": [ @@ -111,5 +141,6 @@ def mock_get_projects_services(*args, **kwargs): }] monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) - with pytest.raises(ValueError): + # A ValueError should be raised saying there's no tape resources. + with pytest.raises(ValueError, match="No tape resources could be found for test_service"): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') \ No newline at end of file From 2cd919a1f051e91d40a6c5844348b8af87d3b609 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 31 Jul 2024 15:45:07 +0100 Subject: [PATCH 04/26] Test json error --- tests/nlds/utils/test_get_quotas.py | 81 ++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index 7bdd1c4a..02a21621 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -1,4 +1,5 @@ import pytest +import json import requests import re from nlds.utils.get_quotas import Quotas @@ -22,7 +23,7 @@ def mock_load_config(): """Mock the load_config function to make it return the test config.""" return { 'authentication': { - 'authenticator':{ + 'jasmin_authenticator':{ 'user_services_url': user_services_url } } @@ -31,7 +32,7 @@ def mock_load_config(): def mock_construct_url(*args, **kwargs): """Mock the construct_url function to make it return the test url.""" - return f'https://example.com/services?name=test_service' + return url monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) class MockResponse: @@ -55,13 +56,13 @@ def mock_get(*args, **kwargs): def test_get_projects_services_connection_error(monkeypatch, quotas): - """Test an unsuccessful instance of get_projects_services""" + """Test an unsuccessful instance of get_projects_services due to connection error.""" def mock_load_config(): """Mock the load_config function to make it return the test config.""" return { 'authentication': { - 'authenticator': { + 'jasmin_authenticator': { 'user_services_url': user_services_url } } @@ -70,11 +71,11 @@ def mock_load_config(): def mock_construct_url(*args, **kwargs): """Mock the construct_url function to make it return the test url.""" - return f'https://example.com/services?name=test_service' + return url monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) def mock_get(*args, **kwargs): - """Mock the get function to give the MockResponse.""" + """Mock the get function to give a ConnectionError.""" raise requests.exceptions.ConnectionError monkeypatch.setattr(requests, 'get', mock_get) @@ -83,8 +84,74 @@ def mock_get(*args, **kwargs): quotas.get_projects_services('dummy_oauth_token', 'test_service') +def test_get_projects_services_key_error(monkeypatch, quotas): + """Test an unsuccessful instance of get_projects_services due to a key error.""" + + def mock_load_config(): + """Mock the load_config function to make it return the test config with no user_services_key""" + return { + 'authentication': { + 'jasmin_authenticator': { + 'other_url': 'test.com' + } + } + } + monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) + + def mock_construct_url(*args, **kwargs): + """Mock the construct_url function to make it return the test url.""" + return url + monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + def mock_get(*args, **kwargs): + """Mock the get function to give the KeyError.""" + raise KeyError + monkeypatch.setattr(requests, 'get', mock_get) + + # Check that the KeyError in the get triggers a RuntimeError with the right text. + with pytest.raises(RuntimeError, match=f"Could not find 'user_services_url' key in the jasmin_authenticator section of the .server_config file."): + quotas.get_projects_services('dummy_oauth_token', 'test_service') + + +def test_get_projects_services_json_error(monkeypatch, quotas): + """Test an unsuccessful instance of get_projects_services due to a JSON error.""" + + def mock_load_config(): + """Mock the load_config function to make it return the test config.""" + return { + 'authentication': { + 'jasmin_authenticator': { + 'user_services_url': user_services_url + } + } + } + + def mock_construct_url(*args, **kwargs): + """Mock the construct url function to make it return the test url.""" + return url + monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + class MockResponse: + """Mock the response to return a 200 status code and the test text.""" + status_code = 200 + text = 'invalid json' + + def json(self): + raise json.JSONDecodeError("Expecting value", "invalid json", 0) + + def mock_get(*args, **kwargs): + """Mock the get function to give the JSON error.""" + return MockResponse() + monkeypatch.setattr(requests, 'get', mock_get) + + # Check that the JSONDecodeError triggers a RuntimeError with the right text. + with pytest.raises(RuntimeError, match=re.escape(f"Invalid JSON returned from the user services url: {url}")): + quotas.get_projects_services('dummy_oauth_token', 'test_service') + + + def test_extract_tape_quota_success(monkeypatch, quotas): - """Test a succesful instance of extract_tape_quota""" + """Test a successful instance of extract_tape_quota""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give the response for From 700027f883c4bc4801b0054bd7398542d755d519 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 5 Aug 2024 15:45:09 +0100 Subject: [PATCH 05/26] Add a test to check for runtime errors with get_projects_services --- nlds/utils/get_quotas.py | 2 +- tests/nlds/utils/test_get_quotas.py | 60 ++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py index 49dd694a..1691ce73 100644 --- a/nlds/utils/get_quotas.py +++ b/nlds/utils/get_quotas.py @@ -44,7 +44,7 @@ def get_projects_services(self, oauth_token: str, service_name): except json.JSONDecodeError: raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") else: - raise RuntimeError(f"Error getting data for {service_name}.") + raise RuntimeError(f"Error getting data for {service_name}") def extract_tape_quota(self, oauth_token: str, service_name): diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index 02a21621..2b3968ce 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -147,6 +147,42 @@ def mock_get(*args, **kwargs): # Check that the JSONDecodeError triggers a RuntimeError with the right text. with pytest.raises(RuntimeError, match=re.escape(f"Invalid JSON returned from the user services url: {url}")): quotas.get_projects_services('dummy_oauth_token', 'test_service') + + +def test_get_projects_services_404_error(monkeypatch, quotas): + """Test an unsuccessful instance of get_projects_services due to a 404 error.""" + + def mock_load_config(): + """Mock the load_config function to make it return the test config.""" + return { + 'authentication': { + 'jasmin_authenticator': { + 'user_services_url': user_services_url + } + } + } + + def mock_construct_url(*args, **kwargs): + """Mock the construct url function to make it return the test url.""" + return url + monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + class MockResponse: + """Mock the response to return a 401 status code and the relevant text.""" + status_code = 401 + text = 'Unauthorized' + + def json(self): + return 'Unauthorized' + + def mock_get(*args, **kwargs): + """Mock the get function to give the 401 error.""" + return MockResponse() + monkeypatch.setattr(requests, 'get', mock_get) + + # Check that the 401 error triggers a RuntimeError with the right text. + with pytest.raises(RuntimeError, match=f"Error getting data for test_service"): + quotas.get_projects_services('dummy_oauth_token', 'test_service') @@ -210,4 +246,26 @@ def mock_get_projects_services(*args, **kwargs): # A ValueError should be raised saying there's no tape resources. with pytest.raises(ValueError, match="No tape resources could be found for test_service"): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') \ No newline at end of file + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + + +def test_extract_tape_quota_services_runtime_error(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to a runtime error getting services from the projects portal.""" + def mock_get_projects_services(*args, **kwargs): + raise RuntimeError('Runtime error occurred') + + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + with pytest.raises(RuntimeError, match="Error getting information for test_service: Runtime error occurred"): + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + + +# def test_extract_tape_quota_services_value_error(monkeypatch, quotas): +# """Test an unsuccessful instance of extract_tape_quota due to a value error getting services from the projects portal.""" + + +# test 'cannot find a gws with the name ....' + +# test 'issue getting tape quota for ...' + +# test 'no provisioned requirements ....' \ No newline at end of file From 0a69c5de828d2a0c0544b2c9a29781c07e9dd702 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 5 Aug 2024 16:19:03 +0100 Subject: [PATCH 06/26] Add extra tests for if the service isn't a gws --- tests/nlds/utils/test_get_quotas.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index 2b3968ce..07e4f090 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -260,11 +260,30 @@ def mock_get_projects_services(*args, **kwargs): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') -# def test_extract_tape_quota_services_value_error(monkeypatch, quotas): -# """Test an unsuccessful instance of extract_tape_quota due to a value error getting services from the projects portal.""" +def test_extract_tape_quota_services_value_error(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to a value error getting services from the projects portal.""" + def mock_get_projects_services(*args, **kwargs): + raise ValueError('Value error occurred') + + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + with pytest.raises(ValueError, match="Error getting information for test_service: Value error occurred"): + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + +def test_extract_tape_quota_no_gws(monkeypatch, quotas): + """Test an unsuccesful instance of extract_tape_quota due to the given service not being a gws.""" + def mock_get_projects_services(*args, **kwargs): + return [ + {"category": 2, "requirements": []}, + {"category": 3, "requirements": []} + ] + + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + with pytest.raises(ValueError, match="Cannot find a Group Workspace with the name test_service. Check the category."): + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') -# test 'cannot find a gws with the name ....' # test 'issue getting tape quota for ...' From 80c10d83a5777c9dd42dc450fc6a6ad9fa408785 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 5 Aug 2024 16:33:33 +0100 Subject: [PATCH 07/26] Catch key error with quota and test --- nlds/utils/get_quotas.py | 11 +++++--- tests/nlds/utils/test_get_quotas.py | 40 +++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py index 1691ce73..64ed6c52 100644 --- a/nlds/utils/get_quotas.py +++ b/nlds/utils/get_quotas.py @@ -73,11 +73,14 @@ def extract_tape_quota(self, oauth_token: str, service_name): if requirement["status"] == 50: # Find the tape resource and get its quota if requirement["resource"]["short_name"] == "tape": - if requirement["amount"]: + try: tape_quota = requirement["amount"] - return tape_quota - else: - raise ValueError(f"Issue getting tape quota for {service_name}. Either quota is zero or couldn't be found.") + if tape_quota: + return tape_quota + else: + raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") + except KeyError: + raise KeyError(f"Issue getting tape quota for {service_name}. No quota field exists.") else: raise ValueError(f"No tape resources could be found for {service_name}") else: diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index 07e4f090..24a08971 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -272,7 +272,7 @@ def mock_get_projects_services(*args, **kwargs): def test_extract_tape_quota_no_gws(monkeypatch, quotas): - """Test an unsuccesful instance of extract_tape_quota due to the given service not being a gws.""" + """Test an unsuccessful instance of extract_tape_quota due to the given service not being a gws.""" def mock_get_projects_services(*args, **kwargs): return [ {"category": 2, "requirements": []}, @@ -285,6 +285,42 @@ def mock_get_projects_services(*args, **kwargs): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') -# test 'issue getting tape quota for ...' +def test_extract_tape_quota_zero_quota(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to the quota being zero.""" + def mock_get_projects_services(*args, **kwargs): + return [{ + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + "amount": 0, + } + ], + }] + + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + with pytest.raises(ValueError, match="Issue getting tape quota for test_service. Quota is zero."): + quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + + +def test_extract_tape_quota_no_quota(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to there being no quota value.""" + def mock_get_projects_services(*args, **kwargs): + return [{ + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + } + ], + }] + + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + with pytest.raises(KeyError, match="Issue getting tape quota for test_service. No quota field exists."): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") # test 'no provisioned requirements ....' \ No newline at end of file From 7a5e3be59841b42dff370bc27f7419adc6c5377d Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 5 Aug 2024 16:48:51 +0100 Subject: [PATCH 08/26] Comment tests. --- nlds/utils/get_quotas.py | 2 +- tests/nlds/utils/test_get_quotas.py | 38 +++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py index 64ed6c52..cfa4a12d 100644 --- a/nlds/utils/get_quotas.py +++ b/nlds/utils/get_quotas.py @@ -80,7 +80,7 @@ def extract_tape_quota(self, oauth_token: str, service_name): else: raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") except KeyError: - raise KeyError(f"Issue getting tape quota for {service_name}. No quota field exists.") + raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") else: raise ValueError(f"No tape resources could be found for {service_name}") else: diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index 24a08971..86840573 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -79,7 +79,7 @@ def mock_get(*args, **kwargs): raise requests.exceptions.ConnectionError monkeypatch.setattr(requests, 'get', mock_get) - # Check that the ConnectionError in the get triggers a RuntimeError with the right text. + # Check that the ConnectionError in the 'get' triggers a RuntimeError with the right text. with pytest.raises(RuntimeError, match=re.escape(f"User services url {url} could not be reached.")): quotas.get_projects_services('dummy_oauth_token', 'test_service') @@ -185,7 +185,6 @@ def mock_get(*args, **kwargs): quotas.get_projects_services('dummy_oauth_token', 'test_service') - def test_extract_tape_quota_success(monkeypatch, quotas): """Test a successful instance of extract_tape_quota""" @@ -252,10 +251,12 @@ def mock_get_projects_services(*args, **kwargs): def test_extract_tape_quota_services_runtime_error(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to a runtime error getting services from the projects portal.""" def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give a RuntimeError.""" raise RuntimeError('Runtime error occurred') monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + # A RuntimeError should be raised saying a runtime error occurred. with pytest.raises(RuntimeError, match="Error getting information for test_service: Runtime error occurred"): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') @@ -263,10 +264,12 @@ def mock_get_projects_services(*args, **kwargs): def test_extract_tape_quota_services_value_error(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to a value error getting services from the projects portal.""" def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give a ValueError.""" raise ValueError('Value error occurred') monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + # A ValueError should be raised saying a value error occurred. with pytest.raises(ValueError, match="Error getting information for test_service: Value error occurred"): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') @@ -274,6 +277,7 @@ def mock_get_projects_services(*args, **kwargs): def test_extract_tape_quota_no_gws(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to the given service not being a gws.""" def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give results with the wrong category (a GWS is 1).""" return [ {"category": 2, "requirements": []}, {"category": 3, "requirements": []} @@ -281,6 +285,7 @@ def mock_get_projects_services(*args, **kwargs): monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + # A ValueError should be raised saying it cannot find a GWS and to check the category. with pytest.raises(ValueError, match="Cannot find a Group Workspace with the name test_service. Check the category."): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') @@ -288,6 +293,7 @@ def mock_get_projects_services(*args, **kwargs): def test_extract_tape_quota_zero_quota(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to the quota being zero.""" def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give a quota of 0.""" return [{ "category": 1, "requirements": [ @@ -301,13 +307,15 @@ def mock_get_projects_services(*args, **kwargs): monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + # A ValueError should be raised saying there was an issue getting tape quota as it is zero. with pytest.raises(ValueError, match="Issue getting tape quota for test_service. Quota is zero."): quotas.extract_tape_quota('dummy_oauth_token', 'test_service') def test_extract_tape_quota_no_quota(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to there being no quota value.""" + """Test an unsuccessful instance of extract_tape_quota due to there being no quota field.""" def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give no 'amount' field.""" return [{ "category": 1, "requirements": [ @@ -320,7 +328,27 @@ def mock_get_projects_services(*args, **kwargs): monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) - with pytest.raises(KeyError, match="Issue getting tape quota for test_service. No quota field exists."): + # A KeyError should be raised saying there was an issue getting tape quota as no value field exists. + with pytest.raises(KeyError, match="Issue getting tape quota for test_service. No 'value' field exists."): quotas.extract_tape_quota("dummy_oauth_token", "test_service") -# test 'no provisioned requirements ....' \ No newline at end of file + +def test_extract_tape_quota_no_provisioned_resources(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to there being no provisioned resources.""" + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give no provisioned resources (status 50).""" + return [{ + "category": 1, + "requirements": [ + { + "status": 1, + "resource": {"short_name": "tape"}, + } + ], + }] + + monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + # A ValueError should be raised saying there were no provisioned requirements found and to check the status of requested resources. + with pytest.raises(ValueError, match="No provisioned requirements found for test_service. Check the status of your requested resources."): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") \ No newline at end of file From 8f96f1300c0dc7814f8b824c5b787df6831642fa Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 5 Aug 2024 16:49:36 +0100 Subject: [PATCH 09/26] Black formatting. --- nlds/utils/get_quotas.py | 49 +++-- tests/nlds/utils/test_get_quotas.py | 321 ++++++++++++++++------------ 2 files changed, 213 insertions(+), 157 deletions(-) diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py index cfa4a12d..5854554e 100644 --- a/nlds/utils/get_quotas.py +++ b/nlds/utils/get_quotas.py @@ -5,9 +5,7 @@ import json - - -class Quotas(): +class Quotas: _timeout = 10.0 @@ -26,7 +24,7 @@ def get_projects_services(self, oauth_token: str, service_name): "Authorization": f"Bearer {oauth_token}", } # Contact the user_services_url to get the information about the services - url = construct_url([config["user_services_url"]], {"name":{service_name}}) + url = construct_url([config["user_services_url"]], {"name": {service_name}}) try: response = requests.get( url, @@ -36,25 +34,28 @@ def get_projects_services(self, oauth_token: str, service_name): except requests.exceptions.ConnectionError: raise RuntimeError(f"User services url {url} could not be reached.") except KeyError: - raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") - if response.status_code == requests.codes.ok: # status code 200 + raise RuntimeError( + f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file." + ) + if response.status_code == requests.codes.ok: # status code 200 try: response_json = json.loads(response.text) return response_json except json.JSONDecodeError: - raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") + raise RuntimeError( + f"Invalid JSON returned from the user services url: {url}" + ) else: raise RuntimeError(f"Error getting data for {service_name}") - - + def extract_tape_quota(self, oauth_token: str, service_name): """Get the service information then process it to extract the quota for the service.""" # Try to get the service information and throw an exception if an error is encountered try: result = self.get_projects_services(self, oauth_token, service_name) - except(RuntimeError, ValueError) as e: + except (RuntimeError, ValueError) as e: raise type(e)(f"Error getting information for {service_name}: {e}") - + # Process the result to get the requirements for attr in result: # Check that the category is Group Workspace @@ -63,10 +64,14 @@ def extract_tape_quota(self, oauth_token: str, service_name): if attr["requirements"]: requirements = attr["requirements"] else: - raise ValueError(f"Cannot find any requirements for {service_name}.") + raise ValueError( + f"Cannot find any requirements for {service_name}." + ) else: - raise ValueError(f"Cannot find a Group Workspace with the name {service_name}. Check the category.") - + raise ValueError( + f"Cannot find a Group Workspace with the name {service_name}. Check the category." + ) + # Go through the requirements to find the tape resource requirement for requirement in requirements: # Only return provisioned requirements @@ -78,10 +83,18 @@ def extract_tape_quota(self, oauth_token: str, service_name): if tape_quota: return tape_quota else: - raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") + raise ValueError( + f"Issue getting tape quota for {service_name}. Quota is zero." + ) except KeyError: - raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") + raise KeyError( + f"Issue getting tape quota for {service_name}. No 'value' field exists." + ) else: - raise ValueError(f"No tape resources could be found for {service_name}") + raise ValueError( + f"No tape resources could be found for {service_name}" + ) else: - raise ValueError(f"No provisioned requirements found for {service_name}. Check the status of your requested resources.") \ No newline at end of file + raise ValueError( + f"No provisioned requirements found for {service_name}. Check the status of your requested resources." + ) diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py index 86840573..079d8bab 100644 --- a/tests/nlds/utils/test_get_quotas.py +++ b/tests/nlds/utils/test_get_quotas.py @@ -12,7 +12,7 @@ def quotas(): # Consts needed in the tests -user_services_url= "https://example.com/services" +user_services_url = "https://example.com/services" url = f"{user_services_url}?name=test_service" @@ -22,34 +22,36 @@ def test_get_projects_services_success(monkeypatch, quotas): def mock_load_config(): """Mock the load_config function to make it return the test config.""" return { - 'authentication': { - 'jasmin_authenticator':{ - 'user_services_url': user_services_url - } + "authentication": { + "jasmin_authenticator": {"user_services_url": user_services_url} } } - monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) - + + monkeypatch.setattr("nlds.utils.get_quotas.load_config", mock_load_config) + def mock_construct_url(*args, **kwargs): """Mock the construct_url function to make it return the test url.""" return url - monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) class MockResponse: """Mock the response to return a 200 status code and the test text.""" + status_code = 200 text = '{"key": "value"}' def json(self): return {"key": "value"} - + def mock_get(*args, **kwargs): """Mock the get function to give the MockResponse.""" return MockResponse() - monkeypatch.setattr(requests, 'get', mock_get) + + monkeypatch.setattr(requests, "get", mock_get) # Call the get_projects_services function with the mocked functions - result = quotas.get_projects_services('dummy_oauth_token', 'test_service') + result = quotas.get_projects_services("dummy_oauth_token", "test_service") # It should succeed and give the {"key":"value"} dict. assert result == {"key": "value"} @@ -61,27 +63,30 @@ def test_get_projects_services_connection_error(monkeypatch, quotas): def mock_load_config(): """Mock the load_config function to make it return the test config.""" return { - 'authentication': { - 'jasmin_authenticator': { - 'user_services_url': user_services_url - } + "authentication": { + "jasmin_authenticator": {"user_services_url": user_services_url} } } - monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) - + + monkeypatch.setattr("nlds.utils.get_quotas.load_config", mock_load_config) + def mock_construct_url(*args, **kwargs): """Mock the construct_url function to make it return the test url.""" return url - monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) def mock_get(*args, **kwargs): """Mock the get function to give a ConnectionError.""" raise requests.exceptions.ConnectionError - monkeypatch.setattr(requests, 'get', mock_get) + + monkeypatch.setattr(requests, "get", mock_get) # Check that the ConnectionError in the 'get' triggers a RuntimeError with the right text. - with pytest.raises(RuntimeError, match=re.escape(f"User services url {url} could not be reached.")): - quotas.get_projects_services('dummy_oauth_token', 'test_service') + with pytest.raises( + RuntimeError, match=re.escape(f"User services url {url} could not be reached.") + ): + quotas.get_projects_services("dummy_oauth_token", "test_service") def test_get_projects_services_key_error(monkeypatch, quotas): @@ -89,28 +94,28 @@ def test_get_projects_services_key_error(monkeypatch, quotas): def mock_load_config(): """Mock the load_config function to make it return the test config with no user_services_key""" - return { - 'authentication': { - 'jasmin_authenticator': { - 'other_url': 'test.com' - } - } - } - monkeypatch.setattr('nlds.utils.get_quotas.load_config', mock_load_config) + return {"authentication": {"jasmin_authenticator": {"other_url": "test.com"}}} + + monkeypatch.setattr("nlds.utils.get_quotas.load_config", mock_load_config) def mock_construct_url(*args, **kwargs): """Mock the construct_url function to make it return the test url.""" return url - monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) def mock_get(*args, **kwargs): """Mock the get function to give the KeyError.""" raise KeyError - monkeypatch.setattr(requests, 'get', mock_get) + + monkeypatch.setattr(requests, "get", mock_get) # Check that the KeyError in the get triggers a RuntimeError with the right text. - with pytest.raises(RuntimeError, match=f"Could not find 'user_services_url' key in the jasmin_authenticator section of the .server_config file."): - quotas.get_projects_services('dummy_oauth_token', 'test_service') + with pytest.raises( + RuntimeError, + match=f"Could not find 'user_services_url' key in the jasmin_authenticator section of the .server_config file.", + ): + quotas.get_projects_services("dummy_oauth_token", "test_service") def test_get_projects_services_json_error(monkeypatch, quotas): @@ -119,34 +124,38 @@ def test_get_projects_services_json_error(monkeypatch, quotas): def mock_load_config(): """Mock the load_config function to make it return the test config.""" return { - 'authentication': { - 'jasmin_authenticator': { - 'user_services_url': user_services_url - } + "authentication": { + "jasmin_authenticator": {"user_services_url": user_services_url} } } - + def mock_construct_url(*args, **kwargs): """Mock the construct url function to make it return the test url.""" return url - monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) class MockResponse: """Mock the response to return a 200 status code and the test text.""" + status_code = 200 - text = 'invalid json' + text = "invalid json" def json(self): raise json.JSONDecodeError("Expecting value", "invalid json", 0) - + def mock_get(*args, **kwargs): """Mock the get function to give the JSON error.""" return MockResponse() - monkeypatch.setattr(requests, 'get', mock_get) + + monkeypatch.setattr(requests, "get", mock_get) # Check that the JSONDecodeError triggers a RuntimeError with the right text. - with pytest.raises(RuntimeError, match=re.escape(f"Invalid JSON returned from the user services url: {url}")): - quotas.get_projects_services('dummy_oauth_token', 'test_service') + with pytest.raises( + RuntimeError, + match=re.escape(f"Invalid JSON returned from the user services url: {url}"), + ): + quotas.get_projects_services("dummy_oauth_token", "test_service") def test_get_projects_services_404_error(monkeypatch, quotas): @@ -155,35 +164,36 @@ def test_get_projects_services_404_error(monkeypatch, quotas): def mock_load_config(): """Mock the load_config function to make it return the test config.""" return { - 'authentication': { - 'jasmin_authenticator': { - 'user_services_url': user_services_url - } + "authentication": { + "jasmin_authenticator": {"user_services_url": user_services_url} } } - + def mock_construct_url(*args, **kwargs): """Mock the construct url function to make it return the test url.""" return url - monkeypatch.setattr('nlds.utils.get_quotas.construct_url', mock_construct_url) + + monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) class MockResponse: """Mock the response to return a 401 status code and the relevant text.""" + status_code = 401 - text = 'Unauthorized' + text = "Unauthorized" def json(self): - return 'Unauthorized' - + return "Unauthorized" + def mock_get(*args, **kwargs): """Mock the get function to give the 401 error.""" return MockResponse() - monkeypatch.setattr(requests, 'get', mock_get) + + monkeypatch.setattr(requests, "get", mock_get) # Check that the 401 error triggers a RuntimeError with the right text. with pytest.raises(RuntimeError, match=f"Error getting data for test_service"): - quotas.get_projects_services('dummy_oauth_token', 'test_service') - + quotas.get_projects_services("dummy_oauth_token", "test_service") + def test_extract_tape_quota_success(monkeypatch, quotas): """Test a successful instance of extract_tape_quota""" @@ -191,20 +201,21 @@ def test_extract_tape_quota_success(monkeypatch, quotas): def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give the response for a GWS with a provisioned tape requirement.""" - return[{ - "category": 1, - "requirements": [ + return [ { - "status": 50, - "resource": {"short_name": "tape"}, - "amount": 100 + "category": 1, + "requirements": [ + {"status": 50, "resource": {"short_name": "tape"}, "amount": 100} + ], } ] - }] - monkeypatch.setattr('nlds.utils.get_quotas.Quotas.get_projects_services', mock_get_projects_services) + + monkeypatch.setattr( + "nlds.utils.get_quotas.Quotas.get_projects_services", mock_get_projects_services + ) # extract_tape_quota should return the quota value of 100 - result = quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + result = quotas.extract_tape_quota("dummy_oauth_token", "test_service") assert result == 100 @@ -214,15 +225,17 @@ def test_extract_tape_quota_no_requirements(monkeypatch, quotas): def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give the response for a GWS with no requirements.""" - return[{ - "category": 1, - "requirements": [] - }] - monkeypatch.setattr('nlds.utils.get_quotas.Quotas.get_projects_services', mock_get_projects_services) + return [{"category": 1, "requirements": []}] + + monkeypatch.setattr( + "nlds.utils.get_quotas.Quotas.get_projects_services", mock_get_projects_services + ) # A ValueError should be raised saying there's no requirements found. - with pytest.raises(ValueError, match="Cannot find any requirements for test_service"): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + with pytest.raises( + ValueError, match="Cannot find any requirements for test_service" + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_no_tape_resource(monkeypatch, quotas): @@ -231,124 +244,154 @@ def test_extract_tape_quota_no_tape_resource(monkeypatch, quotas): def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give the response for a GWS with a requirement that isn't tape.""" - return [{ - "category": 1, - "requirements": [ - { - "status": 50, - "resource": {"short_name": "other"}, - "amount": 100 - } - ] - }] - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + return [ + { + "category": 1, + "requirements": [ + {"status": 50, "resource": {"short_name": "other"}, "amount": 100} + ], + } + ] + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A ValueError should be raised saying there's no tape resources. - with pytest.raises(ValueError, match="No tape resources could be found for test_service"): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + with pytest.raises( + ValueError, match="No tape resources could be found for test_service" + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_services_runtime_error(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to a runtime error getting services from the projects portal.""" + def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give a RuntimeError.""" - raise RuntimeError('Runtime error occurred') - - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + raise RuntimeError("Runtime error occurred") + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A RuntimeError should be raised saying a runtime error occurred. - with pytest.raises(RuntimeError, match="Error getting information for test_service: Runtime error occurred"): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + with pytest.raises( + RuntimeError, + match="Error getting information for test_service: Runtime error occurred", + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_services_value_error(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to a value error getting services from the projects portal.""" + def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give a ValueError.""" - raise ValueError('Value error occurred') - - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + raise ValueError("Value error occurred") + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A ValueError should be raised saying a value error occurred. - with pytest.raises(ValueError, match="Error getting information for test_service: Value error occurred"): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + with pytest.raises( + ValueError, + match="Error getting information for test_service: Value error occurred", + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_no_gws(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to the given service not being a gws.""" + def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give results with the wrong category (a GWS is 1).""" return [ {"category": 2, "requirements": []}, - {"category": 3, "requirements": []} + {"category": 3, "requirements": []}, ] - - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A ValueError should be raised saying it cannot find a GWS and to check the category. - with pytest.raises(ValueError, match="Cannot find a Group Workspace with the name test_service. Check the category."): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + with pytest.raises( + ValueError, + match="Cannot find a Group Workspace with the name test_service. Check the category.", + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_zero_quota(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to the quota being zero.""" + def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give a quota of 0.""" - return [{ - "category": 1, - "requirements": [ - { - "status": 50, - "resource": {"short_name": "tape"}, - "amount": 0, - } - ], - }] - - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + return [ + { + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + "amount": 0, + } + ], + } + ] + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A ValueError should be raised saying there was an issue getting tape quota as it is zero. - with pytest.raises(ValueError, match="Issue getting tape quota for test_service. Quota is zero."): - quotas.extract_tape_quota('dummy_oauth_token', 'test_service') + with pytest.raises( + ValueError, match="Issue getting tape quota for test_service. Quota is zero." + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_no_quota(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to there being no quota field.""" + def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give no 'amount' field.""" - return [{ - "category": 1, - "requirements": [ - { - "status": 50, - "resource": {"short_name": "tape"}, - } - ], - }] - - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + return [ + { + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + } + ], + } + ] + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A KeyError should be raised saying there was an issue getting tape quota as no value field exists. - with pytest.raises(KeyError, match="Issue getting tape quota for test_service. No 'value' field exists."): + with pytest.raises( + KeyError, + match="Issue getting tape quota for test_service. No 'value' field exists.", + ): quotas.extract_tape_quota("dummy_oauth_token", "test_service") def test_extract_tape_quota_no_provisioned_resources(monkeypatch, quotas): """Test an unsuccessful instance of extract_tape_quota due to there being no provisioned resources.""" + def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give no provisioned resources (status 50).""" - return [{ - "category": 1, - "requirements": [ - { - "status": 1, - "resource": {"short_name": "tape"}, - } - ], - }] - - monkeypatch.setattr(Quotas, 'get_projects_services', mock_get_projects_services) + return [ + { + "category": 1, + "requirements": [ + { + "status": 1, + "resource": {"short_name": "tape"}, + } + ], + } + ] + + monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) # A ValueError should be raised saying there were no provisioned requirements found and to check the status of requested resources. - with pytest.raises(ValueError, match="No provisioned requirements found for test_service. Check the status of your requested resources."): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") \ No newline at end of file + with pytest.raises( + ValueError, + match="No provisioned requirements found for test_service. Check the status of your requested resources.", + ): + quotas.extract_tape_quota("dummy_oauth_token", "test_service") From a3fda9b6651056612d779e7fce5d65c2c944f0c0 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Tue, 17 Sep 2024 14:50:47 +0100 Subject: [PATCH 10/26] Push changes for quota stuff --- nlds/main.py | 7 +- nlds/rabbit/publisher.py | 1 + nlds/routers/quota.py | 62 +++++++++++++++++ nlds_processors/catalog/catalog.py | 81 ++++++++++++++++++++++- nlds_processors/catalog/catalog_worker.py | 22 +++++- 5 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 nlds/routers/quota.py diff --git a/nlds/main.py b/nlds/main.py index 18823a04..76892e6a 100644 --- a/nlds/main.py +++ b/nlds/main.py @@ -12,7 +12,7 @@ from .nlds_setup import API_VERSION -from .routers import list, files, probe, status, find, meta, system, init +from .routers import list, files, probe, status, find, meta, system, init, quota nlds = FastAPI() @@ -56,4 +56,9 @@ init.router, tags = ["init", ], prefix = PREFIX + "/init" +) +nlds.include_router( + quota.router, + tags = ["quota", ], + prefix = PREFIX + "/quota" ) \ No newline at end of file diff --git a/nlds/rabbit/publisher.py b/nlds/rabbit/publisher.py index 7f39e29d..6f7ad09c 100644 --- a/nlds/rabbit/publisher.py +++ b/nlds/rabbit/publisher.py @@ -50,6 +50,7 @@ class RabbitMQPublisher(): RK_STAT = "stat" RK_FIND = "find" RK_META = "meta" + RK_QUOTA = "quota" # Exchange routing key parts – root RK_ROOT = "nlds-api" diff --git a/nlds/routers/quota.py b/nlds/routers/quota.py new file mode 100644 index 00000000..3c74b387 --- /dev/null +++ b/nlds/routers/quota.py @@ -0,0 +1,62 @@ +""" + +""" + +from fastapi import Depends, APIRouter, status +from fastapi.exceptions import HTTPException +from fastapi.responses import JSONResponse +from pydantic import BaseModel +import json + +from typing import Optional, List, Dict + +from ..rabbit.publisher import RabbitMQPublisher as RMQP +from ..errors import ResponseError +from ..authenticators.authenticate_methods import authenticate_token, \ + authenticate_group, \ + authenticate_user + +router = APIRouter() + +class QuotaResponse(BaseModel): + quota: int + +############################ GET METHOD ############################ +@router.get("/", + status_code = status.HTTP_202_ACCEPTED, + responses = { + status.HTTPS_202_ACCEPTED: {"model": QuotaResponse}, + status.HTTP_400_BAD_REQUEST: {"model": ResponseError}, + status.HTTP_401_UNAUTHORIZED: {"model": ResponseError}, + status.HTTP_403_FORBIDDEN: {"model": ResponseError}, + status.HTTP_404_NOT_FOUND: {"model": ResponseError}, + status.HTTP_504_GATEWAY: {"model": ResponseError}, + } + ) +async def get(token: str = Depends(authenticate_token), + user: str = Depends(authenticate_user), + group: str = Depends(authenticate_group), + label: Optional[str] = None, + holding_id: Optional[int] = None, + tag: Optional[str] = None + ): + # create the message dictionary + + routing_key = f"{RMQP.RK_ROOT}.{RMQP.RK_ROUTE}.{RMQP.RK_QUOTA}" + api_action = f"{RMQP.RK_QUOTA}" + msg_dict = { + RMQP.MSG_DETAILS: { + RMQP.MSG_USER: user, + RMQP.MSG_GROUP: group, + }, + RMQP.MSG_DATA: {}, + RMQP.MSG_TYPE: RMQP.MSG_TYPE_STANDARD + } + # add the metadata + meta_dict = {} + if (label): + meta_dict[RMQP.MSG_LABEL] = label + if (holding_id): + meta_dict[RMQP.MSG_HOLDING_ID] = holding_id + if (tag): + # convert the string into a dictionary \ No newline at end of file diff --git a/nlds_processors/catalog/catalog.py b/nlds_processors/catalog/catalog.py index f65a8027..dd44721c 100644 --- a/nlds_processors/catalog/catalog.py +++ b/nlds_processors/catalog/catalog.py @@ -4,6 +4,11 @@ from sqlalchemy import func, Enum from sqlalchemy.exc import IntegrityError, OperationalError, ArgumentError, \ NoResultFound +from retry import retry +import requests +import json +from nlds.server_config import load_config +from nlds.utils.construct_url import construct_url from nlds_processors.catalog.catalog_models import CatalogBase, File, Holding,\ Location, Transaction, Aggregation, Storage, Tag @@ -874,4 +879,78 @@ def get_unarchived_files(self, holding: Holding) -> List[File]: f"Couldn't find unarchived files for holding with " f"id:{holding.id}" ) - return unarchived_files \ No newline at end of file + return unarchived_files + + @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) + def get_projects_services(self, oauth_token: str, service_name): + """Make a call to the JASMIN Projects Portal to get the service information.""" + self.config = load_config() + self.name = "jasmin_authenticator" + self.auth_name = "authentication" + self._timeout = 10.0 + + config = self.config[self.auth_name][self.name] + token_headers = { + "Content-Type": "application/x-ww-form-urlencoded", + "cache-control": "no-cache", + "Authorization": f"Bearer {oauth_token}", + } + # Contact the user_services_url to get the information about the services + url = construct_url([config["user_services_urk"]], {"name": {service_name}}) + try: + response = requests.get( + url, + headers=token_headers, + timeout=self._timeout, + ) + except requests.exceptions.ConnectionError: + raise RuntimeError(f"User services url {url} could not be reached.") + except KeyError: + raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") + if response.status_code == requests.codes.ok: # status code 200 + try: + response_json = json.loads(response.text) + return response_json + except json.JSONDecodeError: + raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") + else: + raise RuntimeError(f"Error getting data for {service_name}") + + def extract_tape_quota(self, oauth_token: str, service_name): + """Get the service information then process it to extract the quota for the service.""" + try: + result = self.get_projects_services(self, oauth_token, service_name) + except (RuntimeError, ValueError) as e: + raise type(e)(f"Error getting information for {service_name}: {e}") + + # Process the result to get the requirements + for attr in result: + # Check that the category is Group Workspace + if attr["category"] == 1: + # Check that there are requirements, otherwise throw an error + if attr["requirements"]: + requirements = attr["requirements"] + else: + raise ValueError(f"Cannot find any requirements for {service_name}.") + else: + raise ValueError(f"Cannot find a Group Workspace with the name {service_name}. Check the category.") + + # Go through the requirements to find the tape resource requirement + for requirement in requirements: + # Only return provisioned requirements + if requirement["status"] == 50: + # Find the tape resource and get its quota + if requirement["resource"]["short_name"] == "tape": + try: + tape_quota = requirement["amount"] + if tape_quota: + return tape_quota + else: + raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") + except KeyError: + raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") + else: + raise ValueError(f"No tape resources could be found for {service_name}") + else: + raise ValueError(f"No provisioned requirements found for {service_name}.Check the status of your requested resources.") + \ No newline at end of file diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index 403fcaf7..5eff399b 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -1850,7 +1850,23 @@ def _catalog_meta(self, body: Dict, properties: Header) -> None: msg_dict=body, exchange={'name': ''}, correlation_id=properties.correlation_id - ) + ) + + def _catalog_quota(self, body: Dict, properties: Header) -> None: + """Return the users quota for the given service.""" + message_vars = self._parse_user_vars(body) + if message_vars is None: + # Check if any problems have occured in the parsing of the message + # body and exit if necessary + self.log("Could not parse one or more mandatory variables, exiting" + "callback", self.RK_LOG_ERROR) + return + else: + # Unpack if no problems found in parsing + user, group = message_vars + + try: + group_quota = def attach_database(self, create_db_fl: bool = True): @@ -2024,6 +2040,10 @@ def callback(self, ch: Channel, method: Method, properties: Header, elif (api_method == self.RK_STAT): self._catalog_stat(body, properties) + elif (api_method == self.RK_QUOTA): + # don't need to split any routing key for an RPC method + self._catalog_quota(body, properties) + # If received system test message, reply to it (this is for system status check) elif api_method == "system_stat": if properties.correlation_id is not None and properties.correlation_id != self.channel.consumer_tags[0]: From 8bb1efd58b9cec50a6d2ce4aca12f4d9e53cae87 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 9 Oct 2024 14:11:29 +0100 Subject: [PATCH 11/26] Add quota api endpoint and tests --- nlds/authenticators/authenticate_methods.py | 76 +++- nlds/authenticators/base_authenticator.py | 21 + nlds/authenticators/jasmin_authenticator.py | 129 +++++- nlds/rabbit/publisher.py | 1 + nlds/routers/quota.py | 56 ++- nlds/utils/get_quotas.py | 100 ----- nlds_processors/catalog/catalog.py | 134 +----- nlds_processors/catalog/catalog_worker.py | 28 +- tests/nlds/test_jasmin_authenticator.py | 361 ++++++++++++++++ tests/nlds/utils/test_get_quotas.py | 397 ------------------ tests/nlds_processors/catalog/test_catalog.py | 31 +- 11 files changed, 672 insertions(+), 662 deletions(-) delete mode 100644 nlds/utils/get_quotas.py delete mode 100644 tests/nlds/utils/test_get_quotas.py diff --git a/nlds/authenticators/authenticate_methods.py b/nlds/authenticators/authenticate_methods.py index e60cd9f5..b8b1feea 100644 --- a/nlds/authenticators/authenticate_methods.py +++ b/nlds/authenticators/authenticate_methods.py @@ -11,8 +11,9 @@ """Authentication functions for use by the routers.""" from fastapi import Depends, status from fastapi.security import OAuth2PasswordBearer -from .jasmin_authenticator import JasminAuthenticator as Authenticator -from ..errors import ResponseError +from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator +from nlds_processors.catalog.catalog_models import File, Holding +from nlds.errors import ResponseError from fastapi.exceptions import HTTPException oauth2_scheme = OAuth2PasswordBearer(tokenUrl="", auto_error=False) @@ -95,3 +96,74 @@ async def authenticate_group(group: str, token: str = Depends(oauth2_scheme)): detail = response_error.json() ) return group + + +async def authenticate_user_group_role(user: str, group: str, token: str = Depends(oauth2_scheme)): + """Check the user's role in the group by calling the authenticator's authenticate_user_group_role + method.""" + if token is None: + response_error = ResponseError( + loc = ["authenticate_methods", "authenticate_group"], + msg = "Oauth token not supplied.", + type = "Forbidden." + ) + raise HTTPException( + status_code = status.HTTP_403_FORBIDDEN, + detail = response_error.json() + ) + elif not authenticator.authenticate_user_group_role(token, user, group): + response_error = ResponseError( + loc = ["authenticate_methods", "authenticate_user_group_role"], + msg = f"User is not a manager or deputy of the group {group}.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True + + +async def user_has_get_holding_permission(user: str, group: str, holding: Holding): + """Check whether a user has permission to view this holding.""" + if not authenticator.user_has_get_holding_permission(user, group, holding): + response_error = ResponseError( + loc = ["authenticate_methods", "user_has_get_holding_permission"], + msg = f"User does not have get holding permission for {holding}.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True + + +async def user_has_get_file_permission(session, user: str, group: str, file: File): + """Check whether a user has permission to access a file.""" + if not authenticator.user_has_get_file_permission(session, user, group, file): + response_error = ResponseError( + loc = ["authenticate_methods", "user_has_get_file_permission"], + msg = f"User does not have get file permission.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True + + +async def user_has_delete_from_holding_permission(user: str, group: str, holding: Holding): + """Check whether a user has permission to delete files from this holding.""" + if not authenticator.user_has_delete_from_holding_permission(user, group, holding): + response_error = ResponseError( + loc = ["authenticate_methods", "user_has_delete_from_holding_permission"], + msg = f"User does not have delete from holding permission.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True \ No newline at end of file diff --git a/nlds/authenticators/base_authenticator.py b/nlds/authenticators/base_authenticator.py index 06db103f..dff4980a 100644 --- a/nlds/authenticators/base_authenticator.py +++ b/nlds/authenticators/base_authenticator.py @@ -11,6 +11,7 @@ """Base class used to authenticate / authorise the users, groups, collections, etc. """ +from nlds_processors.catalog.catalog_models import File, Holding from abc import ABC @@ -31,3 +32,23 @@ def authenticate_group(self, oauth_token: str, group: str): def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): """Validate whether the user has manager/deputy permissions in the group.""" return NotImplementedError + + def user_has_get_holding_permission(self, user: str, group: str, holding: Holding) -> bool: + """Check whether a user has permission to view this holding.""" + return NotImplementedError + + def user_has_get_file_permission(self, session, user: str, group: str, file: File) -> bool: + """Check whether a user has permission to access a file.""" + return NotImplementedError + + def user_has_delete_from_holding_permission(self, user: str, group: str, holding: Holding) -> bool: + """Check whether a user has permission to delete files from this holding.""" + return NotImplementedError + + def get_service_information(self, oauth_token: str, service_name: str): + """Get the information about the given service.""" + return NotImplementedError + + def extract_tape_quota(self, oauth_token: str, service_name: str): + """Process the service inforrmation to return the tape quota value.""" + return NotImplementedError diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index ea71d0cb..3132f326 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -8,9 +8,10 @@ __license__ = "BSD - see LICENSE file in top-level package directory" __contact__ = "neil.massey@stfc.ac.uk" -from .base_authenticator import BaseAuthenticator -from ..server_config import load_config -from ..utils.construct_url import construct_url +from nlds.authenticators.base_authenticator import BaseAuthenticator +from nlds.server_config import load_config +from nlds.utils.construct_url import construct_url +from nlds_processors.catalog.catalog_models import File, Holding, Transaction from retry import retry import requests import json @@ -237,3 +238,125 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): ) else: return False + + + @staticmethod + def user_has_get_holding_permission(user: str, + group: str, + holding: Holding) -> bool: + """Check whether a user has permission to view this holding. + When we implement ROLES this will be more complicated.""" + permitted = True + #Users can view / get all holdings in their group + #permitted &= holding.user == user + permitted &= holding.group == group + return permitted + + + def user_has_get_file_permission(session, + user: str, + group: str, + file: File) -> bool: + """Check whether a user has permission to access a file. + Later, when we implement the ROLES this function will be a lot more + complicated!""" + assert(session != None) + holding = session.query(Holding).filter( + Transaction.id == file.transaction_id, + Holding.id == Transaction.holding_id + ).all() + permitted = True + for h in holding: + # users have get file permission if in group + # permitted &= h.user == user + permitted &= h.group == group + + return permitted + + @staticmethod + def user_has_delete_from_holding_permission(self, user: str, + group: str, + holding: Holding) -> bool: + """Check whether a user has permission to delete files from this holding. + When we implement ROLES this will be more complicated.""" + # is_admin == whether the user is an administrator of the group + # i.e. a DEPUTY or MANAGER + # this gives them delete permissions for all files in the group + is_admin = self.authenticate_user_group_role(user, group) + permitted = True + # Currently, only users can delete files from their owned holdings + permitted &= (holding.user == user or is_admin) + permitted &= holding.group == group + return permitted + + + @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) + def get_service_information(self, oauth_token: str, service_name: str): + """Make a call to the JASMIN Projects Portal to get the service information.""" + + config = self.config[self.auth_name][self.name] + token_headers = { + "Content-Type": "application/x-ww-form-urlencoded", + "cache-control": "no-cache", + "Authorization": f"Bearer {oauth_token}", + } + # Contact the user_services_url to get the information about the services + url = construct_url([config["user_services_urk"]], {"name": {service_name}}) + try: + response = requests.get( + url, + headers=token_headers, + timeout=JasminAuthenticator._timeout, + ) + except requests.exceptions.ConnectionError: + raise RuntimeError(f"User services url {url} could not be reached.") + except KeyError: + raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") + if response.status_code == requests.codes.ok: # status code 200 + try: + response_json = json.loads(response.text) + return response_json + except json.JSONDecodeError: + raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") + else: + raise RuntimeError(f"Error getting data for {service_name}") + + + def extract_tape_quota(self, oauth_token: str, service_name: str): + """Get the service information then process it to extract the quota for the service.""" + try: + result = self.get_service_information(self, oauth_token, service_name) + except (RuntimeError, ValueError) as e: + raise type(e)(f"Error getting information for {service_name}: {e}") + + # Process the result to get the requirements + for attr in result: + # Check that the category is Group Workspace + if attr["category"] == 1: + # Check that there are requirements, otherwise throw an error + if attr["requirements"]: + requirements = attr["requirements"] + else: + raise ValueError(f"Cannot find any requirements for {service_name}.") + else: + raise ValueError(f"Cannot find a Group Workspace with the name {service_name}. Check the category.") + + # Go through the requirements to find the tape resource requirement + for requirement in requirements: + # Only return provisioned requirements + if requirement["status"] == 50: + # Find the tape resource and get its quota + if requirement["resource"]["short_name"] == "tape": + try: + tape_quota = requirement["amount"] + if tape_quota: + return tape_quota + else: + raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") + except KeyError: + raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") + else: + raise ValueError(f"No tape resources could be found for {service_name}") + else: + raise ValueError(f"No provisioned requirements found for {service_name}.Check the status of your requested resources.") + diff --git a/nlds/rabbit/publisher.py b/nlds/rabbit/publisher.py index 6f7ad09c..46938466 100644 --- a/nlds/rabbit/publisher.py +++ b/nlds/rabbit/publisher.py @@ -119,6 +119,7 @@ class RabbitMQPublisher(): MSG_TENANCY = "tenancy" MSG_ACCESS_KEY = "access_key" MSG_SECRET_KEY = "secret_key" + MSG_TOKEN = "token" MSG_API_ACTION = "api_action" MSG_JOB_LABEL = "job_label" MSG_DATA = "data" diff --git a/nlds/routers/quota.py b/nlds/routers/quota.py index 3c74b387..ad292562 100644 --- a/nlds/routers/quota.py +++ b/nlds/routers/quota.py @@ -11,11 +11,15 @@ from typing import Optional, List, Dict from ..rabbit.publisher import RabbitMQPublisher as RMQP +from ..routers import rpc_publisher from ..errors import ResponseError from ..authenticators.authenticate_methods import authenticate_token, \ authenticate_group, \ authenticate_user +from ..utils.process_tag import process_tag + + router = APIRouter() class QuotaResponse(BaseModel): @@ -25,12 +29,12 @@ class QuotaResponse(BaseModel): @router.get("/", status_code = status.HTTP_202_ACCEPTED, responses = { - status.HTTPS_202_ACCEPTED: {"model": QuotaResponse}, + status.HTTP_202_ACCEPTED: {"model": QuotaResponse}, status.HTTP_400_BAD_REQUEST: {"model": ResponseError}, status.HTTP_401_UNAUTHORIZED: {"model": ResponseError}, status.HTTP_403_FORBIDDEN: {"model": ResponseError}, status.HTTP_404_NOT_FOUND: {"model": ResponseError}, - status.HTTP_504_GATEWAY: {"model": ResponseError}, + status.HTTP_504_GATEWAY_TIMEOUT: {"model": ResponseError}, } ) async def get(token: str = Depends(authenticate_token), @@ -38,6 +42,7 @@ async def get(token: str = Depends(authenticate_token), group: str = Depends(authenticate_group), label: Optional[str] = None, holding_id: Optional[int] = None, + transaction_id: Optional[str] = None, tag: Optional[str] = None ): # create the message dictionary @@ -48,6 +53,7 @@ async def get(token: str = Depends(authenticate_token), RMQP.MSG_DETAILS: { RMQP.MSG_USER: user, RMQP.MSG_GROUP: group, + RMQP.MSG_TOKEN: token, }, RMQP.MSG_DATA: {}, RMQP.MSG_TYPE: RMQP.MSG_TYPE_STANDARD @@ -58,5 +64,49 @@ async def get(token: str = Depends(authenticate_token), meta_dict[RMQP.MSG_LABEL] = label if (holding_id): meta_dict[RMQP.MSG_HOLDING_ID] = holding_id + if (transaction_id): + meta_dict[RMQP.MSG_TRANSACT_ID] = transaction_id + if (tag): - # convert the string into a dictionary \ No newline at end of file + tag_dict = {} + # convert the string into a dictionary + try: + tag_dict = process_tag(tag) + except ValueError: + response_error = ResponseError( + loc = ["quota", "get"], + msg = "tag cannot be processed.", + type = "Incomplete request." + ) + raise HTTPException( + status_code = status.HTTP_400_BAD_REQUEST, + detail = response_error.json() + ) + else: + meta_dict[RMQP.MSG_TAG] = tag_dict + + if (len(meta_dict) > 0): + msg_dict[RMQP.MSG_META] = meta_dict + + # Call RPC function + routing_key = "catalog_q" + response = await rpc_publisher.call( + msg_dict=msg_dict, routing_key=routing_key + ) + # Check if response is valid or whether the request timed out + if response is not None: + # convert byte response to str + response = response.decode() + + return JSONResponse(status_code = status.HTTP_202_ACCEPTED, + content = response) + else: + response_error = ResponseError( + loc = ["status", "get"], + msg = "Catalog service could not be reached in time.", + type = "Incomplete request." + ) + raise HTTPException( + status_code = status.HTTP_504_GATEWAY_TIMEOUT, + detail = response_error.json() + ) diff --git a/nlds/utils/get_quotas.py b/nlds/utils/get_quotas.py deleted file mode 100644 index 5854554e..00000000 --- a/nlds/utils/get_quotas.py +++ /dev/null @@ -1,100 +0,0 @@ -from ..server_config import load_config -from .construct_url import construct_url -from retry import retry -import requests -import json - - -class Quotas: - - _timeout = 10.0 - - def __init__(self): - self.config = load_config() - self.name = "jasmin_authenticator" - self.auth_name = "authentication" - - @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) - def get_projects_services(self, oauth_token: str, service_name): - """Make a call to the JASMIN Projects Portal to get the service information.""" - config = self.config[self.auth_name][self.name] - token_headers = { - "Content-Type": "application/x-ww-form-urlencoded", - "cache-control": "no-cache", - "Authorization": f"Bearer {oauth_token}", - } - # Contact the user_services_url to get the information about the services - url = construct_url([config["user_services_url"]], {"name": {service_name}}) - try: - response = requests.get( - url, - headers=token_headers, - timeout=Quotas._timeout, - ) - except requests.exceptions.ConnectionError: - raise RuntimeError(f"User services url {url} could not be reached.") - except KeyError: - raise RuntimeError( - f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file." - ) - if response.status_code == requests.codes.ok: # status code 200 - try: - response_json = json.loads(response.text) - return response_json - except json.JSONDecodeError: - raise RuntimeError( - f"Invalid JSON returned from the user services url: {url}" - ) - else: - raise RuntimeError(f"Error getting data for {service_name}") - - def extract_tape_quota(self, oauth_token: str, service_name): - """Get the service information then process it to extract the quota for the service.""" - # Try to get the service information and throw an exception if an error is encountered - try: - result = self.get_projects_services(self, oauth_token, service_name) - except (RuntimeError, ValueError) as e: - raise type(e)(f"Error getting information for {service_name}: {e}") - - # Process the result to get the requirements - for attr in result: - # Check that the category is Group Workspace - if attr["category"] == 1: - # If there are no requirements, throw an error - if attr["requirements"]: - requirements = attr["requirements"] - else: - raise ValueError( - f"Cannot find any requirements for {service_name}." - ) - else: - raise ValueError( - f"Cannot find a Group Workspace with the name {service_name}. Check the category." - ) - - # Go through the requirements to find the tape resource requirement - for requirement in requirements: - # Only return provisioned requirements - if requirement["status"] == 50: - # Find the tape resource and get its quota - if requirement["resource"]["short_name"] == "tape": - try: - tape_quota = requirement["amount"] - if tape_quota: - return tape_quota - else: - raise ValueError( - f"Issue getting tape quota for {service_name}. Quota is zero." - ) - except KeyError: - raise KeyError( - f"Issue getting tape quota for {service_name}. No 'value' field exists." - ) - else: - raise ValueError( - f"No tape resources could be found for {service_name}" - ) - else: - raise ValueError( - f"No provisioned requirements found for {service_name}. Check the status of your requested resources." - ) diff --git a/nlds_processors/catalog/catalog.py b/nlds_processors/catalog/catalog.py index dd44721c..f7e0e2ca 100644 --- a/nlds_processors/catalog/catalog.py +++ b/nlds_processors/catalog/catalog.py @@ -13,7 +13,7 @@ from nlds_processors.catalog.catalog_models import CatalogBase, File, Holding,\ Location, Transaction, Aggregation, Storage, Tag from nlds_processors.db_mixin import DBMixin -from nlds.authenticators.jasmin_authenticator import JasminAuthenticator +from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator class CatalogError(Exception): def __init__(self, message, *args): super().__init__(args) @@ -32,19 +32,6 @@ def __init__(self, db_engine: str, db_options: str): self.session = None - @staticmethod - def _user_has_get_holding_permission(user: str, - group: str, - holding: Holding) -> bool: - """Check whether a user has permission to view this holding. - When we implement ROLES this will be more complicated.""" - permitted = True - #Users can view / get all holdings in their group - #permitted &= holding.user == user - permitted &= holding.group == group - return permitted - - def get_holding(self, user: str, group: str, @@ -109,7 +96,7 @@ def get_holding(self, raise KeyError # check the user has permission to view the holding(s) for h in holding: - if not self._user_has_get_holding_permission(user, group, h): + if not Authenticator.user_has_get_holding_permission(user, group, h): raise CatalogError( f"User:{user} in group:{group} does not have permission " f"to access the holding with label:{h.label}." @@ -301,26 +288,6 @@ def delete_transaction(self, f"Transaction with transaction_id:{transaction_id} could not " "be added to the database" ) - - def _user_has_get_file_permission(self, - user: str, - group: str, - file: File) -> bool: - """Check whether a user has permission to access a file. - Later, when we implement the ROLES this function will be a lot more - complicated!""" - assert(self.session != None) - holding = self.session.query(Holding).filter( - Transaction.id == file.transaction_id, - Holding.id == Transaction.holding_id - ).all() - permitted = True - for h in holding: - # users have get file permission if in group - # permitted &= h.user == user - permitted &= h.group == group - - return permitted def get_file(self, @@ -425,7 +392,7 @@ def get_files(self, # check user has permission to access this file # NRM - 12/10/2023, is this necessary? if (f and - not self._user_has_get_file_permission(user, group, f) + not Authenticator.user_has_get_file_permission(self.session, user, group, f) ): raise CatalogError( f"User:{user} in group:{group} does not have permission to " @@ -483,23 +450,6 @@ def create_file(self, " the database" ) return new_file - - - @staticmethod - def _user_has_delete_from_holding_permission(user: str, - group: str, - holding: Holding) -> bool: - """Check whether a user has permission to delete files from this holding. - When we implement ROLES this will be more complicated.""" - # is_admin == whether the user is an administrator of the group - # i.e. a DEPUTY or MANAGER - # this gives them delete permissions for all files in the group - is_admin = JasminAuthenticator.authenticate_user_group_role(user, group) - permitted = True - # Currently, only users can delete files from their owned holdings - permitted &= (holding.user == user or is_admin) - permitted &= holding.group == group - return permitted def delete_files(self, @@ -520,7 +470,7 @@ def delete_files(self, holding_id=holding_id, original_path=path, tag=tag) holding = self.get_holding(user, group, holding_id=holding_id)[0] - if not Catalog._user_has_delete_from_holding_permission( + if not Authenticator.user_has_delete_from_holding_permission( user, group, holding): # No admins at the moment! raise CatalogError( @@ -879,78 +829,4 @@ def get_unarchived_files(self, holding: Holding) -> List[File]: f"Couldn't find unarchived files for holding with " f"id:{holding.id}" ) - return unarchived_files - - @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) - def get_projects_services(self, oauth_token: str, service_name): - """Make a call to the JASMIN Projects Portal to get the service information.""" - self.config = load_config() - self.name = "jasmin_authenticator" - self.auth_name = "authentication" - self._timeout = 10.0 - - config = self.config[self.auth_name][self.name] - token_headers = { - "Content-Type": "application/x-ww-form-urlencoded", - "cache-control": "no-cache", - "Authorization": f"Bearer {oauth_token}", - } - # Contact the user_services_url to get the information about the services - url = construct_url([config["user_services_urk"]], {"name": {service_name}}) - try: - response = requests.get( - url, - headers=token_headers, - timeout=self._timeout, - ) - except requests.exceptions.ConnectionError: - raise RuntimeError(f"User services url {url} could not be reached.") - except KeyError: - raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") - if response.status_code == requests.codes.ok: # status code 200 - try: - response_json = json.loads(response.text) - return response_json - except json.JSONDecodeError: - raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") - else: - raise RuntimeError(f"Error getting data for {service_name}") - - def extract_tape_quota(self, oauth_token: str, service_name): - """Get the service information then process it to extract the quota for the service.""" - try: - result = self.get_projects_services(self, oauth_token, service_name) - except (RuntimeError, ValueError) as e: - raise type(e)(f"Error getting information for {service_name}: {e}") - - # Process the result to get the requirements - for attr in result: - # Check that the category is Group Workspace - if attr["category"] == 1: - # Check that there are requirements, otherwise throw an error - if attr["requirements"]: - requirements = attr["requirements"] - else: - raise ValueError(f"Cannot find any requirements for {service_name}.") - else: - raise ValueError(f"Cannot find a Group Workspace with the name {service_name}. Check the category.") - - # Go through the requirements to find the tape resource requirement - for requirement in requirements: - # Only return provisioned requirements - if requirement["status"] == 50: - # Find the tape resource and get its quota - if requirement["resource"]["short_name"] == "tape": - try: - tape_quota = requirement["amount"] - if tape_quota: - return tape_quota - else: - raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") - except KeyError: - raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") - else: - raise ValueError(f"No tape resources could be found for {service_name}") - else: - raise ValueError(f"No provisioned requirements found for {service_name}.Check the status of your requested resources.") - \ No newline at end of file + return unarchived_files \ No newline at end of file diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index 5eff399b..e8370ad9 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -46,6 +46,8 @@ from nlds.details import PathDetails, PathType from nlds_processors.db_mixin import DBError +from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator + class Metadata(): """Container class for the meta section of the message body.""" @@ -1863,10 +1865,32 @@ def _catalog_quota(self, body: Dict, properties: Header) -> None: return else: # Unpack if no problems found in parsing - user, group = message_vars + user, group, token = message_vars try: - group_quota = + group_quota = Authenticator.extract_tape_quota(oauth_token=token, service_name=group) + except CatalogError as e: + # failed to get the holdings - send a return message saying so + self.log(e.message, self.RK_LOG_ERROR) + body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message + body[self.MSG_DATA][self.MSG_HOLDING_LIST] = [] + else: + # fill the return message with a dictionary of the holding(s) + body[self.MSG_DATA][self.MSG_HOLDING_LIST] = group_quota + self.log( + f"Quota from CATALOG_QUOTA {group_quota}", + self.RK_LOG_DEBUG + ) + + self.catalog.end_session() + + # return message to complete RPC + self.publish_message( + properties.reply_to, + msg_dict=body, + exchange={'name': ''}, + correlation_id=properties.correlation_id + ) def attach_database(self, create_db_fl: bool = True): diff --git a/tests/nlds/test_jasmin_authenticator.py b/tests/nlds/test_jasmin_authenticator.py index 1275ba1c..b07e9e65 100644 --- a/tests/nlds/test_jasmin_authenticator.py +++ b/tests/nlds/test_jasmin_authenticator.py @@ -2,8 +2,10 @@ import pytest import json import urllib +import re from nlds.authenticators.jasmin_authenticator import JasminAuthenticator +from nlds_processors.catalog.catalog_models import Holding from nlds.utils.construct_url import construct_url @@ -244,3 +246,362 @@ def test_authenticate_user_group_role( # The authenticate_user_group_role method will use the monkeypatch has_role = auth.authenticate_user_group_role(oauth_token, user, group) assert has_role == expected_result + +class TestUserPermissions: + """Test the functions that assign permissions to get holdings, get files and to delete from holding.""" + + @pytest.fixture() + def mock_holding(): + return Holding( + label='test-label', + user='test-user', + group='test-group', + ) + + def test_user_has_get_holding_permission(self): + # Leaving this for now until it's a bit more fleshed out + pass + + def test_user_has_get_file_permission(self): + # Leaving this for now until it's a bit more fleshed out + pass + + @pytest.mark.parametrize("user, group, mock_is_admin, expected", [ + ("test-user", "test-group", False, True), # User owns the holding + ("user2", "test-group", False, False), # User does not own holding and is not admin + ("user2", "test-group", True, True), # User is admin of the group + ("test-user", "group2", False, False), # User is owner of different holding + ]) + def test_user_has_delete_from_holiding_permission(self, monkeypatch, user, group, mock_is_admin, expected, mock_holding): + # Mock the authenticate_user_group_role method + def mock_authenticate_user_group_role(user, group): + return mock_is_admin + + monkeypatch.setattr(JasminAuthenticator, "authenticate_user_group_role", mock_authenticate_user_group_role) + result = JasminAuthenticator.user_has_delete_from_holding_permission(user, group, mock_holding) + assert result == expected + + + +class TestGetProjectsServices: + """Get the projects for a service from the JASMIN Projects Portal.""" + user_services_url = "https://example.com/services" + url = f"{user_services_url}?name=test_service" + + @pytest.fixture() + def mock_construct_url(self, *args, **kwargs): + """Mock the construct_url function to make it return the test url.""" + return self.url + + def test_get_projects_services_success(self,monkeypatch): + """Test a successful instance of get_projects_services.""" + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + + class MockResponse: + """Mock the response to return a 200 status code and the test text.""" + + status_code = 200 + text = '{"key": "value"}' + + def json(self): + return {"key": "value"} + + def mock_get(*args, **kwargs): + """Mock the get function to give MockResponse.""" + return MockResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + # Call the get_projects_services function with the mocked functions + result = JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + + # It should succeed and give the {"key":"value"} dict. + assert result == {"key":"value"} + + def test_get_projects_services_connection_error(self,monkeypatch, quotas): + """Test an unsuccessful instance of get_projects_services due to connection error.""" + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + + def mock_get(*args, **kwargs): + """Mock the get function to give a ConnectionError.""" + raise requests.exceptions.ConnectionError + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the ConnectionError in the 'get' triggers a RuntimeError with the right text. + with pytest.raises( + RuntimeError, match=re.escape(f"User services url {self.url} could not be reached.") + ): + JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + + def test_get_projects_services_key_error(self,monkeypatch): + """Test an unsuccessful instance of get_projects_services due to a key error.""" + + def mock_load_config_key_error(): + """Mock the load_config function to make it return the test config with no user_services_key""" + return {"authentication": {"jasmin_authenticator": {"other_url": "test.com"}}} + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config_key_error) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + + def mock_get(*args, **kwargs): + """Mock the get function to give the KeyError.""" + raise KeyError + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the KeyError in the 'get' triggers a RuntimeError with the right text. + with pytest.raises( + RuntimeError, + match=f"Could not find 'user_services_url' key in the jasmin_authenticator section of the .server_config file.", + ): + JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + + def test_get_projects_services_json_error(self, monkeypatch): + """Test an unsuccessful instance of get_projects_services due to a JSON error.""" + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + + class MockInvalidJSONResponse: + """Mock the response to return a 200 status code and the JSON decode error.""" + status_code = 200 + text = "invalid json" + + def json(self): + raise json.JSONDecodeError("Expecting value", "invalid json", 0) + + def mock_get(*args, **kwargs): + """Mock the 'get' function to give the JSON error.""" + return MockInvalidJSONResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the JSONDecodeError triggers a RuntimeError with the right text. + with pytest.raises( + RuntimeError, + match=re.escape(f"Invalid JSON returned from the user services url: {self.url}"), + ): + JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + + def test_get_projects_services_404_error(self,monkeypatch): + """Test an unsuccessful instance of get_projects_services due to a 404 error.""" + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + + class MockResponse: + """Mock the response to return a 401 status code and the relevant text.""" + status_code = 401 + text = "Unauthorized" + + def json(self): + return "Unauthorized" + + def mock_get(*args, **kwargs): + """Mock the get function to give the 401 error.""" + return MockResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the 401 error triggers a RuntimeError with the right text. + with pytest.raises(RuntimeError, match=f"Error getting data for test_service"): + JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + + +class TestExtractTapeQuota: + """Get the tape quota from the list of projects services.""" + + def test_extract_tape_quota_success(monkeypatch): + """Test a successful instance of extract_tape_quota""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to gvie the response for + a GWS with a provisioned tape requirement.""" + return [ + { + "category": 1, + "requirements": [ + {"status": 50, "resource": {"short_name": "tape"}, "amount": 100} + ], + } + ] + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.get_projects_services", mock_get_projects_services) + + # extract_tape_quota should return the quota value of 100 + result = JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + assert result == 100 + + def test_extract_tape_quota_no_requirements(monkeypatch, quotas): + """Test an unsuccessful instance of extract_tape_quota due to no requirements.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with no requirements.""" + return [{"category": 1, "requirements": []}] + + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.get_projects_setvices", mock_get_projects_services) + + # A ValueError should be raised saying there's no requirements found. + with pytest.raises(ValueError, match="Cannot find any requirements for test_service"): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_tape_quota_no_tape_resource(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to no tape resources.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with a requirement that isn't tape.""" + return [ + { + "category": 1, + "requirements": [ + {"status": 50, "resource": {"short_name": "other"}, "amount": 100} + ], + } + ] + + monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A ValueError should be raised saying there's no tape resources. + with pytest.raises( + ValueError, match="No tape resources could be found for test_service" + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_tape_quota_services_runtime_error(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to a runtime error when + getting services from the projects portal.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give a RuntimeError.""" + raise RuntimeError("Runtime error occurred.") + + monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A RuntimeError should be raised saying a runtime error occurred. + with pytest.raises( + RuntimeError, + match="Error getting information for test_service: Runtime error occurred", + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_tape_quota_services_value_error(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to a value error + getting services from the projects portal.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give a ValueError.""" + raise ValueError("Value error occurred") + + monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A ValueError should be raised saying a value error occurred. + with pytest.raises( + ValueError, + match="Error getting information for test_service: Value error occurred", + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_tape_quota_no_gws(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to the given service + not being a GWS.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give results with the wrong category (a GWS is 1).""" + return [ + {"category": 2, "requirements": []}, + {"category": 3, "requirements": []}, + ] + + monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A ValueError should be raised saying it cannot find a GWS and to check the category. + with pytest.raises( + ValueError, + match="Cannot find a Group Workspace with the name test_service. Check the category.", + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_quota_zero_quota(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to the quota being zero.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give a quota of 0.""" + return [ + { + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + "amount": 0, + } + ], + } + ] + + monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A ValueError should be raised saying there was an issue getting tape quota as it was zero. + with pytest.raises( + ValueError, match="Issue getting tape quota for test_service. Quota is zero." + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_tape_quota_no_quota(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to there being no quota field.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give no 'amount field.""" + return [ + { + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + } + ], + } + ] + + monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A key error should be raised saying there was an issue getting tape quota as no value field exists. + with pytest.raises( + KeyError, + match="Issue getting tape quota for test_service. No 'value' field exists.", + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + + def test_extract_tape_quota_no_provisioned_resources(monkeypatch): + """Test an unsuccessful instance of extract_tape_quota due to there being no provisioned resources.""" + + def mock_get_projects_services(*args, **kwargs): + """Mock the response from get_projects_services to give no provisioned resources (status 50).""" + return [ + { + "category": 1, + "requirements": [ + { + "status": 1, + "resource": {"short_name": "tape"}, + } + ], + } + ] + + monkeypatch,setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + + # A value error should be raised saying there were no provisioned requirements found and to check the status of requested resources. + with pytest.raises( + ValueError, + match="No provisioned requirements found for test_service. Check the status of your requested resources.", + ): + JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") diff --git a/tests/nlds/utils/test_get_quotas.py b/tests/nlds/utils/test_get_quotas.py deleted file mode 100644 index 079d8bab..00000000 --- a/tests/nlds/utils/test_get_quotas.py +++ /dev/null @@ -1,397 +0,0 @@ -import pytest -import json -import requests -import re -from nlds.utils.get_quotas import Quotas - - -# Create an instance of Quotas -@pytest.fixture -def quotas(): - return Quotas() - - -# Consts needed in the tests -user_services_url = "https://example.com/services" -url = f"{user_services_url}?name=test_service" - - -def test_get_projects_services_success(monkeypatch, quotas): - """Test a successful instance of get_projects_services.""" - - def mock_load_config(): - """Mock the load_config function to make it return the test config.""" - return { - "authentication": { - "jasmin_authenticator": {"user_services_url": user_services_url} - } - } - - monkeypatch.setattr("nlds.utils.get_quotas.load_config", mock_load_config) - - def mock_construct_url(*args, **kwargs): - """Mock the construct_url function to make it return the test url.""" - return url - - monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) - - class MockResponse: - """Mock the response to return a 200 status code and the test text.""" - - status_code = 200 - text = '{"key": "value"}' - - def json(self): - return {"key": "value"} - - def mock_get(*args, **kwargs): - """Mock the get function to give the MockResponse.""" - return MockResponse() - - monkeypatch.setattr(requests, "get", mock_get) - - # Call the get_projects_services function with the mocked functions - result = quotas.get_projects_services("dummy_oauth_token", "test_service") - - # It should succeed and give the {"key":"value"} dict. - assert result == {"key": "value"} - - -def test_get_projects_services_connection_error(monkeypatch, quotas): - """Test an unsuccessful instance of get_projects_services due to connection error.""" - - def mock_load_config(): - """Mock the load_config function to make it return the test config.""" - return { - "authentication": { - "jasmin_authenticator": {"user_services_url": user_services_url} - } - } - - monkeypatch.setattr("nlds.utils.get_quotas.load_config", mock_load_config) - - def mock_construct_url(*args, **kwargs): - """Mock the construct_url function to make it return the test url.""" - return url - - monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) - - def mock_get(*args, **kwargs): - """Mock the get function to give a ConnectionError.""" - raise requests.exceptions.ConnectionError - - monkeypatch.setattr(requests, "get", mock_get) - - # Check that the ConnectionError in the 'get' triggers a RuntimeError with the right text. - with pytest.raises( - RuntimeError, match=re.escape(f"User services url {url} could not be reached.") - ): - quotas.get_projects_services("dummy_oauth_token", "test_service") - - -def test_get_projects_services_key_error(monkeypatch, quotas): - """Test an unsuccessful instance of get_projects_services due to a key error.""" - - def mock_load_config(): - """Mock the load_config function to make it return the test config with no user_services_key""" - return {"authentication": {"jasmin_authenticator": {"other_url": "test.com"}}} - - monkeypatch.setattr("nlds.utils.get_quotas.load_config", mock_load_config) - - def mock_construct_url(*args, **kwargs): - """Mock the construct_url function to make it return the test url.""" - return url - - monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) - - def mock_get(*args, **kwargs): - """Mock the get function to give the KeyError.""" - raise KeyError - - monkeypatch.setattr(requests, "get", mock_get) - - # Check that the KeyError in the get triggers a RuntimeError with the right text. - with pytest.raises( - RuntimeError, - match=f"Could not find 'user_services_url' key in the jasmin_authenticator section of the .server_config file.", - ): - quotas.get_projects_services("dummy_oauth_token", "test_service") - - -def test_get_projects_services_json_error(monkeypatch, quotas): - """Test an unsuccessful instance of get_projects_services due to a JSON error.""" - - def mock_load_config(): - """Mock the load_config function to make it return the test config.""" - return { - "authentication": { - "jasmin_authenticator": {"user_services_url": user_services_url} - } - } - - def mock_construct_url(*args, **kwargs): - """Mock the construct url function to make it return the test url.""" - return url - - monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) - - class MockResponse: - """Mock the response to return a 200 status code and the test text.""" - - status_code = 200 - text = "invalid json" - - def json(self): - raise json.JSONDecodeError("Expecting value", "invalid json", 0) - - def mock_get(*args, **kwargs): - """Mock the get function to give the JSON error.""" - return MockResponse() - - monkeypatch.setattr(requests, "get", mock_get) - - # Check that the JSONDecodeError triggers a RuntimeError with the right text. - with pytest.raises( - RuntimeError, - match=re.escape(f"Invalid JSON returned from the user services url: {url}"), - ): - quotas.get_projects_services("dummy_oauth_token", "test_service") - - -def test_get_projects_services_404_error(monkeypatch, quotas): - """Test an unsuccessful instance of get_projects_services due to a 404 error.""" - - def mock_load_config(): - """Mock the load_config function to make it return the test config.""" - return { - "authentication": { - "jasmin_authenticator": {"user_services_url": user_services_url} - } - } - - def mock_construct_url(*args, **kwargs): - """Mock the construct url function to make it return the test url.""" - return url - - monkeypatch.setattr("nlds.utils.get_quotas.construct_url", mock_construct_url) - - class MockResponse: - """Mock the response to return a 401 status code and the relevant text.""" - - status_code = 401 - text = "Unauthorized" - - def json(self): - return "Unauthorized" - - def mock_get(*args, **kwargs): - """Mock the get function to give the 401 error.""" - return MockResponse() - - monkeypatch.setattr(requests, "get", mock_get) - - # Check that the 401 error triggers a RuntimeError with the right text. - with pytest.raises(RuntimeError, match=f"Error getting data for test_service"): - quotas.get_projects_services("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_success(monkeypatch, quotas): - """Test a successful instance of extract_tape_quota""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give the response for - a GWS with a provisioned tape requirement.""" - return [ - { - "category": 1, - "requirements": [ - {"status": 50, "resource": {"short_name": "tape"}, "amount": 100} - ], - } - ] - - monkeypatch.setattr( - "nlds.utils.get_quotas.Quotas.get_projects_services", mock_get_projects_services - ) - - # extract_tape_quota should return the quota value of 100 - result = quotas.extract_tape_quota("dummy_oauth_token", "test_service") - assert result == 100 - - -def test_extract_tape_quota_no_requirements(monkeypatch, quotas): - """Test an unsuccesful instance of extract_tape_quota due to no requirements.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give the response for - a GWS with no requirements.""" - return [{"category": 1, "requirements": []}] - - monkeypatch.setattr( - "nlds.utils.get_quotas.Quotas.get_projects_services", mock_get_projects_services - ) - - # A ValueError should be raised saying there's no requirements found. - with pytest.raises( - ValueError, match="Cannot find any requirements for test_service" - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_no_tape_resource(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to no tape resources.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give the response for - a GWS with a requirement that isn't tape.""" - return [ - { - "category": 1, - "requirements": [ - {"status": 50, "resource": {"short_name": "other"}, "amount": 100} - ], - } - ] - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A ValueError should be raised saying there's no tape resources. - with pytest.raises( - ValueError, match="No tape resources could be found for test_service" - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_services_runtime_error(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to a runtime error getting services from the projects portal.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give a RuntimeError.""" - raise RuntimeError("Runtime error occurred") - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A RuntimeError should be raised saying a runtime error occurred. - with pytest.raises( - RuntimeError, - match="Error getting information for test_service: Runtime error occurred", - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_services_value_error(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to a value error getting services from the projects portal.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give a ValueError.""" - raise ValueError("Value error occurred") - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A ValueError should be raised saying a value error occurred. - with pytest.raises( - ValueError, - match="Error getting information for test_service: Value error occurred", - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_no_gws(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to the given service not being a gws.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give results with the wrong category (a GWS is 1).""" - return [ - {"category": 2, "requirements": []}, - {"category": 3, "requirements": []}, - ] - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A ValueError should be raised saying it cannot find a GWS and to check the category. - with pytest.raises( - ValueError, - match="Cannot find a Group Workspace with the name test_service. Check the category.", - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_zero_quota(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to the quota being zero.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give a quota of 0.""" - return [ - { - "category": 1, - "requirements": [ - { - "status": 50, - "resource": {"short_name": "tape"}, - "amount": 0, - } - ], - } - ] - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A ValueError should be raised saying there was an issue getting tape quota as it is zero. - with pytest.raises( - ValueError, match="Issue getting tape quota for test_service. Quota is zero." - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_no_quota(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to there being no quota field.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give no 'amount' field.""" - return [ - { - "category": 1, - "requirements": [ - { - "status": 50, - "resource": {"short_name": "tape"}, - } - ], - } - ] - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A KeyError should be raised saying there was an issue getting tape quota as no value field exists. - with pytest.raises( - KeyError, - match="Issue getting tape quota for test_service. No 'value' field exists.", - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") - - -def test_extract_tape_quota_no_provisioned_resources(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to there being no provisioned resources.""" - - def mock_get_projects_services(*args, **kwargs): - """Mock the response from get_projects_services to give no provisioned resources (status 50).""" - return [ - { - "category": 1, - "requirements": [ - { - "status": 1, - "resource": {"short_name": "tape"}, - } - ], - } - ] - - monkeypatch.setattr(Quotas, "get_projects_services", mock_get_projects_services) - - # A ValueError should be raised saying there were no provisioned requirements found and to check the status of requested resources. - with pytest.raises( - ValueError, - match="No provisioned requirements found for test_service. Check the status of your requested resources.", - ): - quotas.extract_tape_quota("dummy_oauth_token", "test_service") diff --git a/tests/nlds_processors/catalog/test_catalog.py b/tests/nlds_processors/catalog/test_catalog.py index ca0410d4..3e2ea649 100644 --- a/tests/nlds_processors/catalog/test_catalog.py +++ b/tests/nlds_processors/catalog/test_catalog.py @@ -1,6 +1,10 @@ import uuid import time +import requests +import re +import json + import pytest from sqlalchemy import func @@ -9,7 +13,6 @@ ) from nlds_processors.catalog.catalog import Catalog, CatalogError from nlds.details import PathType -from nlds.authenticators.jasmin_authenticator import JasminAuthenticator test_uuid = '00a246cf-e2a8-46f0-baca-be3972fc4034' @@ -350,30 +353,6 @@ def test_create_transaction(self, mock_catalog, mock_holding): transaction_3 = mock_catalog.create_transaction(holding, test_uuid) - def test_user_has_get_holding_permission(self): - # Leaving this for now until it's a bit more fleshed out - pass - - def test_user_has_get_file_permission(self): - # Leaving this for now until it's a bit more fleshed out - pass - - @pytest.mark.parametrize("user, group, mock_is_admin, expected", [ - ("test-user", "test-group", False, True), # User owns the holding - ("user2", "test-group", False, False), # User does not own holding and is not admin - ("user2", "test-group", True, True), # User is admin of the group - ("test-user", "group2", False, False), # User is owner of different holding - ]) - def test_user_has_delete_from_holiding_permission(self, monkeypatch, user, group, mock_is_admin, expected, mock_holding): - # Mock the authenticate_user_group_role method - def mock_authenticate_user_group_role(user, group): - return mock_is_admin - - monkeypatch.setattr(JasminAuthenticator, "authenticate_user_group_role", mock_authenticate_user_group_role) - result = Catalog._user_has_delete_from_holding_permission(user, group, mock_holding) - assert result == expected - - def test_get_files(self, mock_catalog, mock_holding, mock_transaction, mock_file): test_uuid = str(uuid.uuid4()) @@ -495,4 +474,4 @@ def test_modify_tag(self): pass def test_del_tag(self): - pass + pass \ No newline at end of file From 2db7bbffccd127e48cf67caf245e710d60ddf13a Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 9 Oct 2024 17:02:38 +0100 Subject: [PATCH 12/26] Edit path to quotas endpoint to be /catalog/quota --- nlds/main.py | 10 +++++----- nlds/routers/quota.py | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nlds/main.py b/nlds/main.py index 76892e6a..8d26579e 100644 --- a/nlds/main.py +++ b/nlds/main.py @@ -32,6 +32,11 @@ tags = ["find",], prefix = PREFIX + "/catalog/find" ) +nlds.include_router( + quota.router, + tags = ["quota", ], + prefix = PREFIX + "/catalog/quota" +) nlds.include_router( status.router, tags = ["status",], @@ -57,8 +62,3 @@ tags = ["init", ], prefix = PREFIX + "/init" ) -nlds.include_router( - quota.router, - tags = ["quota", ], - prefix = PREFIX + "/quota" -) \ No newline at end of file diff --git a/nlds/routers/quota.py b/nlds/routers/quota.py index ad292562..3f4eb35c 100644 --- a/nlds/routers/quota.py +++ b/nlds/routers/quota.py @@ -54,6 +54,7 @@ async def get(token: str = Depends(authenticate_token), RMQP.MSG_USER: user, RMQP.MSG_GROUP: group, RMQP.MSG_TOKEN: token, + RMQP.MSG_API_ACTION: api_action }, RMQP.MSG_DATA: {}, RMQP.MSG_TYPE: RMQP.MSG_TYPE_STANDARD From f1e551309fbb9204ee0ae1af23d9de24491eb0c5 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 21 Oct 2024 11:48:27 +0100 Subject: [PATCH 13/26] Edit name of extract_tape_quota to get_tape_quota --- nlds/authenticators/base_authenticator.py | 2 +- nlds/authenticators/jasmin_authenticator.py | 2 +- nlds_processors/catalog/catalog_worker.py | 2 +- tests/nlds/test_jasmin_authenticator.py | 58 ++++++++++----------- 4 files changed, 32 insertions(+), 32 deletions(-) diff --git a/nlds/authenticators/base_authenticator.py b/nlds/authenticators/base_authenticator.py index dff4980a..6d49319e 100644 --- a/nlds/authenticators/base_authenticator.py +++ b/nlds/authenticators/base_authenticator.py @@ -49,6 +49,6 @@ def get_service_information(self, oauth_token: str, service_name: str): """Get the information about the given service.""" return NotImplementedError - def extract_tape_quota(self, oauth_token: str, service_name: str): + def get_tape_quota(self, oauth_token: str, service_name: str): """Process the service inforrmation to return the tape quota value.""" return NotImplementedError diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index 3132f326..4f4f56e3 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -322,7 +322,7 @@ def get_service_information(self, oauth_token: str, service_name: str): raise RuntimeError(f"Error getting data for {service_name}") - def extract_tape_quota(self, oauth_token: str, service_name: str): + def get_tape_quota(self, oauth_token: str, service_name: str): """Get the service information then process it to extract the quota for the service.""" try: result = self.get_service_information(self, oauth_token, service_name) diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index e8370ad9..12a24dd0 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -1868,7 +1868,7 @@ def _catalog_quota(self, body: Dict, properties: Header) -> None: user, group, token = message_vars try: - group_quota = Authenticator.extract_tape_quota(oauth_token=token, service_name=group) + group_quota = Authenticator.get_tape_quota(oauth_token=token, service_name=group) except CatalogError as e: # failed to get the holdings - send a return message saying so self.log(e.message, self.RK_LOG_ERROR) diff --git a/tests/nlds/test_jasmin_authenticator.py b/tests/nlds/test_jasmin_authenticator.py index b07e9e65..0d34fe67 100644 --- a/tests/nlds/test_jasmin_authenticator.py +++ b/tests/nlds/test_jasmin_authenticator.py @@ -413,11 +413,11 @@ def mock_get(*args, **kwargs): JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") -class TestExtractTapeQuota: +class TestGetTapeQuota: """Get the tape quota from the list of projects services.""" - def test_extract_tape_quota_success(monkeypatch): - """Test a successful instance of extract_tape_quota""" + def test_get_tape_quota_success(monkeypatch): + """Test a successful instance of get_tape_quota""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to gvie the response for @@ -433,12 +433,12 @@ def mock_get_projects_services(*args, **kwargs): monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.get_projects_services", mock_get_projects_services) - # extract_tape_quota should return the quota value of 100 - result = JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + # get_tape_quota should return the quota value of 100 + result = JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") assert result == 100 - def test_extract_tape_quota_no_requirements(monkeypatch, quotas): - """Test an unsuccessful instance of extract_tape_quota due to no requirements.""" + def test_get_tape_quota_no_requirements(monkeypatch, quotas): + """Test an unsuccessful instance of get_tape_quota due to no requirements.""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give the response for @@ -449,10 +449,10 @@ def mock_get_projects_services(*args, **kwargs): # A ValueError should be raised saying there's no requirements found. with pytest.raises(ValueError, match="Cannot find any requirements for test_service"): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_tape_quota_no_tape_resource(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to no tape resources.""" + def test_get_tape_quota_no_tape_resource(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to no tape resources.""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give the response for @@ -472,10 +472,10 @@ def mock_get_projects_services(*args, **kwargs): with pytest.raises( ValueError, match="No tape resources could be found for test_service" ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_tape_quota_services_runtime_error(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to a runtime error when + def test_get_tape_quota_services_runtime_error(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to a runtime error when getting services from the projects portal.""" def mock_get_projects_services(*args, **kwargs): @@ -489,10 +489,10 @@ def mock_get_projects_services(*args, **kwargs): RuntimeError, match="Error getting information for test_service: Runtime error occurred", ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_tape_quota_services_value_error(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to a value error + def test_get_tape_quota_services_value_error(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to a value error getting services from the projects portal.""" def mock_get_projects_services(*args, **kwargs): @@ -506,10 +506,10 @@ def mock_get_projects_services(*args, **kwargs): ValueError, match="Error getting information for test_service: Value error occurred", ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_tape_quota_no_gws(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to the given service + def test_get_tape_quota_no_gws(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to the given service not being a GWS.""" def mock_get_projects_services(*args, **kwargs): @@ -526,10 +526,10 @@ def mock_get_projects_services(*args, **kwargs): ValueError, match="Cannot find a Group Workspace with the name test_service. Check the category.", ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_quota_zero_quota(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to the quota being zero.""" + def get_quota_zero_quota(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to the quota being zero.""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give a quota of 0.""" @@ -552,10 +552,10 @@ def mock_get_projects_services(*args, **kwargs): with pytest.raises( ValueError, match="Issue getting tape quota for test_service. Quota is zero." ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_tape_quota_no_quota(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to there being no quota field.""" + def test_get_tape_quota_no_quota(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to there being no quota field.""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give no 'amount field.""" @@ -578,10 +578,10 @@ def mock_get_projects_services(*args, **kwargs): KeyError, match="Issue getting tape quota for test_service. No 'value' field exists.", ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_extract_tape_quota_no_provisioned_resources(monkeypatch): - """Test an unsuccessful instance of extract_tape_quota due to there being no provisioned resources.""" + def test_get_tape_quota_no_provisioned_resources(monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to there being no provisioned resources.""" def mock_get_projects_services(*args, **kwargs): """Mock the response from get_projects_services to give no provisioned resources (status 50).""" @@ -604,4 +604,4 @@ def mock_get_projects_services(*args, **kwargs): ValueError, match="No provisioned requirements found for test_service. Check the status of your requested resources.", ): - JasminAuthenticator.extract_tape_quota("dummy_oauth_token", "test_service") + JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") From 299135cfb5d32c36b9d9fcd01fafdfb20396d02a Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 4 Dec 2024 15:30:22 +0000 Subject: [PATCH 14/26] Fix mistakes with authenticate_user_group_role --- nlds/authenticators/jasmin_authenticator.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index 4f4f56e3..05d1c569 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -219,11 +219,10 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): if response.status_code == requests.codes.ok: # status code 200 try: response_json = json.loads(response.text) - user_role = response_json["group_workspaces"] # is_manager is False by default and only changes if user has a manager or deputy role. is_manager = False - for role in user_role: - if role in ["MANAGER", "DEPUTY"]: + for role in response_json: + if role["role"]["name"] in ["MANAGER", "DEPUTY"]: is_manager = True return is_manager except KeyError: @@ -291,17 +290,20 @@ def user_has_delete_from_holding_permission(self, user: str, @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) - def get_service_information(self, oauth_token: str, service_name: str): + def get_service_information(self, service_name: str): """Make a call to the JASMIN Projects Portal to get the service information.""" config = self.config[self.auth_name][self.name] token_headers = { "Content-Type": "application/x-ww-form-urlencoded", "cache-control": "no-cache", - "Authorization": f"Bearer {oauth_token}", + # WORK THIS OUT + "Authorization": f"Bearer {config["client_token"]}", } # Contact the user_services_url to get the information about the services - url = construct_url([config["user_services_urk"]], {"name": {service_name}}) + url = construct_url([config["project_services_url"]], {"name": {service_name}}) + print("jasmin_authenticator.py URL:", url) + print("jasmin_authenticator.py token_headers:", token_headers) try: response = requests.get( url, @@ -313,6 +315,7 @@ def get_service_information(self, oauth_token: str, service_name: str): except KeyError: raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") if response.status_code == requests.codes.ok: # status code 200 + print("jasmin_authenticator.py RESPONSE:", response) try: response_json = json.loads(response.text) return response_json @@ -322,10 +325,11 @@ def get_service_information(self, oauth_token: str, service_name: str): raise RuntimeError(f"Error getting data for {service_name}") - def get_tape_quota(self, oauth_token: str, service_name: str): + def get_tape_quota(self, service_name: str): """Get the service information then process it to extract the quota for the service.""" + print("REACHED GET_TPE_QUOTA") try: - result = self.get_service_information(self, oauth_token, service_name) + result = self.get_service_information(self, service_name) except (RuntimeError, ValueError) as e: raise type(e)(f"Error getting information for {service_name}: {e}") From 570143b625f1c868a0b61f56bbed305928dc5538 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 4 Dec 2024 16:02:02 +0000 Subject: [PATCH 15/26] Rename construct_url to format_url --- nlds/authenticators/jasmin_authenticator.py | 6 +++--- nlds/routers/quota.py | 3 +++ nlds/utils/{construct_url.py => format_url.py} | 2 +- nlds_processors/catalog/catalog.py | 6 ------ nlds_processors/catalog/catalog_worker.py | 6 +++++- tests/nlds/test_jasmin_authenticator.py | 18 +++++++++--------- ...est_construct_url.py => test_format_url.py} | 14 +++++++------- 7 files changed, 28 insertions(+), 27 deletions(-) rename nlds/utils/{construct_url.py => format_url.py} (93%) rename tests/nlds/utils/{test_construct_url.py => test_format_url.py} (67%) diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index 05d1c569..7c3bd47e 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -10,7 +10,7 @@ from nlds.authenticators.base_authenticator import BaseAuthenticator from nlds.server_config import load_config -from nlds.utils.construct_url import construct_url +from nlds.utils.format_url import format_url from nlds_processors.catalog.catalog_models import File, Holding, Transaction from retry import retry import requests @@ -195,7 +195,7 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): "Authorization": f"Bearer {oauth_token}", } # Construct the URL - url = construct_url( + url = format_url( [config["user_grants_url"], user, "grants"], {"category": "GWS", "service": group}, ) @@ -301,7 +301,7 @@ def get_service_information(self, service_name: str): "Authorization": f"Bearer {config["client_token"]}", } # Contact the user_services_url to get the information about the services - url = construct_url([config["project_services_url"]], {"name": {service_name}}) + url = format_url([config["project_services_url"]], {"name": service_name}) print("jasmin_authenticator.py URL:", url) print("jasmin_authenticator.py token_headers:", token_headers) try: diff --git a/nlds/routers/quota.py b/nlds/routers/quota.py index 3f4eb35c..d4639a50 100644 --- a/nlds/routers/quota.py +++ b/nlds/routers/quota.py @@ -59,6 +59,7 @@ async def get(token: str = Depends(authenticate_token), RMQP.MSG_DATA: {}, RMQP.MSG_TYPE: RMQP.MSG_TYPE_STANDARD } + print("quota.py MESSAGE DICT:", msg_dict) # add the metadata meta_dict = {} if (label): @@ -94,8 +95,10 @@ async def get(token: str = Depends(authenticate_token), response = await rpc_publisher.call( msg_dict=msg_dict, routing_key=routing_key ) + print("quota.py REACHED RESPONSE", response) # Check if response is valid or whether the request timed out if response is not None: + print('quota.py RESPONSE:', response) # convert byte response to str response = response.decode() diff --git a/nlds/utils/construct_url.py b/nlds/utils/format_url.py similarity index 93% rename from nlds/utils/construct_url.py rename to nlds/utils/format_url.py index 2444890d..e7c434b3 100644 --- a/nlds/utils/construct_url.py +++ b/nlds/utils/format_url.py @@ -1,7 +1,7 @@ from urllib.parse import urljoin, urlencode -def construct_url(url_parts, query_params=None): +def format_url(url_parts, query_params=None): """ Constructs a URL from a list of parts. diff --git a/nlds_processors/catalog/catalog.py b/nlds_processors/catalog/catalog.py index f7e0e2ca..f4301f8a 100644 --- a/nlds_processors/catalog/catalog.py +++ b/nlds_processors/catalog/catalog.py @@ -4,12 +4,6 @@ from sqlalchemy import func, Enum from sqlalchemy.exc import IntegrityError, OperationalError, ArgumentError, \ NoResultFound -from retry import retry -import requests -import json -from nlds.server_config import load_config -from nlds.utils.construct_url import construct_url - from nlds_processors.catalog.catalog_models import CatalogBase, File, Holding,\ Location, Transaction, Aggregation, Storage, Tag from nlds_processors.db_mixin import DBMixin diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index 12a24dd0..035bc51c 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -148,6 +148,7 @@ def __init__(self, queue=DEFAULT_QUEUE_NAME): self.catalog = None self.reroutelist = [] self.retrievedict = {} + self.authenticator = Authenticator() @property @@ -1865,10 +1866,12 @@ def _catalog_quota(self, body: Dict, properties: Header) -> None: return else: # Unpack if no problems found in parsing + print("MESSAGE VARS", message_vars) user, group, token = message_vars try: - group_quota = Authenticator.get_tape_quota(oauth_token=token, service_name=group) + group_quota = self.authenticator.get_tape_quota(service_name=group) + print("catalog_worker.py GROUP_QUOTA:", group_quota) except CatalogError as e: # failed to get the holdings - send a return message saying so self.log(e.message, self.RK_LOG_ERROR) @@ -2066,6 +2069,7 @@ def callback(self, ch: Channel, method: Method, properties: Header, elif (api_method == self.RK_QUOTA): # don't need to split any routing key for an RPC method + print("catalog_worker.py ELIF LOOP !!") self._catalog_quota(body, properties) # If received system test message, reply to it (this is for system status check) diff --git a/tests/nlds/test_jasmin_authenticator.py b/tests/nlds/test_jasmin_authenticator.py index 0d34fe67..f7789430 100644 --- a/tests/nlds/test_jasmin_authenticator.py +++ b/tests/nlds/test_jasmin_authenticator.py @@ -6,7 +6,7 @@ from nlds.authenticators.jasmin_authenticator import JasminAuthenticator from nlds_processors.catalog.catalog_models import Holding -from nlds.utils.construct_url import construct_url +from nlds.utils.format_url import format_url @pytest.fixture(autouse=True) @@ -227,7 +227,7 @@ def test_authenticate_user_group_role( ): """Check whether the user has a manager/deputy role within the specified group.""" # Create the URL - url = construct_url( + url = format_url( ["https://mock.url/api/v1/users", user, "grants"], {"category": "GWS", "service": group}, ) @@ -289,15 +289,15 @@ class TestGetProjectsServices: url = f"{user_services_url}?name=test_service" @pytest.fixture() - def mock_construct_url(self, *args, **kwargs): - """Mock the construct_url function to make it return the test url.""" + def mock_format_url(self, *args, **kwargs): + """Mock the format_url function to make it return the test url.""" return self.url def test_get_projects_services_success(self,monkeypatch): """Test a successful instance of get_projects_services.""" monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) class MockResponse: """Mock the response to return a 200 status code and the test text.""" @@ -324,7 +324,7 @@ def test_get_projects_services_connection_error(self,monkeypatch, quotas): """Test an unsuccessful instance of get_projects_services due to connection error.""" monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) def mock_get(*args, **kwargs): """Mock the get function to give a ConnectionError.""" @@ -346,7 +346,7 @@ def mock_load_config_key_error(): return {"authentication": {"jasmin_authenticator": {"other_url": "test.com"}}} monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config_key_error) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) def mock_get(*args, **kwargs): """Mock the get function to give the KeyError.""" @@ -365,7 +365,7 @@ def test_get_projects_services_json_error(self, monkeypatch): """Test an unsuccessful instance of get_projects_services due to a JSON error.""" monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) class MockInvalidJSONResponse: """Mock the response to return a 200 status code and the JSON decode error.""" @@ -392,7 +392,7 @@ def test_get_projects_services_404_error(self,monkeypatch): """Test an unsuccessful instance of get_projects_services due to a 404 error.""" monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.construct_url", self.mock_construct_url) + monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) class MockResponse: """Mock the response to return a 401 status code and the relevant text.""" diff --git a/tests/nlds/utils/test_construct_url.py b/tests/nlds/utils/test_format_url.py similarity index 67% rename from tests/nlds/utils/test_construct_url.py rename to tests/nlds/utils/test_format_url.py index 8d813b48..97f436a5 100644 --- a/tests/nlds/utils/test_construct_url.py +++ b/tests/nlds/utils/test_format_url.py @@ -1,36 +1,36 @@ -from nlds.utils.construct_url import construct_url +from nlds.utils.format_url import format_url def test_no_parts(): - assert construct_url([]) == "" + assert format_url([]) == "" def test_single_part(): - assert construct_url(["http://example.com"]) == "http://example.com" + assert format_url(["http://example.com"]) == "http://example.com" def test_multiple_parts(): url_parts = ["http://example.com", "path", "to", "resource"] expected_url = "http://example.com/path/to/resource" - assert construct_url(url_parts) == expected_url + assert format_url(url_parts) == expected_url def test_with_query_params(): url_parts = ["http://example.com", "path", "to", "resource"] query_params = {"key1": "value1", "key2": "value2"} expected_url = "http://example.com/path/to/resource?key1=value1&key2=value2" - assert construct_url(url_parts, query_params) == expected_url + assert format_url(url_parts, query_params) == expected_url def test_empty_query_params(): url_parts = ["http://example.com", "path", "to", "resource"] query_params = {} expected_url = "http://example.com/path/to/resource" - assert construct_url(url_parts, query_params) == expected_url + assert format_url(url_parts, query_params) == expected_url def test_complex_query_params(): url_parts = ["http://example.com", "search"] query_params = {"q": "test search", "page": "1", "sort": "asc"} expected_url = "http://example.com/search?q=test+search&page=1&sort=asc" - assert construct_url(url_parts, query_params) == expected_url + assert format_url(url_parts, query_params) == expected_url From ee3e71d5ab62fd1c2c5ba6c6692d5ed328494124 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 18 Dec 2024 10:27:25 +0000 Subject: [PATCH 16/26] Fix issues with quota --- nlds/authenticators/jasmin_authenticator.py | 24 ++++++++++----------- nlds/routers/quota.py | 3 --- nlds_processors/catalog/catalog_worker.py | 13 +++++------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index 7c3bd47e..4702b47e 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -26,6 +26,7 @@ def __init__(self): self.config = load_config() self.name = "jasmin_authenticator" self.auth_name = "authentication" + self.default_quota = 0 @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) def authenticate_token(self, oauth_token: str): @@ -292,7 +293,6 @@ def user_has_delete_from_holding_permission(self, user: str, @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) def get_service_information(self, service_name: str): """Make a call to the JASMIN Projects Portal to get the service information.""" - config = self.config[self.auth_name][self.name] token_headers = { "Content-Type": "application/x-ww-form-urlencoded", @@ -302,8 +302,6 @@ def get_service_information(self, service_name: str): } # Contact the user_services_url to get the information about the services url = format_url([config["project_services_url"]], {"name": service_name}) - print("jasmin_authenticator.py URL:", url) - print("jasmin_authenticator.py token_headers:", token_headers) try: response = requests.get( url, @@ -315,7 +313,6 @@ def get_service_information(self, service_name: str): except KeyError: raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") if response.status_code == requests.codes.ok: # status code 200 - print("jasmin_authenticator.py RESPONSE:", response) try: response_json = json.loads(response.text) return response_json @@ -327,9 +324,8 @@ def get_service_information(self, service_name: str): def get_tape_quota(self, service_name: str): """Get the service information then process it to extract the quota for the service.""" - print("REACHED GET_TPE_QUOTA") try: - result = self.get_service_information(self, service_name) + result = self.get_service_information(service_name) except (RuntimeError, ValueError) as e: raise type(e)(f"Error getting information for {service_name}: {e}") @@ -355,12 +351,14 @@ def get_tape_quota(self, service_name: str): tape_quota = requirement["amount"] if tape_quota: return tape_quota - else: - raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") + # else: + # raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") except KeyError: - raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") - else: - raise ValueError(f"No tape resources could be found for {service_name}") - else: - raise ValueError(f"No provisioned requirements found for {service_name}.Check the status of your requested resources.") + # raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") + continue + # else: + # raise ValueError(f"No tape resources could be found for {service_name}") + # else: + # raise ValueError(f"No provisioned requirements found for {service_name}.Check the status of your requested resources.") + return self.default_quota diff --git a/nlds/routers/quota.py b/nlds/routers/quota.py index d4639a50..3f4eb35c 100644 --- a/nlds/routers/quota.py +++ b/nlds/routers/quota.py @@ -59,7 +59,6 @@ async def get(token: str = Depends(authenticate_token), RMQP.MSG_DATA: {}, RMQP.MSG_TYPE: RMQP.MSG_TYPE_STANDARD } - print("quota.py MESSAGE DICT:", msg_dict) # add the metadata meta_dict = {} if (label): @@ -95,10 +94,8 @@ async def get(token: str = Depends(authenticate_token), response = await rpc_publisher.call( msg_dict=msg_dict, routing_key=routing_key ) - print("quota.py REACHED RESPONSE", response) # Check if response is valid or whether the request timed out if response is not None: - print('quota.py RESPONSE:', response) # convert byte response to str response = response.decode() diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index 035bc51c..b9df4aed 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -1866,26 +1866,27 @@ def _catalog_quota(self, body: Dict, properties: Header) -> None: return else: # Unpack if no problems found in parsing - print("MESSAGE VARS", message_vars) - user, group, token = message_vars + print("catalog_worker.py message_vars in _catalog_quota:", message_vars) + user, group = message_vars try: group_quota = self.authenticator.get_tape_quota(service_name=group) - print("catalog_worker.py GROUP_QUOTA:", group_quota) + print("catalog_worker.py _catalog_quota group_quota:", group_quota) except CatalogError as e: + print("THROWIN A CATALOGERROR IN catalog_worker.py") # failed to get the holdings - send a return message saying so self.log(e.message, self.RK_LOG_ERROR) body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message body[self.MSG_DATA][self.MSG_HOLDING_LIST] = [] else: + print("THROWING ANOTHER ERROR IN catalog_worker.py???") # fill the return message with a dictionary of the holding(s) body[self.MSG_DATA][self.MSG_HOLDING_LIST] = group_quota self.log( f"Quota from CATALOG_QUOTA {group_quota}", self.RK_LOG_DEBUG ) - - self.catalog.end_session() + print("Gets through else block in catalog_Worker.py") # return message to complete RPC self.publish_message( @@ -2069,7 +2070,7 @@ def callback(self, ch: Channel, method: Method, properties: Header, elif (api_method == self.RK_QUOTA): # don't need to split any routing key for an RPC method - print("catalog_worker.py ELIF LOOP !!") + print("catalog_worker.py calling self._catalog_quota") self._catalog_quota(body, properties) # If received system test message, reply to it (this is for system status check) From ab8dd3bfbf2fad32e6b1f9422bb44f57914fe100 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 18 Dec 2024 10:41:28 +0000 Subject: [PATCH 17/26] Refactor get_tape_quota function --- nlds/authenticators/jasmin_authenticator.py | 58 +++++++++------------ 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index 4702b47e..1a4f7e9e 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -329,36 +329,28 @@ def get_tape_quota(self, service_name: str): except (RuntimeError, ValueError) as e: raise type(e)(f"Error getting information for {service_name}: {e}") - # Process the result to get the requirements - for attr in result: - # Check that the category is Group Workspace - if attr["category"] == 1: - # Check that there are requirements, otherwise throw an error - if attr["requirements"]: - requirements = attr["requirements"] - else: - raise ValueError(f"Cannot find any requirements for {service_name}.") - else: - raise ValueError(f"Cannot find a Group Workspace with the name {service_name}. Check the category.") - - # Go through the requirements to find the tape resource requirement - for requirement in requirements: - # Only return provisioned requirements - if requirement["status"] == 50: - # Find the tape resource and get its quota - if requirement["resource"]["short_name"] == "tape": - try: - tape_quota = requirement["amount"] - if tape_quota: - return tape_quota - # else: - # raise ValueError(f"Issue getting tape quota for {service_name}. Quota is zero.") - except KeyError: - # raise KeyError(f"Issue getting tape quota for {service_name}. No 'value' field exists.") - continue - # else: - # raise ValueError(f"No tape resources could be found for {service_name}") - # else: - # raise ValueError(f"No provisioned requirements found for {service_name}.Check the status of your requested resources.") - return self.default_quota - + try: + # Filter for Group Workspace category + group_workspace = next( + service for service in result if service.get("category") == 1 + ) + except StopIteration: + raise ValueError(f"Cannot find a Group workspace with the name {service_name}. Check the category.") + + requirements = group_workspace.get("requirements") + if not requirements: + raise ValueError(f"Cannot find any requirements for {service_name}.") + + tape_quota = next( + ( + req.get("amount") + for req in requirements + if req.get("status") == 50 and req.get("resource", {}).get("short_name") == "tape" + ), + None, + ) + + if tape_quota is not None: + return tape_quota + else: + return self.default_quota \ No newline at end of file From cd98bd4cd5736388d1b7859b2ff3c8f4103823d5 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Wed, 18 Dec 2024 11:45:13 +0000 Subject: [PATCH 18/26] Rename dict key for quota response --- nlds/rabbit/publisher.py | 1 + nlds_processors/catalog/catalog_worker.py | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/nlds/rabbit/publisher.py b/nlds/rabbit/publisher.py index 46938466..3e2624c0 100644 --- a/nlds/rabbit/publisher.py +++ b/nlds/rabbit/publisher.py @@ -112,6 +112,7 @@ class RabbitMQPublisher(): MSG_TIMESTAMP = "timestamp" MSG_USER = "user" MSG_GROUP = "group" + MSG_QUOTA = "quota" MSG_GROUPALL = "groupall" MSG_TARGET = "target" MSG_ROUTE = "route" diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index b9df4aed..3e117cd8 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -1866,27 +1866,22 @@ def _catalog_quota(self, body: Dict, properties: Header) -> None: return else: # Unpack if no problems found in parsing - print("catalog_worker.py message_vars in _catalog_quota:", message_vars) user, group = message_vars try: group_quota = self.authenticator.get_tape_quota(service_name=group) - print("catalog_worker.py _catalog_quota group_quota:", group_quota) except CatalogError as e: - print("THROWIN A CATALOGERROR IN catalog_worker.py") # failed to get the holdings - send a return message saying so self.log(e.message, self.RK_LOG_ERROR) body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message - body[self.MSG_DATA][self.MSG_HOLDING_LIST] = [] + body[self.MSG_DATA][self.MSG_QUOTA] = None else: - print("THROWING ANOTHER ERROR IN catalog_worker.py???") # fill the return message with a dictionary of the holding(s) - body[self.MSG_DATA][self.MSG_HOLDING_LIST] = group_quota + body[self.MSG_DATA][self.MSG_QUOTA] = group_quota self.log( f"Quota from CATALOG_QUOTA {group_quota}", self.RK_LOG_DEBUG ) - print("Gets through else block in catalog_Worker.py") # return message to complete RPC self.publish_message( @@ -2070,7 +2065,6 @@ def callback(self, ch: Channel, method: Method, properties: Header, elif (api_method == self.RK_QUOTA): # don't need to split any routing key for an RPC method - print("catalog_worker.py calling self._catalog_quota") self._catalog_quota(body, properties) # If received system test message, reply to it (this is for system status check) From 6fc4a65779baa4a11b420ff2107ef00da50284cd Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Thu, 19 Dec 2024 11:25:11 +0000 Subject: [PATCH 19/26] Add extra fields to server_config docs --- docs/source/server-config/server-config.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/server-config/server-config.rst b/docs/source/server-config/server-config.rst index f04ff033..c95d98dd 100644 --- a/docs/source/server-config/server-config.rst +++ b/docs/source/server-config/server-config.rst @@ -28,6 +28,8 @@ client. The following fields are required in the dictionary:: "jasmin_authenticator" : { "user_profile_url" : "{{ user_profile_url }}", "user_services_url" : "{{ user_services_url }}", + "user_grants_url" : "{{ }}", + "project_services_url" : "{{ }}", "oauth_token_introspect_url" : "{{ token_introspect_url }}" } } @@ -40,7 +42,7 @@ other industry standard authenticators like google and microsoft. The authenticator setup is then specified in a separate dictionary named after the authenticator, which is specific to each authenticator. The ``jasmin_authenticator`` requires, as above, values for ``user_profile_url``, -``user_services_url``, and ``oauth_token_introspect_url``. This cannot be +``user_services_url``, ``user_grants_url``, ``project_services_url`` and ``oauth_token_introspect_url``. This cannot be divulged publicly on github for JASMIN, so please get in contact for the actual values to use. From 39e1f3ddfba198c39f49edc14a396bc0e3c8d5aa Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Thu, 19 Dec 2024 11:39:28 +0000 Subject: [PATCH 20/26] Add extra keys to server-congig example docs --- docs/source/server-config/examples.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/server-config/examples.rst b/docs/source/server-config/examples.rst index 62fc8130..ed03dc0f 100644 --- a/docs/source/server-config/examples.rst +++ b/docs/source/server-config/examples.rst @@ -16,6 +16,8 @@ machine - likely a laptop or single vm. This file would be saved at "jasmin_authenticator" : { "user_profile_url" : "[REDACTED]", "user_services_url" : "[REDACTED]", + "user_grants_url" : "[REDACTED]", + "projects_services_url" : "[REDACTED]", "oauth_token_introspect_url" : "[REDACTED]" } }, From 567c2cc8781574f44c0150aaf9e903b3b5a8fcdd Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 13 Jan 2025 10:44:14 +0000 Subject: [PATCH 21/26] Add quotas docs --- docs/source/quotas.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/source/quotas.rst diff --git a/docs/source/quotas.rst b/docs/source/quotas.rst new file mode 100644 index 00000000..749aee00 --- /dev/null +++ b/docs/source/quotas.rst @@ -0,0 +1,17 @@ +NLDS quotas +================================ + +Get the quota for your group in the NLDS. + + +Implementation. +------------------------ + +This is currently only implemented for the `jasmin_authenticator`. + +To get the quota, the `project_services_url` and `user_services_url` need to be present in the `jasmin_authenticator` section of the `server-config.rst` file. + +First, there is a call to the JASMIN Projects Portal to get information about the service. This call is made to the `project_services_url`. +This is authorized on behalf of the NLDS using a client token, supplied in the config. +The tape quota is then extracted from the service information. Only quota for allocated tape resource in a Group Workspace is returned, no other categories or status of resource (such as pending requests) are included. +The quota command can be called from the `nlds client`. \ No newline at end of file From 6a5732beadede3df5aee07ec90f9281a8e382c48 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Mon, 13 Jan 2025 10:59:48 +0000 Subject: [PATCH 22/26] Add tests for format_url.py --- nlds/utils/format_url.py | 4 ++++ tests/nlds/utils/test_format_url.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/nlds/utils/format_url.py b/nlds/utils/format_url.py index e7c434b3..964ea603 100644 --- a/nlds/utils/format_url.py +++ b/nlds/utils/format_url.py @@ -12,6 +12,10 @@ def format_url(url_parts, query_params=None): Returns: base (str): The constructed URL. """ + + if not isinstance(url_parts, list): + raise TypeError("url_parts must be a list") + if not url_parts: return "" diff --git a/tests/nlds/utils/test_format_url.py b/tests/nlds/utils/test_format_url.py index 97f436a5..59346fd1 100644 --- a/tests/nlds/utils/test_format_url.py +++ b/tests/nlds/utils/test_format_url.py @@ -1,4 +1,5 @@ from nlds.utils.format_url import format_url +import pytest def test_no_parts(): @@ -34,3 +35,23 @@ def test_complex_query_params(): query_params = {"q": "test search", "page": "1", "sort": "asc"} expected_url = "http://example.com/search?q=test+search&page=1&sort=asc" assert format_url(url_parts, query_params) == expected_url + + +def test_string(): + with pytest.raises(TypeError): + format_url("not-a-list") + + +def test_int(): + with pytest.raises(TypeError): + format_url(1) + + +def test_float(): + with pytest.raises(TypeError): + format_url(1.0) + + +def test_dict(): + with pytest.raises(TypeError): + format_url({"url_parts": "www.example.com"}) \ No newline at end of file From 827c40bb829ae2001bb4eb4e9ca32e53bac042a2 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Tue, 14 Jan 2025 14:38:28 +0000 Subject: [PATCH 23/26] Add jasmin_authenticator tests --- nlds/authenticators/jasmin_authenticator.py | 7 +- tests/nlds/test_jasmin_authenticator.py | 180 +++++++++++--------- 2 files changed, 100 insertions(+), 87 deletions(-) diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index 1a4f7e9e..c08ff911 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -222,8 +222,8 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): response_json = json.loads(response.text) # is_manager is False by default and only changes if user has a manager or deputy role. is_manager = False - for role in response_json: - if role["role"]["name"] in ["MANAGER", "DEPUTY"]: + for role in response_json['group_workspaces']: + if role in ["MANAGER", "DEPUTY"]: is_manager = True return is_manager except KeyError: @@ -273,7 +273,7 @@ def user_has_get_file_permission(session, return permitted - @staticmethod + def user_has_delete_from_holding_permission(self, user: str, group: str, holding: Holding) -> bool: @@ -302,6 +302,7 @@ def get_service_information(self, service_name: str): } # Contact the user_services_url to get the information about the services url = format_url([config["project_services_url"]], {"name": service_name}) + print(url) try: response = requests.get( url, diff --git a/tests/nlds/test_jasmin_authenticator.py b/tests/nlds/test_jasmin_authenticator.py index f7789430..9543e708 100644 --- a/tests/nlds/test_jasmin_authenticator.py +++ b/tests/nlds/test_jasmin_authenticator.py @@ -37,6 +37,7 @@ def mock_load_config(monkeypatch): "user_profile_url": "https://mock.url/api/profile/", "user_services_url": "https://mock.url/api/services/", "user_grants_url": "https://mock.url/api/v1/users/", + "projects_services_url": "https://mock.url/api/services", }, } } @@ -247,11 +248,12 @@ def test_authenticate_user_group_role( has_role = auth.authenticate_user_group_role(oauth_token, user, group) assert has_role == expected_result + class TestUserPermissions: """Test the functions that assign permissions to get holdings, get files and to delete from holding.""" @pytest.fixture() - def mock_holding(): + def mock_holding(self): return Holding( label='test-label', user='test-user', @@ -277,27 +279,40 @@ def test_user_has_delete_from_holiding_permission(self, monkeypatch, user, group def mock_authenticate_user_group_role(user, group): return mock_is_admin - monkeypatch.setattr(JasminAuthenticator, "authenticate_user_group_role", mock_authenticate_user_group_role) - result = JasminAuthenticator.user_has_delete_from_holding_permission(user, group, mock_holding) + auth = JasminAuthenticator() + + monkeypatch.setattr(auth, "authenticate_user_group_role", mock_authenticate_user_group_role) + result = auth.user_has_delete_from_holding_permission(user=user, group=group, holding=mock_holding) assert result == expected class TestGetProjectsServices: """Get the projects for a service from the JASMIN Projects Portal.""" - user_services_url = "https://example.com/services" + user_services_url = "https://mock.url/api/services/" url = f"{user_services_url}?name=test_service" + auth = JasminAuthenticator() + config = { + "authentication": { + "jasmin_authenticator": { + "project_services_url": "https://mock.url/api/services/", + "client_token": "test_token" + } + } + } + @pytest.fixture() def mock_format_url(self, *args, **kwargs): """Mock the format_url function to make it return the test url.""" return self.url + - def test_get_projects_services_success(self,monkeypatch): + def test_get_service_information_success(self, monkeypatch): """Test a successful instance of get_projects_services.""" - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) class MockResponse: """Mock the response to return a 200 status code and the test text.""" @@ -315,16 +330,17 @@ def mock_get(*args, **kwargs): monkeypatch.setattr(requests, "get", mock_get) # Call the get_projects_services function with the mocked functions - result = JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + result = self.auth.get_service_information("test_service") # It should succeed and give the {"key":"value"} dict. assert result == {"key":"value"} - def test_get_projects_services_connection_error(self,monkeypatch, quotas): + + def test_get_projects_services_connection_error(self,monkeypatch): """Test an unsuccessful instance of get_projects_services due to connection error.""" - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) def mock_get(*args, **kwargs): """Mock the get function to give a ConnectionError.""" @@ -336,17 +352,16 @@ def mock_get(*args, **kwargs): with pytest.raises( RuntimeError, match=re.escape(f"User services url {self.url} could not be reached.") ): - JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + self.auth.get_service_information("test_service") + def test_get_projects_services_key_error(self,monkeypatch): """Test an unsuccessful instance of get_projects_services due to a key error.""" - def mock_load_config_key_error(): - """Mock the load_config function to make it return the test config with no user_services_key""" - return {"authentication": {"jasmin_authenticator": {"other_url": "test.com"}}} + config = {"authentication": {"jasmin_authenticator": {"other_url": "test.com", "client_token":"test_token"}}} - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config_key_error) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) + monkeypatch.setattr(self.auth, "config", config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) def mock_get(*args, **kwargs): """Mock the get function to give the KeyError.""" @@ -356,16 +371,16 @@ def mock_get(*args, **kwargs): # Check that the KeyError in the 'get' triggers a RuntimeError with the right text. with pytest.raises( - RuntimeError, - match=f"Could not find 'user_services_url' key in the jasmin_authenticator section of the .server_config file.", + KeyError, + match=f"project_services_url", ): - JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + self.auth.get_service_information("test_service") def test_get_projects_services_json_error(self, monkeypatch): """Test an unsuccessful instance of get_projects_services due to a JSON error.""" - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) class MockInvalidJSONResponse: """Mock the response to return a 200 status code and the JSON decode error.""" @@ -386,13 +401,14 @@ def mock_get(*args, **kwargs): RuntimeError, match=re.escape(f"Invalid JSON returned from the user services url: {self.url}"), ): - JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + self.auth.get_service_information("test_service") + def test_get_projects_services_404_error(self,monkeypatch): """Test an unsuccessful instance of get_projects_services due to a 404 error.""" - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.load_config", mock_load_config) - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.format_url", self.mock_format_url) + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) class MockResponse: """Mock the response to return a 401 status code and the relevant text.""" @@ -410,16 +426,17 @@ def mock_get(*args, **kwargs): # Check that the 401 error triggers a RuntimeError with the right text. with pytest.raises(RuntimeError, match=f"Error getting data for test_service"): - JasminAuthenticator.get_projects_services("dummy_oauth_token", "test_service") + self.auth.get_service_information("test_service") class TestGetTapeQuota: """Get the tape quota from the list of projects services.""" + auth = JasminAuthenticator() - def test_get_tape_quota_success(monkeypatch): + def test_get_tape_quota_success(self, monkeypatch): """Test a successful instance of get_tape_quota""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to gvie the response for a GWS with a provisioned tape requirement.""" return [ @@ -431,30 +448,32 @@ def mock_get_projects_services(*args, **kwargs): } ] - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) # get_tape_quota should return the quota value of 100 - result = JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + result = self.auth.get_tape_quota("test_service") assert result == 100 - def test_get_tape_quota_no_requirements(monkeypatch, quotas): + + def test_get_tape_quota_no_requirements(self, monkeypatch): """Test an unsuccessful instance of get_tape_quota due to no requirements.""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give the response for a GWS with no requirements.""" return [{"category": 1, "requirements": []}] - monkeypatch.setattr("jasmin_authenticator.JasminAuthenticator.get_projects_setvices", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) # A ValueError should be raised saying there's no requirements found. with pytest.raises(ValueError, match="Cannot find any requirements for test_service"): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + self.auth.get_tape_quota("test_service") - def test_get_tape_quota_no_tape_resource(monkeypatch): - """Test an unsuccessful instance of get_tape_quota due to no tape resources.""" - def mock_get_projects_services(*args, **kwargs): + def test_get_tape_quota_no_tape_resource(self, monkeypatch): + """Test an instance of no tape resources.""" + + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give the response for a GWS with a requirement that isn't tape.""" return [ @@ -466,72 +485,74 @@ def mock_get_projects_services(*args, **kwargs): } ] - monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) # A ValueError should be raised saying there's no tape resources. - with pytest.raises( - ValueError, match="No tape resources could be found for test_service" - ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + result = self.auth.get_tape_quota("test_service") + assert result == 0 + - def test_get_tape_quota_services_runtime_error(monkeypatch): + def test_get_tape_quota_services_runtime_error(self, monkeypatch): """Test an unsuccessful instance of get_tape_quota due to a runtime error when getting services from the projects portal.""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give a RuntimeError.""" raise RuntimeError("Runtime error occurred.") - monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) # A RuntimeError should be raised saying a runtime error occurred. with pytest.raises( RuntimeError, match="Error getting information for test_service: Runtime error occurred", ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + self.auth.get_tape_quota("test_service") - def test_get_tape_quota_services_value_error(monkeypatch): + + def test_get_tape_quota_services_value_error(self, monkeypatch): """Test an unsuccessful instance of get_tape_quota due to a value error getting services from the projects portal.""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give a ValueError.""" raise ValueError("Value error occurred") - monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) # A ValueError should be raised saying a value error occurred. with pytest.raises( ValueError, match="Error getting information for test_service: Value error occurred", ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + self.auth.get_tape_quota("test_service") + - def test_get_tape_quota_no_gws(monkeypatch): + def test_get_tape_quota_no_gws(self, monkeypatch): """Test an unsuccessful instance of get_tape_quota due to the given service not being a GWS.""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give results with the wrong category (a GWS is 1).""" return [ {"category": 2, "requirements": []}, {"category": 3, "requirements": []}, ] - monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) # A ValueError should be raised saying it cannot find a GWS and to check the category. with pytest.raises( ValueError, - match="Cannot find a Group Workspace with the name test_service. Check the category.", + match="Cannot find a Group workspace with the name test_service. Check the category.", ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + self.auth.get_tape_quota("test_service") - def get_quota_zero_quota(monkeypatch): - """Test an unsuccessful instance of get_tape_quota due to the quota being zero.""" - def mock_get_projects_services(*args, **kwargs): + def test_get_quota_zero_quota(self, monkeypatch): + """Test an instance of the quota being zero.""" + + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give a quota of 0.""" return [ { @@ -546,18 +567,16 @@ def mock_get_projects_services(*args, **kwargs): } ] - monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + result = self.auth.get_tape_quota("test_service") + assert result == 0 - # A ValueError should be raised saying there was an issue getting tape quota as it was zero. - with pytest.raises( - ValueError, match="Issue getting tape quota for test_service. Quota is zero." - ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_get_tape_quota_no_quota(monkeypatch): - """Test an unsuccessful instance of get_tape_quota due to there being no quota field.""" + def test_get_tape_quota_no_quota(self, monkeypatch): + """Test an instance of zero quota due to there being no quota field.""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give no 'amount field.""" return [ { @@ -571,19 +590,16 @@ def mock_get_projects_services(*args, **kwargs): } ] - monkeypatch.setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + result = self.auth.get_tape_quota("test_service") + assert result == 0 - # A key error should be raised saying there was an issue getting tape quota as no value field exists. - with pytest.raises( - KeyError, - match="Issue getting tape quota for test_service. No 'value' field exists.", - ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") - def test_get_tape_quota_no_provisioned_resources(monkeypatch): - """Test an unsuccessful instance of get_tape_quota due to there being no provisioned resources.""" + def test_get_tape_quota_no_provisioned_resources(self, monkeypatch): + """Test an instance of zero quota due to there being no provisioned resources.""" - def mock_get_projects_services(*args, **kwargs): + def mock_get_service_information(*args, **kwargs): """Mock the response from get_projects_services to give no provisioned resources (status 50).""" return [ { @@ -597,11 +613,7 @@ def mock_get_projects_services(*args, **kwargs): } ] - monkeypatch,setattr(JasminAuthenticator, "get_projects_services", mock_get_projects_services) + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) - # A value error should be raised saying there were no provisioned requirements found and to check the status of requested resources. - with pytest.raises( - ValueError, - match="No provisioned requirements found for test_service. Check the status of your requested resources.", - ): - JasminAuthenticator.get_tape_quota("dummy_oauth_token", "test_service") + result = self.auth.get_tape_quota("test_service") + assert result == 0 From 8376647f19eb2c22b61179a3e5462fd9d42e374b Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Tue, 14 Jan 2025 14:44:12 +0000 Subject: [PATCH 24/26] Edit docs for consistency --- docs/source/server-config/server-config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/server-config/server-config.rst b/docs/source/server-config/server-config.rst index c95d98dd..5954ed5f 100644 --- a/docs/source/server-config/server-config.rst +++ b/docs/source/server-config/server-config.rst @@ -28,8 +28,8 @@ client. The following fields are required in the dictionary:: "jasmin_authenticator" : { "user_profile_url" : "{{ user_profile_url }}", "user_services_url" : "{{ user_services_url }}", - "user_grants_url" : "{{ }}", - "project_services_url" : "{{ }}", + "user_grants_url" : "{{ user_grants_url }}", + "project_services_url" : "{{ project_services_url }}", "oauth_token_introspect_url" : "{{ token_introspect_url }}" } } From c75f9f7eeed1b64ae7de1b5a3182f07241e251cc Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Tue, 25 Feb 2025 14:38:04 +0000 Subject: [PATCH 25/26] Add function to calculate used diskspace --- nlds_processors/catalog/catalog.py | 33 +++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/nlds_processors/catalog/catalog.py b/nlds_processors/catalog/catalog.py index f4301f8a..31fe2d24 100644 --- a/nlds_processors/catalog/catalog.py +++ b/nlds_processors/catalog/catalog.py @@ -823,4 +823,35 @@ def get_unarchived_files(self, holding: Holding) -> List[File]: f"Couldn't find unarchived files for holding with " f"id:{holding.id}" ) - return unarchived_files \ No newline at end of file + return unarchived_files + + + def get_used_diskspace(self, user: str, group: str) -> float: + """Return the total amount of diskspace used by the group.""" + if group is None: + raise ValueError("Group cannot be none.") + + total_diskspace = 0.0 + + try: + # Get the holdings + holdings = self.get_holding(user, group, groupall = True) + print(holdings) + + # Loop through the holdings + for holding in holdings: + print(holding) + + # Loop through the transactions: + for transaction in holding.transactions: + print(transaction) + + # Loop through the files: + for file in transaction.files: + + # Add file size to total diskspace + total_diskspace += file.size + + except Exception as e: + raise RuntimeError(f"An error occured while calculating the disk space: {e}") + return total_diskspace \ No newline at end of file From b4eb34cb4d637b9c7ec2792a8538a3a298fc3918 Mon Sep 17 00:00:00 2001 From: Nicola Farmer Date: Tue, 25 Feb 2025 15:40:44 +0000 Subject: [PATCH 26/26] Add in used_diskspace to catalog_worker --- nlds/rabbit/publisher.py | 1 + nlds_processors/catalog/catalog_worker.py | 25 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/nlds/rabbit/publisher.py b/nlds/rabbit/publisher.py index 3e2624c0..c5d8c884 100644 --- a/nlds/rabbit/publisher.py +++ b/nlds/rabbit/publisher.py @@ -113,6 +113,7 @@ class RabbitMQPublisher(): MSG_USER = "user" MSG_GROUP = "group" MSG_QUOTA = "quota" + MSG_DISKSPACE = "diskspace" MSG_GROUPALL = "groupall" MSG_TARGET = "target" MSG_ROUTE = "route" diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index 3e117cd8..b6375f6c 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -1856,7 +1856,7 @@ def _catalog_meta(self, body: Dict, properties: Header) -> None: ) def _catalog_quota(self, body: Dict, properties: Header) -> None: - """Return the users quota for the given service.""" + """Return the user's quota for the given group.""" message_vars = self._parse_user_vars(body) if message_vars is None: # Check if any problems have occured in the parsing of the message @@ -1871,18 +1871,37 @@ def _catalog_quota(self, body: Dict, properties: Header) -> None: try: group_quota = self.authenticator.get_tape_quota(service_name=group) except CatalogError as e: - # failed to get the holdings - send a return message saying so + # failed to get the tape quota - send a return message saying so self.log(e.message, self.RK_LOG_ERROR) body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message body[self.MSG_DATA][self.MSG_QUOTA] = None else: - # fill the return message with a dictionary of the holding(s) + # fill the return message with the group quota body[self.MSG_DATA][self.MSG_QUOTA] = group_quota self.log( f"Quota from CATALOG_QUOTA {group_quota}", self.RK_LOG_DEBUG ) + self.catalog.start_session() + + try: + used_diskspace = self.catalog.get_used_diskspace(user=user, group=group) + except CatalogError as e: + # failed to get the used diskspace - send a return message saying so + self.log(e.message, self.RK_LOG_ERROR) + body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message + body[self.MSG_DATA][self.MSG_DISKSPACE] = None + else: + # fill the return message with the used diskspace + body[self.MSG_DATA][self.MSG_DISKSPACE] = used_diskspace + self.log( + f"Used diskspace from CATALOG_QUOTA {used_diskspace}", + self.RK_LOG_DEBUG + ) + + self.catalog.end_session() + # return message to complete RPC self.publish_message( properties.reply_to,