diff --git a/docs/source/quotas.rst b/docs/source/quotas.rst new file mode 100644 index 00000000..749aee00 --- /dev/null +++ b/docs/source/quotas.rst @@ -0,0 +1,17 @@ +NLDS quotas +================================ + +Get the quota for your group in the NLDS. + + +Implementation. +------------------------ + +This is currently only implemented for the `jasmin_authenticator`. + +To get the quota, the `project_services_url` and `user_services_url` need to be present in the `jasmin_authenticator` section of the `server-config.rst` file. + +First, there is a call to the JASMIN Projects Portal to get information about the service. This call is made to the `project_services_url`. +This is authorized on behalf of the NLDS using a client token, supplied in the config. +The tape quota is then extracted from the service information. Only quota for allocated tape resource in a Group Workspace is returned, no other categories or status of resource (such as pending requests) are included. +The quota command can be called from the `nlds client`. \ No newline at end of file diff --git a/docs/source/server-config/examples.rst b/docs/source/server-config/examples.rst index 62fc8130..ed03dc0f 100644 --- a/docs/source/server-config/examples.rst +++ b/docs/source/server-config/examples.rst @@ -16,6 +16,8 @@ machine - likely a laptop or single vm. This file would be saved at "jasmin_authenticator" : { "user_profile_url" : "[REDACTED]", "user_services_url" : "[REDACTED]", + "user_grants_url" : "[REDACTED]", + "projects_services_url" : "[REDACTED]", "oauth_token_introspect_url" : "[REDACTED]" } }, diff --git a/docs/source/server-config/server-config.rst b/docs/source/server-config/server-config.rst index f04ff033..5954ed5f 100644 --- a/docs/source/server-config/server-config.rst +++ b/docs/source/server-config/server-config.rst @@ -28,6 +28,8 @@ client. The following fields are required in the dictionary:: "jasmin_authenticator" : { "user_profile_url" : "{{ user_profile_url }}", "user_services_url" : "{{ user_services_url }}", + "user_grants_url" : "{{ user_grants_url }}", + "project_services_url" : "{{ project_services_url }}", "oauth_token_introspect_url" : "{{ token_introspect_url }}" } } @@ -40,7 +42,7 @@ other industry standard authenticators like google and microsoft. The authenticator setup is then specified in a separate dictionary named after the authenticator, which is specific to each authenticator. The ``jasmin_authenticator`` requires, as above, values for ``user_profile_url``, -``user_services_url``, and ``oauth_token_introspect_url``. This cannot be +``user_services_url``, ``user_grants_url``, ``project_services_url`` and ``oauth_token_introspect_url``. This cannot be divulged publicly on github for JASMIN, so please get in contact for the actual values to use. diff --git a/nlds/authenticators/authenticate_methods.py b/nlds/authenticators/authenticate_methods.py index e60cd9f5..b8b1feea 100644 --- a/nlds/authenticators/authenticate_methods.py +++ b/nlds/authenticators/authenticate_methods.py @@ -11,8 +11,9 @@ """Authentication functions for use by the routers.""" from fastapi import Depends, status from fastapi.security import OAuth2PasswordBearer -from .jasmin_authenticator import JasminAuthenticator as Authenticator -from ..errors import ResponseError +from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator +from nlds_processors.catalog.catalog_models import File, Holding +from nlds.errors import ResponseError from fastapi.exceptions import HTTPException oauth2_scheme = OAuth2PasswordBearer(tokenUrl="", auto_error=False) @@ -95,3 +96,74 @@ async def authenticate_group(group: str, token: str = Depends(oauth2_scheme)): detail = response_error.json() ) return group + + +async def authenticate_user_group_role(user: str, group: str, token: str = Depends(oauth2_scheme)): + """Check the user's role in the group by calling the authenticator's authenticate_user_group_role + method.""" + if token is None: + response_error = ResponseError( + loc = ["authenticate_methods", "authenticate_group"], + msg = "Oauth token not supplied.", + type = "Forbidden." + ) + raise HTTPException( + status_code = status.HTTP_403_FORBIDDEN, + detail = response_error.json() + ) + elif not authenticator.authenticate_user_group_role(token, user, group): + response_error = ResponseError( + loc = ["authenticate_methods", "authenticate_user_group_role"], + msg = f"User is not a manager or deputy of the group {group}.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True + + +async def user_has_get_holding_permission(user: str, group: str, holding: Holding): + """Check whether a user has permission to view this holding.""" + if not authenticator.user_has_get_holding_permission(user, group, holding): + response_error = ResponseError( + loc = ["authenticate_methods", "user_has_get_holding_permission"], + msg = f"User does not have get holding permission for {holding}.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True + + +async def user_has_get_file_permission(session, user: str, group: str, file: File): + """Check whether a user has permission to access a file.""" + if not authenticator.user_has_get_file_permission(session, user, group, file): + response_error = ResponseError( + loc = ["authenticate_methods", "user_has_get_file_permission"], + msg = f"User does not have get file permission.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True + + +async def user_has_delete_from_holding_permission(user: str, group: str, holding: Holding): + """Check whether a user has permission to delete files from this holding.""" + if not authenticator.user_has_delete_from_holding_permission(user, group, holding): + response_error = ResponseError( + loc = ["authenticate_methods", "user_has_delete_from_holding_permission"], + msg = f"User does not have delete from holding permission.", + type = "Resource not found." + ) + raise HTTPException( + status_code = status.HTTP_404_NOT_FOUND, + detail = response_error.json() + ) + return True \ No newline at end of file diff --git a/nlds/authenticators/base_authenticator.py b/nlds/authenticators/base_authenticator.py index 06db103f..6d49319e 100644 --- a/nlds/authenticators/base_authenticator.py +++ b/nlds/authenticators/base_authenticator.py @@ -11,6 +11,7 @@ """Base class used to authenticate / authorise the users, groups, collections, etc. """ +from nlds_processors.catalog.catalog_models import File, Holding from abc import ABC @@ -31,3 +32,23 @@ def authenticate_group(self, oauth_token: str, group: str): def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): """Validate whether the user has manager/deputy permissions in the group.""" return NotImplementedError + + def user_has_get_holding_permission(self, user: str, group: str, holding: Holding) -> bool: + """Check whether a user has permission to view this holding.""" + return NotImplementedError + + def user_has_get_file_permission(self, session, user: str, group: str, file: File) -> bool: + """Check whether a user has permission to access a file.""" + return NotImplementedError + + def user_has_delete_from_holding_permission(self, user: str, group: str, holding: Holding) -> bool: + """Check whether a user has permission to delete files from this holding.""" + return NotImplementedError + + def get_service_information(self, oauth_token: str, service_name: str): + """Get the information about the given service.""" + return NotImplementedError + + def get_tape_quota(self, oauth_token: str, service_name: str): + """Process the service inforrmation to return the tape quota value.""" + return NotImplementedError diff --git a/nlds/authenticators/jasmin_authenticator.py b/nlds/authenticators/jasmin_authenticator.py index ea71d0cb..c08ff911 100644 --- a/nlds/authenticators/jasmin_authenticator.py +++ b/nlds/authenticators/jasmin_authenticator.py @@ -8,9 +8,10 @@ __license__ = "BSD - see LICENSE file in top-level package directory" __contact__ = "neil.massey@stfc.ac.uk" -from .base_authenticator import BaseAuthenticator -from ..server_config import load_config -from ..utils.construct_url import construct_url +from nlds.authenticators.base_authenticator import BaseAuthenticator +from nlds.server_config import load_config +from nlds.utils.format_url import format_url +from nlds_processors.catalog.catalog_models import File, Holding, Transaction from retry import retry import requests import json @@ -25,6 +26,7 @@ def __init__(self): self.config = load_config() self.name = "jasmin_authenticator" self.auth_name = "authentication" + self.default_quota = 0 @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) def authenticate_token(self, oauth_token: str): @@ -194,7 +196,7 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): "Authorization": f"Bearer {oauth_token}", } # Construct the URL - url = construct_url( + url = format_url( [config["user_grants_url"], user, "grants"], {"category": "GWS", "service": group}, ) @@ -218,10 +220,9 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): if response.status_code == requests.codes.ok: # status code 200 try: response_json = json.loads(response.text) - user_role = response_json["group_workspaces"] # is_manager is False by default and only changes if user has a manager or deputy role. is_manager = False - for role in user_role: + for role in response_json['group_workspaces']: if role in ["MANAGER", "DEPUTY"]: is_manager = True return is_manager @@ -237,3 +238,120 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str): ) else: return False + + + @staticmethod + def user_has_get_holding_permission(user: str, + group: str, + holding: Holding) -> bool: + """Check whether a user has permission to view this holding. + When we implement ROLES this will be more complicated.""" + permitted = True + #Users can view / get all holdings in their group + #permitted &= holding.user == user + permitted &= holding.group == group + return permitted + + + def user_has_get_file_permission(session, + user: str, + group: str, + file: File) -> bool: + """Check whether a user has permission to access a file. + Later, when we implement the ROLES this function will be a lot more + complicated!""" + assert(session != None) + holding = session.query(Holding).filter( + Transaction.id == file.transaction_id, + Holding.id == Transaction.holding_id + ).all() + permitted = True + for h in holding: + # users have get file permission if in group + # permitted &= h.user == user + permitted &= h.group == group + + return permitted + + + def user_has_delete_from_holding_permission(self, user: str, + group: str, + holding: Holding) -> bool: + """Check whether a user has permission to delete files from this holding. + When we implement ROLES this will be more complicated.""" + # is_admin == whether the user is an administrator of the group + # i.e. a DEPUTY or MANAGER + # this gives them delete permissions for all files in the group + is_admin = self.authenticate_user_group_role(user, group) + permitted = True + # Currently, only users can delete files from their owned holdings + permitted &= (holding.user == user or is_admin) + permitted &= holding.group == group + return permitted + + + @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2) + def get_service_information(self, service_name: str): + """Make a call to the JASMIN Projects Portal to get the service information.""" + config = self.config[self.auth_name][self.name] + token_headers = { + "Content-Type": "application/x-ww-form-urlencoded", + "cache-control": "no-cache", + # WORK THIS OUT + "Authorization": f"Bearer {config["client_token"]}", + } + # Contact the user_services_url to get the information about the services + url = format_url([config["project_services_url"]], {"name": service_name}) + print(url) + try: + response = requests.get( + url, + headers=token_headers, + timeout=JasminAuthenticator._timeout, + ) + except requests.exceptions.ConnectionError: + raise RuntimeError(f"User services url {url} could not be reached.") + except KeyError: + raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.") + if response.status_code == requests.codes.ok: # status code 200 + try: + response_json = json.loads(response.text) + return response_json + except json.JSONDecodeError: + raise RuntimeError(f"Invalid JSON returned from the user services url: {url}") + else: + raise RuntimeError(f"Error getting data for {service_name}") + + + def get_tape_quota(self, service_name: str): + """Get the service information then process it to extract the quota for the service.""" + try: + result = self.get_service_information(service_name) + except (RuntimeError, ValueError) as e: + raise type(e)(f"Error getting information for {service_name}: {e}") + + try: + # Filter for Group Workspace category + group_workspace = next( + service for service in result if service.get("category") == 1 + ) + except StopIteration: + raise ValueError(f"Cannot find a Group workspace with the name {service_name}. Check the category.") + + requirements = group_workspace.get("requirements") + if not requirements: + raise ValueError(f"Cannot find any requirements for {service_name}.") + + tape_quota = next( + ( + req.get("amount") + for req in requirements + if req.get("status") == 50 and req.get("resource", {}).get("short_name") == "tape" + ), + None, + ) + + if tape_quota is not None: + return tape_quota + else: + return self.default_quota \ No newline at end of file diff --git a/nlds/main.py b/nlds/main.py index 18823a04..8d26579e 100644 --- a/nlds/main.py +++ b/nlds/main.py @@ -12,7 +12,7 @@ from .nlds_setup import API_VERSION -from .routers import list, files, probe, status, find, meta, system, init +from .routers import list, files, probe, status, find, meta, system, init, quota nlds = FastAPI() @@ -32,6 +32,11 @@ tags = ["find",], prefix = PREFIX + "/catalog/find" ) +nlds.include_router( + quota.router, + tags = ["quota", ], + prefix = PREFIX + "/catalog/quota" +) nlds.include_router( status.router, tags = ["status",], @@ -56,4 +61,4 @@ init.router, tags = ["init", ], prefix = PREFIX + "/init" -) \ No newline at end of file +) diff --git a/nlds/rabbit/publisher.py b/nlds/rabbit/publisher.py index 7f39e29d..c5d8c884 100644 --- a/nlds/rabbit/publisher.py +++ b/nlds/rabbit/publisher.py @@ -50,6 +50,7 @@ class RabbitMQPublisher(): RK_STAT = "stat" RK_FIND = "find" RK_META = "meta" + RK_QUOTA = "quota" # Exchange routing key parts – root RK_ROOT = "nlds-api" @@ -111,6 +112,8 @@ class RabbitMQPublisher(): MSG_TIMESTAMP = "timestamp" MSG_USER = "user" MSG_GROUP = "group" + MSG_QUOTA = "quota" + MSG_DISKSPACE = "diskspace" MSG_GROUPALL = "groupall" MSG_TARGET = "target" MSG_ROUTE = "route" @@ -118,6 +121,7 @@ class RabbitMQPublisher(): MSG_TENANCY = "tenancy" MSG_ACCESS_KEY = "access_key" MSG_SECRET_KEY = "secret_key" + MSG_TOKEN = "token" MSG_API_ACTION = "api_action" MSG_JOB_LABEL = "job_label" MSG_DATA = "data" diff --git a/nlds/routers/quota.py b/nlds/routers/quota.py new file mode 100644 index 00000000..3f4eb35c --- /dev/null +++ b/nlds/routers/quota.py @@ -0,0 +1,113 @@ +""" + +""" + +from fastapi import Depends, APIRouter, status +from fastapi.exceptions import HTTPException +from fastapi.responses import JSONResponse +from pydantic import BaseModel +import json + +from typing import Optional, List, Dict + +from ..rabbit.publisher import RabbitMQPublisher as RMQP +from ..routers import rpc_publisher +from ..errors import ResponseError +from ..authenticators.authenticate_methods import authenticate_token, \ + authenticate_group, \ + authenticate_user + +from ..utils.process_tag import process_tag + + +router = APIRouter() + +class QuotaResponse(BaseModel): + quota: int + +############################ GET METHOD ############################ +@router.get("/", + status_code = status.HTTP_202_ACCEPTED, + responses = { + status.HTTP_202_ACCEPTED: {"model": QuotaResponse}, + status.HTTP_400_BAD_REQUEST: {"model": ResponseError}, + status.HTTP_401_UNAUTHORIZED: {"model": ResponseError}, + status.HTTP_403_FORBIDDEN: {"model": ResponseError}, + status.HTTP_404_NOT_FOUND: {"model": ResponseError}, + status.HTTP_504_GATEWAY_TIMEOUT: {"model": ResponseError}, + } + ) +async def get(token: str = Depends(authenticate_token), + user: str = Depends(authenticate_user), + group: str = Depends(authenticate_group), + label: Optional[str] = None, + holding_id: Optional[int] = None, + transaction_id: Optional[str] = None, + tag: Optional[str] = None + ): + # create the message dictionary + + routing_key = f"{RMQP.RK_ROOT}.{RMQP.RK_ROUTE}.{RMQP.RK_QUOTA}" + api_action = f"{RMQP.RK_QUOTA}" + msg_dict = { + RMQP.MSG_DETAILS: { + RMQP.MSG_USER: user, + RMQP.MSG_GROUP: group, + RMQP.MSG_TOKEN: token, + RMQP.MSG_API_ACTION: api_action + }, + RMQP.MSG_DATA: {}, + RMQP.MSG_TYPE: RMQP.MSG_TYPE_STANDARD + } + # add the metadata + meta_dict = {} + if (label): + meta_dict[RMQP.MSG_LABEL] = label + if (holding_id): + meta_dict[RMQP.MSG_HOLDING_ID] = holding_id + if (transaction_id): + meta_dict[RMQP.MSG_TRANSACT_ID] = transaction_id + + if (tag): + tag_dict = {} + # convert the string into a dictionary + try: + tag_dict = process_tag(tag) + except ValueError: + response_error = ResponseError( + loc = ["quota", "get"], + msg = "tag cannot be processed.", + type = "Incomplete request." + ) + raise HTTPException( + status_code = status.HTTP_400_BAD_REQUEST, + detail = response_error.json() + ) + else: + meta_dict[RMQP.MSG_TAG] = tag_dict + + if (len(meta_dict) > 0): + msg_dict[RMQP.MSG_META] = meta_dict + + # Call RPC function + routing_key = "catalog_q" + response = await rpc_publisher.call( + msg_dict=msg_dict, routing_key=routing_key + ) + # Check if response is valid or whether the request timed out + if response is not None: + # convert byte response to str + response = response.decode() + + return JSONResponse(status_code = status.HTTP_202_ACCEPTED, + content = response) + else: + response_error = ResponseError( + loc = ["status", "get"], + msg = "Catalog service could not be reached in time.", + type = "Incomplete request." + ) + raise HTTPException( + status_code = status.HTTP_504_GATEWAY_TIMEOUT, + detail = response_error.json() + ) diff --git a/nlds/utils/construct_url.py b/nlds/utils/format_url.py similarity index 82% rename from nlds/utils/construct_url.py rename to nlds/utils/format_url.py index 2444890d..964ea603 100644 --- a/nlds/utils/construct_url.py +++ b/nlds/utils/format_url.py @@ -1,7 +1,7 @@ from urllib.parse import urljoin, urlencode -def construct_url(url_parts, query_params=None): +def format_url(url_parts, query_params=None): """ Constructs a URL from a list of parts. @@ -12,6 +12,10 @@ def construct_url(url_parts, query_params=None): Returns: base (str): The constructed URL. """ + + if not isinstance(url_parts, list): + raise TypeError("url_parts must be a list") + if not url_parts: return "" diff --git a/nlds_processors/catalog/catalog.py b/nlds_processors/catalog/catalog.py index f65a8027..31fe2d24 100644 --- a/nlds_processors/catalog/catalog.py +++ b/nlds_processors/catalog/catalog.py @@ -4,11 +4,10 @@ from sqlalchemy import func, Enum from sqlalchemy.exc import IntegrityError, OperationalError, ArgumentError, \ NoResultFound - from nlds_processors.catalog.catalog_models import CatalogBase, File, Holding,\ Location, Transaction, Aggregation, Storage, Tag from nlds_processors.db_mixin import DBMixin -from nlds.authenticators.jasmin_authenticator import JasminAuthenticator +from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator class CatalogError(Exception): def __init__(self, message, *args): super().__init__(args) @@ -27,19 +26,6 @@ def __init__(self, db_engine: str, db_options: str): self.session = None - @staticmethod - def _user_has_get_holding_permission(user: str, - group: str, - holding: Holding) -> bool: - """Check whether a user has permission to view this holding. - When we implement ROLES this will be more complicated.""" - permitted = True - #Users can view / get all holdings in their group - #permitted &= holding.user == user - permitted &= holding.group == group - return permitted - - def get_holding(self, user: str, group: str, @@ -104,7 +90,7 @@ def get_holding(self, raise KeyError # check the user has permission to view the holding(s) for h in holding: - if not self._user_has_get_holding_permission(user, group, h): + if not Authenticator.user_has_get_holding_permission(user, group, h): raise CatalogError( f"User:{user} in group:{group} does not have permission " f"to access the holding with label:{h.label}." @@ -296,26 +282,6 @@ def delete_transaction(self, f"Transaction with transaction_id:{transaction_id} could not " "be added to the database" ) - - def _user_has_get_file_permission(self, - user: str, - group: str, - file: File) -> bool: - """Check whether a user has permission to access a file. - Later, when we implement the ROLES this function will be a lot more - complicated!""" - assert(self.session != None) - holding = self.session.query(Holding).filter( - Transaction.id == file.transaction_id, - Holding.id == Transaction.holding_id - ).all() - permitted = True - for h in holding: - # users have get file permission if in group - # permitted &= h.user == user - permitted &= h.group == group - - return permitted def get_file(self, @@ -420,7 +386,7 @@ def get_files(self, # check user has permission to access this file # NRM - 12/10/2023, is this necessary? if (f and - not self._user_has_get_file_permission(user, group, f) + not Authenticator.user_has_get_file_permission(self.session, user, group, f) ): raise CatalogError( f"User:{user} in group:{group} does not have permission to " @@ -478,23 +444,6 @@ def create_file(self, " the database" ) return new_file - - - @staticmethod - def _user_has_delete_from_holding_permission(user: str, - group: str, - holding: Holding) -> bool: - """Check whether a user has permission to delete files from this holding. - When we implement ROLES this will be more complicated.""" - # is_admin == whether the user is an administrator of the group - # i.e. a DEPUTY or MANAGER - # this gives them delete permissions for all files in the group - is_admin = JasminAuthenticator.authenticate_user_group_role(user, group) - permitted = True - # Currently, only users can delete files from their owned holdings - permitted &= (holding.user == user or is_admin) - permitted &= holding.group == group - return permitted def delete_files(self, @@ -515,7 +464,7 @@ def delete_files(self, holding_id=holding_id, original_path=path, tag=tag) holding = self.get_holding(user, group, holding_id=holding_id)[0] - if not Catalog._user_has_delete_from_holding_permission( + if not Authenticator.user_has_delete_from_holding_permission( user, group, holding): # No admins at the moment! raise CatalogError( @@ -874,4 +823,35 @@ def get_unarchived_files(self, holding: Holding) -> List[File]: f"Couldn't find unarchived files for holding with " f"id:{holding.id}" ) - return unarchived_files \ No newline at end of file + return unarchived_files + + + def get_used_diskspace(self, user: str, group: str) -> float: + """Return the total amount of diskspace used by the group.""" + if group is None: + raise ValueError("Group cannot be none.") + + total_diskspace = 0.0 + + try: + # Get the holdings + holdings = self.get_holding(user, group, groupall = True) + print(holdings) + + # Loop through the holdings + for holding in holdings: + print(holding) + + # Loop through the transactions: + for transaction in holding.transactions: + print(transaction) + + # Loop through the files: + for file in transaction.files: + + # Add file size to total diskspace + total_diskspace += file.size + + except Exception as e: + raise RuntimeError(f"An error occured while calculating the disk space: {e}") + return total_diskspace \ No newline at end of file diff --git a/nlds_processors/catalog/catalog_worker.py b/nlds_processors/catalog/catalog_worker.py index 403fcaf7..b6375f6c 100644 --- a/nlds_processors/catalog/catalog_worker.py +++ b/nlds_processors/catalog/catalog_worker.py @@ -46,6 +46,8 @@ from nlds.details import PathDetails, PathType from nlds_processors.db_mixin import DBError +from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator + class Metadata(): """Container class for the meta section of the message body.""" @@ -146,6 +148,7 @@ def __init__(self, queue=DEFAULT_QUEUE_NAME): self.catalog = None self.reroutelist = [] self.retrievedict = {} + self.authenticator = Authenticator() @property @@ -1850,7 +1853,62 @@ def _catalog_meta(self, body: Dict, properties: Header) -> None: msg_dict=body, exchange={'name': ''}, correlation_id=properties.correlation_id - ) + ) + + def _catalog_quota(self, body: Dict, properties: Header) -> None: + """Return the user's quota for the given group.""" + message_vars = self._parse_user_vars(body) + if message_vars is None: + # Check if any problems have occured in the parsing of the message + # body and exit if necessary + self.log("Could not parse one or more mandatory variables, exiting" + "callback", self.RK_LOG_ERROR) + return + else: + # Unpack if no problems found in parsing + user, group = message_vars + + try: + group_quota = self.authenticator.get_tape_quota(service_name=group) + except CatalogError as e: + # failed to get the tape quota - send a return message saying so + self.log(e.message, self.RK_LOG_ERROR) + body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message + body[self.MSG_DATA][self.MSG_QUOTA] = None + else: + # fill the return message with the group quota + body[self.MSG_DATA][self.MSG_QUOTA] = group_quota + self.log( + f"Quota from CATALOG_QUOTA {group_quota}", + self.RK_LOG_DEBUG + ) + + self.catalog.start_session() + + try: + used_diskspace = self.catalog.get_used_diskspace(user=user, group=group) + except CatalogError as e: + # failed to get the used diskspace - send a return message saying so + self.log(e.message, self.RK_LOG_ERROR) + body[self.MSG_DETAILS][self.MSG_FAILURE] = e.message + body[self.MSG_DATA][self.MSG_DISKSPACE] = None + else: + # fill the return message with the used diskspace + body[self.MSG_DATA][self.MSG_DISKSPACE] = used_diskspace + self.log( + f"Used diskspace from CATALOG_QUOTA {used_diskspace}", + self.RK_LOG_DEBUG + ) + + self.catalog.end_session() + + # return message to complete RPC + self.publish_message( + properties.reply_to, + msg_dict=body, + exchange={'name': ''}, + correlation_id=properties.correlation_id + ) def attach_database(self, create_db_fl: bool = True): @@ -2024,6 +2082,10 @@ def callback(self, ch: Channel, method: Method, properties: Header, elif (api_method == self.RK_STAT): self._catalog_stat(body, properties) + elif (api_method == self.RK_QUOTA): + # don't need to split any routing key for an RPC method + self._catalog_quota(body, properties) + # If received system test message, reply to it (this is for system status check) elif api_method == "system_stat": if properties.correlation_id is not None and properties.correlation_id != self.channel.consumer_tags[0]: diff --git a/tests/nlds/test_jasmin_authenticator.py b/tests/nlds/test_jasmin_authenticator.py index 1275ba1c..9543e708 100644 --- a/tests/nlds/test_jasmin_authenticator.py +++ b/tests/nlds/test_jasmin_authenticator.py @@ -2,9 +2,11 @@ import pytest import json import urllib +import re from nlds.authenticators.jasmin_authenticator import JasminAuthenticator -from nlds.utils.construct_url import construct_url +from nlds_processors.catalog.catalog_models import Holding +from nlds.utils.format_url import format_url @pytest.fixture(autouse=True) @@ -35,6 +37,7 @@ def mock_load_config(monkeypatch): "user_profile_url": "https://mock.url/api/profile/", "user_services_url": "https://mock.url/api/services/", "user_grants_url": "https://mock.url/api/v1/users/", + "projects_services_url": "https://mock.url/api/services", }, } } @@ -225,7 +228,7 @@ def test_authenticate_user_group_role( ): """Check whether the user has a manager/deputy role within the specified group.""" # Create the URL - url = construct_url( + url = format_url( ["https://mock.url/api/v1/users", user, "grants"], {"category": "GWS", "service": group}, ) @@ -244,3 +247,373 @@ def test_authenticate_user_group_role( # The authenticate_user_group_role method will use the monkeypatch has_role = auth.authenticate_user_group_role(oauth_token, user, group) assert has_role == expected_result + + +class TestUserPermissions: + """Test the functions that assign permissions to get holdings, get files and to delete from holding.""" + + @pytest.fixture() + def mock_holding(self): + return Holding( + label='test-label', + user='test-user', + group='test-group', + ) + + def test_user_has_get_holding_permission(self): + # Leaving this for now until it's a bit more fleshed out + pass + + def test_user_has_get_file_permission(self): + # Leaving this for now until it's a bit more fleshed out + pass + + @pytest.mark.parametrize("user, group, mock_is_admin, expected", [ + ("test-user", "test-group", False, True), # User owns the holding + ("user2", "test-group", False, False), # User does not own holding and is not admin + ("user2", "test-group", True, True), # User is admin of the group + ("test-user", "group2", False, False), # User is owner of different holding + ]) + def test_user_has_delete_from_holiding_permission(self, monkeypatch, user, group, mock_is_admin, expected, mock_holding): + # Mock the authenticate_user_group_role method + def mock_authenticate_user_group_role(user, group): + return mock_is_admin + + auth = JasminAuthenticator() + + monkeypatch.setattr(auth, "authenticate_user_group_role", mock_authenticate_user_group_role) + result = auth.user_has_delete_from_holding_permission(user=user, group=group, holding=mock_holding) + assert result == expected + + + +class TestGetProjectsServices: + """Get the projects for a service from the JASMIN Projects Portal.""" + user_services_url = "https://mock.url/api/services/" + url = f"{user_services_url}?name=test_service" + auth = JasminAuthenticator() + config = { + "authentication": { + "jasmin_authenticator": { + "project_services_url": "https://mock.url/api/services/", + "client_token": "test_token" + } + } + } + + + @pytest.fixture() + def mock_format_url(self, *args, **kwargs): + """Mock the format_url function to make it return the test url.""" + return self.url + + + def test_get_service_information_success(self, monkeypatch): + """Test a successful instance of get_projects_services.""" + + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) + + class MockResponse: + """Mock the response to return a 200 status code and the test text.""" + + status_code = 200 + text = '{"key": "value"}' + + def json(self): + return {"key": "value"} + + def mock_get(*args, **kwargs): + """Mock the get function to give MockResponse.""" + return MockResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + # Call the get_projects_services function with the mocked functions + result = self.auth.get_service_information("test_service") + + # It should succeed and give the {"key":"value"} dict. + assert result == {"key":"value"} + + + def test_get_projects_services_connection_error(self,monkeypatch): + """Test an unsuccessful instance of get_projects_services due to connection error.""" + + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) + + def mock_get(*args, **kwargs): + """Mock the get function to give a ConnectionError.""" + raise requests.exceptions.ConnectionError + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the ConnectionError in the 'get' triggers a RuntimeError with the right text. + with pytest.raises( + RuntimeError, match=re.escape(f"User services url {self.url} could not be reached.") + ): + self.auth.get_service_information("test_service") + + + def test_get_projects_services_key_error(self,monkeypatch): + """Test an unsuccessful instance of get_projects_services due to a key error.""" + + config = {"authentication": {"jasmin_authenticator": {"other_url": "test.com", "client_token":"test_token"}}} + + monkeypatch.setattr(self.auth, "config", config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) + + def mock_get(*args, **kwargs): + """Mock the get function to give the KeyError.""" + raise KeyError + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the KeyError in the 'get' triggers a RuntimeError with the right text. + with pytest.raises( + KeyError, + match=f"project_services_url", + ): + self.auth.get_service_information("test_service") + + def test_get_projects_services_json_error(self, monkeypatch): + """Test an unsuccessful instance of get_projects_services due to a JSON error.""" + + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) + + class MockInvalidJSONResponse: + """Mock the response to return a 200 status code and the JSON decode error.""" + status_code = 200 + text = "invalid json" + + def json(self): + raise json.JSONDecodeError("Expecting value", "invalid json", 0) + + def mock_get(*args, **kwargs): + """Mock the 'get' function to give the JSON error.""" + return MockInvalidJSONResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the JSONDecodeError triggers a RuntimeError with the right text. + with pytest.raises( + RuntimeError, + match=re.escape(f"Invalid JSON returned from the user services url: {self.url}"), + ): + self.auth.get_service_information("test_service") + + + def test_get_projects_services_404_error(self,monkeypatch): + """Test an unsuccessful instance of get_projects_services due to a 404 error.""" + + monkeypatch.setattr(self.auth, "config", self.config) + monkeypatch.setattr("nlds.utils.format_url", self.mock_format_url) + + class MockResponse: + """Mock the response to return a 401 status code and the relevant text.""" + status_code = 401 + text = "Unauthorized" + + def json(self): + return "Unauthorized" + + def mock_get(*args, **kwargs): + """Mock the get function to give the 401 error.""" + return MockResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + # Check that the 401 error triggers a RuntimeError with the right text. + with pytest.raises(RuntimeError, match=f"Error getting data for test_service"): + self.auth.get_service_information("test_service") + + +class TestGetTapeQuota: + """Get the tape quota from the list of projects services.""" + auth = JasminAuthenticator() + + def test_get_tape_quota_success(self, monkeypatch): + """Test a successful instance of get_tape_quota""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to gvie the response for + a GWS with a provisioned tape requirement.""" + return [ + { + "category": 1, + "requirements": [ + {"status": 50, "resource": {"short_name": "tape"}, "amount": 100} + ], + } + ] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + # get_tape_quota should return the quota value of 100 + result = self.auth.get_tape_quota("test_service") + assert result == 100 + + + def test_get_tape_quota_no_requirements(self, monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to no requirements.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with no requirements.""" + return [{"category": 1, "requirements": []}] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + # A ValueError should be raised saying there's no requirements found. + with pytest.raises(ValueError, match="Cannot find any requirements for test_service"): + self.auth.get_tape_quota("test_service") + + + def test_get_tape_quota_no_tape_resource(self, monkeypatch): + """Test an instance of no tape resources.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give the response for + a GWS with a requirement that isn't tape.""" + return [ + { + "category": 1, + "requirements": [ + {"status": 50, "resource": {"short_name": "other"}, "amount": 100} + ], + } + ] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + # A ValueError should be raised saying there's no tape resources. + result = self.auth.get_tape_quota("test_service") + assert result == 0 + + + def test_get_tape_quota_services_runtime_error(self, monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to a runtime error when + getting services from the projects portal.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give a RuntimeError.""" + raise RuntimeError("Runtime error occurred.") + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + # A RuntimeError should be raised saying a runtime error occurred. + with pytest.raises( + RuntimeError, + match="Error getting information for test_service: Runtime error occurred", + ): + self.auth.get_tape_quota("test_service") + + + def test_get_tape_quota_services_value_error(self, monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to a value error + getting services from the projects portal.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give a ValueError.""" + raise ValueError("Value error occurred") + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + # A ValueError should be raised saying a value error occurred. + with pytest.raises( + ValueError, + match="Error getting information for test_service: Value error occurred", + ): + self.auth.get_tape_quota("test_service") + + + def test_get_tape_quota_no_gws(self, monkeypatch): + """Test an unsuccessful instance of get_tape_quota due to the given service + not being a GWS.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give results with the wrong category (a GWS is 1).""" + return [ + {"category": 2, "requirements": []}, + {"category": 3, "requirements": []}, + ] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + # A ValueError should be raised saying it cannot find a GWS and to check the category. + with pytest.raises( + ValueError, + match="Cannot find a Group workspace with the name test_service. Check the category.", + ): + self.auth.get_tape_quota("test_service") + + + def test_get_quota_zero_quota(self, monkeypatch): + """Test an instance of the quota being zero.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give a quota of 0.""" + return [ + { + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + "amount": 0, + } + ], + } + ] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + result = self.auth.get_tape_quota("test_service") + assert result == 0 + + + def test_get_tape_quota_no_quota(self, monkeypatch): + """Test an instance of zero quota due to there being no quota field.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give no 'amount field.""" + return [ + { + "category": 1, + "requirements": [ + { + "status": 50, + "resource": {"short_name": "tape"}, + } + ], + } + ] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + result = self.auth.get_tape_quota("test_service") + assert result == 0 + + + def test_get_tape_quota_no_provisioned_resources(self, monkeypatch): + """Test an instance of zero quota due to there being no provisioned resources.""" + + def mock_get_service_information(*args, **kwargs): + """Mock the response from get_projects_services to give no provisioned resources (status 50).""" + return [ + { + "category": 1, + "requirements": [ + { + "status": 1, + "resource": {"short_name": "tape"}, + } + ], + } + ] + + monkeypatch.setattr(self.auth, "get_service_information", mock_get_service_information) + + result = self.auth.get_tape_quota("test_service") + assert result == 0 diff --git a/tests/nlds/utils/test_construct_url.py b/tests/nlds/utils/test_format_url.py similarity index 52% rename from tests/nlds/utils/test_construct_url.py rename to tests/nlds/utils/test_format_url.py index 8d813b48..59346fd1 100644 --- a/tests/nlds/utils/test_construct_url.py +++ b/tests/nlds/utils/test_format_url.py @@ -1,36 +1,57 @@ -from nlds.utils.construct_url import construct_url +from nlds.utils.format_url import format_url +import pytest def test_no_parts(): - assert construct_url([]) == "" + assert format_url([]) == "" def test_single_part(): - assert construct_url(["http://example.com"]) == "http://example.com" + assert format_url(["http://example.com"]) == "http://example.com" def test_multiple_parts(): url_parts = ["http://example.com", "path", "to", "resource"] expected_url = "http://example.com/path/to/resource" - assert construct_url(url_parts) == expected_url + assert format_url(url_parts) == expected_url def test_with_query_params(): url_parts = ["http://example.com", "path", "to", "resource"] query_params = {"key1": "value1", "key2": "value2"} expected_url = "http://example.com/path/to/resource?key1=value1&key2=value2" - assert construct_url(url_parts, query_params) == expected_url + assert format_url(url_parts, query_params) == expected_url def test_empty_query_params(): url_parts = ["http://example.com", "path", "to", "resource"] query_params = {} expected_url = "http://example.com/path/to/resource" - assert construct_url(url_parts, query_params) == expected_url + assert format_url(url_parts, query_params) == expected_url def test_complex_query_params(): url_parts = ["http://example.com", "search"] query_params = {"q": "test search", "page": "1", "sort": "asc"} expected_url = "http://example.com/search?q=test+search&page=1&sort=asc" - assert construct_url(url_parts, query_params) == expected_url + assert format_url(url_parts, query_params) == expected_url + + +def test_string(): + with pytest.raises(TypeError): + format_url("not-a-list") + + +def test_int(): + with pytest.raises(TypeError): + format_url(1) + + +def test_float(): + with pytest.raises(TypeError): + format_url(1.0) + + +def test_dict(): + with pytest.raises(TypeError): + format_url({"url_parts": "www.example.com"}) \ No newline at end of file diff --git a/tests/nlds_processors/catalog/test_catalog.py b/tests/nlds_processors/catalog/test_catalog.py index ca0410d4..3e2ea649 100644 --- a/tests/nlds_processors/catalog/test_catalog.py +++ b/tests/nlds_processors/catalog/test_catalog.py @@ -1,6 +1,10 @@ import uuid import time +import requests +import re +import json + import pytest from sqlalchemy import func @@ -9,7 +13,6 @@ ) from nlds_processors.catalog.catalog import Catalog, CatalogError from nlds.details import PathType -from nlds.authenticators.jasmin_authenticator import JasminAuthenticator test_uuid = '00a246cf-e2a8-46f0-baca-be3972fc4034' @@ -350,30 +353,6 @@ def test_create_transaction(self, mock_catalog, mock_holding): transaction_3 = mock_catalog.create_transaction(holding, test_uuid) - def test_user_has_get_holding_permission(self): - # Leaving this for now until it's a bit more fleshed out - pass - - def test_user_has_get_file_permission(self): - # Leaving this for now until it's a bit more fleshed out - pass - - @pytest.mark.parametrize("user, group, mock_is_admin, expected", [ - ("test-user", "test-group", False, True), # User owns the holding - ("user2", "test-group", False, False), # User does not own holding and is not admin - ("user2", "test-group", True, True), # User is admin of the group - ("test-user", "group2", False, False), # User is owner of different holding - ]) - def test_user_has_delete_from_holiding_permission(self, monkeypatch, user, group, mock_is_admin, expected, mock_holding): - # Mock the authenticate_user_group_role method - def mock_authenticate_user_group_role(user, group): - return mock_is_admin - - monkeypatch.setattr(JasminAuthenticator, "authenticate_user_group_role", mock_authenticate_user_group_role) - result = Catalog._user_has_delete_from_holding_permission(user, group, mock_holding) - assert result == expected - - def test_get_files(self, mock_catalog, mock_holding, mock_transaction, mock_file): test_uuid = str(uuid.uuid4()) @@ -495,4 +474,4 @@ def test_modify_tag(self): pass def test_del_tag(self): - pass + pass \ No newline at end of file