Merge pull request #117 from cedadev/quotas

Quotas initial implementation.

docs/source/quotas.rst

@@ -0,0 +1,17 @@
+NLDS quotas
+================================
+
+Get the quota for your group in the NLDS.
+
+
+Implementation.
+------------------------
+
+This is currently only implemented for the `jasmin_authenticator`.
+
+To get the quota, the `project_services_url` and `user_services_url` need to be present in the `jasmin_authenticator` section of the `server-config.rst` file.
+
+First, there is a call to the JASMIN Projects Portal to get information about the service. This call is made to the `project_services_url`.
+This is authorized on behalf of the NLDS using a client token, supplied in the config.
+The tape quota is then extracted from the service information. Only quota for allocated tape resource in a Group Workspace is returned, no other categories or status of resource (such as pending requests) are included.
+The quota command can be called from the `nlds client`. 

docs/source/server-config/examples.rst

@@ -16,6 +16,8 @@ machine - likely a laptop or single vm. This file would be saved at
             "jasmin_authenticator" : {
                 "user_profile_url" : "[REDACTED]",
                 "user_services_url" : "[REDACTED]",
+                "user_grants_url" : "[REDACTED]",
+                "projects_services_url" : "[REDACTED]",
                 "oauth_token_introspect_url" : "[REDACTED]"
             }
         },

docs/source/server-config/server-config.rst

@@ -28,6 +28,8 @@ client. The following fields are required in the dictionary::
         "jasmin_authenticator" : {
             "user_profile_url" : "{{ user_profile_url }}",
             "user_services_url" : "{{ user_services_url }}",
+            "user_grants_url" : "{{ user_grants_url }}",
+            "project_services_url" : "{{ project_services_url }}",
             "oauth_token_introspect_url" : "{{ token_introspect_url }}"
         }
     }
@@ -40,7 +42,7 @@ other industry standard authenticators like google and microsoft.
 The authenticator setup is then specified in a separate dictionary named after 
 the authenticator, which is specific to each authenticator. The 
 ``jasmin_authenticator`` requires, as above, values for ``user_profile_url``, 
-``user_services_url``, and ``oauth_token_introspect_url``. This cannot be 
+``user_services_url``, ``user_grants_url``, ``project_services_url`` and ``oauth_token_introspect_url``. This cannot be 
 divulged publicly on github for JASMIN, so please get in contact for the actual 
 values to use. 
 

nlds/authenticators/authenticate_methods.py

@@ -11,8 +11,9 @@
 """Authentication functions for use by the routers."""
 from fastapi import Depends, status
 from fastapi.security import OAuth2PasswordBearer
-from .jasmin_authenticator import JasminAuthenticator as Authenticator
-from ..errors import ResponseError
+from nlds.authenticators.jasmin_authenticator import JasminAuthenticator as Authenticator
+from nlds_processors.catalog.catalog_models import File, Holding
+from nlds.errors import ResponseError
 from fastapi.exceptions import HTTPException
 
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="", auto_error=False)
@@ -95,3 +96,74 @@ async def authenticate_group(group: str, token: str = Depends(oauth2_scheme)):
             detail = response_error.json()
         )
     return group
+
+
+async def authenticate_user_group_role(user: str, group: str, token: str = Depends(oauth2_scheme)):
+    """Check the user's role in the group by calling the authenticator's authenticate_user_group_role 
+    method."""
+    if token is None:
+        response_error = ResponseError(
+            loc = ["authenticate_methods", "authenticate_group"],
+            msg = "Oauth token not supplied.",
+            type = "Forbidden."
+        )
+        raise HTTPException(
+            status_code = status.HTTP_403_FORBIDDEN,
+            detail = response_error.json()
+        )
+    elif not authenticator.authenticate_user_group_role(token, user, group):
+        response_error = ResponseError(
+            loc = ["authenticate_methods", "authenticate_user_group_role"],
+            msg = f"User is not a manager or deputy of the group {group}.",
+            type = "Resource not found."
+        )
+        raise HTTPException(
+            status_code = status.HTTP_404_NOT_FOUND,
+            detail = response_error.json()
+        )
+    return True
+
+
+async def user_has_get_holding_permission(user: str, group: str, holding: Holding):
+    """Check whether a user has permission to view this holding."""
+    if not authenticator.user_has_get_holding_permission(user, group, holding):
+        response_error = ResponseError(
+            loc = ["authenticate_methods", "user_has_get_holding_permission"],
+            msg = f"User does not have get holding permission for {holding}.",
+            type = "Resource not found."
+        )
+        raise HTTPException(
+            status_code = status.HTTP_404_NOT_FOUND,
+            detail = response_error.json()
+        )
+    return True
+
+
+async def user_has_get_file_permission(session, user: str, group: str, file: File):
+    """Check whether a user has permission to access a file."""
+    if not authenticator.user_has_get_file_permission(session, user, group, file):
+        response_error = ResponseError(
+            loc = ["authenticate_methods", "user_has_get_file_permission"],
+            msg = f"User does not have get file permission.",
+            type = "Resource not found."
+        )
+        raise HTTPException(
+            status_code = status.HTTP_404_NOT_FOUND,
+            detail = response_error.json()
+        )
+    return True
+
+
+async def user_has_delete_from_holding_permission(user: str, group: str, holding: Holding):
+    """Check whether a user has permission to delete files from this holding."""
+    if not authenticator.user_has_delete_from_holding_permission(user, group, holding):
+        response_error = ResponseError(
+            loc = ["authenticate_methods", "user_has_delete_from_holding_permission"],
+            msg = f"User does not have delete from holding permission.",
+            type = "Resource not found."
+        )
+        raise HTTPException(
+            status_code = status.HTTP_404_NOT_FOUND,
+            detail = response_error.json()
+        )
+    return True

nlds/authenticators/base_authenticator.py

@@ -11,6 +11,7 @@
 """Base class used to authenticate / authorise the users, groups, collections,
    etc.
 """
+from nlds_processors.catalog.catalog_models import File, Holding
 from abc import ABC
 
 
@@ -31,3 +32,23 @@ def authenticate_group(self, oauth_token: str, group: str):
     def authenticate_user_group_role(self, oauth_token: str, user: str, group: str):
         """Validate whether the user has manager/deputy permissions in the group."""
         return NotImplementedError
+
+    def user_has_get_holding_permission(self, user: str, group: str, holding: Holding) -> bool:
+        """Check whether a user has permission to view this holding."""
+        return NotImplementedError
+
+    def user_has_get_file_permission(self, session, user: str, group: str, file: File) -> bool:
+        """Check whether a user has permission to access a file."""
+        return NotImplementedError
+
+    def user_has_delete_from_holding_permission(self, user: str, group: str, holding: Holding) ->  bool:
+        """Check whether a user has permission to delete files from this holding."""
+        return NotImplementedError
+
+    def get_service_information(self, oauth_token: str, service_name: str):
+        """Get the information about the given service."""
+        return NotImplementedError
+
+    def get_tape_quota(self, oauth_token: str, service_name: str):
+        """Process the service inforrmation to return the tape quota value."""
+        return NotImplementedError

nlds/authenticators/jasmin_authenticator.py

@@ -8,9 +8,10 @@
 __license__ = "BSD - see LICENSE file in top-level package directory"
 __contact__ = "[email protected]"
 
-from .base_authenticator import BaseAuthenticator
-from ..server_config import load_config
-from ..utils.construct_url import construct_url
+from nlds.authenticators.base_authenticator import BaseAuthenticator
+from nlds.server_config import load_config
+from nlds.utils.format_url import format_url
+from nlds_processors.catalog.catalog_models import File, Holding, Transaction
 from retry import retry
 import requests
 import json
@@ -25,6 +26,7 @@ def __init__(self):
         self.config = load_config()
         self.name = "jasmin_authenticator"
         self.auth_name = "authentication"
+        self.default_quota = 0
 
     @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2)
     def authenticate_token(self, oauth_token: str):
@@ -194,7 +196,7 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str):
             "Authorization": f"Bearer {oauth_token}",
         }
         # Construct the URL
-        url = construct_url(
+        url = format_url(
             [config["user_grants_url"], user, "grants"],
             {"category": "GWS", "service": group},
         )
@@ -218,10 +220,9 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str):
         if response.status_code == requests.codes.ok:  # status code 200
             try:
                 response_json = json.loads(response.text)
-                user_role = response_json["group_workspaces"]
                 # is_manager is False by default and only changes if user has a manager or deputy role.
                 is_manager = False
-                for role in user_role:
+                for role in response_json['group_workspaces']:
                     if role in ["MANAGER", "DEPUTY"]:
                         is_manager = True
                 return is_manager
@@ -237,3 +238,120 @@ def authenticate_user_group_role(self, oauth_token: str, user: str, group: str):
                 )
         else:
             return False
+
+
+    @staticmethod
+    def user_has_get_holding_permission(user: str, 
+                                         group: str,
+                                         holding: Holding) -> bool:
+        """Check whether a user has permission to view this holding.
+        When we implement ROLES this will be more complicated."""
+        permitted = True
+        #Users can view / get all holdings in their group
+        #permitted &= holding.user == user
+        permitted &= holding.group == group
+        return permitted
+
+
+    def user_has_get_file_permission(session, 
+                                      user: str, 
+                                      group: str,
+                                      file: File) -> bool:
+        """Check whether a user has permission to access a file.
+        Later, when we implement the ROLES this function will be a lot more
+        complicated!"""
+        assert(session != None)
+        holding = session.query(Holding).filter(
+            Transaction.id == file.transaction_id,
+            Holding.id == Transaction.holding_id
+        ).all()
+        permitted = True
+        for h in holding:
+            # users have get file permission if in group
+            # permitted &= h.user == user
+            permitted &= h.group == group
+
+        return permitted
+
+
+    def user_has_delete_from_holding_permission(self, user: str, 
+                                                 group: str,
+                                                 holding: Holding) -> bool:
+        """Check whether a user has permission to delete files from this holding.
+        When we implement ROLES this will be more complicated."""
+        # is_admin == whether the user is an administrator of the group
+        # i.e. a DEPUTY or MANAGER
+        # this gives them delete permissions for all files in the group
+        is_admin = self.authenticate_user_group_role(user, group)
+        permitted = True
+        # Currently, only users can delete files from their owned holdings
+        permitted &= (holding.user == user or is_admin)
+        permitted &= holding.group == group
+        return permitted
+
+
+    @retry(requests.ConnectTimeout, tries=5, delay=1, backoff=2)
+    def get_service_information(self, service_name: str):
+        """Make a call to the JASMIN Projects Portal to get the service information."""
+        config = self.config[self.auth_name][self.name]
+        token_headers = {
+            "Content-Type": "application/x-ww-form-urlencoded",
+            "cache-control": "no-cache",
+            # WORK THIS OUT
+            "Authorization": f"Bearer {config["client_token"]}",
+        }
+        # Contact the user_services_url to get the information about the services
+        url = format_url([config["project_services_url"]], {"name": service_name})
+        print(url)
+        try:
+            response = requests.get(
+                url,
+                headers=token_headers,
+                timeout=JasminAuthenticator._timeout,
+            )
+        except requests.exceptions.ConnectionError:
+            raise RuntimeError(f"User services url {url} could not be reached.")
+        except KeyError:
+            raise RuntimeError(f"Could not find 'user_services_url' key in the {self.name} section of the .server_config file.")
+        if response.status_code == requests.codes.ok: # status code 200
+            try:
+                response_json = json.loads(response.text)
+                return response_json
+            except json.JSONDecodeError:
+                raise RuntimeError(f"Invalid JSON returned from the user services url: {url}")
+        else:
+            raise RuntimeError(f"Error getting data for {service_name}")
+
+
+    def get_tape_quota(self, service_name: str):
+        """Get the service information then process it to extract the quota for the service."""
+        try:
+            result = self.get_service_information(service_name)
+        except (RuntimeError, ValueError) as e:
+            raise type(e)(f"Error getting information for {service_name}: {e}")
+
+        try:
+            # Filter for Group Workspace category
+            group_workspace = next(
+                service for service in result if service.get("category") == 1
+            )
+        except StopIteration:
+            raise ValueError(f"Cannot find a Group workspace with the name {service_name}. Check the category.")
+
+        requirements = group_workspace.get("requirements")
+        if not requirements:
+            raise ValueError(f"Cannot find any requirements for {service_name}.")
+
+        tape_quota = next(
+            (
+                req.get("amount")
+                for req in requirements
+                if req.get("status") == 50 and req.get("resource", {}).get("short_name") == "tape"
+            ),
+            None,
+        )
+
+        if tape_quota is not None:
+            return tape_quota
+        else:
+            return self.default_quota

nlds/main.py

@@ -12,7 +12,7 @@
 
 from .nlds_setup import API_VERSION
 
-from .routers import list, files, probe, status, find, meta, system, init
+from .routers import list, files, probe, status, find, meta, system, init, quota
 
 nlds = FastAPI()
 
@@ -32,6 +32,11 @@
     tags = ["find",],
     prefix = PREFIX + "/catalog/find"
 )
+nlds.include_router(
+    quota.router,
+    tags = ["quota", ],
+    prefix = PREFIX + "/catalog/quota"
+)
 nlds.include_router(
     status.router,
     tags = ["status",],
@@ -56,4 +61,4 @@
     init.router,
     tags = ["init", ],
     prefix = PREFIX + "/init"
-)
+)

nlds/rabbit/publisher.py

@@ -50,6 +50,7 @@ class RabbitMQPublisher():
     RK_STAT = "stat"
     RK_FIND = "find"
     RK_META = "meta"
+    RK_QUOTA = "quota"
 
     # Exchange routing key parts – root
     RK_ROOT = "nlds-api"
@@ -111,13 +112,16 @@ class RabbitMQPublisher():
     MSG_TIMESTAMP = "timestamp"
     MSG_USER = "user"
     MSG_GROUP = "group"
+    MSG_QUOTA = "quota"
+    MSG_DISKSPACE = "diskspace"
     MSG_GROUPALL = "groupall"
     MSG_TARGET = "target"
     MSG_ROUTE = "route"
     MSG_ERROR = "error"
     MSG_TENANCY = "tenancy"
     MSG_ACCESS_KEY = "access_key"
     MSG_SECRET_KEY = "secret_key"
+    MSG_TOKEN = "token"
     MSG_API_ACTION = "api_action"
     MSG_JOB_LABEL = "job_label"
     MSG_DATA = "data"