From 466616e7809f9748ded04601c1080092ad0a85ef Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 2 Nov 2022 10:24:35 +0100 Subject: [PATCH] Issue #78 Implementing GET /services --- src/openeo_aggregator/backend.py | 33 +- tests/test_backend.py | 719 +++++++++++++++++++++++++++++++ 2 files changed, 747 insertions(+), 5 deletions(-) diff --git a/src/openeo_aggregator/backend.py b/src/openeo_aggregator/backend.py index 8efc48ee..37ff8293 100644 --- a/src/openeo_aggregator/backend.py +++ b/src/openeo_aggregator/backend.py @@ -24,7 +24,7 @@ from openeo_aggregator.utils import MultiDictGetter, subdict, dict_merge, normalize_issuer_url from openeo_driver.ProcessGraphDeserializer import SimpleProcessing from openeo_driver.backend import OpenEoBackendImplementation, AbstractCollectionCatalog, LoadParameters, Processing, \ - OidcProvider, BatchJobs, BatchJobMetadata, SecondaryServices + OidcProvider, BatchJobs, BatchJobMetadata, SecondaryServices, ServiceMetadata from openeo_driver.datacube import DriverDataCube from openeo_driver.errors import CollectionNotFoundException, OpenEOApiException, ProcessGraphMissingException, \ JobNotFoundException, JobNotFinishedException, ProcessGraphInvalidException, PermissionsInsufficientException, \ @@ -808,10 +808,30 @@ def merge(formats: dict, to_add: dict): return service_types - # next one to implement - # def list_services(self, user_id: str) -> List[ServiceMetadata]: - # """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" - # return [] + def list_services(self, user_id: str) -> List[ServiceMetadata]: + """https://openeo.org/documentation/1.0/developers/api/reference.html#operation/list-services""" + + all_services = [] + def merge(services, to_add): + # For now ignore the links + services_to_add = to_add.get("services") + if services_to_add: + services_metadata = [ServiceMetadata.from_dict(s) for s in services_to_add] + services.extend(services_metadata) + + # Get stuff from backends + for con in self._backends: + services_json = None + try: + services_json = con.get("/services").json() + except Exception as e: + _log.warning("Failed to get services from {con.id}: {e!r}", exc_info=True) + continue + + if services_json: + merge(all_services, services_json) + + return all_services class AggregatorBackendImplementation(OpenEoBackendImplementation): # No basic auth: OIDC auth is required (to get EGI Check-in eduperson_entitlement data) @@ -1000,3 +1020,6 @@ def postprocess_capabilities(self, capabilities: dict) -> dict: def service_types(self) -> dict: return self.secondary_services.service_types() + + def list_services(self, user_id: str) -> List[ServiceMetadata]: + return self.secondary_services.list_services(user_id=user_id) diff --git a/tests/test_backend.py b/tests/test_backend.py index 255b2db4..f3e47269 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,9 +1,13 @@ +from sys import implementation +from datetime import datetime + import pytest from openeo_aggregator.backend import AggregatorCollectionCatalog, AggregatorProcessing, \ AggregatorBackendImplementation, _InternalCollectionMetadata, JobIdMapping from openeo_aggregator.caching import DictMemoizer from openeo_aggregator.testing import clock_mock +from openeo_driver.backend import ServiceMetadata from openeo_driver.errors import OpenEOApiException, CollectionNotFoundException, JobNotFoundException from openeo_driver.testing import DictSubSet from openeo_driver.users.oidc import OidcProvider @@ -180,6 +184,721 @@ def test_service_types_merging(self, multi_backend_connection, config, backend1, expected.update(service_2) assert service_types == expected + TEST_SERVICES = { + "services": [{ + "id": "wms-a3cca9", + "title": "NDVI based on Sentinel 2", + "description": "Deriving minimum NDVI measurements over pixel time series of Sentinel 2", + "url": "https://example.openeo.org/wms/wms-a3cca9", + "type": "wms", + "enabled": True, + "process": { + "id": "ndvi", + "summary": "string", + "description": "string", + "parameters": [{ + "schema": { + "parameters": [{ + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + } + }, + "categories": ["string"], + "deprecated": False, + "experimental": False, + "exceptions": { + "Error Code1": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + }, + "Error Code2": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + } + }, + "examples": [{ + "title": "string", + "description": "string", + "arguments": { + "property1": { + "from_parameter": None, + "from_node": None, + "process_graph": None + }, + "property2": { + "from_parameter": None, + "from_node": None, + "process_graph": None + } + }, + "returns": None + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }], + "process_graph": { + "dc": { + "process_id": "load_collection", + "arguments": { + "id": "Sentinel-2", + "spatial_extent": { + "west": 16.1, + "east": 16.6, + "north": 48.6, + "south": 47.2 + }, + "temporal_extent": ["2018-01-01", "2018-02-01"] + } + }, + "bands": { + "process_id": "filter_bands", + "description": + "Filter and order the bands. The order is important for the following reduce operation.", + "arguments": { + "data": { + "from_node": "dc" + }, + "bands": ["B08", "B04", "B02"] + } + }, + "evi": { + "process_id": "reduce", + "description": + "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)", + "arguments": { + "data": { + "from_node": "bands" + }, + "dimension": "bands", + "reducer": { + "process_graph": { + "nir": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 0 + } + }, + "red": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 1 + } + }, + "blue": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 2 + } + }, + "sub": { + "process_id": "subtract", + "arguments": { + "data": [{ + "from_node": "nir" + }, { + "from_node": "red" + }] + } + }, + "p1": { + "process_id": "product", + "arguments": { + "data": [6, { + "from_node": "red" + }] + } + }, + "p2": { + "process_id": "product", + "arguments": { + "data": + [-7.5, { + "from_node": "blue" + }] + } + }, + "sum": { + "process_id": "sum", + "arguments": { + "data": [ + 1, { + "from_node": "nir" + }, { + "from_node": "p1" + }, { + "from_node": "p2" + } + ] + } + }, + "div": { + "process_id": "divide", + "arguments": { + "data": [{ + "from_node": "sub" + }, { + "from_node": "sum" + }] + } + }, + "p3": { + "process_id": "product", + "arguments": { + "data": + [2.5, { + "from_node": "div" + }] + }, + "result": True + } + } + } + } + }, + "mintime": { + "process_id": "reduce", + "description": + "Compute a minimum time composite by reducing the temporal dimension", + "arguments": { + "data": { + "from_node": "evi" + }, + "dimension": "temporal", + "reducer": { + "process_graph": { + "min": { + "process_id": "min", + "arguments": { + "data": { + "from_parameter": "data" + } + }, + "result": True + } + } + } + } + }, + "save": { + "process_id": "save_result", + "arguments": { + "data": { + "from_node": "mintime" + }, + "format": "GTiff" + }, + "result": True + } + } + }, + "configuration": { + "version": "1.3.0" + }, + "attributes": { + "layers": ["ndvi", "evi"] + }, + "created": "2017-01-01T09:32:12Z", + "plan": "free", + "costs": 12.98, + "budget": 100, + "usage": { + "cpu": { + "value": 40668, + "unit": "cpu-seconds" + }, + "duration": { + "value": 2611, + "unit": "seconds" + }, + "memory": { + "value": 108138811, + "unit": "mb-seconds" + }, + "network": { + "value": 0, + "unit": "kb" + }, + "storage": { + "value": 55, + "unit": "mb" + } + } + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }] + } + + TEST_SERVICES2 = { + "services": [{ + "id": "wms-a3cca9", + "title": "TEST COPY -- NDVI based on Sentinel 2", + "description": "TEST COPY Deriving minimum NDVI measurements over pixel time series of Sentinel 2", + "url": "https://example.openeo.org/wms/wms-a3cca9", + "type": "wms", + "enabled": True, + "process": { + "id": "ndvi", + "summary": "string", + "description": "string", + "parameters": [{ + "schema": { + "parameters": [{ + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + }, + "name": "string", + "description": "string", + "optional": False, + "deprecated": False, + "experimental": False, + "default": None + }], + "returns": { + "description": "string", + "schema": { + "type": "array", + "subtype": "string", + "pattern": "/regex/", + "enum": [None], + "minimum": 0, + "maximum": 0, + "minItems": 0, + "maxItems": 0, + "items": [{}], + "deprecated": False + } + }, + "categories": ["string"], + "deprecated": False, + "experimental": False, + "exceptions": { + "Error Code1": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + }, + "Error Code2": { + "description": "string", + "message": "The value specified for the process argument '{argument}' in process '{process}' is invalid: {reason}", + "http": 400 + } + }, + "examples": [{ + "title": "string", + "description": "string", + "arguments": { + "property1": { + "from_parameter": None, + "from_node": None, + "process_graph": None + }, + "property2": { + "from_parameter": None, + "from_node": None, + "process_graph": None + } + }, + "returns": None + }], + "links": [{ + "rel": "related", + "href": "https://example.openeo.org", + "type": "text/html", + "title": "openEO" + }], + "process_graph": { + "dc": { + "process_id": "load_collection", + "arguments": { + "id": "Sentinel-2", + "spatial_extent": { + "west": 16.1, + "east": 16.6, + "north": 48.6, + "south": 47.2 + }, + "temporal_extent": ["2018-01-01", "2018-02-01"] + } + }, + "bands": { + "process_id": "filter_bands", + "description": + "Filter and order the bands. The order is important for the following reduce operation.", + "arguments": { + "data": { + "from_node": "dc" + }, + "bands": ["B08", "B04", "B02"] + } + }, + "evi": { + "process_id": "reduce", + "description": + "Compute the EVI. Formula: 2.5 * (NIR - RED) / (1 + NIR + 6*RED + -7.5*BLUE)", + "arguments": { + "data": { + "from_node": "bands" + }, + "dimension": "bands", + "reducer": { + "process_graph": { + "nir": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 0 + } + }, + "red": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 1 + } + }, + "blue": { + "process_id": "array_element", + "arguments": { + "data": { + "from_parameter": "data" + }, + "index": 2 + } + }, + "sub": { + "process_id": "subtract", + "arguments": { + "data": [{ + "from_node": "nir" + }, { + "from_node": "red" + }] + } + }, + "p1": { + "process_id": "product", + "arguments": { + "data": [6, { + "from_node": "red" + }] + } + }, + "p2": { + "process_id": "product", + "arguments": { + "data": + [-7.5, { + "from_node": "blue" + }] + } + }, + "sum": { + "process_id": "sum", + "arguments": { + "data": [ + 1, { + "from_node": "nir" + }, { + "from_node": "p1" + }, { + "from_node": "p2" + } + ] + } + }, + "div": { + "process_id": "divide", + "arguments": { + "data": [{ + "from_node": "sub" + }, { + "from_node": "sum" + }] + } + }, + "p3": { + "process_id": "product", + "arguments": { + "data": + [2.5, { + "from_node": "div" + }] + }, + "result": True + } + } + } + } + }, + "mintime": { + "process_id": "reduce", + "description": + "Compute a minimum time composite by reducing the temporal dimension", + "arguments": { + "data": { + "from_node": "evi" + }, + "dimension": "temporal", + "reducer": { + "process_graph": { + "min": { + "process_id": "min", + "arguments": { + "data": { + "from_parameter": "data" + } + }, + "result": True + } + } + } + } + }, + "save": { + "process_id": "save_result", + "arguments": { + "data": { + "from_node": "mintime" + }, + "format": "GTiff" + }, + "result": True + } + } + }, + "configuration": { + "version": "1.3.0" + }, + "attributes": { + "layers": ["ndvi", "evi"] + }, + "created": "2017-01-01T09:32:12Z", + "plan": "free", + "costs": 12.98, + "budget": 100, + "usage": { + "cpu": { + "value": 40668, + "unit": "cpu-seconds" + }, + "duration": { + "value": 2611, + "unit": "seconds" + }, + "memory": { + "value": 108138811, + "unit": "mb-seconds" + }, + "network": { + "value": 0, + "unit": "kb" + }, + "storage": { + "value": 55, + "unit": "mb" + } + } + }], + "links": [] + } + + def test_list_services_simple(self, multi_backend_connection, config, backend1, backend2, requests_mock): + services1 = self.TEST_SERVICES + services2 = {} + test_user_id = "fakeuser" + requests_mock.get(backend1 + "/services", json=services1) + requests_mock.get(backend2 + "/services", json=services2) + implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + actual_services = implementation.list_services(user_id=test_user_id) + + # Construct expected result. We have get just data from the service in services1 + # (there is only one) for conversion to a ServiceMetadata. + the_service = services1["services"][0] + expected_services = [ + ServiceMetadata.from_dict(the_service) + ] + assert actual_services == expected_services + + def test_list_services_merged(self, multi_backend_connection, config, backend1, backend2, requests_mock): + services1 = self.TEST_SERVICES + serv_metadata_wmts_foo = ServiceMetadata( + id="wmts-foo", + process={"process_graph": {"foo": {"process_id": "foo", "arguments": {}}}}, + url='https://oeo.net/wmts/foo', + type="WMTS", + enabled=True, + configuration={"version": "0.5.8"}, + attributes={}, + title="Test service", + created=datetime(2020, 4, 9, 15, 5, 8) + ) + services2 = {"services": [serv_metadata_wmts_foo.prepare_for_json()], "links": []} + test_user_id = "fakeuser" + requests_mock.get(backend1 + "/services", json=services1) + requests_mock.get(backend2 + "/services", json=services2) + implementation = AggregatorBackendImplementation(backends=multi_backend_connection, config=config) + + actual_services = implementation.list_services(user_id=test_user_id) + + # Construct expected result. We have get just data from the service in + # services1 (there is only one) for conversion to a ServiceMetadata. + # For now we still ignore the key "links" in the outer dictionary. + service1 = services1["services"][0] + service1_md = ServiceMetadata.from_dict(service1) + service2 = services2["services"][0] + service2_md = ServiceMetadata.from_dict(service2) + expected_services = [service1_md, service2_md] + + assert actual_services == expected_services + class TestInternalCollectionMetadata: