From 746e3ec7c63735c8c084db1aee748b7185a0ebc8 Mon Sep 17 00:00:00 2001 From: Argyrios Samourkasidis Date: Mon, 15 Aug 2022 17:42:12 +0300 Subject: [PATCH] fix: Refactor SOS service Refactor SOS service to support the updated EDAM. Fix #46 --- edam/reader/models/measurement.py | 17 +- edam/reader/models/station.py | 3 +- .../flask_related/templates/SOS_Examples.html | 16 +- .../flask_related/templates/base.html | 1 + .../templates/sos/DescribeSensor.xml | 12 +- .../templates/sos/DescribeSensorException.xml | 2 +- .../templates/sos/GetCapabilities.xml | 8 +- .../templates/sos/GetObservation.xml | 30 ++-- .../templates/sos/GetObservationException.xml | 2 +- edam/services/__init__.py | 0 edam/services/sos/__init__.py | 0 edam/services/sos/sos.py | 150 ++++++++++++++++++ edam/viewer/app/OgcSos.py | 102 ------------ edam/viewer/app/__init__.py | 21 --- edam/viewer/app/views.py | 59 +++---- tests/test_sos.py | 25 +++ tests/tests/__init__.py | 0 17 files changed, 248 insertions(+), 200 deletions(-) create mode 100644 edam/services/__init__.py create mode 100644 edam/services/sos/__init__.py create mode 100755 edam/services/sos/sos.py delete mode 100755 edam/viewer/app/OgcSos.py create mode 100644 tests/test_sos.py create mode 100644 tests/tests/__init__.py diff --git a/edam/reader/models/measurement.py b/edam/reader/models/measurement.py index 736e30b..2206463 100644 --- a/edam/reader/models/measurement.py +++ b/edam/reader/models/measurement.py @@ -1,12 +1,7 @@ -class Measurement(object): - def __init__(self, value, timestamp=None, observable=None, - uom=None, station=None, junction=None): - self.value = value - self.timestamp = timestamp - self.observable = observable - self.uom = uom - self.station = station - self.junction = junction +from dataclasses import dataclass - def __repr__(self): - return f"{self.value}" + +@dataclass +class Measurement: + timestamp: str + value: str diff --git a/edam/reader/models/station.py b/edam/reader/models/station.py index 20fa401..0159e50 100644 --- a/edam/reader/models/station.py +++ b/edam/reader/models/station.py @@ -148,7 +148,8 @@ def data(self) -> pd.DataFrame: if dataframes: dataframe = pd.concat(dataframes, join='inner', axis=1).fillna( "empty") # type: pd.DataFrame - dataframe.set_index(keys=["timestamp"], drop=False, inplace=True) + dataframe['my_index'] = pd.to_datetime(dataframe['timestamp']) + dataframe.set_index('my_index', drop=True, inplace=True) return dataframe logger.warning(f"{self.name} does not have any data associated") return None diff --git a/edam/resources/flask_related/templates/SOS_Examples.html b/edam/resources/flask_related/templates/SOS_Examples.html index d0ed424..edb3044 100644 --- a/edam/resources/flask_related/templates/SOS_Examples.html +++ b/edam/resources/flask_related/templates/SOS_Examples.html @@ -36,8 +36,8 @@ procedure = URN of sensor (see getCapabilities output for sensor's URN)
  • Example: {{ - request.url_root }}SOS/?request=DescribeSensor&procedure=2:13:TX + href="{{ request.url_root }}SOS/{{ example_describe }}">{{ + request.url_root }}SOS/{{ example_describe }}
    • @@ -49,10 +49,10 @@ time frame
  • Parameters: request = GetObservation
    • -
  • offering = URN of sensor (see getCapabilities output for - sensor's URN)
    • -
  • observedProperty = (One of : Temperature, Pressure, - Nitrogen_Dioxide, Carbon_Monoxide) +
  • offering = Station name (see getCapabilities output for + available stations)
    • +
  • observedProperty = (see getCapabilities output for + available stations))
  • eventTime = (YYYY-MM-DD HH:MM:SS/YYYY-MM-DD HH:MM:SS). When eventTime = Datetime/ @@ -60,8 +60,8 @@ measurements from Datetime until now.
  • Example: {{ - request.url_root }}SOS/?request=GetObservation&procedure=2:13:TX + href="{{ request.url_root }}SOS/{{ example_observation }}">{{ + request.url_root }}SOS/{{ example_observation }}
    • diff --git a/edam/resources/flask_related/templates/base.html b/edam/resources/flask_related/templates/base.html index 0e10370..fd190c1 100644 --- a/edam/resources/flask_related/templates/base.html +++ b/edam/resources/flask_related/templates/base.html @@ -28,6 +28,7 @@ {% set navigation_bar = [ ('/maps/', 'maps', 'Maps'), ('/about/', 'about', 'API'), +('/SensorObservationService/', 'sos', 'OGC SOS'), ('/home/', 'home', 'Home'), ('/graphs', 'graphs', 'Graphs') ] -%} diff --git a/edam/resources/flask_related/templates/sos/DescribeSensor.xml b/edam/resources/flask_related/templates/sos/DescribeSensor.xml index ef57ba5..1e69131 100644 --- a/edam/resources/flask_related/templates/sos/DescribeSensor.xml +++ b/edam/resources/flask_related/templates/sos/DescribeSensor.xml @@ -4,26 +4,24 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/sensorML/1.0.1 http://schemas.opengis.net/sensorML/1.0.1/sensorML.xsd" version="1.0.1"> - {% set helper = sensor %} - {{ helper.sensor.name }} observes {{ - helper.observable.name }}. It is manufactured - by {{ helper.sensor.manufacturer }} + {{ junction.sensor.name }} observes {{ + junction.observable.name }}. It is manufactured + by {{ junction.sensor.manufacturer }} - {{ helper.sensor.name }} + {{ junction.sensor.name }} - {{ helper.station_id }}:{{ helper.sensor_id}}:{{ - helper.template_id }} + {{ station.name }}:{{ junction.sensor.name}} diff --git a/edam/resources/flask_related/templates/sos/DescribeSensorException.xml b/edam/resources/flask_related/templates/sos/DescribeSensorException.xml index 5a4919a..178683c 100644 --- a/edam/resources/flask_related/templates/sos/DescribeSensorException.xml +++ b/edam/resources/flask_related/templates/sos/DescribeSensorException.xml @@ -4,6 +4,6 @@ version="1.0.0" xml:lang="en"> - {{ procedure }} + {{ exception_text }} \ No newline at end of file diff --git a/edam/resources/flask_related/templates/sos/GetCapabilities.xml b/edam/resources/flask_related/templates/sos/GetCapabilities.xml index 3d9b227..48e78b3 100644 --- a/edam/resources/flask_related/templates/sos/GetCapabilities.xml +++ b/edam/resources/flask_related/templates/sos/GetCapabilities.xml @@ -115,13 +115,11 @@ - {% for helper in helpers %}{% if helper.station_id == station.id - %} + {% for junction in station.junctions %} + xlink:href="{{ station.name }}:{{ junction.sensor.name }}"/> {% endif %}{% - endfor %} + xlink:href="{{ junction.observable.observable_id }}"/>{% endfor %} text/xml;subtype="om/1.0.0" diff --git a/edam/resources/flask_related/templates/sos/GetObservation.xml b/edam/resources/flask_related/templates/sos/GetObservation.xml index ed8c4d8..7f4472d 100644 --- a/edam/resources/flask_related/templates/sos/GetObservation.xml +++ b/edam/resources/flask_related/templates/sos/GetObservation.xml @@ -6,36 +6,36 @@ xmlns:om="http://www.opengis.net/om/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/om/1.0 http://schemas.opengis.net/om/1.0.0/om.xsd"> - {{ results[0].observable.name}} measurements - from {{ results[0].station.name}} station + **{{ junction.observable.name}}** measurements observed with + **{{ junction.sensor.name }}** from **{{ station.name}}** station - {{ results[0].station.id}}:{{ results[0].helper.sensor.id }} + {{ station.name}}:{{ junction.observable.name }} - {{ results[0].timestamp }} + {{ measurements[0].timestamp }} - {{ results[-1].timestamp }} + {{ measurements[-1].timestamp }} + - {% for measurement in results %} + {% for measurement in measurements %} - {#+ hash(helper.observable.name)#} - + - {# {{ hash(d2s(measurement.timestamp)) }}#} - + {{ measurement.timestamp }} - {# {{ hash(d2s(measurement.timestamp)) }}#} - - {{ results[0].helper.sensor.name}} + + {{ station.name }}:{{ junction.sensor.name }} + - {{ measurement.value + xlink:href="{{ junction.observable.ontology }}"/> + {{ measurement.value }} diff --git a/edam/resources/flask_related/templates/sos/GetObservationException.xml b/edam/resources/flask_related/templates/sos/GetObservationException.xml index 770c0e4..c40d668 100644 --- a/edam/resources/flask_related/templates/sos/GetObservationException.xml +++ b/edam/resources/flask_related/templates/sos/GetObservationException.xml @@ -5,6 +5,6 @@ xml:lang="en"> - + {{ exception_text }} \ No newline at end of file diff --git a/edam/services/__init__.py b/edam/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edam/services/sos/__init__.py b/edam/services/sos/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/edam/services/sos/sos.py b/edam/services/sos/sos.py new file mode 100755 index 0000000..a0f5642 --- /dev/null +++ b/edam/services/sos/sos.py @@ -0,0 +1,150 @@ +import datetime + +from flask import render_template + +from edam.reader.models.measurement import Measurement +from edam import OGC_SOS_CONFIGURATION + + +class OgcSos: + def __init__(self, stations=None): + + self.stations = stations + self.info = OGC_SOS_CONFIGURATION + self.keywords = [f"{station.name}:{junction.observable.name}" for + station in stations for junction in station.junctions] + + def resolve_request(self, request=None, **kwargs): + """ + Determines the type or request and sets the appropriate template + """ + if request is None: + return + if request == "GetCapabilities": + return self.get_capabilities(**kwargs) + elif request == "DescribeSensor": + return self.describe_sensor(**kwargs) + elif request == "GetObservation": + return self.get_observation(**kwargs) + + def get_capabilities(self, **kwargs): + template = "sos/GetCapabilities.xml" + template_related = { + "template_name_or_list": template, + "info": self.info, + "keywords": self.keywords, + "stations": self.stations + } + return template_related + + def describe_sensor(self, **kwargs): + + procedure = kwargs['procedure'] + station_name, sensor_name = procedure.split(':') + station = filter(lambda st: st.name == station_name, self.stations) + try: + station = next(station) + except StopIteration: + template_related = { + "template_name_or_list": "sos/DescribeSensorException.xml", + "exception_text": f"*{procedure}* does not exist. " + f"*{station_name}* is not a station.", + } + return template_related + junction = filter(lambda sen: sen.sensor.name == sensor_name, + station.junctions) + try: + junction = next(junction) + except StopIteration: + template_related = { + "template_name_or_list": "sos/DescribeSensorException.xml", + "exception_text": f"*{procedure}* does not exist. " + f"*{sensor_name}* is not a sensor name.", + } + return template_related + + template_related = { + "template_name_or_list": "sos/DescribeSensor.xml", + "station": station, + "junction": junction, + } + return template_related + + def get_observation(self, **kwargs): + offering = kwargs['offering'] + station_name = offering + station = filter(lambda st: st.name == station_name, self.stations) + try: + station = next(station) + except StopIteration: + template_related = { + "template_name_or_list": "sos/GetObservationException.xml", + "exception_text": f"*{station_name}* station does not exist." + } + return template_related + observed_property = kwargs['observed_property'] + if observed_property is None: + template_related = { + "template_name_or_list": "sos/GetObservationException.xml", + "exception_text": f"observedProperty does not exist. " + } + return template_related + else: + junction = filter( + lambda sen: sen.observable.observable_id == observed_property, + station.junctions) + try: + junction = next(junction) + except StopIteration: + template_related = { + "template_name_or_list": "sos/GetObservationException.xml", + "exception_text": f"*{observed_property}* observed property" + f" does not exist." + } + return template_related + + # observed_property exists + event_time = kwargs['event_time'] + if event_time: + + start, end = event_time.split('/') + try: + start_date = datetime.datetime.strptime(start, + "%Y-%m-%d %H:%M:%S") + except ValueError: + start_exists = False + else: + start_exists = True + try: + end_date = datetime.datetime.strptime(end, + "%Y-%m-%d %H:%M:%S") + except ValueError: + end_exists = False + else: + end_exists = True + + if start_exists and end_exists: + raw = station.data.loc[lambda df: df.index >= start_date and df.index <= end_date,:] + elif start_exists and not end_exists: + raw = station.data.loc[lambda df: df.index >= start_date, :] + elif not start_exists and end_exists: + raw = station.data.loc[lambda df: df.index <= end_date, :] + + raw = raw[observed_property] + raw = raw.to_dict() + else: + raw = station.data[observed_property].to_dict() + + measurements = list() + + for timestamp, value in raw.items(): + measurements.append( + Measurement(timestamp=timestamp, value=value)) + + template_related = { + "template_name_or_list": "sos/GetObservation.xml", + "junction": junction, + "station": station, + "measurements": measurements + } + return template_related diff --git a/edam/viewer/app/OgcSos.py b/edam/viewer/app/OgcSos.py deleted file mode 100755 index a11ea42..0000000 --- a/edam/viewer/app/OgcSos.py +++ /dev/null @@ -1,102 +0,0 @@ -from edam.reader.models.measurement import Measurement -from edam import OGC_SOS_CONFIGURATION - - -class OgcSos: - def __init__(self, request, procedure=None, offering=None, - event_time=None, observed_property=None, page=None): - self.available_requests = [ - 'GetCapabilities', - 'DescribeSensor', - 'GetObservation'] - self.offering = offering - self.eventTime = event_time - self.observedProperty = observed_property - self.info = OGC_SOS_CONFIGURATION - self.keywords = list() - self.stations = list() - self.results = list() - self.metadata = list() - self.procedure = procedure - self.sensor = None - self.page = page - self.template = None - - self.exception = False - self.helper_object = None - - self.exceptionDetails = {} - - self.data = None - - if request in self.available_requests: - self.request = request - self.determine_request() - - def determine_request(self): - """ - Determines the type or request and sets the appropriate template - """ - if self.request == "GetCapabilities": - self.find_keywords() - self.find_stations() - self.find_metadata() - self.template = "sos/GetCapabilities.xml" - elif self.request == "DescribeSensor": - # self.procedure is like: station_name:sensor_name:template_id - - try: - station_id, sensor_id, template_id = self.procedure.split(':') - exists = self.data.get_helper_for_describe_sensor( - station_id=station_id, - sensor_id=sensor_id, - template_id=template_id) - if exists: - - self.sensor = exists - self.template = "sos/DescribeSensor.xml" - else: - self.template = "sos/DescribeSensorException.xml" - except BaseException: - self.template = "sos/DescribeSensorException.xml" - elif self.request == "GetObservation": - try: - station_id, sensor_id, template_id = self.procedure.split(':') - exists = self.data.get_helper_for_describe_sensor( - station_id=station_id, sensor_id=sensor_id, - template_id=template_id) - if exists: - # from_time, to_time = self.eventTime.split('/') - # from_time = pd.to_datetime(from_time) - # to_time = pd.to_datetime(to_time) - - self.helper_object = exists - results = self.data.get_observations_by_helper_id( - self.helper_object.id) - for row in results: - self.results.append(Measurement(value=row.value, - timestamp=row.timestamp, - observable=self.helper_object.observable, - uom=self.helper_object.uom, - station=self.helper_object.station, - helper=self.helper_object)) - # self.results = [Measurement(value=row.value, timestamp=row.timestamp) for row in results] - - self.template = "sos/GetObservation.xml" - else: - self.template = "sos/GetObservationException.xml" - except Exception as inst: - print(inst) - self.template = "sos/GetObservationException.xml" - - def find_keywords(self): - [self.keywords.append(quantity.name) - for quantity in self.data.get_all_observables()] - - def find_stations(self): - [self.stations.append(station) - for station in self.data.get_all_stations()] - - def find_metadata(self): - [self.metadata.append(helper) - for helper in self.data.get_all_helper_observable_ids()] diff --git a/edam/viewer/app/__init__.py b/edam/viewer/app/__init__.py index 3ddc003..c501131 100755 --- a/edam/viewer/app/__init__.py +++ b/edam/viewer/app/__init__.py @@ -79,25 +79,4 @@ def no_cache(*args, **kwargs): return update_wrapper(no_cache, view) -def hash(anything): - return '_' + hashlib.md5(anything).hexdigest() - - app.jinja_env.globals.update(hash=hash) - - -def d2s(datetime_object): - return datetime_object.strftime("%D %H %M") - - -app.jinja_env.globals.update(d2s=d2s) - - -def same_timestamp(*args): - if args: - return args[0] - else: - return None - - -app.jinja_env.globals.update(same_timestamp=same_timestamp) diff --git a/edam/viewer/app/views.py b/edam/viewer/app/views.py index 77da9b0..8085c31 100755 --- a/edam/viewer/app/views.py +++ b/edam/viewer/app/views.py @@ -15,7 +15,7 @@ from edam.utilities.exceptions import InvalidUsage from edam.viewer.app import app, stations, nocache, templates, \ render_data -from edam.viewer.app.OgcSos import OgcSos +from edam.services.sos.sos import OgcSos logger = get_logger('edam.viewer.app.views') @@ -209,39 +209,42 @@ def line_plotter(metrics, st): @app.route('/SensorObservationService/') def sensor_info(): - return render_template('SOS_Examples.html') + all_stations = stations() + if all_stations: + first_station = all_stations[0] # type: Station + station = first_station.name + + first_junction = first_station.junctions[0] + procedure = f"{station}:{first_junction.sensor.name}" + observed = f"{first_junction.observable.observable_id}" + else: + procedure = "STATION_NAME:SENSOR_NAME" + station = "STATION_NAME" + observed = "OBSERVABLE_ID" + example_describe = f"?request=DescribeSensor&procedure={procedure}" + example_observation = f"?request=GetObservation&offering={station}&observedProperty={observed}" + return render_template('SOS_Examples.html', + example_describe=example_describe, + example_observation=example_observation) @app.route('/sos/') @app.route('/Sos/') @app.route('/sos/') @app.route('/SOS/') -def sos(): - sos_request = request.args.get('request', '') - sos_procedure = request.args.get('procedure', '') - sos_offering = request.args.get('offering', '') - sos_observed_property = request.args.get('observedProperty', '') - sos_event_time = request.args.get('eventTime', '') - - sos = OgcSos( - sos_request, - sos_procedure, - sos_offering, - sos_event_time, - sos_observed_property) - sos_response = sos - - response = make_response( - render_template( - sos_response.template, - info=sos.info, - keywords=sos.keywords, - stations=sos.stations, - helpers=sos.metadata, - procedure=sos.procedure, - sensor=sos.sensor, - results=sos.results, - helper=sos.helper_object)) +def sensor_observation_service(): + sos = OgcSos(stations=stations()) + response = sos.resolve_request(request=request.args.get('request', None), + procedure=request.args.get('procedure', + None), + offering=request.args.get('offering', None), + observed_property=request.args.get( + 'observedProperty', + None), + event_time=request.args.get('eventTime', + None)) + + response = make_response(render_template(**response)) response.headers["Content-Type"] = "application/xml" return response diff --git a/tests/test_sos.py b/tests/test_sos.py new file mode 100644 index 0000000..ef89b62 --- /dev/null +++ b/tests/test_sos.py @@ -0,0 +1,25 @@ +import pytest + +from edam.services.sos.sos import OgcSos + + +@pytest.fixture +def sos_object() -> OgcSos: + pass + # return OgcSos() + + +def test_resolve_request_correct(): + assert True + + +def test_get_capabilities(): + assert True + + +def test_describe_sensor(): + assert True + + +def test_get_observation(): + assert True diff --git a/tests/tests/__init__.py b/tests/tests/__init__.py new file mode 100644 index 0000000..e69de29