From 522873e5ce48bb9cbd4e7e8168ca881ce709c222 Mon Sep 17 00:00:00 2001 From: Tom Kralidis Date: Tue, 18 Oct 2016 08:29:49 -0400 Subject: [PATCH] implement CQL to OGC filter transforms --- pycsw/cql.py | 114 ++++++++++++++++++ pycsw/server.py | 41 +++++-- ...etRecords-filter-cql-title-or-abstract.xml | 23 ++++ ...efault_get_GetRecords-filter-cql-title.xml | 23 ++++ ...post_GetRecords-cql-title-and-abstract.xml | 12 ++ tests/suites/default/get/requests.txt | 2 + .../GetRecords-cql-title-and-abstract.xml | 9 ++ 7 files changed, 213 insertions(+), 11 deletions(-) create mode 100644 pycsw/cql.py create mode 100644 tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml create mode 100644 tests/expected/suites_default_get_GetRecords-filter-cql-title.xml create mode 100644 tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml create mode 100644 tests/suites/default/post/GetRecords-cql-title-and-abstract.xml diff --git a/pycsw/cql.py b/pycsw/cql.py new file mode 100644 index 000000000..cd97ae2f0 --- /dev/null +++ b/pycsw/cql.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +# ================================================================= +# +# Authors: Tom Kralidis +# +# Copyright (c) 2016 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging + +from lxml import etree +from pycsw import util +from pycsw.fes import MODEL as fes1_model + +LOGGER = logging.getLogger(__name__) + + +def cql2fes1(cql, namespaces): + """transforms Common Query Language (CQL) query into OGC fes1 syntax""" + + filters = [] + tmp_list = [] + logical_op = None + + LOGGER.debug('CQL: %s', cql) + + if ' or ' in cql: + logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces)) + tmp_list = cql.split(' or ') + elif ' OR ' in cql: + logical_op = etree.Element(util.nspath_eval('ogc:Or', namespaces)) + tmp_list = cql.split(' OR ') + elif ' and ' in cql: + logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces)) + tmp_list = cql.split(' and ') + elif ' AND ' in cql: + logical_op = etree.Element(util.nspath_eval('ogc:And', namespaces)) + tmp_list = cql.split(' AND ') + + if tmp_list: + LOGGER.debug('Logical operator found (AND/OR)') + else: + tmp_list.append(cql) + + for t in tmp_list: + filters.append(_parse_condition(t)) + + root = etree.Element(util.nspath_eval('ogc:Filter', namespaces)) + + if logical_op is not None: + root.append(logical_op) + + for flt in filters: + condition = etree.Element(util.nspath_eval(flt[0], namespaces)) + + etree.SubElement( + condition, + util.nspath_eval('ogc:PropertyName', namespaces)).text = flt[1] + + etree.SubElement( + condition, + util.nspath_eval('ogc:Literal', namespaces)).text = flt[2] + + if logical_op is not None: + logical_op.append(condition) + else: + root.append(condition) + + LOGGER.debug('Resulting OGC Filter: %s', + etree.tostring(root, pretty_print=1)) + + return root + + +def _parse_condition(condition): + """parses a single condition""" + + LOGGER.debug('condition: %s', condition) + + property_name, operator, literal = condition.split() + + literal = literal.replace('"', '').replace('\'', '') + + for k, v in fes1_model['ComparisonOperators'].items(): + if v['opvalue'] == operator: + fes1_predicate = k + + LOGGER.debug('parsed condition: %s %s %s', property_name, fes1_predicate, + literal) + + return (fes1_predicate, property_name, literal) + diff --git a/pycsw/server.py b/pycsw/server.py index c68b7f3a0..11eb7d020 100644 --- a/pycsw/server.py +++ b/pycsw/server.py @@ -40,6 +40,7 @@ from pycsw.plugins.profiles import profile as pprofile import pycsw.plugins.outputschemas from pycsw import config, fes, log, metadata, util, sru, oaipmh, opensearch +from pycsw.cql import cql2fes1 import logging LOGGER = logging.getLogger(__name__) @@ -1281,12 +1282,20 @@ def getrecords(self): % self.kvp['constraintlanguage']) if self.kvp['constraintlanguage'] == 'CQL_TEXT': tmp = self.kvp['constraint'] - self.kvp['constraint'] = {} - self.kvp['constraint']['type'] = 'cql' - self.kvp['constraint']['where'] = \ - self._cql_update_queryables_mappings(tmp, - self.repository.queryables['_all']) - self.kvp['constraint']['values'] = {} + try: + LOGGER.debug('Transforming CQL into fes1') + LOGGER.debug('CQL: %s', tmp) + self.kvp['constraint'] = {} + self.kvp['constraint']['type'] = 'filter' + cql = cql2fes1(tmp, self.context.namespaces) + self.kvp['constraint']['where'], self.kvp['constraint']['values'] = fes.parse(cql, + self.repository.queryables['_all'], self.repository.dbtype, + self.context.namespaces, self.orm, self.language['text'], self.repository.fts) + except Exception as err: + LOGGER.error('Invalid CQL query %s', tmp) + LOGGER.error('Error message: %s', err, exc_info=True) + return self.exceptionreport('InvalidParameterValue', + 'constraint', 'Invalid Filter syntax') elif self.kvp['constraintlanguage'] == 'FILTER': # validate filter XML try: @@ -1364,8 +1373,10 @@ def getrecords(self): maxrecords=self.kvp['maxrecords'], startposition=int(self.kvp['startposition'])-1) except Exception as err: + LOGGER.debug('Invalid query syntax. Query: %s', self.kvp['constraint']) + LOGGER.debug('Invalid query syntax. Result: %s', err) return self.exceptionreport('InvalidParameterValue', 'constraint', - 'Invalid query: %s' % err) + 'Invalid query syntax') dsresults = [] @@ -2479,13 +2490,21 @@ def _parse_constraint(self, element): self.context.namespaces, self.orm, self.language['text'], self.repository.fts) except Exception as err: return 'Invalid Filter request: %s' % err + tmp = element.find(util.nspath_eval('csw:CqlText', self.context.namespaces)) if tmp is not None: LOGGER.debug('CQL specified: %s.' % tmp.text) - query['type'] = 'cql' - query['where'] = self._cql_update_queryables_mappings(tmp.text, - self.repository.queryables['_all']) - query['values'] = {} + try: + LOGGER.debug('Transforming CQL into OGC Filter') + query['type'] = 'filter' + cql = cql2fes1(tmp.text, self.context.namespaces) + query['where'], query['values'] = fes.parse(cql, + self.repository.queryables['_all'], self.repository.dbtype, + self.context.namespaces, self.orm, self.language['text'], self.repository.fts) + except Exception as err: + LOGGER.error('Invalid CQL request: %s', tmp.text) + LOGGER.error('Error message: %s', err, exc_info=True) + return 'Invalid CQL request' return query def _test_manager(self): diff --git a/tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml b/tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml new file mode 100644 index 000000000..30f9498a9 --- /dev/null +++ b/tests/expected/suites_default_get_GetRecords-filter-cql-title-or-abstract.xml @@ -0,0 +1,23 @@ + + + + + + + urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f + http://purl.org/dc/dcmitype/Image + image/svg+xml + Lorem ipsum + GR-22 + Tourism--Greece + Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu. + + + urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2 + http://purl.org/dc/dcmitype/Image + Lorem ipsum dolor sit amet + image/jpeg + IT-FI + + + diff --git a/tests/expected/suites_default_get_GetRecords-filter-cql-title.xml b/tests/expected/suites_default_get_GetRecords-filter-cql-title.xml new file mode 100644 index 000000000..30f9498a9 --- /dev/null +++ b/tests/expected/suites_default_get_GetRecords-filter-cql-title.xml @@ -0,0 +1,23 @@ + + + + + + + urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f + http://purl.org/dc/dcmitype/Image + image/svg+xml + Lorem ipsum + GR-22 + Tourism--Greece + Quisque lacus diam, placerat mollis, pharetra in, commodo sed, augue. Duis iaculis arcu vel arcu. + + + urn:uuid:a06af396-3105-442d-8b40-22b57a90d2f2 + http://purl.org/dc/dcmitype/Image + Lorem ipsum dolor sit amet + image/jpeg + IT-FI + + + diff --git a/tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml b/tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml new file mode 100644 index 000000000..6a9d17517 --- /dev/null +++ b/tests/expected/suites_default_post_GetRecords-cql-title-and-abstract.xml @@ -0,0 +1,12 @@ + + + + + + + urn:uuid:19887a8a-f6b0-4a63-ae56-7fba0e17801f + Lorem ipsum + http://purl.org/dc/dcmitype/Image + + + diff --git a/tests/suites/default/get/requests.txt b/tests/suites/default/get/requests.txt index 5897393c6..b039f845b 100644 --- a/tests/suites/default/get/requests.txt +++ b/tests/suites/default/get/requests.txt @@ -9,3 +9,5 @@ GetRecords-filter,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=C Exception-GetRepositoryItem-service-invalid1,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00&version=2.0.2&request=GetRepositoryItem&id=123 Exception-GetRepositoryItem-service-invalid2,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW%00'&version=2.0.2&request=GetRepositoryItem&id=123 Exception-GetRepositoryItem-version-invalid,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2'&request=GetRepositoryItem&id=123 +GetRecords-filter-cql-title,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27 +GetRecords-filter-cql-title-or-abstract,PYCSW_SERVER?config=tests/suites/default/default.cfg&service=CSW&version=2.0.2&request=GetRecords&typenames=csw:Record&elementsetname=full&resulttype=results&constraintlanguage=CQL_TEXT&constraint=dc%3Atitle%20like%20%27%25lor%25%27%20or%20dct%3Aabstract%20like%20%27%25pharetra%25%27 diff --git a/tests/suites/default/post/GetRecords-cql-title-and-abstract.xml b/tests/suites/default/post/GetRecords-cql-title-and-abstract.xml new file mode 100644 index 000000000..9bdb5795c --- /dev/null +++ b/tests/suites/default/post/GetRecords-cql-title-and-abstract.xml @@ -0,0 +1,9 @@ + + + + brief + + dc:title like '%ips%' and dct:abstract like '%pharetra%' + + +