From 6b9987ad871dce8d7446f9fc90f8715bd691a1bc Mon Sep 17 00:00:00 2001 From: Ben Marchant <13385275+bmarchant@users.noreply.github.com> Date: Mon, 23 Jan 2023 16:36:02 -0600 Subject: [PATCH] `Remark` tags cause Hoot to parse Overpass JSON incorrectly (#5555) * Fix error checking to ignore 'remark' tags and only catch 'remark' error codes * Copyright --- .../core/io/ParallelBoundedApiReaderTest.cpp | 28 ++++++++++++++----- .../src/main/cpp/hoot/core/io/OgrWriter.cpp | 2 +- .../hoot/core/io/ParallelBoundedApiReader.cpp | 2 +- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/hoot-core-test/src/test/cpp/hoot/core/io/ParallelBoundedApiReaderTest.cpp b/hoot-core-test/src/test/cpp/hoot/core/io/ParallelBoundedApiReaderTest.cpp index caa9329d22..a10422ffbf 100644 --- a/hoot-core-test/src/test/cpp/hoot/core/io/ParallelBoundedApiReaderTest.cpp +++ b/hoot-core-test/src/test/cpp/hoot/core/io/ParallelBoundedApiReaderTest.cpp @@ -46,23 +46,23 @@ class ParallelBoundedApiReaderTest : public HootTestFixture { QString expected_result = "runtime error: Query ran out of memory in \"query\" at line 1. It would need at least 95 MB of RAM to continue."; ParallelBoundedApiReader uut; - QString json_result = + QString json_error_result = "{" "\"version\": 0.6," "\"generator\": \"NOME Overpass API 0.7.57 c954ae26\"," "\"osm3s\": {" - " \"timestamp_osm_base\": \"2021-09-09T17:21:17Z\"," - " \"copyright\": \"The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf\"" + "\"timestamp_osm_base\": \"2021-09-09T17:21:17Z\"," + "\"copyright\": \"The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf\"" "}," "\"elements\": [" "]," "\"remark\": \"runtime error: Query ran out of memory in \"query\" at line 1. It would need at least 95 MB of RAM to continue.\"" "}"; QString error; - CPPUNIT_ASSERT(uut._isQueryError(json_result, error)); + CPPUNIT_ASSERT(uut._isQueryError(json_error_result, error)); HOOT_STR_EQUALS(expected_result, error); - QString xml_result = + QString xml_error_result = "" "" "The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf" @@ -70,8 +70,23 @@ class ParallelBoundedApiReaderTest : public HootTestFixture " runtime error: Query ran out of memory in \"query\" at line 1. It would need at least 95 MB of RAM to continue. " ""; error = ""; - CPPUNIT_ASSERT(uut._isQueryError(xml_result, error)); + CPPUNIT_ASSERT(uut._isQueryError(xml_error_result, error)); HOOT_STR_EQUALS(expected_result, error); + + /** This data caused infinite splitting in JOSM because of the 'remark' tag */ + QString json_success = + "{" + "\"version\": 0.6," + "\"generator\": \"NOME Overpass API 0.7.57 c954ae26\"," + "\"osm3s\": {" + "\"timestamp_osm_base\": \"2023-01-20T20:40:19Z\"," + "\"copyright\": \"The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf\"" + "}," + "\"elements\": [" + "\"{\"type\": \"node\",\"id\": 3767576613,\"lat\": 47.8026036,\"lon\": 18.9633523,\"tags\": {\"name\": \"Nagyne Marika\",\"remark\": \"2012-2014\"}}]}}"; + error = ""; + CPPUNIT_ASSERT(!uut._isQueryError(json_success, error)); + HOOT_STR_EQUALS("", error); } }; @@ -79,4 +94,3 @@ class ParallelBoundedApiReaderTest : public HootTestFixture CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(ParallelBoundedApiReaderTest, "quick"); } - diff --git a/hoot-core/src/main/cpp/hoot/core/io/OgrWriter.cpp b/hoot-core/src/main/cpp/hoot/core/io/OgrWriter.cpp index f8bffa29f4..1cd128bf15 100644 --- a/hoot-core/src/main/cpp/hoot/core/io/OgrWriter.cpp +++ b/hoot-core/src/main/cpp/hoot/core/io/OgrWriter.cpp @@ -645,7 +645,7 @@ OGRLayer* OgrWriter::_getLayer(const QString& layerName) void OgrWriter::_addFeature(OGRLayer* layer, const std::shared_ptr& f, const std::shared_ptr& g) const { - OGRFeature* poFeature = OGRFeature::CreateFeature( layer->GetLayerDefn() ); + OGRFeature* poFeature = OGRFeature::CreateFeature(layer->GetLayerDefn()); // set all the column values. const QVariantMap& vm = f->getValues(); diff --git a/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.cpp b/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.cpp index 1e8dbfdbea..8a02c9cc05 100644 --- a/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.cpp +++ b/hoot-core/src/main/cpp/hoot/core/io/ParallelBoundedApiReader.cpp @@ -427,7 +427,7 @@ bool ParallelBoundedApiReader::_isQueryError(const QString& result, QString& err static QRegularExpression regexXml(" (.*) ", QRegularExpression::OptimizeOnFirstUsageOption); // JSON handling regular expression // "remark": "runtime error: Query ran out of memory in \"query\" at line 10. It would need at least 0 MB of RAM to continue." - static QRegularExpression regexJson("[\"|\']remark[\"|\'|>]: *[\"|\']?(.*)[\"|\']", QRegularExpression::OptimizeOnFirstUsageOption); + static QRegularExpression regexJson("[\"|\']remark[\"|\'|>]: *[\"|\']?(runtime error.*)[\"|\']", QRegularExpression::OptimizeOnFirstUsageOption); static std::vector regexes({regexXml, regexJson}); for (const auto& regex : regexes)