Skip to content

Commit

Permalink
Remark tags cause Hoot to parse Overpass JSON incorrectly (#5555)
Browse files Browse the repository at this point in the history
* Fix error checking to ignore 'remark' tags and only catch 'remark' error codes
* Copyright
  • Loading branch information
bmarchant authored and jordanmurray35 committed Feb 1, 2023
1 parent adff894 commit 6b9987a
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,37 +46,51 @@ class ParallelBoundedApiReaderTest : public HootTestFixture
{
QString expected_result = "runtime error: Query ran out of memory in \"query\" at line 1. It would need at least 95 MB of RAM to continue.";
ParallelBoundedApiReader uut;
QString json_result =
QString json_error_result =
"{"
"\"version\": 0.6,"
"\"generator\": \"NOME Overpass API 0.7.57 c954ae26\","
"\"osm3s\": {"
" \"timestamp_osm_base\": \"2021-09-09T17:21:17Z\","
" \"copyright\": \"The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf\""
"\"timestamp_osm_base\": \"2021-09-09T17:21:17Z\","
"\"copyright\": \"The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf\""
"},"
"\"elements\": ["
"],"
"\"remark\": \"runtime error: Query ran out of memory in \"query\" at line 1. It would need at least 95 MB of RAM to continue.\""
"}";
QString error;
CPPUNIT_ASSERT(uut._isQueryError(json_result, error));
CPPUNIT_ASSERT(uut._isQueryError(json_error_result, error));
HOOT_STR_EQUALS(expected_result, error);

QString xml_result =
QString xml_error_result =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<osm version=\"0.6\" generator=\"NOME Overpass API 0.7.57 c954ae26\">"
"<note>The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf</note>"
"<meta osm_base=\"2021-09-09T17:21:17Z\"/>"
"<remark> runtime error: Query ran out of memory in \"query\" at line 1. It would need at least 95 MB of RAM to continue. </remark>"
"</osm>";
error = "";
CPPUNIT_ASSERT(uut._isQueryError(xml_result, error));
CPPUNIT_ASSERT(uut._isQueryError(xml_error_result, error));
HOOT_STR_EQUALS(expected_result, error);

/** This data caused infinite splitting in JOSM because of the 'remark' tag */
QString json_success =
"{"
"\"version\": 0.6,"
"\"generator\": \"NOME Overpass API 0.7.57 c954ae26\","
"\"osm3s\": {"
"\"timestamp_osm_base\": \"2023-01-20T20:40:19Z\","
"\"copyright\": \"The data included in this document is from NOME and OpenStreetMap. The data is made available under ODbL. data included in this document falls under NOME Terms of Use https://nome.vgihub.geointservices.io/disclaimer/NOME_Enclave_Disclaimer_V4.pdf\""
"},"
"\"elements\": ["
"\"{\"type\": \"node\",\"id\": 3767576613,\"lat\": 47.8026036,\"lon\": 18.9633523,\"tags\": {\"name\": \"Nagyne Marika\",\"remark\": \"2012-2014\"}}]}}";
error = "";
CPPUNIT_ASSERT(!uut._isQueryError(json_success, error));
HOOT_STR_EQUALS("", error);
}

};

CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(ParallelBoundedApiReaderTest, "quick");

}

2 changes: 1 addition & 1 deletion hoot-core/src/main/cpp/hoot/core/io/OgrWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -645,7 +645,7 @@ OGRLayer* OgrWriter::_getLayer(const QString& layerName)

void OgrWriter::_addFeature(OGRLayer* layer, const std::shared_ptr<Feature>& f, const std::shared_ptr<Geometry>& g) const
{
OGRFeature* poFeature = OGRFeature::CreateFeature( layer->GetLayerDefn() );
OGRFeature* poFeature = OGRFeature::CreateFeature(layer->GetLayerDefn());

// set all the column values.
const QVariantMap& vm = f->getValues();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -427,7 +427,7 @@ bool ParallelBoundedApiReader::_isQueryError(const QString& result, QString& err
static QRegularExpression regexXml("<remark> (.*) </remark>", QRegularExpression::OptimizeOnFirstUsageOption);
// JSON handling regular expression
// "remark": "runtime error: Query ran out of memory in \"query\" at line 10. It would need at least 0 MB of RAM to continue."
static QRegularExpression regexJson("[\"|\']remark[\"|\'|>]: *[\"|\']?(.*)[\"|\']", QRegularExpression::OptimizeOnFirstUsageOption);
static QRegularExpression regexJson("[\"|\']remark[\"|\'|>]: *[\"|\']?(runtime error.*)[\"|\']", QRegularExpression::OptimizeOnFirstUsageOption);
static std::vector<QRegularExpression> regexes({regexXml, regexJson});

for (const auto& regex : regexes)
Expand Down

0 comments on commit 6b9987a

Please sign in to comment.