Skip to content

Commit

Permalink
Thematic and FCODE GeoJSON output is not translated (#5517)
Browse files Browse the repository at this point in the history 
* Translate GeoJSON that has schema translation script applied and ignore empty fields in output.
* Allow for translating of single files also.
* Updated unit test output for translated features
  • Loading branch information
@bmarchant
bmarchant authored Dec 6, 2022
1 parent 7e5a59d commit 8835748
Show file tree
Hide file tree
Showing 55 changed files with 6,413 additions and 4,279 deletions.
33 changes: 30 additions & 3 deletions hoot-core-test/src/test/cpp/hoot/core/io/OsmGeoJsonWriterTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class OsmGeoJsonWriterTest : public HootTestFixture
CPPUNIT_TEST(runAllDataTypesTest);
CPPUNIT_TEST(runDcTigerTest);
CPPUNIT_TEST(runBostonSubsetRoadBuildingTest);
CPPUNIT_TEST(runBostonSubsetRoadBuildingTranslationTest);
CPPUNIT_TEST(runSplitThematicBostonRoadBuildingTest);
CPPUNIT_TEST(runSplitFcodeBostonRoadBuildingTest);
CPPUNIT_TEST(runObjectGeoJsonTest);
Expand Down Expand Up @@ -78,6 +79,24 @@ class OsmGeoJsonWriterTest : public HootTestFixture
Log::getInstance().setLevel(logLevel);
}

void runBostonSubsetRoadBuildingTranslationTest()
{
// Suppress the warning from the OsmXmlReader about missing nodes for ways by temporarily changing
// the log level. We expect the nodes to be missing since the Boston data has issues
Log::WarningLevel logLevel = Log::getInstance().getLevel();
if (Log::getInstance().getLevel() >= Log::Info)
Log::getInstance().setLevel(Log::Error);
// Set global settings
conf().set(ConfigOptions::getOgrAddUuidKey(), false);
// Set local settings
Settings s;
s.set(ConfigOptions::getOgrAddUuidKey(), false);
s.set(ConfigOptions::getSchemaTranslationScriptKey(), "translations/TDSv70.js");

runTest("test-files/BostonSubsetRoadBuilding_FromOsm.osm", "BostonSubsetRoadBuilding-Translated.geojson", &s);
Log::getInstance().setLevel(logLevel);
}

void runSplitThematicBostonRoadBuildingTest()
{
// The Boston data splits out into 18 different thematic files
Expand Down Expand Up @@ -108,10 +127,15 @@ class OsmGeoJsonWriterTest : public HootTestFixture
Log::WarningLevel logLevel = Log::getInstance().getLevel();
if (Log::getInstance().getLevel() >= Log::Info)
Log::getInstance().setLevel(Log::Error);
// Set global settings
conf().set(ConfigOptions::getOgrAddUuidKey(), false);
// Set local settings
Settings s;
s.set(ConfigOptions::getGeojsonWriteSplitFileStructureKey(), true);
s.set(ConfigOptions::getWriterThematicStructureKey(), true);
s.set(ConfigOptions::getOgrAddUuidKey(), false);
s.set(ConfigOptions::getSchemaTranslationScriptKey(), "translations/TDSv70.js");

runTest("test-files/BostonSubsetRoadBuilding_FromOsm.osm", "BostonSubsetRoadBuildingSplit.geojson", &s, multi_files);
Log::getInstance().setLevel(logLevel);
}
Expand Down Expand Up @@ -149,7 +173,6 @@ class OsmGeoJsonWriterTest : public HootTestFixture
"TRAFFIC_LIGHT_P",
"TRAIL_C",
"TRANSPORTATION_STATION_P",
"UNKNOWN_C",
"VEHICLE_LOT_S",
"WALL_C"
});
Expand All @@ -160,12 +183,16 @@ class OsmGeoJsonWriterTest : public HootTestFixture
Log::WarningLevel logLevel = Log::getInstance().getLevel();
if (Log::getInstance().getLevel() >= Log::Info)
Log::getInstance().setLevel(Log::Error);

// Set global settings
conf().set(ConfigOptions::getWriterThematicStructureKey(), false);
conf().set(ConfigOptions::getOgrAddUuidKey(), false);
// Set local settings
Settings s;
s.set(ConfigOptions::getGeojsonWriteSplitFileStructureKey(), true);
s.set(ConfigOptions::getWriterThematicStructureKey(), false);
conf().set(ConfigOptions::getWriterThematicStructureKey(), false);
s.set(ConfigOptions::getOgrAddUuidKey(), false);
s.set(ConfigOptions::getSchemaTranslationScriptKey(), "translations/GGDMv30.js");

runTest("test-files/BostonSubsetRoadBuilding_FromOsm.osm", "BostonSubsetRoadBuildingFcode.geojson", &s, multi_files);
Log::getInstance().setLevel(logLevel);
}
Expand Down
162 changes: 97 additions & 65 deletions hoot-core/src/main/cpp/hoot/core/io/OsmGeoJsonWriter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,8 @@ void OsmGeoJsonWriter::open(const QString& url)
{
// Open the writer
_writer.open(url);
// Initialize the translator for splitting files
if (_writeSplitFiles)
// Initialize the translator if we have one
if (!_scriptPath.isEmpty())
initTranslator();
}

Expand Down Expand Up @@ -118,30 +118,37 @@ void OsmGeoJsonWriter::setConfiguration(const Settings& conf)
_writeHootFormat = false;
// Set the MultiFileWriter settings
MultiFileWriter::MultiFileWriterType t = MultiFileWriter::SingleFile;
bool try_split = options.getGeojsonWriteSplitFileStructure();

if (options.getGeojsonWriteSplitFileStructure())
// Thematic structure requires a translator
QString script = options.getSchemaTranslationScript();
if (script.isEmpty())
{
// Thematic structure requires a translator
QString script = options.getSchemaTranslationScript();
if (script.isEmpty())
if (try_split)
{
LOG_ERROR("OsmGeoJsonWriter requires schema translation script when used with '" <<
ConfigOptions::getGeojsonWriteSplitFileStructureKey() << "'. Reverting to single file output.");
}
else
}
else
{
try
{
try
// Set the Translation
SchemaUtils::validateTranslationUrl(script);
setSchemaTranslationScript(script);
// Translated files shouldn't include circular error values
setIncludeCircularError(false);
// Spliting files requires a translation
if (try_split)
{
// Set the Translation
SchemaUtils::validateTranslationUrl(script);
setSchemaTranslationScript(script);
t = MultiFileWriter::MultiThematic;
_writeSplitFiles = true;
}
catch (const IllegalArgumentException& e)
{
LOG_ERROR(e.getWhat() << " Reverting to single file output.");
}
}
catch (const IllegalArgumentException& e)
{
LOG_ERROR(e.getWhat() << " Reverting to single file output.");
}
}
_writer.setWriterType(t);
Expand Down Expand Up @@ -183,12 +190,13 @@ void OsmGeoJsonWriter::_writeNodes()

for (auto node_id : nids)
{
ConstNodePtr n = _map->getNode(node_id);
_setWriterIndex(n);
ConstNodePtr n = std::dynamic_pointer_cast<const Node>(_translateElement(_map->getNode(node_id)));
if (!n)
continue;
if (_writer.isCurrentIndexWritten())
_write(",", true);

_writeNode(n);
_writeElement(n);

_numWritten++;
if (_numWritten % _statusUpdateInterval == 0)
Expand All @@ -203,8 +211,8 @@ void OsmGeoJsonWriter::_writeWays()
const WayMap& ways = _map->getWays();
for (auto it = ways.begin(); it != ways.end(); ++it)
{
ConstWayPtr w = it->second;
if (w.get() == nullptr)
ConstWayPtr w = std::dynamic_pointer_cast<const Way>(_translateElement(it->second));
if (!w)
continue;
// Skip any ways that have parents
set<ElementId> parents = _map->getParents(w->getElementId());
Expand All @@ -217,8 +225,9 @@ void OsmGeoJsonWriter::_writeWays()
{
for (auto node_id : nodes)
{
// No need to translate the node, it is for validation only
ConstNodePtr node = _map->getNode(node_id);
if (node.get() == nullptr)
if (!node)
{
valid = false;
break;
Expand All @@ -228,22 +237,20 @@ void OsmGeoJsonWriter::_writeWays()
// Write out the way in geojson if valid
if (valid)
{
_setWriterIndex(w);
if (_writer.isCurrentIndexWritten())
_write(",", true);
_writeWay(w);
_writeElement(w);
}
else
{
for (auto node_id : nodes)
{
ConstNodePtr node = _map->getNode(node_id);
if (node.get() != nullptr)
ConstNodePtr node = std::dynamic_pointer_cast<const Node>(_translateElement(_map->getNode(node_id)));
if (!node)
{
_setWriterIndex(node);
if (_writer.isCurrentIndexWritten())
_write(",", true);
_writeNode(node);
_writeElement(node);
}
}
}
Expand All @@ -261,20 +268,14 @@ void OsmGeoJsonWriter::_writeRelations()
const RelationMap& relations = _map->getRelations();
for (auto it = relations.begin(); it != relations.end(); ++it)
{
ConstRelationPtr r = it->second;

_setWriterIndex(r);
ConstRelationPtr r = std::dynamic_pointer_cast<const Relation>(_translateElement(it->second));
if (!r)
continue;

if (_writer.isCurrentIndexWritten())
_write(",", true);
// Write out the relation (and all children) in geojson
_write("{");
_writeFeature(r);
_write(",");
_write("\"geometry\": {");
_writeGeometry(r);
_write("}");
_write("}", false);

_writeElement(r);

_numWritten++;
if (_numWritten % (_statusUpdateInterval) == 0)
Expand Down Expand Up @@ -313,7 +314,7 @@ void OsmGeoJsonWriter::_writeGeometry(ConstRelationPtr r)
for (const auto& member : members)
{
ConstElementPtr e = _map->getElement(member.getElementId());
if (e.get() == nullptr)
if (!e)
continue;
if (first)
first = false;
Expand Down Expand Up @@ -413,34 +414,20 @@ void OsmGeoJsonWriter::_writeFeature(ConstElementPtr e)
}
}

void OsmGeoJsonWriter::_writeNode(ConstNodePtr node)
void OsmGeoJsonWriter::_writeElement(ConstElementPtr element)
{
if (node.get() == nullptr)
if (!element)
return;
// Write out the element in geojson
_write("{");
_writeFeature(node);
_writeFeature(element);
_write(",");
_write("\"geometry\": {");
_writeGeometry(node);
_writeGeometry(element);
_write("}");
_write("}", false);
}

void OsmGeoJsonWriter::_writeWay(ConstWayPtr way)
{
if (way.get() == nullptr)
return;
// Write out the way in geojson
_write("{");
_writeFeature(way);
_write(",");
_write("\"geometry\": {");
_writeGeometry(way);
_write("}");
_write("}", false);

}

void OsmGeoJsonWriter::_writeRelationInfo(ConstRelationPtr r)
{
_writeKvp("relation-type", r->getType());
Expand All @@ -463,7 +450,7 @@ string OsmGeoJsonWriter::_buildRoles(ConstRelationPtr r, bool& first)
for (const auto& member : members)
{
ConstElementPtr e = _map->getElement(member.getElementId());
if (e.get() == nullptr)
if (!e)
continue;
if (first)
first = false;
Expand All @@ -477,18 +464,63 @@ string OsmGeoJsonWriter::_buildRoles(ConstRelationPtr r, bool& first)
return ss.str();
}

void OsmGeoJsonWriter::_setWriterIndex(const ConstElementPtr& e)
ConstElementPtr OsmGeoJsonWriter::_translateElement(const ConstElementPtr& e)
{
// Don't translate without a translator
if (!_translator)
return e;
// Translate the element and get the
ElementPtr c = e->clone();
std::shared_ptr<geos::geom::Geometry> geometry;
std::vector<ScriptToOgrSchemaTranslator::TranslatedFeature> feature;
ElementProviderPtr provider(std::const_pointer_cast<ElementProvider>(std::dynamic_pointer_cast<const ElementProvider>(_map)));
// Translate the feature
translateToFeatures(provider, e, geometry, feature);
if (feature.empty())
return e;
// Convert the feature values to tags for output
Tags t;
const QVariantMap& vm = feature[0].feature->getValues();
for (auto it = vm.constBegin(); it != vm.constEnd(); ++it)
{
const QVariant& v = it.value();
QByteArray ba = it.key().toUtf8();
// Convert the variant values, don't insert "empty" values (i.e. -999999 or 'No Information')
switch (v.type())
{
case QVariant::Invalid:
break;
case QVariant::Int:
if (v.toInt() != -999999)
t.set(ba.constData(), v.toInt());
break;
case QVariant::LongLong:
if (v.toLongLong() != -999999)
t.set(ba.constData(), QString::number(v.toLongLong()));
break;
case QVariant::Double:
if (v.toDouble() != -999999.0)
t.set(ba.constData(), v.toDouble());
break;
case QVariant::String:
{
QString value = v.toString();
if (value != "No Information" && value != "noInformation" && !value.isEmpty())
t.set(ba.constData(), v.toString());
break;
}
default:
LOG_WARN("Can't convert the provided value into an OGR value. (" << v.toString() << ")");
break;
}
}
c->setTags(t);
if (_writeSplitFiles)
{
std::shared_ptr<geos::geom::Geometry> geometry;
std::vector<ScriptToOgrSchemaTranslator::TranslatedFeature> feature;
ElementProviderPtr provider(std::const_pointer_cast<ElementProvider>(std::dynamic_pointer_cast<const ElementProvider>(_map)));
// Translate the feature
translateToFeatures(provider, e, geometry, feature);
QString layer = _getLayerName(feature, geometry);
_writer.setCurrentFileIndex(layer);
}
return c;
}

QString OsmGeoJsonWriter::_getLayerName(const std::vector<ScriptToOgrSchemaTranslator::TranslatedFeature>& feature,
Expand Down
18 changes: 5 additions & 13 deletions hoot-core/src/main/cpp/hoot/core/io/OsmGeoJsonWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,15 +95,10 @@ class OsmGeoJsonWriter : public OsmJsonWriter, public TranslationInterface
*/
QString _getBbox() const;
/**
* @brief _writeNode Writes a single node; metadata, tags, and geometry
* @param node
*/
void _writeNode(ConstNodePtr node);
/**
* @brief _writeWay Writes a single way; metadata, tags, and geometry
* @param way
* @brief _writeNode Writes a single element; metadata, tags, and geometry
* @param element
*/
void _writeWay(ConstWayPtr way);
void _writeElement(ConstElementPtr element);
/**
* @brief _writeRelationInfo Writes relation specific information, relation-type and roles
* @param relation
Expand Down Expand Up @@ -155,11 +150,8 @@ class OsmGeoJsonWriter : public OsmJsonWriter, public TranslationInterface
* @return Semicolon separated list of roles
*/
std::string _buildRoles(ConstRelationPtr relation, bool& first);
/**
* @brief _setWriterIndex Set the writer index for MultiFile objects
* @param e Element dictating the index
*/
void _setWriterIndex(const ConstElementPtr& e) override;

ConstElementPtr _translateElement(const ConstElementPtr& e);

QString _getLayerName(const std::vector<ScriptToOgrSchemaTranslator::TranslatedFeature>& feature,
const std::shared_ptr<geos::geom::Geometry>& geometry) const;
Expand Down
Loading

0 comments on commit 8835748

Please sign in to comment.