From 6feae82fdd50279397167abca65c1555e3620dae Mon Sep 17 00:00:00 2001 From: Dimitry Kh Date: Sun, 22 Aug 2021 23:07:47 +0200 Subject: [PATCH] sort data only on export --- retesteth/TestSuite.cpp | 4 +- retesteth/dataObject/ConvertYaml.cpp | 4 +- retesteth/dataObject/DataObject.cpp | 65 ++++++++++++++++--- retesteth/dataObject/DataObject.h | 11 ++-- retesteth/testStructures/Common.cpp | 12 ++++ retesteth/testStructures/Common.h | 1 + .../types/Ethereum/TransactionLegacy.cpp | 1 + .../Base/TransactionInGeneralSection.cpp | 4 +- retesteth/testSuites/StateTests.cpp | 1 - retesteth/unitTests/dataObjectTests.cpp | 24 ++++++- 10 files changed, 105 insertions(+), 22 deletions(-) diff --git a/retesteth/TestSuite.cpp b/retesteth/TestSuite.cpp index 298203bd3..6661c5f90 100644 --- a/retesteth/TestSuite.cpp +++ b/retesteth/TestSuite.cpp @@ -51,7 +51,7 @@ TestFileData readTestFile(fs::path const& _testFileName) ETH_LOG("Read json structure " + string(_testFileName.filename().c_str()), 5); TestFileData testData; if (_testFileName.extension() == ".json") - testData.data = test::readJsonData(_testFileName, string(), true); + testData.data = test::readJsonData(_testFileName, string(), false); else if (_testFileName.extension() == ".yml") testData.data = test::readYamlData(_testFileName); else @@ -272,6 +272,7 @@ void TestSuite::runTestWithoutFiller(boost::filesystem::path const& _file) const opt.allowInvalidBlocks = true; spDataObject output = doTests(testData.data, opt); addClientInfo(output.getContent(), _file, testData.hash); + (*output).performModifier(mod_sortKeys); writeFile(outPath, asBytes(output->asJson())); } else @@ -634,6 +635,7 @@ void TestSuite::executeTest(string const& _testFolder, fs::path const& _testFile spDataObject output = doTests(testData.data, opt); // Add client info for all of the tests in output addClientInfo(output.getContent(), boostRelativeTestPath, testData.hash); + (*output).performModifier(mod_sortKeys); writeFile(boostTestPath.path(), asBytes(output->asJson())); if (!Options::get().getGStateTransactionFilter().empty()) diff --git a/retesteth/dataObject/ConvertYaml.cpp b/retesteth/dataObject/ConvertYaml.cpp index d8e370f71..2fdae6502 100644 --- a/retesteth/dataObject/ConvertYaml.cpp +++ b/retesteth/dataObject/ConvertYaml.cpp @@ -41,7 +41,7 @@ spDataObject ConvertYamlToData(YAML::Node const& _node) if (_node.IsMap()) { spDataObject jObject(new DataObject(DataType::Object)); - (*jObject).setAutosort(true); + //(*jObject).setAutosort(true); for (auto const& i : _node) (*jObject).addSubObject(i.first.as(), ConvertYamlToData(i.second)); return jObject; @@ -50,7 +50,7 @@ spDataObject ConvertYamlToData(YAML::Node const& _node) if (_node.IsSequence()) { spDataObject jArray(new DataObject(DataType::Array)); - (*jArray).setAutosort(true); + //(*jArray).setAutosort(true); for (size_t i = 0; i < _node.size(); i++) (*jArray).addArrayObject(ConvertYamlToData(_node[i])); return jArray; diff --git a/retesteth/dataObject/DataObject.cpp b/retesteth/dataObject/DataObject.cpp index 9f102a081..936db8060 100644 --- a/retesteth/dataObject/DataObject.cpp +++ b/retesteth/dataObject/DataObject.cpp @@ -1,7 +1,8 @@ #include +#include #include -#include #include +#include using namespace dataobject; /// Default dataobject is null @@ -128,7 +129,8 @@ bool DataObject::asBool() const /// Set position in vector of the subobject with _key void DataObject::setKeyPos(std::string const& _key, size_t _pos) { - _assert(_pos < m_subObjects.size(), "_pos < m_subObjects.size()"); + //_assert(_pos < m_subObjects.size(), "DataObject::setKeyPos(`" + _key + "`, `" + to_string(_pos) + "`) _pos < + //m_subObjects.size() "); _assert(count(_key), "count(_key) _key = " + _key + " (DataObject::setKeyPos)"); _assert(!_key.empty(), "!_key.empty() (DataObject::setKeyPos)"); @@ -148,7 +150,10 @@ void DataObject::setKeyPos(std::string const& _key, size_t _pos) setOverwrite(true); spDataObject data = m_subObjects.at(elementPos); m_subObjects.erase(m_subObjects.begin() + elementPos); - m_subObjects.insert(m_subObjects.begin() + _pos, 1, data); + if (_pos >= m_subObjects.size()) + m_subObjects.push_back(data); + else + m_subObjects.insert(m_subObjects.begin() + _pos, 1, data); setOverwrite(false); } @@ -185,9 +190,9 @@ DataObjectK& DataObjectK::operator=(spDataObject const& _value) if (m_data.count(m_key)) { m_data.removeKey(m_key); - if(_value.getCContent().getKey().empty()) + if (_value->getKey().empty()) throw DataObjectException("DataObjectK::operator=(spDataObject const& _value) _value without key, but key required!"); - m_data.addSubObject(_value.getCContent().getKey(), _value); + m_data.addSubObject(_value->getKey(), _value); } else m_data.addSubObject(m_key, _value); @@ -518,24 +523,64 @@ std::string DataObject::dataTypeAsString(DataType _type) size_t dataobject::findOrderedKeyPosition(string const& _key, vector const& _objects) { + static size_t used = 0; + used++; + std::cerr << used << std::endl; + if (_objects.size() == 0) return 0; - size_t step = _objects.size() / 2; + + size_t m = 0; + int L = 0; + int R = _objects.size() - 1; + while (L <= R) + { + m = floor((L + R) / 2); + if (_objects.at(m)->getKey() < _key) + L = m + 1; + else if (_objects.at(m)->getKey() > _key) + R = m - 1; + else + return m; + } + + if (_objects.at(m)->getKey() > _key) + return m; + else + return m + 1; + + /*size_t step = floor((double)_objects.size() / 2); size_t guess = step; - while (step > 0) + while (step > 0 && guess > 0 && guess < _objects.size()) { - step = step / 2; if (_objects.at(guess)->getKey() > _key) + { + step = floor((double)(guess) / 2); guess -= std::max(step, (size_t)1); + } else + { + step = floor((double)(_objects.size() - guess) / 2); guess += std::max(step, (size_t)1); + } } + if (guess == _objects.size()) + { + std::cerr << _key << " - " << guess << std::endl; return guess; - guess = max(0, (int)guess - 5); - while (guess < _objects.size() && _objects.at(guess)->getKey() <= _key) + } + + if (_objects.at(guess)->getKey() <= _key) guess++; + + // //guess = max(0, (int)guess - 5); + // while (guess < _objects.size() && _objects.at(guess)->getKey() <= _key) + // guess++; + + std::cerr << _key << " - " << guess << std::endl; return guess; +*/ } DataObject& DataObject::_addSubObject(spDataObject const& _obj, string const& _keyOverwrite) diff --git a/retesteth/dataObject/DataObject.h b/retesteth/dataObject/DataObject.h index 56d395d91..8a2fcfa91 100644 --- a/retesteth/dataObject/DataObject.h +++ b/retesteth/dataObject/DataObject.h @@ -112,8 +112,11 @@ class DataObject : public GCP_SPointerBase DataObject& _addSubObject(spDataObject const& _obj, string const& _keyOverwrite = string()); void _assert(bool _flag, std::string const& _comment = "") const; + // Use vector here to be able to quickly find insert position + // of objects to be ordered by it's key with findOrderedKeyPosition std::vector m_subObjects; std::map m_subObjectKeys; + DataType m_type; std::string m_strKey; bool m_allowOverwrite = false; // allow overwrite elements @@ -128,7 +131,7 @@ class DataObject : public GCP_SPointerBase typedef GCP_SPointer spDataObject; -// The key assigner +// The key assigner, assign left pointer to DataObjectK's host m_data[key] class DataObjectK { public: @@ -145,7 +148,7 @@ class DataObjectK // Can help to keep incapsulation -// DataObject move requester +// DataObject move requester, require the memory pointer to be irreversably moved into it class spDataObjectMove { public: @@ -158,10 +161,10 @@ class spDataObjectMove spDataObject m_obj; }; -// Move +// Move memory from _obj to spDataObjectMove and flush _obj pointer spDataObjectMove move(spDataObject& _obj); -// TODO refactor to use linked lists so no need to insert into vector // Find index that _key should take place in when being added to ordered _objects by key +// Heavy function, use only on export when need to construct json with sorted keys size_t findOrderedKeyPosition(string const& _key, vector const& _objects); } diff --git a/retesteth/testStructures/Common.cpp b/retesteth/testStructures/Common.cpp index 4a066e5c1..7aff59f8e 100644 --- a/retesteth/testStructures/Common.cpp +++ b/retesteth/testStructures/Common.cpp @@ -166,6 +166,18 @@ void mod_valueInsertZeroXPrefix(DataObject& _obj) _obj.asStringUnsafe().insert(0, "0x"); } +void mod_sortKeys(DataObject& _obj) +{ + std::map const map = _obj.getSubObjectKeys(); + if (map.size() > 1) + { + _obj.clearSubobjects(); + _obj.setAutosort(true); + for (auto const& el : map) + _obj.atKeyPointer(el.second->getKey()) = el.second; + } +} + long long int hexOrDecStringToInt(string const& _str) { long long int res; diff --git a/retesteth/testStructures/Common.h b/retesteth/testStructures/Common.h index 9ac005d5b..5028e3bd8 100644 --- a/retesteth/testStructures/Common.h +++ b/retesteth/testStructures/Common.h @@ -22,6 +22,7 @@ void mod_keyToLowerCase(DataObject&); void mod_valueToCompactEvenHexPrefixed(DataObject&); void mod_keyToCompactEvenHexPrefixed(DataObject&); void mod_valueInsertZeroXPrefix(DataObject&); +void mod_sortKeys(DataObject&); long long int hexOrDecStringToInt(string const& _str); // Check the presents of fields in a DataObject with a validation map diff --git a/retesteth/testStructures/types/Ethereum/TransactionLegacy.cpp b/retesteth/testStructures/types/Ethereum/TransactionLegacy.cpp index 338692f38..2966f287d 100644 --- a/retesteth/testStructures/types/Ethereum/TransactionLegacy.cpp +++ b/retesteth/testStructures/types/Ethereum/TransactionLegacy.cpp @@ -165,6 +165,7 @@ void TransactionLegacy::buildVRS(VALUE const& _secret) spDataObject TransactionLegacy::asDataObject(ExportOrder _order) const { + // TODO Memory cache? spDataObject out(new DataObject()); (*out)["data"] = m_data->asString(); (*out)["gasLimit"] = m_gasLimit->asString(); diff --git a/retesteth/testStructures/types/StateTests/Base/TransactionInGeneralSection.cpp b/retesteth/testStructures/types/StateTests/Base/TransactionInGeneralSection.cpp index 41ad7640f..3411cd3b9 100644 --- a/retesteth/testStructures/types/StateTests/Base/TransactionInGeneralSection.cpp +++ b/retesteth/testStructures/types/StateTests/Base/TransactionInGeneralSection.cpp @@ -12,8 +12,8 @@ TransactionInGeneralSection::TransactionInGeneralSection( m_skipped(false) { m_tr = readTransaction(_tr); - m_tr.getContent().setDataLabel(_dataLabel); - m_tr.getContent().setDataRawPreview(_dataRawPreview); + (*m_tr).setDataLabel(_dataLabel); + (*m_tr).setDataRawPreview(_dataRawPreview); } void TransactionInGeneralSection::assignTransactionLabel(string const& _label) diff --git a/retesteth/testSuites/StateTests.cpp b/retesteth/testSuites/StateTests.cpp index 0542c8ec9..a96c5df44 100644 --- a/retesteth/testSuites/StateTests.cpp +++ b/retesteth/testSuites/StateTests.cpp @@ -213,7 +213,6 @@ spDataObject FillTestAsBlockchain(StateTestInFiller const& _test) spDataObject FillTest(StateTestInFiller const& _test) { spDataObject filledTest(new DataObject()); - (*filledTest).setAutosort(true); TestOutputHelper::get().setCurrentTestName(_test.testName()); SessionInterface& session = RPCSession::instance(TestOutputHelper::getThreadID()); diff --git a/retesteth/unitTests/dataObjectTests.cpp b/retesteth/unitTests/dataObjectTests.cpp index e17b7a964..9da47186d 100644 --- a/retesteth/unitTests/dataObjectTests.cpp +++ b/retesteth/unitTests/dataObjectTests.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include using namespace std; @@ -30,6 +31,25 @@ using namespace dataobject; BOOST_FIXTURE_TEST_SUITE(DataObjectTestSuite, TestOutputHelperFixture) +BOOST_AUTO_TEST_CASE(dataobject_sort) +{ + DataObject obj; + obj.setAutosort(true); + obj["indexes"] = "1"; + obj["hash"] = "2"; + obj["txbytes"] = "3"; + obj["logs"] = "4"; + + DataObject obj2; + obj2["indexes"] = "1"; + obj2["hash"] = "2"; + obj2["txbytes"] = "3"; + obj2["logs"] = "4"; + obj2.performModifier(mod_sortKeys); + + BOOST_CHECK_EQUAL(obj.asJson(0, false), obj2.asJson(0, false)); +} + BOOST_AUTO_TEST_CASE(dataobject_bracers) { string data = R"( @@ -507,7 +527,6 @@ BOOST_AUTO_TEST_CASE(dataobject_readJson15) BOOST_CHECK(dObj->asJson(0, false) == res); } - BOOST_AUTO_TEST_CASE(dataobject_findOrderedKeyPosition_before1_of3) { string const key = "aab0"; @@ -787,6 +806,7 @@ BOOST_AUTO_TEST_CASE(dataobject_jsonOrder) data["aa2"] = "2"; data["aa70"] = "7"; data["aa8"] = "8"; + std::cerr << data.asJson(0) << std::endl; BOOST_CHECK(data.asJson(0, false) == "{\"aa1\":\"1\",\"aa2\":\"2\",\"aa3\":\"3\",\"aa31\":\"3\",\"aa5\":\"5\",\"aa7\":" "\"7\",\"aa70\":\"7\",\"aa8\":\"8\"}"); @@ -855,9 +875,9 @@ BOOST_AUTO_TEST_CASE(dataobject_besuresponse) } })"; spDataObject dObj = ConvertJsoncppStringToData(data, string(), true); - string const expectedParse = R"({"result":{"transactions":[{"blockHash":"0xac7b82af234ef01bf4d24a3b9c22c2de091c6f71ec04d51ff23bd780533d999f","blockNumber":"0x1","chainId":null,"from":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b","gas":"0x7a120","gasPrice":"0xa","hash":"0x225117089dee26945644798e2c64d3117f55c95c7cf5509f7176de4b3af5202d","input":"0x604b80600c6000396000f3007c01000000000000000000000000000000000000000000000000000000006000350463cbf0b0c08114602d57005b60006004358073ffffffffffffffffffffffffffffffffffffffff16ff","nonce":"0x0","publicKey":"0x3a514176466fa815ed481ffad09110a2d344f6c9b78c1d14afc351c3a51be33d8072e77939dc03ba44790779b7a1025baf3003f6732430e20cd9b76d953391b3","r":"0xe7d3c664c49aa9f5ce4eb76c8547450466262a78bd093160f492ea0853c68e9","raw":"0xf8a5800a8307a1208081ffb857604b80600c6000396000f3007c01000000000000000000000000000000000000000000000000000000006000350463cbf0b0c08114602d57005b60006004358073ffffffffffffffffffffffffffffffffffffffff16ff1ca00e7d3c664c49aa9f5ce4eb76c8547450466262a78bd093160f492ea0853c68e9a03f843e72210ff1da4fd9e375339872bcf0fad05c014e280ffc755e173700dd62","s":"0x3f843e72210ff1da4fd9e375339872bcf0fad05c014e280ffc755e173700dd62","to":null,"transactionIndex":"0x0","v":"0x1c","value":"0xff"}]}})"; + BOOST_CHECK(dObj->asJson(0, false) == expectedParse); }