sort data only on export

retesteth/TestSuite.cpp

@@ -51,7 +51,7 @@ TestFileData readTestFile(fs::path const& _testFileName)
     ETH_LOG("Read json structure " + string(_testFileName.filename().c_str()), 5);
     TestFileData testData;
     if (_testFileName.extension() == ".json")
-        testData.data = test::readJsonData(_testFileName, string(), true);
+        testData.data = test::readJsonData(_testFileName, string(), false);
     else if (_testFileName.extension() == ".yml")
         testData.data = test::readYamlData(_testFileName);
     else
@@ -272,6 +272,7 @@ void TestSuite::runTestWithoutFiller(boost::filesystem::path const& _file) const
                     opt.allowInvalidBlocks = true;
                     spDataObject output = doTests(testData.data, opt);
                     addClientInfo(output.getContent(), _file, testData.hash);
+                    (*output).performModifier(mod_sortKeys);
                     writeFile(outPath, asBytes(output->asJson()));
                 }
                 else
@@ -634,6 +635,7 @@ void TestSuite::executeTest(string const& _testFolder, fs::path const& _testFile
                     spDataObject output = doTests(testData.data, opt);
                     // Add client info for all of the tests in output
                     addClientInfo(output.getContent(), boostRelativeTestPath, testData.hash);
+                    (*output).performModifier(mod_sortKeys);
                     writeFile(boostTestPath.path(), asBytes(output->asJson()));
 
                     if (!Options::get().getGStateTransactionFilter().empty())

retesteth/dataObject/ConvertYaml.cpp

@@ -41,7 +41,7 @@ spDataObject ConvertYamlToData(YAML::Node const& _node)
     if (_node.IsMap())
     {
         spDataObject jObject(new DataObject(DataType::Object));
-        (*jObject).setAutosort(true);
+        //(*jObject).setAutosort(true);
         for (auto const& i : _node)
             (*jObject).addSubObject(i.first.as<string>(), ConvertYamlToData(i.second));
         return jObject;
@@ -50,7 +50,7 @@ spDataObject ConvertYamlToData(YAML::Node const& _node)
     if (_node.IsSequence())
     {
         spDataObject jArray(new DataObject(DataType::Array));
-        (*jArray).setAutosort(true);
+        //(*jArray).setAutosort(true);
         for (size_t i = 0; i < _node.size(); i++)
             (*jArray).addArrayObject(ConvertYamlToData(_node[i]));
         return jArray;

retesteth/dataObject/DataObject.cpp

@@ -1,7 +1,8 @@
 #include <dataObject/DataObject.h>
+#include <math.h>
 #include <algorithm>
-#include <sstream>
 #include <iostream>
+#include <sstream>
 using namespace dataobject;
 
 /// Default dataobject is null
@@ -128,7 +129,8 @@ bool DataObject::asBool() const
 /// Set position in vector of the subobject with _key
 void DataObject::setKeyPos(std::string const& _key, size_t _pos)
 {
-    _assert(_pos < m_subObjects.size(), "_pos < m_subObjects.size()");
+    //_assert(_pos < m_subObjects.size(), "DataObject::setKeyPos(`" + _key + "`, `" + to_string(_pos) + "`) _pos <
+    //m_subObjects.size() ");
     _assert(count(_key), "count(_key) _key = " + _key + " (DataObject::setKeyPos)");
     _assert(!_key.empty(), "!_key.empty() (DataObject::setKeyPos)");
 
@@ -148,7 +150,10 @@ void DataObject::setKeyPos(std::string const& _key, size_t _pos)
     setOverwrite(true);
     spDataObject data = m_subObjects.at(elementPos);
     m_subObjects.erase(m_subObjects.begin() + elementPos);
-    m_subObjects.insert(m_subObjects.begin() + _pos, 1, data);
+    if (_pos >= m_subObjects.size())
+        m_subObjects.push_back(data);
+    else
+        m_subObjects.insert(m_subObjects.begin() + _pos, 1, data);
     setOverwrite(false);
 }
 
@@ -185,9 +190,9 @@ DataObjectK& DataObjectK::operator=(spDataObject const& _value)
     if (m_data.count(m_key))
     {
         m_data.removeKey(m_key);
-        if(_value.getCContent().getKey().empty())
+        if (_value->getKey().empty())
             throw DataObjectException("DataObjectK::operator=(spDataObject const& _value)  _value without key, but key required!");
-        m_data.addSubObject(_value.getCContent().getKey(), _value);
+        m_data.addSubObject(_value->getKey(), _value);
     }
     else
         m_data.addSubObject(m_key, _value);
@@ -518,24 +523,64 @@ std::string DataObject::dataTypeAsString(DataType _type)
 
 size_t dataobject::findOrderedKeyPosition(string const& _key, vector<spDataObject> const& _objects)
 {
+    static size_t used = 0;
+    used++;
+    std::cerr << used << std::endl;
+
     if (_objects.size() == 0)
         return 0;
-    size_t step = _objects.size() / 2;
+
+    size_t m = 0;
+    int L = 0;
+    int R = _objects.size() - 1;
+    while (L <= R)
+    {
+        m = floor((L + R) / 2);
+        if (_objects.at(m)->getKey() < _key)
+            L = m + 1;
+        else if (_objects.at(m)->getKey() > _key)
+            R = m - 1;
+        else
+            return m;
+    }
+
+    if (_objects.at(m)->getKey() > _key)
+        return m;
+    else
+        return m + 1;
+
+    /*size_t step =  floor((double)_objects.size() / 2);
     size_t guess = step;
-    while (step > 0)
+    while (step > 0 && guess > 0 && guess < _objects.size())
     {
-        step = step / 2;
         if (_objects.at(guess)->getKey() > _key)
+        {
+            step = floor((double)(guess) / 2);
             guess -= std::max(step, (size_t)1);
+        }
         else
+        {
+            step = floor((double)(_objects.size() - guess) / 2);
             guess += std::max(step, (size_t)1);
+        }
     }
+
     if (guess == _objects.size())
+    {
+        std::cerr << _key << " - " << guess << std::endl;
         return guess;
-    guess = max(0, (int)guess - 5);
-    while (guess < _objects.size() && _objects.at(guess)->getKey() <= _key)
+    }
+
+    if (_objects.at(guess)->getKey() <= _key)
         guess++;
+
+   // //guess = max(0, (int)guess - 5);
+  //  while (guess < _objects.size() && _objects.at(guess)->getKey() <= _key)
+  //      guess++;
+
+    std::cerr << _key << " - " << guess << std::endl;
     return guess;
+*/
 }
 
 DataObject& DataObject::_addSubObject(spDataObject const& _obj, string const& _keyOverwrite)

retesteth/dataObject/DataObject.h

@@ -112,8 +112,11 @@ class DataObject : public GCP_SPointerBase
     DataObject& _addSubObject(spDataObject const& _obj, string const& _keyOverwrite = string());
     void _assert(bool _flag, std::string const& _comment = "") const;
 
+    // Use vector here to be able to quickly find insert position
+    // of objects to be ordered by it's key with findOrderedKeyPosition
     std::vector<spDataObject> m_subObjects;
     std::map<string, spDataObject> m_subObjectKeys;
+
     DataType m_type;
     std::string m_strKey;
     bool m_allowOverwrite = false;  // allow overwrite elements
@@ -128,7 +131,7 @@ class DataObject : public GCP_SPointerBase
 
 typedef GCP_SPointer<DataObject> spDataObject;
 
-// The key assigner
+// The key assigner, assign left pointer to DataObjectK's host m_data[key]
 class DataObjectK
 {
 public:
@@ -145,7 +148,7 @@ class DataObjectK
 
 
 // Can help to keep incapsulation
-// DataObject move requester
+// DataObject move requester, require the memory pointer to be irreversably moved into it
 class spDataObjectMove
 {
 public:
@@ -158,10 +161,10 @@ class spDataObjectMove
     spDataObject m_obj;
 };
 
-// Move
+// Move memory from _obj to spDataObjectMove and flush _obj pointer
 spDataObjectMove move(spDataObject& _obj);
 
-// TODO refactor to use linked lists so no need to insert into vector
 // Find index that _key should take place in when being added to ordered _objects by key
+// Heavy function, use only on export when need to construct json with sorted keys
 size_t findOrderedKeyPosition(string const& _key, vector<spDataObject> const& _objects);
 }

retesteth/testStructures/Common.cpp

@@ -166,6 +166,18 @@ void mod_valueInsertZeroXPrefix(DataObject& _obj)
         _obj.asStringUnsafe().insert(0, "0x");
 }
 
+void mod_sortKeys(DataObject& _obj)
+{
+    std::map<string, spDataObject> const map = _obj.getSubObjectKeys();
+    if (map.size() > 1)
+    {
+        _obj.clearSubobjects();
+        _obj.setAutosort(true);
+        for (auto const& el : map)
+            _obj.atKeyPointer(el.second->getKey()) = el.second;
+    }
+}
+
 long long int hexOrDecStringToInt(string const& _str)
 {
     long long int res;

retesteth/testStructures/Common.h

@@ -22,6 +22,7 @@ void mod_keyToLowerCase(DataObject&);
 void mod_valueToCompactEvenHexPrefixed(DataObject&);
 void mod_keyToCompactEvenHexPrefixed(DataObject&);
 void mod_valueInsertZeroXPrefix(DataObject&);
+void mod_sortKeys(DataObject&);
 long long int hexOrDecStringToInt(string const& _str);
 
 // Check the presents of fields in a DataObject with a validation map

retesteth/testStructures/types/Ethereum/TransactionLegacy.cpp

@@ -165,6 +165,7 @@ void TransactionLegacy::buildVRS(VALUE const& _secret)
 
 spDataObject TransactionLegacy::asDataObject(ExportOrder _order) const
 {
+    // TODO Memory cache?
     spDataObject out(new DataObject());
     (*out)["data"] = m_data->asString();
     (*out)["gasLimit"] = m_gasLimit->asString();

retesteth/testStructures/types/StateTests/Base/TransactionInGeneralSection.cpp

@@ -12,8 +12,8 @@ TransactionInGeneralSection::TransactionInGeneralSection(
     m_skipped(false)
 {
     m_tr = readTransaction(_tr);
-    m_tr.getContent().setDataLabel(_dataLabel);
-    m_tr.getContent().setDataRawPreview(_dataRawPreview);
+    (*m_tr).setDataLabel(_dataLabel);
+    (*m_tr).setDataRawPreview(_dataRawPreview);
 }
 
 void TransactionInGeneralSection::assignTransactionLabel(string const& _label)

retesteth/testSuites/StateTests.cpp

@@ -213,7 +213,6 @@ spDataObject FillTestAsBlockchain(StateTestInFiller const& _test)
 spDataObject FillTest(StateTestInFiller const& _test)
 {
     spDataObject filledTest(new DataObject());
-    (*filledTest).setAutosort(true);
     TestOutputHelper::get().setCurrentTestName(_test.testName());
 
     SessionInterface& session = RPCSession::instance(TestOutputHelper::getThreadID());

retesteth/unitTests/dataObjectTests.cpp

@@ -21,6 +21,7 @@
 #include <dataObject/ConvertFile.h>
 #include <dataObject/DataObject.h>
 #include <retesteth/TestOutputHelper.h>
+#include <retesteth/testSuites/Common.h>
 #include <boost/test/unit_test.hpp>
 
 using namespace std;
@@ -30,6 +31,25 @@ using namespace dataobject;
 
 BOOST_FIXTURE_TEST_SUITE(DataObjectTestSuite, TestOutputHelperFixture)
 
+BOOST_AUTO_TEST_CASE(dataobject_sort)
+{
+    DataObject obj;
+    obj.setAutosort(true);
+    obj["indexes"] = "1";
+    obj["hash"] = "2";
+    obj["txbytes"] = "3";
+    obj["logs"] = "4";
+
+    DataObject obj2;
+    obj2["indexes"] = "1";
+    obj2["hash"] = "2";
+    obj2["txbytes"] = "3";
+    obj2["logs"] = "4";
+    obj2.performModifier(mod_sortKeys);
+
+    BOOST_CHECK_EQUAL(obj.asJson(0, false), obj2.asJson(0, false));
+}
+
 BOOST_AUTO_TEST_CASE(dataobject_bracers)
 {
     string data = R"(
@@ -507,7 +527,6 @@ BOOST_AUTO_TEST_CASE(dataobject_readJson15)
     BOOST_CHECK(dObj->asJson(0, false) == res);
 }
 
-
 BOOST_AUTO_TEST_CASE(dataobject_findOrderedKeyPosition_before1_of3)
 {
     string const key = "aab0";
@@ -787,6 +806,7 @@ BOOST_AUTO_TEST_CASE(dataobject_jsonOrder)
     data["aa2"] = "2";
     data["aa70"] = "7";
     data["aa8"] = "8";
+    std::cerr << data.asJson(0) << std::endl;
     BOOST_CHECK(data.asJson(0, false) ==
                 "{\"aa1\":\"1\",\"aa2\":\"2\",\"aa3\":\"3\",\"aa31\":\"3\",\"aa5\":\"5\",\"aa7\":"
                 "\"7\",\"aa70\":\"7\",\"aa8\":\"8\"}");
@@ -855,9 +875,9 @@ BOOST_AUTO_TEST_CASE(dataobject_besuresponse)
           }
         })";
     spDataObject dObj = ConvertJsoncppStringToData(data, string(), true);
-
     string const expectedParse =
         R"({"result":{"transactions":[{"blockHash":"0xac7b82af234ef01bf4d24a3b9c22c2de091c6f71ec04d51ff23bd780533d999f","blockNumber":"0x1","chainId":null,"from":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b","gas":"0x7a120","gasPrice":"0xa","hash":"0x225117089dee26945644798e2c64d3117f55c95c7cf5509f7176de4b3af5202d","input":"0x604b80600c6000396000f3007c01000000000000000000000000000000000000000000000000000000006000350463cbf0b0c08114602d57005b60006004358073ffffffffffffffffffffffffffffffffffffffff16ff","nonce":"0x0","publicKey":"0x3a514176466fa815ed481ffad09110a2d344f6c9b78c1d14afc351c3a51be33d8072e77939dc03ba44790779b7a1025baf3003f6732430e20cd9b76d953391b3","r":"0xe7d3c664c49aa9f5ce4eb76c8547450466262a78bd093160f492ea0853c68e9","raw":"0xf8a5800a8307a1208081ffb857604b80600c6000396000f3007c01000000000000000000000000000000000000000000000000000000006000350463cbf0b0c08114602d57005b60006004358073ffffffffffffffffffffffffffffffffffffffff16ff1ca00e7d3c664c49aa9f5ce4eb76c8547450466262a78bd093160f492ea0853c68e9a03f843e72210ff1da4fd9e375339872bcf0fad05c014e280ffc755e173700dd62","s":"0x3f843e72210ff1da4fd9e375339872bcf0fad05c014e280ffc755e173700dd62","to":null,"transactionIndex":"0x0","v":"0x1c","value":"0xff"}]}})";
+
     BOOST_CHECK(dObj->asJson(0, false) == expectedParse);
 }