state test fillers parsing memory WIP

retesteth/dataObject/DataObject.cpp

@@ -52,6 +52,7 @@ void DataObject::setKey(std::string const& _key) { m_strKey = _key; }
 
 /// Get key of the dataobject
 std::string const& DataObject::getKey() const { return m_strKey; }
+std::string& DataObject::getKeyUnsafe() { return m_strKey; }
 
 /// Get vector of subobjects
 std::vector<spDataObject> const& DataObject::getSubObjects() const
@@ -104,6 +105,11 @@ std::string const& DataObject::asString() const
     _assert(m_type == DataType::String, "m_type == DataType::String (DataObject::asString())");
     return m_strVal;
 }
+std::string& DataObject::asStringUnsafe()
+{
+    _assert(m_type == DataType::String, "m_type == DataType::String (DataObject::asStringUnsafe())");
+    return m_strVal;
+}
 
 /// Get int value
 int DataObject::asInt() const

retesteth/dataObject/DataObject.h

@@ -39,6 +39,7 @@ class DataObject : public GCP_SPointerBase
     DataType type() const;
     void setKey(std::string const& _key);
     std::string const& getKey() const;
+    std::string& getKeyUnsafe();
 
     std::vector<spDataObject> const& getSubObjects() const;
     std::map<string, spDataObject> const& getSubObjectKeys() const;
@@ -54,6 +55,8 @@ class DataObject : public GCP_SPointerBase
     bool count(std::string const& _key) const;
 
     std::string const& asString() const;
+    std::string& asStringUnsafe();
+
     int asInt() const;
     bool asBool() const;
 

retesteth/testStructures/Common.cpp

@@ -47,14 +47,22 @@ namespace test
 {
 namespace teststruct
 {
-// DataObject modifiers. Convert all json values to lower case
-void mod_valuesToLowerCase(DataObject& _obj)
+// DataObject modifiers
+void mod_valueToLowerCase(DataObject& _obj)
 {
     if (_obj.type() == DataType::String)
     {
-        string value = _obj.asString();
+        string& value = _obj.asStringUnsafe();
+        std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); });
+    }
+}
+
+void mod_keyToLowerCase(DataObject& _obj)
+{
+    if (!_obj.getKey().empty())
+    {
+        string& value = _obj.getKeyUnsafe();
         std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); });
-        _obj = value;
     }
 }
 
@@ -97,6 +105,8 @@ void mod_keyToCompactEvenHexPrefixed(DataObject& _obj)
     {
         if (!(_obj.getKey()[0] == '0' && _obj.getKey()[1] == 'x'))
             _obj.setKey(toCompactHexPrefixed(_obj.getKey(), 1));
+        if (_obj.getKey().size() == 3)
+            _obj.getKeyUnsafe().insert(2, "0");
     }
     catch (std::exception const& _ex)
     {
@@ -249,7 +259,7 @@ void requireJsonFields(DataObject const& _o, std::string const& _config, std::ma
 
 // Compile LLL in code
 // Convert dec fields to hex, add 0x prefix to accounts and storage keys
-void convertDecStateToHex(spDataObject& _data, solContracts const& _preSolidity)
+void convertDecStateToHex(spDataObject& _data, solContracts const& _preSolidity, StateToHex _compileCode)
 {
     // -- Compile LLL in pre state into byte code if not already
     // -- Convert State::Storage keys/values into hex
@@ -258,13 +268,27 @@ void convertDecStateToHex(spDataObject& _data, solContracts const& _preSolidity)
         DataObject& acc = acc2.getContent();
         if (acc.getKey()[1] != 'x')
             acc.setKey("0x" + acc.getKey());
-        acc["code"].setString(test::compiler::replaceCode(acc.atKey("code").asString(), _preSolidity));
-        acc["nonce"].performModifier(mod_valueToCompactEvenHexPrefixed);
-        acc["balance"].performModifier(mod_valueToCompactEvenHexPrefixed);
-        for (auto& rec : acc["storage"].getSubObjectsUnsafe())
+        acc.performModifier(mod_keyToLowerCase);
+
+        if (acc.count("code") && _compileCode == StateToHex::COMPILECODE)
+            acc["code"].setString(test::compiler::replaceCode(acc.atKey("code").asString(), _preSolidity));
+        if (acc.count("code") && acc.atKey("code").asString().empty())
+            acc["code"] = "0x" + acc.atKey("code").asString();
+
+        if (acc.count("nonce"))
+            acc["nonce"].performModifier(mod_valueToCompactEvenHexPrefixed);
+        if (acc.count("balance"))
+            acc["balance"].performModifier(mod_valueToCompactEvenHexPrefixed);
+        if (acc.count("storage"))
         {
-            rec.getContent().performModifier(mod_keyToCompactEvenHexPrefixed);
-            rec.getContent().performModifier(mod_valueToCompactEvenHexPrefixed);
+            for (auto& rec : acc["storage"].getSubObjectsUnsafe())
+            {
+                rec.getContent().performModifier(mod_keyToCompactEvenHexPrefixed);
+                rec.getContent().performModifier(mod_valueToCompactEvenHexPrefixed);
+                rec.getContent().performModifier(mod_removeLeadingZerosFromHexValuesEVEN);
+                rec.getContent().performModifier(mod_valueToLowerCase);
+                rec.getContent().performModifier(mod_keyToLowerCase);
+            }
         }
     }
 }

retesteth/testStructures/Common.h

@@ -16,7 +16,8 @@ void mod_removeLeadingZerosFromHexValues(DataObject&);
 void mod_removeLeadingZerosFromHexValuesEVEN(DataObject&);
 void mod_removeLeadingZerosFromHexKeysEVEN(DataObject&);
 void mod_removeComments(DataObject& _obj);
-void mod_valuesToLowerCase(DataObject&);
+void mod_valueToLowerCase(DataObject&);
+void mod_keyToLowerCase(DataObject&);
 void mod_valueToCompactEvenHexPrefixed(DataObject&);
 void mod_keyToCompactEvenHexPrefixed(DataObject&);
 long long int hexOrDecStringToInt(string const& _str);
@@ -45,7 +46,13 @@ void requireJsonFields(
 
 // Compile LLL in code, solidity in code
 // Convert dec fields to hex, add 0x prefix to accounts and storage keys
-void convertDecStateToHex(spDataObject& _data, solContracts const& _preSolidity = solContracts());
+enum class StateToHex
+{
+    COMPILECODE,
+    NOCOMPILECODE
+};
+void convertDecStateToHex(
+    spDataObject& _data, solContracts const& _preSolidity = solContracts(), StateToHex _compileCode = StateToHex::COMPILECODE);
 
 // Convert dec fields to hex, add 0x prefix to accounts and storage keys
 spDataObject convertDecBlockheaderIncompleteToHex(DataObject const& _data);

retesteth/testStructures/types/BlockchainTests/BlockchainTestFiller.cpp

@@ -9,6 +9,15 @@ BlockchainTestInFiller::BlockchainTestInFiller(spDataObject& _data)
 {
     try
     {
+        requireJsonFields(_data, "BlockchainTestInFiller " + _data->getKey(),
+            {{"_info", {{DataType::Object}, jsonField::Optional}},
+                {"sealEngine", {{DataType::String}, jsonField::Optional}},
+                {"genesisBlockHeader", {{DataType::Object}, jsonField::Required}},
+                {"expect", {{DataType::Array}, jsonField::Required}},
+                {"exceptions", {{DataType::Array}, jsonField::Optional}},
+                {"pre", {{DataType::Object}, jsonField::Required}},
+                {"blocks", {{DataType::Array}, jsonField::Required}}});
+
         m_hasAtLeastOneUncle = false;
         m_name = _data->getKey();
         if (_data->count("_info"))
@@ -37,10 +46,9 @@ BlockchainTestInFiller::BlockchainTestInFiller(spDataObject& _data)
 
         // Process expect section
         std::set<FORK> knownForks;
-        for (auto const& el2 : _data->atKey("expect").getSubObjects())
+        for (auto& el2 : (*_data).atKeyUnsafe("expect").getSubObjectsUnsafe())
         {
-            DataObject const& el = el2.getCContent();
-            m_expects.push_back(el);
+            m_expects.push_back(el2);
             BlockchainTestFillerExpectSection const& expect = m_expects.at(m_expects.size() - 1);
             for (auto const& fork : expect.forks())
             {
@@ -65,15 +73,6 @@ BlockchainTestInFiller::BlockchainTestInFiller(spDataObject& _data)
             if (m_blocks.at(m_blocks.size() - 1).uncles().size() > 0)
                 m_hasAtLeastOneUncle = true;
         }
-
-        requireJsonFields(_data, "BlockchainTestInFiller " + _data->getKey(),
-            {{"_info", {{DataType::Object}, jsonField::Optional}},
-                {"sealEngine", {{DataType::String}, jsonField::Optional}},
-                {"genesisBlockHeader", {{DataType::Object}, jsonField::Required}},
-                {"expect", {{DataType::Array}, jsonField::Required}},
-                {"exceptions", {{DataType::Array}, jsonField::Optional}},
-                {"pre", {{DataType::Object}, jsonField::Required}},
-                {"blocks", {{DataType::Array}, jsonField::Required}}});
     }
     catch (std::exception const& _ex)
     {

retesteth/testStructures/types/BlockchainTests/Filler/BlockchainTestFillerExpectSection.cpp

@@ -7,22 +7,25 @@ namespace test
 {
 namespace teststruct
 {
-BlockchainTestFillerExpectSection::BlockchainTestFillerExpectSection(DataObject const& _data)
+BlockchainTestFillerExpectSection::BlockchainTestFillerExpectSection(spDataObject& _data)
 {
     try
     {
+        requireJsonFields(_data, "BlockchainTestFillerExpectSection " + _data->getKey(),
+            {{"network", {{DataType::Array, DataType::String}, jsonField::Required}},
+                {"result", {{DataType::Object}, jsonField::Required}}});
+
         // get allowed networks for this expect section
         std::set<string> forks;
-        parseJsonStrValueIntoSet(_data.atKey("network"), forks);
+        parseJsonStrValueIntoSet(_data->atKey("network"), forks);
         ETH_ERROR_REQUIRE_MESSAGE(forks.size() > 0, "Expect section `network` is empty!");
 
         // Parse >=Frontier into  Frontier, Homestead, ... Constantinople according to current config
         ClientConfig const& cfg = Options::get().getDynamicOptions().getCurrentConfig();
         m_forks = cfg.translateNetworks(forks);
-        m_result = GCP_SPointer<StateIncomplete>(new StateIncomplete(_data.atKey("result"), DataRequier::ALLOWDEC));
-        requireJsonFields(_data, "BlockchainTestFillerExpectSection " + _data.getKey(),
-            {{"network", {{DataType::Array, DataType::String}, jsonField::Required}},
-                {"result", {{DataType::Object}, jsonField::Required}}});
+
+        convertDecStateToHex((*_data).atKeyPointerUnsafe("result"), solContracts(), StateToHex::NOCOMPILECODE);
+        m_result = GCP_SPointer<StateIncomplete>(new StateIncomplete(MOVE(_data, "result")));
     }
     catch (std::exception const& _ex)
     {

retesteth/testStructures/types/BlockchainTests/Filler/BlockchainTestFillerExpectSection.h

@@ -12,7 +12,7 @@ namespace teststruct
 {
 struct BlockchainTestFillerExpectSection
 {
-    BlockchainTestFillerExpectSection(DataObject const&);
+    BlockchainTestFillerExpectSection(spDataObject&);
     StateIncomplete const& result() const { return m_result; }
     std::vector<FORK> const& forks() const { return m_forks; }
     bool hasFork(FORK const& _fork) const

retesteth/testStructures/types/Ethereum/State.cpp

@@ -56,6 +56,12 @@ bool State::hasAccount(FH20 const& _address) const
 
 spDataObject State::asDataObject(ExportOrder _order) const
 {
+    // As long as we guarantee unmutability of parsed data in the structure
+    // We can return the same data object as we got, not recalculating the whole thing
+    if (!m_raw.isEmpty())
+        return m_raw;
+
+    // TODO move initialization into constructor???
     spDataObject out(new DataObject());
     if (_order == ExportOrder::OldStyle)
     {

retesteth/testStructures/types/Ethereum/StateIncomplete.cpp

@@ -7,49 +7,17 @@ namespace test
 {
 namespace teststruct
 {
-StateIncomplete::StateIncomplete(DataObject const& _data, DataRequier _req)
+StateIncomplete::StateIncomplete(spDataObjectMove _data)
 {
+    m_rawData = _data.getPointer();
     try
     {
-        // The difference to Common.h::ConvertDecStateToHex is that here we don't compile code
-        if (_req == DataRequier::ALLOWDEC)
-        {
-            // Convertion is here so not to repeat convertion in State and Blockchain tests
-            // Convert Expect Section IncompletePostState fields to hex, account keys add `0x` prefix
-            // Code field add `0x` prefix, storage key:value add `0x` prefix, coverted to hex
-            spDataObject tmpD(new DataObject());
-            (*tmpD).copyFrom(_data);
-            for (auto& acc2 : (*tmpD).getSubObjectsUnsafe())
-            {
-                DataObject& acc = acc2.getContent();
-                string const& key = acc2->getKey();
-                if ((key.size() > 2 && (key[0] != '0' || key[1] != 'x')))
-                    acc.setKey("0x" + acc.getKey());
-                if (acc.count("balance"))
-                    acc["balance"].performModifier(mod_valueToCompactEvenHexPrefixed);
-                if (acc.count("nonce"))
-                    acc["nonce"].performModifier(mod_valueToCompactEvenHexPrefixed);
-                if (acc.count("code") && acc.atKey("code").asString().empty())
-                    acc["code"] = "0x" + acc.atKey("code").asString();
-                if (acc.count("storage"))
-                    for (auto& rec : acc["storage"].getSubObjectsUnsafe())
-                    {
-                        rec.getContent().performModifier(mod_keyToCompactEvenHexPrefixed);
-                        rec.getContent().performModifier(mod_valueToCompactEvenHexPrefixed);
-                    }
-            }
-            for (auto const& el : tmpD->getSubObjects())
-                m_accounts[FH20(el->getKey())] = spAccountBase(new AccountIncomplete(el));
-        }
-        else
-        {
-            for (auto const& el : _data.getSubObjects())
-                m_accounts[FH20(el->getKey())] = spAccountBase(new AccountIncomplete(el));
-        }
+        for (auto const& el : m_rawData->getSubObjects())
+            m_accounts[FH20(el->getKey())] = spAccountBase(new AccountIncomplete(el));
     }
     catch (std::exception const& _ex)
     {
-        throw UpwardsException(string("StateIncomplete parse error: ") + _ex.what() + _data.asJson());
+        throw UpwardsException(string("StateIncomplete parse error: ") + _ex.what() + m_rawData->asJson());
     }
 }
 

retesteth/testStructures/types/Ethereum/StateIncomplete.h

@@ -10,20 +10,17 @@ namespace test
 {
 namespace teststruct
 {
-enum class DataRequier
-{
-    ALLOWDEC,
-    ONLYHEX
-};
 // Marks that State is made of AccountIncomplete
 struct StateIncomplete : StateBase
 {
-    StateIncomplete(DataObject const&, DataRequier req = DataRequier::ONLYHEX);
+    StateIncomplete(spDataObjectMove);
     void correctMiningReward(FH20 const& _coinbase, VALUE const& _reward);
     spDataObject asDataObject(ExportOrder _order = ExportOrder::Default) const override;
+    spDataObject const& rawData() const { return m_rawData; }
 
 private:
     StateIncomplete(){};
+    spDataObject m_rawData;
 };
 
 typedef GCP_SPointer<StateIncomplete> spStateIncomplete;

retesteth/testStructures/types/StateTests/Filler/StateTestFillerExpectSection.cpp

@@ -79,45 +79,44 @@ spDataObject ReplaceValueToIndexesInDataList(spStateTestFillerTransaction const&
     return dataIndexes;
 }
 
-StateTestFillerExpectSection::StateTestFillerExpectSection(DataObject const& _data, spStateTestFillerTransaction const& _gtr)
+StateTestFillerExpectSection::StateTestFillerExpectSection(spDataObjectMove _data, spStateTestFillerTransaction const& _gtr)
 {
     try
     {
-        m_initialData = spDataObject(new DataObject());
-        (*m_initialData).atKeyPointer("indexes").getContent().copyFrom(_data.atKey("indexes"));
-        (*m_initialData).atKeyPointer("network").getContent().copyFrom(_data.atKey("network"));
+        m_initialData = _data.getPointer();
+        requireJsonFields(m_initialData, "StateTestFillerExpectSection " + m_initialData->getKey(),
+            {{"indexes", {{DataType::Object}, jsonField::Required}},
+                {"network", {{DataType::Array}, jsonField::Required}},
+                {"expectException", {{DataType::Object}, jsonField::Optional}},
+                {"result", {{DataType::Object}, jsonField::Required}}});
 
-        spDataObject dataIndexes = ReplaceValueToIndexesInDataList(_gtr, _data.atKey("indexes").atKey("data"));
+        spDataObject dataIndexes = ReplaceValueToIndexesInDataList(_gtr, m_initialData->atKey("indexes").atKey("data"));
         parseJsonIntValueIntoSet(dataIndexes, m_dataInd);
-        parseJsonIntValueIntoSet(_data.atKey("indexes").atKey("gas"), m_gasInd);
-        parseJsonIntValueIntoSet(_data.atKey("indexes").atKey("value"), m_valInd);
+        parseJsonIntValueIntoSet(m_initialData->atKey("indexes").atKey("gas"), m_gasInd);
+        parseJsonIntValueIntoSet(m_initialData->atKey("indexes").atKey("value"), m_valInd);
 
         ETH_ERROR_REQUIRE_MESSAGE(m_dataInd.size() > 0, "Expect section `indexes::data` is empty!");
         ETH_ERROR_REQUIRE_MESSAGE(m_gasInd.size() > 0, "Expect section `indexes::gas` is empty!");
         ETH_ERROR_REQUIRE_MESSAGE(m_valInd.size() > 0, "Expect section `indexes::value` is empty!");
 
         // get allowed networks for this expect section
         std::set<string> forks;
-        parseJsonStrValueIntoSet(_data.atKey("network"), forks);
+        parseJsonStrValueIntoSet(m_initialData->atKey("network"), forks);
         ETH_ERROR_REQUIRE_MESSAGE(forks.size() > 0, "Expect section `network` is empty!");
 
         // Parse >=Frontier into  Frontier, Homestead, ... Constantinople according to current config
         ClientConfig const& cfg = Options::get().getDynamicOptions().getCurrentConfig();
         m_forks = cfg.translateNetworks(forks);
-        m_result = GCP_SPointer<StateIncomplete>(new StateIncomplete(_data.atKey("result"), DataRequier::ALLOWDEC));
 
-        if (_data.count("expectException"))
-            readExpectExceptions(_data.atKey("expectException"), m_expectExceptions);
+        convertDecStateToHex((*m_initialData).atKeyPointerUnsafe("result"), solContracts(), StateToHex::NOCOMPILECODE);
+        m_result = GCP_SPointer<StateIncomplete>(new StateIncomplete(MOVE(m_initialData, "result")));
 
-        requireJsonFields(_data, "StateTestFillerExpectSection " + _data.getKey(),
-            {{"indexes", {{DataType::Object}, jsonField::Required}},
-             {"network", {{DataType::Array}, jsonField::Required}},
-             {"expectException", {{DataType::Object}, jsonField::Optional}},
-             {"result", {{DataType::Object}, jsonField::Required}}});
+        if (m_initialData->count("expectException"))
+            readExpectExceptions(m_initialData->atKey("expectException"), m_expectExceptions);
     }
     catch (std::exception const& _ex)
     {
-        throw UpwardsException(string("StateTestFillerExpectSection parse error: ") + _ex.what() + "\n" + _data.asJson());
+        throw UpwardsException(string("StateTestFillerExpectSection parse error: ") + _ex.what() + "\n" + m_initialData->asJson());
     }
 }
 

retesteth/testStructures/types/StateTests/Filler/StateTestFillerExpectSection.h

@@ -13,7 +13,7 @@ namespace teststruct
 {
 struct StateTestFillerExpectSection
 {
-    StateTestFillerExpectSection(DataObject const&, spStateTestFillerTransaction const&);
+    StateTestFillerExpectSection(spDataObjectMove, spStateTestFillerTransaction const&);
     StateIncomplete const& result() const { return m_result; }
     DataObject const& initialData() const { return m_initialData; }
     std::vector<FORK> const& forks() const { return m_forks; }