Try to fix tests and also change by project hints

to CPart pointers.

Author: jamescowens

src/neuralnet/superblock.cpp

@@ -559,9 +559,9 @@ Superblock Superblock::FromConvergence(
     //
     for (const auto& part_pair : stats.Convergence.ConvergedManifestPartPtrsMap) {
         const std::string& project_name = part_pair.first;
-        const CSerializeData& part_data = part_pair.second->data;
+        const CSplitBlob::CPart* part_data_ptr = part_pair.second;
 
-        projects.SetHint(project_name, part_data);
+        projects.SetHint(project_name, part_data_ptr);
     }
 
     return superblock;
@@ -954,7 +954,7 @@ void Superblock::ProjectIndex::Add(std::string name, const ProjectStats& stats)
 
 void Superblock::ProjectIndex::SetHint(
     const std::string& name,
-    const CSerializeData& part_data)
+    const CSplitBlob::CPart* part_data_ptr)
 {
     auto iter = std::lower_bound(
         m_projects.begin(),
@@ -966,7 +966,7 @@ void Superblock::ProjectIndex::SetHint(
         return;
     }
 
-    const uint256 part_hash = Hash(part_data.begin(), part_data.end());
+    const uint256 part_hash = Hash(part_data_ptr->data.begin(), part_data_ptr->data.end());
     iter->second.m_convergence_hint = part_hash.GetUint64() >> 32;
 
     m_converged_by_project = true;

src/neuralnet/superblock.h

@@ -1087,7 +1087,7 @@ class Superblock
         //!
         //! \param part_data The convergence part to create the hint from.
         //!
-        void SetHint(const std::string& name, const CSerializeData& part_data);
+        void SetHint(const std::string& name, const CSplitBlob::CPart *part_data_ptr);
 
         //!
         //! \brief Serialize the object to the provided stream.
@@ -1576,16 +1576,6 @@ struct ConvergedScraperStats
             // SB Hash and Convergence stored will be the LATER one.
 
             PastConvergences[nReducedContentHash] = std::make_pair(NewFormatSuperblock.GetHash(), Convergence);
-
-            /*
-            if (PastConvergences.find(nReducedContentHash) != PastConvergences.end())
-            {
-                PastConvergences.erase(nReducedContentHash);
-            }
-
-            PastConvergences.emplace(std::make_pair(nReducedContentHash, Convergence));
-            */
-
         }
     }
 

src/scraper/fwd.h

@@ -226,8 +226,6 @@ struct ConvergedManifest
 
     std::shared_ptr<CScraperManifest> CScraperConvergedManifest_ptr;
 
-    // mConvergedManifestParts ConvergedManifestPartsMap;
-
     mConvergedManifestPart_ptrs ConvergedManifestPartPtrsMap;
 
     // Used when convergence is at the manifest level (normal)

src/scraper/scraper.cpp

@@ -5745,9 +5745,9 @@ UniValue testnewsb(const UniValue& params, bool fHelp)
             for (const auto& part_pair : RandomPastConvergedManifest.ConvergedManifestPartPtrsMap)
             {
                 const std::string& project_name = part_pair.first;
-                const CSerializeData& part_data = part_pair.second->data;
+                const CSplitBlob::CPart* part_data_ptr = part_pair.second;
 
-                projects.SetHint(project_name, part_data); // This also sets m_converged_by_project to true.
+                projects.SetHint(project_name, part_data_ptr); // This also sets m_converged_by_project to true.
             }
         }
         else
@@ -5804,14 +5804,8 @@ UniValue scraperreport(const UniValue& params, bool fHelp)
 
     uint64_t manifest_map_size = 0;
     uint64_t parts_map_size = 0;
-    uint64_t current_convergence_publishing_scrapers = 0;
-    uint64_t current_convergence_part_pointer_map_size = 0;
-    uint64_t past_convergence_map_size = 0;
-    uint64_t total_past_convergences_part_pointer_maps_size = 0;
-    uint64_t total_past_convergences_part_unique_pointer_maps_size = 0;
-    int64_t part_objects_reduced = 0;
 
-    std::set<uint256> global_cache_unique_parts;
+    std::set<CSplitBlob::CPart*> global_cache_unique_parts;
 
     {
         LOCK(CScraperManifest::cs_mapManifest);
@@ -5835,6 +5829,11 @@ UniValue scraperreport(const UniValue& params, bool fHelp)
 
         if (ConvergedScraperStatsCache.NewFormatSuperblock.WellFormed())
         {
+            uint64_t current_convergence_publishing_scrapers = 0;
+            uint64_t current_convergence_part_pointer_map_size = 0;
+            uint64_t past_convergence_map_size = 0;
+            int64_t part_objects_reduced = 0;
+
             current_convergence_publishing_scrapers =
                     ConvergedScraperStatsCache.Convergence.vIncludedScrapers.size()
                     + ConvergedScraperStatsCache.Convergence.vExcludedScrapers.size();
@@ -5845,32 +5844,36 @@ UniValue scraperreport(const UniValue& params, bool fHelp)
             past_convergence_map_size =
                     ConvergedScraperStatsCache.PastConvergences.size();
 
-            // This next section will form a set of hashs from the pointers in the global cache
+            // This next section will form a set of unique pointers in the global cache
             // and also add the pointers up arithmetically. The difference is the efficiency gain
             // from using pointers rather than copies into the global cache.
             for (const auto& iter : ConvergedScraperStatsCache.Convergence.ConvergedManifestPartPtrsMap)
             {
-                global_cache_unique_parts.insert(iter.second->hash);
+                global_cache_unique_parts.insert(iter.second);
             }
 
+            uint64_t total_convergences_part_pointer_maps_size = current_convergence_part_pointer_map_size;
+
             for (const auto& iter : ConvergedScraperStatsCache.PastConvergences)
             {
                 for (const auto& iter2 : iter.second.second.ConvergedManifestPartPtrsMap)
                 {
-                    global_cache_unique_parts.insert(iter2.second->hash);
+                    global_cache_unique_parts.insert(iter2.second);
                 }
 
-                total_past_convergences_part_pointer_maps_size +=
+                total_convergences_part_pointer_maps_size +=
                         iter.second.second.ConvergedManifestPartPtrsMap.size();
             }
 
-            total_past_convergences_part_unique_pointer_maps_size = global_cache_unique_parts.size();
+            uint64_t total_convergences_part_unique_pointer_maps_size = 0;
 
-            part_objects_reduced = total_past_convergences_part_pointer_maps_size
-                                   - total_past_convergences_part_unique_pointer_maps_size;
+            total_convergences_part_unique_pointer_maps_size = global_cache_unique_parts.size();
 
+            part_objects_reduced = total_convergences_part_pointer_maps_size
+                                   - total_convergences_part_unique_pointer_maps_size;
 
-            converged_scraper_stats_cache.pushKV("current_convergence_included_scrapers",
+
+            converged_scraper_stats_cache.pushKV("current_convergence_publishing_scrapers",
                                                  current_convergence_publishing_scrapers);
 
             converged_scraper_stats_cache.pushKV("current_convergence_part_pointer_map_size",
@@ -5879,11 +5882,13 @@ UniValue scraperreport(const UniValue& params, bool fHelp)
             converged_scraper_stats_cache.pushKV("past_convergence_map_size",
                                                  past_convergence_map_size);
 
-            converged_scraper_stats_cache.pushKV("total_past_convergences_part_pointer_maps_size",
-                                                 total_past_convergences_part_pointer_maps_size);
+            converged_scraper_stats_cache.pushKV("total_convergences_part_pointer_maps_size",
+                                                 total_convergences_part_pointer_maps_size);
+
+            converged_scraper_stats_cache.pushKV("total_convergences_part_unique_pointer_maps_size",
+                                                 total_convergences_part_unique_pointer_maps_size);
 
-            converged_scraper_stats_cache.pushKV("total_past_convergences_part_unique_pointer_maps_size",
-                                                 total_past_convergences_part_unique_pointer_maps_size);
+            converged_scraper_stats_cache.pushKV("part_objects_reduced", part_objects_reduced);
         }
     }
 

src/test/neuralnet/superblock_tests.cpp

@@ -1,6 +1,7 @@
 #include "base58.h"
 #include "compat/endian.h"
 #include "neuralnet/superblock.h"
+#include "scraper_net.h"
 #include "streams.h"
 
 #include <array>
@@ -432,22 +433,46 @@ ConvergedScraperStats GetTestConvergence(
     // Add a verified beacons project part. Technically, this is the second
     // part for a manifest (offset 1). We skipped adding the beacon list part.
     //
-    CDataStream verified_beacons_part(SER_NETWORK, PROTOCOL_VERSION);
-    verified_beacons_part
+    CDataStream verified_beacons_part_data(SER_NETWORK, PROTOCOL_VERSION);
+    verified_beacons_part_data
         << ScraperPendingBeaconMap {
              *stats.mVerifiedMap.begin(),
              *++stats.mVerifiedMap.begin(),
         };
 
-    convergence.Convergence.ConvergedManifestPartsMap.emplace(
-        "VerifiedBeacons",
-        CSerializeData(verified_beacons_part.begin() , verified_beacons_part.end()));
+    CSplitBlob::CPart verified_beacons_part(Hash(verified_beacons_part_data.begin(),
+                                                 verified_beacons_part_data.end()));
+
+    verified_beacons_part.data = CSerializeData(verified_beacons_part_data.begin() , verified_beacons_part_data.end());
+
+    //auto verified_beacons_part_ptr = std::make_shared<CSplitBlob::CPart>(verified_beacons_part);
+
+    convergence.Convergence.ConvergedManifestPartPtrsMap.emplace("VerifiedBeacons",
+                                                              &verified_beacons_part);
+
+    CDataStream project_1_part_data(SER_NETWORK, PROTOCOL_VERSION);
+    project_1_part_data << "foo";
+
+    CSplitBlob::CPart project_1_part(Hash(project_1_part_data.begin(), project_1_part_data.end()));
+    project_1_part.data = CSerializeData(project_1_part_data.begin(), project_1_part_data.end());
+
+    //auto project_1_part_ptr = std::make_shared<CSplitBlob::CPart>(project_1_part);
+
+
+    CDataStream project_2_part_data(SER_NETWORK, PROTOCOL_VERSION);
+    project_2_part_data << "fi";
+
+    CSplitBlob::CPart project_2_part(Hash(project_2_part_data.begin(), project_2_part_data.end()));
+    project_2_part.data = CSerializeData(project_2_part_data.begin(), project_2_part_data.end());
+
+    //auto project_2_part_ptr = std::make_shared<CSplitBlob::CPart>(project_2_part);
+
 
     // Add some project parts with the same names as the projects in the stats.
-    // The part data doesn't matter, so we just add empty containers.
+    // The part data pointers don't matter, so we just add nullptrs.
     //
-    convergence.Convergence.ConvergedManifestPartsMap.emplace("project_1", CSerializeData());
-    convergence.Convergence.ConvergedManifestPartsMap.emplace("project_2", CSerializeData());
+    convergence.Convergence.ConvergedManifestPartPtrsMap.emplace("project_1", &project_1_part);
+    convergence.Convergence.ConvergedManifestPartPtrsMap.emplace("project_2", &project_2_part);
 
     return convergence;
 }
@@ -847,7 +872,16 @@ BOOST_AUTO_TEST_CASE(it_checks_whether_it_was_created_from_fallback_convergence)
     BOOST_CHECK(superblock.ConvergedByProject() == false);
 
     superblock.m_projects.Add("project_name", NN::Superblock::ProjectStats());
-    superblock.m_projects.SetHint("project_name", CSerializeData());
+
+    CDataStream project_part_stream(SER_NETWORK, PROTOCOL_VERSION);
+    project_part_stream << "";
+
+    CSerializeData project_part_data(project_part_stream.begin(), project_part_stream.end());
+
+    CSplitBlob::CPart project_part(Hash(project_part_data.begin(),project_part_data.end()));
+    project_part.data = project_part_data;
+
+    superblock.m_projects.SetHint("project_name", &project_part);
 
     BOOST_CHECK(superblock.ConvergedByProject() == true);
 }
@@ -1613,7 +1647,16 @@ BOOST_AUTO_TEST_CASE(it_sets_a_project_part_convergence_hint)
     NN::Superblock::ProjectIndex projects;
 
     projects.Add("project_name", NN::Superblock::ProjectStats());
-    projects.SetHint("project_name", CSerializeData());
+
+    CDataStream project_part_stream(SER_NETWORK, PROTOCOL_VERSION);
+    project_part_stream << "";
+
+    CSerializeData project_part_data(project_part_stream.begin(), project_part_stream.end());
+
+    CSplitBlob::CPart project_part(Hash(project_part_data.begin(),project_part_data.end()));
+    project_part.data = project_part_data;
+
+    projects.SetHint("project_name", &project_part);
 
     BOOST_CHECK(projects.m_converged_by_project == true);
 
@@ -1804,8 +1847,25 @@ BOOST_AUTO_TEST_CASE(it_serializes_to_a_stream_for_fallback_convergences)
     projects.Add("project_1", NN::Superblock::ProjectStats(1, 2, 3));
     projects.Add("project_2", NN::Superblock::ProjectStats(1, 2, 3));
 
-    projects.SetHint("project_1", CSerializeData());
-    projects.SetHint("project_2", CSerializeData());
+    CDataStream project_1_part_stream(SER_NETWORK, PROTOCOL_VERSION);
+    project_1_part_stream << "foo";
+
+    CSerializeData project_1_part_data(project_1_part_stream.begin(), project_1_part_stream.end());
+
+    CSplitBlob::CPart project_1_part(Hash(project_1_part_data.begin(),project_1_part_data.end()));
+    project_1_part.data = project_1_part_data;
+
+    projects.SetHint("project_1", &project_1_part);
+
+    CDataStream project_2_part_stream(SER_NETWORK, PROTOCOL_VERSION);
+    project_2_part_stream << "fi";
+
+    CSerializeData project_2_part_data(project_2_part_stream.begin(), project_2_part_stream.end());
+
+    CSplitBlob::CPart project_2_part(Hash(project_2_part_data.begin(),project_2_part_data.end()));
+    project_2_part.data = project_2_part_data;
+
+    projects.SetHint("project_2", &project_2_part);
 
     BOOST_CHECK(GetSerializeSize(projects, SER_NETWORK, 1) == expected.size());