99#include " util.h"
1010#include " streams.h"
1111
12+ #include " scraper_net.h"
13+
1214/* ********************
1315* Scraper ENUMS *
1416*********************/
@@ -55,13 +57,80 @@ typedef std::multimap<int64_t, std::pair<uint256, uint256>, std::greater <int64_
5557// See the ScraperID typedef above.
5658typedef std::map<ScraperID, mCSManifest > mmCSManifestsBinnedByScraper;
5759
58- // -------------- Project ---- Converged Part
59- typedef std::map<std::string, CSerializeData> mConvergedManifestParts ;
60- // Note that this IS a copy not a pointer. Since manifests and parts can be deleted because of aging rules,
61- // it is dangerous to save memory and point to the actual part objects themselves.
60+ // Note the CParts pointed to by this map are safe to access, because the pointers are guaranteed valid
61+ // as long as the holding CScraperManifests (both in the CScaperManifest global map, and this cache)
62+ // still exist. So the safety of these pointers is coincident with the lifespan of CScraperManifests
63+ // that have reference to them. If you have questions about this, you should review the CSplitBlob abstract
64+ // class, which is the base class of the CScraperManifest class, and provides the mechanisms for part
65+ // control. Note that two LOCKS are used to protect the integrity of the underlying global maps,
66+ // CScraperManifest::cs_mapManifest and CSplitBlob::cs_mapParts.
67+ // -------------- Project -- Converged Part Pointer
68+ typedef std::map<std::string, CSplitBlob::CPart*> mConvergedManifestPart_ptrs ;
6269
6370struct ConvergedManifest
6471{
72+ // Empty converged manifest constructor
73+ ConvergedManifest ()
74+ {
75+ nContentHash = {};
76+ ConsensusBlock = {};
77+ timestamp = 0 ;
78+ bByParts = false ;
79+
80+ CScraperConvergedManifest_ptr = nullptr ;
81+
82+ ConvergedManifestPartPtrsMap = {};
83+
84+ mIncludedScraperManifests = {};
85+
86+ nUnderlyingManifestContentHash = {};
87+
88+ vIncludedScrapers = {};
89+ vExcludedScrapers = {};
90+ vScrapersNotPublishing = {};
91+
92+ mIncludedScrapersbyProject = {};
93+ mIncludedProjectsbyScraper = {};
94+
95+ mScraperConvergenceCountbyProject = {};
96+
97+ vExcludedProjects = {};
98+ }
99+
100+ // For constructing a dummy converged manifest from a single manifest
101+ ConvergedManifest (CScraperManifest& in)
102+ {
103+ ConsensusBlock = in.ConsensusBlock ;
104+ timestamp = GetAdjustedTime ();
105+ bByParts = false ;
106+
107+ CScraperConvergedManifest_ptr = std::make_shared<CScraperManifest>(in);
108+
109+ PopulateConvergedManifestPartPtrsMap ();
110+
111+ ComputeConvergedContentHash ();
112+
113+ nUnderlyingManifestContentHash = in.nContentHash ;
114+ }
115+
116+ // Call operator to update an already initialized ConvergedManifest with a passed in CScraperManifest
117+ bool operator ()(const CScraperManifest& in)
118+ {
119+ ConsensusBlock = in.ConsensusBlock ;
120+ timestamp = GetAdjustedTime ();
121+ bByParts = false ;
122+
123+ CScraperConvergedManifest_ptr = std::make_shared<CScraperManifest>(in);
124+
125+ bool bConvergedContentHashMatches = PopulateConvergedManifestPartPtrsMap ();
126+
127+ ComputeConvergedContentHash ();
128+
129+ nUnderlyingManifestContentHash = in.nContentHash ;
130+
131+ return bConvergedContentHashMatches;
132+ }
133+
65134 // IMPORTANT... nContentHash is NOT the hash of part hashes in the order of vParts unlike CScraper::manifest.
66135 // It is the hash of the data in the ConvergedManifestPartsMap in the order of the key. It represents
67136 // the composite convergence by taking parts piecewise in the case of the fallback to bByParts (project) level.
@@ -70,7 +139,9 @@ struct ConvergedManifest
70139 int64_t timestamp;
71140 bool bByParts;
72141
73- mConvergedManifestParts ConvergedManifestPartsMap;
142+ std::shared_ptr<CScraperManifest> CScraperConvergedManifest_ptr;
143+
144+ mConvergedManifestPart_ptrs ConvergedManifestPartPtrsMap;
74145
75146 // Used when convergence is at the manifest level (normal)
76147 std::map<ScraperID, uint256> mIncludedScraperManifests ;
@@ -97,6 +168,62 @@ struct ConvergedManifest
97168
98169 // --------- project
99170 std::vector<std::string> vExcludedProjects;
171+
172+ bool PopulateConvergedManifestPartPtrsMap ()
173+ {
174+ if (CScraperConvergedManifest_ptr == nullptr ) return false ;
175+
176+ int iPartNum = 0 ;
177+ CDataStream ss (SER_NETWORK,1 );
178+ WriteCompactSize (ss, CScraperConvergedManifest_ptr->vParts .size ());
179+ uint256 nContentHashCheck;
180+
181+ for (const auto & iter : CScraperConvergedManifest_ptr->vParts )
182+ {
183+ std::string sProject ;
184+
185+ if (iPartNum == 0 )
186+ sProject = " BeaconList"
187+ else
188+ sProject = CScraperConvergedManifest_ptr->projects [iPartNum-1 ].project ;
189+
190+ // Copy the pointer to the CPart into the map. This is ok, because the parts will be held
191+ // until the CScraperManifest in this object is destroyed and all of the manifest refs to the part
192+ // are gone.
193+ ConvergedManifestPartPtrsMap.insert (std::make_pair (sProject , iter));
194+
195+ // Serialize the hash to doublecheck the content hash.
196+ ss << iter->hash ;
197+
198+ iPartNum++;
199+ }
200+
201+ ss << CScraperConvergedManifest_ptr->ConsensusBlock ;
202+
203+ nContentHashCheck = Hash (ss.begin (), ss.end ());
204+
205+ if (nContentHashCheck != CScraperConvergedManifest_ptr->nContentHash )
206+ {
207+ LogPrintf (" ERROR: PopulateConvergedManifestPartPtrsMap(): Selected Manifest content hash check failed! "
208+ " nContentHashCheck = %s and nContentHash = %s."
209+ nContentHashCheck.GetHex (), CScraperConvergedManifest_ptr->nContentHash .GetHex ());
210+ return false ;
211+ }
212+
213+ return true ;
214+ }
215+
216+ void ComputeConvergedContentHash ()
217+ {
218+ CDataStream ss (SER_NETWORK,1 );
219+
220+ for (const auto & iter : ConvergedManifestPartPtrsMap)
221+ {
222+ ss << iter.second ->data ;
223+ }
224+
225+ nContentHash = Hash (ss.begin (), ss.end ());
226+ }
100227};
101228
102229
0 commit comments