Merge pull request #3886 from TrueBlocks/feature/indexManager

Adds a devtool to check disc usage on various computers

bin/.gitignore

@@ -4,6 +4,7 @@ SymbolCache.noindex
 
 chifra*
 goMaker
+indexManager
 sdkFuzzer
 makeClass
 testRunner

scripts/go-mod-tidy.bat

@@ -10,17 +10,18 @@ set SCRIPT_DIR=%script_dir_backslash:~0,-1%
 set ROOT_DIR=%SCRIPT_DIR%\..
 
 :: Find all go.mod files and tidy them
-cd /d %ROOT_DIR%\examples\balanceChart    && go mod tidy
-cd /d %ROOT_DIR%\examples\cancelContext   && go mod tidy
-cd /d %ROOT_DIR%\examples\findFirst       && go mod tidy
-cd /d %ROOT_DIR%\examples\nameManager     && go mod tidy
-cd /d %ROOT_DIR%\examples\simple          && go mod tidy
-cd /d %ROOT_DIR%\examples\withStreaming   && go mod tidy
-cd /d %ROOT_DIR%\src\apps\chifra          && go mod tidy
-cd /d %ROOT_DIR%\src\node                 && go mod tidy
-cd /d %ROOT_DIR%\src\dev_tools\goMaker    && go mod tidy
-cd /d %ROOT_DIR%\src\dev_tools\sdkFuzzer  && go mod tidy
-cd /d %ROOT_DIR%\src\dev_tools\testRunner && go mod tidy
+cd /d %ROOT_DIR%\examples\balanceChart      && go mod tidy
+cd /d %ROOT_DIR%\examples\cancelContext     && go mod tidy
+cd /d %ROOT_DIR%\examples\findFirst         && go mod tidy
+cd /d %ROOT_DIR%\examples\nameManager       && go mod tidy
+cd /d %ROOT_DIR%\examples\simple            && go mod tidy
+cd /d %ROOT_DIR%\examples\withStreaming     && go mod tidy
+cd /d %ROOT_DIR%\src\apps\chifra            && go mod tidy
+cd /d %ROOT_DIR%\src\node                   && go mod tidy
+cd /d %ROOT_DIR%\src\dev_tools\goMaker      && go mod tidy
+cd /d %ROOT_DIR%\src\dev_tools\indexManager && go mod tidy
+cd /d %ROOT_DIR%\src\dev_tools\sdkFuzzer    && go mod tidy
+cd /d %ROOT_DIR%\src\dev_tools\testRunner   && go mod tidy
 
 :: Return to the original directory and exit
 exit /b

scripts/go-work-sync.bat

@@ -24,6 +24,7 @@ cd %ROOT_DIR% && go work use .\examples\withStreaming
 cd %ROOT_DIR% && go work use .\src\apps\chifra
 cd %ROOT_DIR% && go work use .\src\node
 cd %ROOT_DIR% && go work use .\src\dev_tools\goMaker
+cd %ROOT_DIR% && go work use .\src\dev_tools\indexManager
 cd %ROOT_DIR% && go work use .\src\dev_tools\sdkFuzzer
 cd %ROOT_DIR% && go work use .\src\dev_tools\testRunner
 

src/CMakeLists.txt

@@ -87,11 +87,13 @@ add_subdirectory(other/install)
 
 ADD_GO_INSTALLABLE_PROGRAM(chifra ${CMAKE_SOURCE_DIR}/apps/chifra ${BIN_DIR})
 ADD_GO_INSTALLABLE_PROGRAM(goMaker ${CMAKE_SOURCE_DIR}/dev_tools/goMaker ${BIN_DIR})
+ADD_GO_INSTALLABLE_PROGRAM(indexManager ${CMAKE_SOURCE_DIR}/dev_tools/indexManager ${BIN_DIR})
 if (NOT WIN32)
     ADD_GO_INSTALLABLE_PROGRAM(testRunner ${CMAKE_SOURCE_DIR}/dev_tools/testRunner ${BIN_DIR})
     ADD_GO_INSTALLABLE_PROGRAM(sdkFuzzer ${CMAKE_SOURCE_DIR}/dev_tools/sdkFuzzer ${BIN_DIR})
     add_dependencies(testRunner goMaker)
     add_dependencies(sdkFuzzer goMaker)
+    add_dependencies(indexManager goMaker)
     add_dependencies(chifra testRunner)
 endif()
 

src/apps/chifra/internal/chunks/handle_check.go

@@ -155,6 +155,20 @@ func (opts *ChunksOptions) check(rCtx *output.RenderCtx, blockNums []base.Blknum
 	}
 	reports = append(reports, sizes)
 
+	// are all the hashes present?
+	contentCheck := types.ReportCheck{}
+	contentCheck.Reason = "Remote manifest contents"
+	if err := opts.CheckManContents(remoteManifest, &contentCheck); err != nil {
+		return err, false
+	}
+	reports = append(reports, contentCheck)
+
+	contentCheck.Reason = "Local manifest contents"
+	if err := opts.CheckManContents(remoteManifest, &contentCheck); err != nil {
+		return err, false
+	}
+	reports = append(reports, contentCheck)
+
 	// compare with çached manifest with files on disc
 	d2c := types.ReportCheck{Reason: "Disc files to cached manifest"}
 	if err := opts.CheckManifest(fnArray, cacheArray, &d2c); err != nil {

src/apps/chifra/internal/chunks/handle_check_manifest.go

@@ -8,7 +8,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/manifest"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/version"
 )
 
 type CompareState struct {
@@ -22,9 +25,9 @@ type CompareState struct {
 	// failB    int
 }
 
-// CheckManifest takes two arrays (either onDisc vs. LocalManifest, onDisc vs. RemoteManifest, or LocalManifest
-// vs. RemoteManifest) and compares them for equality. If everything is up to date, all three arrays should
-// be identical. Only the block ranges are in the arrays.
+// CheckManifest takes two arrays (either onDisc vs. LocalManifest, onDisc vs. RemoteManifest,
+// or LocalManifest vs. RemoteManifest) and compares them for equality. If everything is up
+// to date, all three arrays should be identical. Only the block ranges are in the arrays.
 func (opts *ChunksOptions) CheckManifest(arrayA, arrayB []string, report *types.ReportCheck) error {
 	comp := CompareState{
 		testMode: opts.Globals.TestMode,
@@ -46,7 +49,6 @@ func (opts *ChunksOptions) CheckManifest(arrayA, arrayB []string, report *types.
 	return comp.checkArrays(report)
 }
 
-// TODO: Can this be made concurrent?
 func (comp *CompareState) checkArrays(report *types.ReportCheck) error {
 	marker := ""
 	if comp.testMode {
@@ -96,3 +98,70 @@ func (comp *CompareState) checkArrays(report *types.ReportCheck) error {
 
 	return nil
 }
+
+// CheckManContents spins through the manifest and makes sure all the
+// bloom and index CIDs are present. It does not check that the CIDs are available.
+func (opts *ChunksOptions) CheckManContents(man *manifest.Manifest, report *types.ReportCheck) error {
+	errs := []string{}
+
+	report.VisitedCnt += 3
+	report.CheckedCnt += 3
+
+	if !version.IsValidVersion(man.Version) {
+		errs = append(errs, "invalid version string "+opts.Tag)
+	} else {
+		report.PassedCnt++
+	}
+
+	if opts.Globals.Chain != man.Chain {
+		errs = append(errs, "different chains: globals("+opts.Globals.Chain+") manifest ("+man.Chain+")")
+	} else {
+		report.PassedCnt++
+	}
+
+	expectedSpec := config.SpecTags["[email protected]"]
+	if expectedSpec != man.Specification.String() {
+		errs = append(errs, "invalid spec "+man.Specification.String())
+	} else {
+		report.PassedCnt++
+	}
+
+	for _, chunk := range man.Chunks {
+		report.VisitedCnt++
+		report.CheckedCnt++
+		if len(chunk.Range) == 0 {
+			errs = append(errs, "empty range")
+		} else {
+			report.PassedCnt++
+		}
+
+		report.VisitedCnt++
+		report.CheckedCnt++
+		hasBloom := len(chunk.BloomHash) > 0
+		hasIndex := len(chunk.IndexHash) > 0
+		bloomSize := chunk.BloomSize > 0
+		indexSize := chunk.IndexSize > 0
+		if hasBloom && hasIndex && bloomSize && indexSize {
+			report.PassedCnt++
+		} else {
+			if !hasBloom {
+				errs = append(errs, chunk.Range+" - missing bloom hash")
+			}
+			if !hasIndex {
+				errs = append(errs, chunk.Range+" - missing index hash")
+			}
+			if !bloomSize {
+				errs = append(errs, chunk.Range+" - missing bloom size")
+			}
+			if !indexSize {
+				errs = append(errs, chunk.Range+" - missing index size")
+			}
+		}
+	}
+
+	if len(errs) > 0 {
+		report.MsgStrings = append(report.MsgStrings, errs...)
+	}
+
+	return nil
+}

src/apps/chifra/internal/chunks/handle_pin.go

@@ -96,40 +96,49 @@ func (opts *ChunksOptions) HandlePin(rCtx *output.RenderCtx, blockNums []base.Bl
 			return rng1.First < rng2.First
 		})
 
-		for _, path := range fileList {
+		failCnt := 1.0
+		for i := 0; i < len(fileList); i++ {
+			sleep := opts.Sleep
+
+			path := fileList[i]
 			if opts.Globals.Verbose {
 				logger.Info("pinning path:", path)
 			}
+
 			local, remote, err := pinning.PinOneChunk(chain, path, opts.Remote)
 			if err != nil {
 				errorChan <- err
 				logger.Error("Pin failed:", path, err)
-			}
-
-			blMatches, idxMatches := matches(&local, &remote)
-			opts.matchReport(blMatches, local.BloomHash, remote.BloomHash)
-			opts.matchReport(idxMatches, local.IndexHash, remote.IndexHash)
+				failCnt *= 2.
+				sleep = failCnt
+				i-- // try again after sleeping for a bit
+				logger.Info(colors.Yellow, "Sleeping for", sleep, "seconds then trying again.", colors.Off)
 
-			if opts.Remote {
-				man.Chunks = append(man.Chunks, remote)
 			} else {
-				man.Chunks = append(man.Chunks, local)
-			}
-			_ = man.SaveManifest(chain, outPath)
+				blMatches, idxMatches := matches(&local, &remote)
+				opts.matchReport(blMatches, local.BloomHash, remote.BloomHash)
+				opts.matchReport(idxMatches, local.IndexHash, remote.IndexHash)
 
-			if opts.Globals.Verbose {
 				if opts.Remote {
-					fmt.Println("result.Remote:", remote.String())
+					man.Chunks = append(man.Chunks, remote)
 				} else {
-					fmt.Println("result.Local:", local.String())
+					man.Chunks = append(man.Chunks, local)
+				}
+				_ = man.SaveManifest(chain, outPath)
+
+				if opts.Globals.Verbose {
+					if opts.Remote {
+						fmt.Println("result.Remote:", remote.String())
+					} else {
+						fmt.Println("result.Local:", local.String())
+					}
 				}
 			}
 
-			sleep := opts.Sleep
 			if sleep > 0 {
 				ms := time.Duration(sleep*1000) * time.Millisecond
 				if !opts.Globals.TestMode {
-					logger.Info(fmt.Sprintf("Sleeping for %g seconds", sleep))
+					logger.Info("Sleeping for", sleep, "seconds")
 				}
 				time.Sleep(ms)
 			}

src/dev_tools/indexManager/.gitignore

@@ -0,0 +1 @@
+disc_usage_report.md

src/dev_tools/indexManager/README.md

@@ -0,0 +1,3 @@
+# indexManager
+
+The `indexManager` tool is for internal use only.