Merge branch 'develop' into further-improvements-2

src/apps/chifra/internal/chunks/handle_check.go

@@ -74,7 +74,7 @@ func (opts *ChunksOptions) check(blockNums []uint64, silent bool) (error, bool)
 		return fileNames[i] < fileNames[j]
 	})
 
-	cacheManifest, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.Cache|manifest.NoUpdate)
+	cacheManifest, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.LocalCache|manifest.NoUpdate)
 	if err != nil {
 		return err, false
 	}

src/apps/chifra/internal/chunks/handle_check_internal.go

@@ -35,7 +35,7 @@ func (opts *ChunksOptions) checkIndexChunkInternal(fileName string, checkVersion
 		if strings.Contains(err.Error(), "no such file or directory") {
 			// This is the case where the user did not download all the index chunks, only blooms
 			report.PassedCnt++
-		} else if !checkVersion && errors.Is(err, index.ErrIndexHeaderDiffHash) {
+		} else if !checkVersion && errors.Is(err, index.ErrIncorrectHash) {
 			report.PassedCnt++
 		} else {
 			report.MsgStrings = append(report.MsgStrings, fmt.Sprintf("%s: %s", err, fileName))

src/apps/chifra/internal/chunks/handle_manifest.go

@@ -13,7 +13,7 @@ import (
 )
 
 var sourceMap = map[bool]manifest.Source{
-	false: manifest.Cache,
+	false: manifest.LocalCache,
 	true:  manifest.Contract,
 }
 

src/apps/chifra/internal/chunks/handle_pin.go

@@ -46,17 +46,18 @@ func (opts *ChunksOptions) HandlePin(blockNums []uint64) error {
 		outPath = config.PathToManifest(chain)
 	}
 
-	man, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.Cache|manifest.NoUpdate)
+	man, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.LocalCache|manifest.NoUpdate)
 	if err != nil {
 		return err
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	fetchData := func(modelChan chan types.Modeler[types.RawModeler], errorChan chan error) {
+		hash := base.BytesToHash(config.SpecVersionKeccak())
 		report := simpleChunkPinReport{
-			Version:  config.SpecVersionText(),
+			Version:  config.VersionTags[hash.Hex()],
 			Chain:    chain,
-			SpecHash: base.IpfsHash(config.GetUnchained().Specification),
+			SpecHash: base.IpfsHash(config.Specification),
 		}
 
 		fileList := make([]string, 0, len(man.Chunks))

src/apps/chifra/internal/chunks/handle_tag.go

@@ -37,7 +37,7 @@ func (opts *ChunksOptions) HandleTag(blockNums []uint64) error {
 
 	fetchData := func(modelChan chan types.Modeler[types.RawModeler], errorChan chan error) {
 		nChunksTagged := 0
-		man, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.Cache|manifest.NoUpdate)
+		man, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.LocalCache|manifest.NoUpdate)
 		if err != nil {
 			return
 		}

src/apps/chifra/internal/chunks/output.go

@@ -13,6 +13,8 @@ import (
 	"net/http"
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/globals"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/identifiers"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
@@ -142,4 +144,12 @@ func (opts *ChunksOptions) shouldShow(obj index.AddressRecord) bool {
 	return false
 }
 
+func FormattedTag(verbose bool, hash base.Hash) string {
+	if tag, ok := config.VersionTags[hash.Hex()]; !ok {
+		return utils.FormattedHash(verbose, hash.Hex())
+	} else {
+		return tag
+	}
+}
+
 // EXISTING_CODE

src/apps/chifra/internal/chunks/types_chunkbloom.go

@@ -41,7 +41,7 @@ func (s *simpleChunkBloom) Model(chain, format string, verbose bool, extraOption
 	model = map[string]interface{}{
 		"range":     s.Range,
 		"magic":     s.Magic,
-		"hash":      utils.FormattedTag(verbose, s.Hash.Hex()),
+		"hash":      FormattedTag(verbose, s.Hash),
 		"nBlooms":   s.NBlooms,
 		"nInserted": s.NInserted,
 		"size":      s.Size,

src/apps/chifra/internal/chunks/types_chunkindex.go

@@ -12,7 +12,6 @@ package chunksPkg
 import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/types"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
 )
 
 // EXISTING_CODE
@@ -40,7 +39,7 @@ func (s *simpleChunkIndex) Model(chain, format string, verbose bool, extraOption
 	model = map[string]interface{}{
 		"range":        s.Range,
 		"magic":        s.Magic,
-		"hash":         utils.FormattedTag(verbose, s.Hash.Hex()),
+		"hash":         FormattedTag(verbose, s.Hash),
 		"nAddresses":   s.NAddresses,
 		"nAppearances": s.NAppearances,
 		"fileSize":     s.Size,

src/apps/chifra/internal/chunks/validate.go

@@ -235,13 +235,13 @@ func (opts *ChunksOptions) validateChunks() error {
 		// }
 	}
 
-	// Note that this does not return if the index is not initialized
-	if err := index.IsInitialized(chain); err != nil {
-		if opts.Globals.IsApiMode() {
-			return err
-		} else {
+	if err := index.IsInitialized(chain, config.HeaderVersion); err != nil {
+		if errors.Is(err, index.ErrNotInitialized) && !opts.Globals.IsApiMode() {
 			logger.Fatal(err)
+		} else if len(opts.Tag) == 0 {
+			return err
 		}
+		// It's okay to mismatch versions if we're tagging
 	}
 
 	return opts.Globals.Validate()

src/apps/chifra/internal/export/validate.go

@@ -5,6 +5,7 @@
 package exportPkg
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 
@@ -193,13 +194,11 @@ func (opts *ExportOptions) validateExport() error {
 		return validate.Usage("The {0} option requires an Etherscan API key.", "--articulate")
 	}
 
-	// Note that this does not return if the index is not initialized
-	if err := index.IsInitialized(chain); err != nil {
-		if opts.Globals.IsApiMode() {
-			return err
-		} else {
+	if err := index.IsInitialized(chain, config.HeaderVersion); err != nil {
+		if errors.Is(err, index.ErrNotInitialized) && !opts.Globals.IsApiMode() {
 			logger.Fatal(err)
 		}
+		return err
 	}
 
 	return opts.Globals.Validate()

src/apps/chifra/internal/init/handle_dryrun.go

@@ -29,7 +29,7 @@ func (opts *InitOptions) HandleDryRun() error {
 		return err
 	}
 
-	spec := config.GetUnchained().Specification
+	spec := config.Specification
 	if opts.Globals.TestMode {
 		nToDownload = utils.Min(10, nToDownload)
 		spec = "--testing-hash--"

src/apps/chifra/internal/init/handle_init.go

@@ -15,6 +15,7 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/history"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/manifest"
@@ -33,7 +34,7 @@ func (opts *InitOptions) HandleInit() error {
 	// scraper starts, it starts on the correct block.
 	_ = file.CleanFolder(chain, config.PathToIndex(chain), []string{"ripe", "unripe", "maps", "staging"})
 
-	existing, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.Cache|manifest.NoUpdate)
+	existing, err := manifest.ReadManifest(chain, opts.PublisherAddr, manifest.LocalCache|manifest.NoUpdate)
 	if err != nil {
 		return err
 	}
@@ -55,15 +56,15 @@ func (opts *InitOptions) HandleInit() error {
 
 	// Tell the user what we're doing
 	logger.InfoTable("Unchained Index:", config.GetUnchained().SmartContract)
-	logger.InfoTable("Specification:", config.GetUnchained().Specification)
+	logger.InfoTable("Specification:", config.Specification)
 	logger.InfoTable("Config Folder:", config.MustGetPathToChainConfig(chain))
 	logger.InfoTable("Index Folder:", config.PathToIndex(chain))
 	logger.InfoTable("Chunks in manifest:", fmt.Sprintf("%d", len(remote.Chunks)))
 	logger.InfoTable("Files deleted:", fmt.Sprintf("%d", nDeleted))
 	logger.InfoTable("Files downloaded:", fmt.Sprintf("%d", nToDownload))
 
-	if opts.All && config.GetHistory().Init != "all" {
-		_ = config.ChangeSetting("history", "init", "all", true /* writeOut */)
+	if opts.All && !history.FromHistoryBool(opts.Globals.Chain, "init") {
+		_ = history.ToHistory(chain, "init", "true")
 	}
 
 	// Open a channel to receive a message when all the blooms have been downloaded...

src/apps/chifra/internal/init/handle_init_prepare.go

@@ -281,7 +281,7 @@ func checkHeader(path string) (InitReason, error) {
 		if err != nil {
 			return FILE_ERROR, err
 		}
-		if hash.Hex() != config.SpecVersionHex() {
+		if hash != base.BytesToHash(config.SpecVersionKeccak()) {
 			return WRONG_HASH, nil
 		}
 
@@ -302,7 +302,7 @@ func checkHeader(path string) (InitReason, error) {
 		if err != nil {
 			return FILE_ERROR, err
 		}
-		if hash.Hex() != config.SpecVersionHex() {
+		if hash != base.BytesToHash(config.SpecVersionKeccak()) {
 			return WRONG_HASH, nil
 		}
 

src/apps/chifra/internal/init/validate.go

@@ -6,6 +6,7 @@ package initPkg
 
 import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/history"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/validate"
 )
 
@@ -33,7 +34,7 @@ func (opts *InitOptions) validateInit() error {
 		}
 	}
 
-	if config.GetHistory().Init == "all" && !opts.All {
+	if history.FromHistoryBool(opts.Globals.Chain, "init") && !opts.All {
 		return validate.Usage("You previously called chifra init --all. You must continue to do so.")
 	}
 

src/apps/chifra/internal/list/handle_freshen.go

@@ -273,7 +273,7 @@ func (updater *MonitorUpdate) visitChunkToFreshenFinal(fileName string, resultCh
 	if !file.FileExists(indexFilename) {
 		chain := updater.Options.Globals.Chain
 		var man *manifest.Manifest
-		man, err = manifest.ReadManifest(chain, updater.Options.PublisherAddr, manifest.Cache)
+		man, err = manifest.ReadManifest(chain, updater.Options.PublisherAddr, manifest.LocalCache)
 		if err != nil {
 			results = append(results, index.AppearanceResult{Range: bl.Range, Err: err})
 			return

src/apps/chifra/internal/list/validate.go

@@ -5,6 +5,7 @@
 package listPkg
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
@@ -74,13 +75,11 @@ func (opts *ListOptions) validateList() error {
 		}
 	}
 
-	// Note that this does not return if the index is not initialized
-	if err := index.IsInitialized(chain); err != nil {
-		if opts.Globals.IsApiMode() {
-			return err
-		} else {
+	if err := index.IsInitialized(chain, config.HeaderVersion); err != nil {
+		if errors.Is(err, index.ErrNotInitialized) && !opts.Globals.IsApiMode() {
 			logger.Fatal(err)
 		}
+		return err
 	}
 
 	return nil

src/apps/chifra/internal/monitors/validate.go

@@ -5,6 +5,7 @@
 package monitorsPkg
 
 import (
+	"errors"
 	"log"
 	"path/filepath"
 
@@ -67,13 +68,11 @@ func (opts *MonitorsOptions) validateMonitors() error {
 				}
 			}
 
-			// Note that this does not return if the index is not initialized
-			if err := index.IsInitialized(chain); err != nil {
-				if opts.Globals.IsApiMode() {
-					return err
-				} else {
+			if err := index.IsInitialized(chain, config.HeaderVersion); err != nil {
+				if errors.Is(err, index.ErrNotInitialized) && !opts.Globals.IsApiMode() {
 					logger.Fatal(err)
 				}
+				return err
 			}
 
 			if opts.BatchSize < 1 {

src/apps/chifra/internal/scrape/validate.go

@@ -72,13 +72,6 @@ func (opts *ScrapeOptions) validateScrape() error {
 		}
 	}
 
-	if len(opts.Publisher) > 0 {
-		err := validate.ValidateExactlyOneAddr([]string{opts.Publisher})
-		if err != nil {
-			return err
-		}
-	}
-
 	ret := opts.Globals.Validate()
 
 	pidPath := filepath.Join(config.PathToCache(chain), "tmp/scrape.pid")

src/apps/chifra/pkg/config/config.go

@@ -28,7 +28,6 @@ type ConfigFile struct {
 	Pinning   pinningGroup          `toml:"pinning"`
 	Unchained unchainedGroup        `toml:"unchained"`
 	Chains    map[string]chainGroup `toml:"chains"`
-	History   historyGroup          `toml:"history"`
 }
 
 // init sets up default values for the given configuration
@@ -40,24 +39,18 @@ func init() {
 	trueBlocksViper.SetDefault("Settings.IndexPath", PathToRootConfig()+"unchained/")
 	// The default chain to use if none is provided
 	trueBlocksViper.SetDefault("Settings.DefaultChain", "mainnet")
-	// Storage for the last call to particular chifra commands
-	trueBlocksViper.SetDefault("History.Comment", Reserved)
 	// The pinning gateway to query when downloading the unchained index
 	trueBlocksViper.SetDefault("Pinning.GatewayURL", defaultIpfsGateway)
 	// The local endpoint for the IPFS daemon
 	trueBlocksViper.SetDefault("Pinning.LocalPinUrl", "http://localhost:5001")
 	// The remote endpoint for pinning on Pinata
 	trueBlocksViper.SetDefault("Pinning.GatewayUrl", "https://api.pinata.cloud/pinning/pinFileToIPFS")
 	// A warning to the user not to edit the [unchained] section of the config file
-	trueBlocksViper.SetDefault("Unchained.Comment", Reserved)
+	trueBlocksViper.SetDefault("Unchained.Comment", "Use this to customize the Unchained Index")
 	// The default publisher of the index of none other is provided
 	trueBlocksViper.SetDefault("Unchained.PreferredPublisher", "0xf503017d7baf7fbc0fff7492b751025c6a78179b")
 	// V2: The address of the current version of the Unchained Index
 	trueBlocksViper.SetDefault("Unchained.SmartContract", "0x0c316b7042b419d07d343f2f4f5bd54ff731183d")
-	// IPFS hash of the specification for the Unchained Index
-	trueBlocksViper.SetDefault("Unchained.Specification", "QmUyyU8wKW57c3CuwphhMdZb2QA5bsjt9vVfTE6LcBKmE9")
-	// The version of the specification and the pre-image of hash in the header of each chunk
-	trueBlocksViper.SetDefault("Unchained.SpecVersion", "[email protected]")
 }
 
 var configMutex sync.Mutex
@@ -205,5 +198,3 @@ func pathFromXDG(envVar string) (string, error) {
 
 	return filepath.Join(xdg, "") + "/", nil
 }
-
-var Reserved = "reserved for system use - do not edit unless instructed to do so"

src/apps/chifra/pkg/config/migrate.go

@@ -81,14 +81,9 @@ func migrate(currentVer version.Version) error {
 		cfg.Pinning = pinning
 
 		unchained := unchainedGroup{
-			Comment: Reserved,
+			Comment: "Use this to customize the Unchained Index",
 		}
 		cfg.Unchained = unchained
-
-		history := historyGroup{
-			Comment: Reserved,
-		}
-		cfg.History = history
 	}
 
 	// Re-write the file (after making a backup) with the new version

src/apps/chifra/pkg/config/pinningGroup.go

@@ -10,7 +10,7 @@ import (
 	shell "github.com/ipfs/go-ipfs-api"
 )
 
-const defaultIpfsGateway = "https://ipfs.unchainedindex.io/ipfs"
+const defaultIpfsGateway = "https://ipfs.unchainedindex.io/ipfs/"
 
 type pinningGroup struct {
 	GatewayURL   string `toml:"gatewayUrl"`