Merge pull request #3470 from TrueBlocks/feature/monitor-update-to-pkg

Moves monitor freshening to monitor package

src/apps/chifra/internal/export/handle_freshen.go

@@ -1,16 +1,10 @@
 package exportPkg
 
 import (
-	listPkg "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/list"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/monitor"
 )
 
 func (opts *ExportOptions) FreshenMonitorsForExport(monitorArray *[]monitor.Monitor) (bool, error) {
-	listOpts := listPkg.ListOptions{
-		Addrs:   opts.Addrs,
-		Silent:  true,
-		Globals: opts.Globals,
-	}
-
-	return listOpts.HandleFreshenMonitors(monitorArray)
+	var updater = monitor.NewUpdater(opts.Globals.Chain, opts.Globals.TestMode, true, opts.Addrs)
+	return updater.FreshenMonitors(monitorArray)
 }

src/apps/chifra/internal/list/handle_count.go

@@ -62,3 +62,5 @@ func (opts *ListOptions) HandleCount(monitorArray []monitor.Monitor) error {
 
 	return output.StreamMany(ctx, fetchData, opts.Globals.OutputOpts())
 }
+
+const maxTestingBlock = 17000000

src/apps/chifra/internal/list/output.go

@@ -55,7 +55,9 @@ func (opts *ListOptions) ListInternal() error {
 	// EXISTING_CODE
 	// We always freshen the monitors. This call fills the monitors array.
 	monitorArray := make([]monitor.Monitor, 0, len(opts.Addrs))
-	if canceled, err := opts.HandleFreshenMonitors(&monitorArray); err != nil || canceled {
+	var updater = monitor.NewUpdater(opts.Globals.Chain, opts.Globals.TestMode, true, opts.Addrs)
+	updater.PublisherAddr = opts.PublisherAddr
+	if canceled, err := updater.FreshenMonitors(&monitorArray); err != nil || canceled {
 		return err
 	}
 

src/apps/chifra/internal/monitors/handle_freshen.go

@@ -1,7 +1,6 @@
 package monitorsPkg
 
 import (
-	listPkg "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/list"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/monitor"
 )
@@ -12,12 +11,7 @@ func (opts *MonitorsOptions) FreshenMonitorsForWatch(addrs []base.Address) (bool
 		strs = append(strs, addr.Hex())
 	}
 
-	listOpts := listPkg.ListOptions{
-		Addrs:   strs,
-		Silent:  true,
-		Globals: opts.Globals,
-	}
-
-	unused := make([]monitor.Monitor, 0, len(addrs))
-	return listOpts.HandleFreshenMonitors(&unused)
+	unusedMonitors := make([]monitor.Monitor, 0, len(addrs))
+	var updater = monitor.NewUpdater(opts.Globals.Chain, opts.Globals.TestMode, true, strs)
+	return updater.FreshenMonitors(&unusedMonitors)
 }

src/apps/chifra/internal/list/handle_freshen.go → src/apps/chifra/pkg/monitor/monitor_freshen.go

@@ -1,4 +1,4 @@
-package listPkg
+package monitor
 
 // Copyright 2021 The TrueBlocks Authors. All rights reserved.
 // Use of this source code is governed by a license that can
@@ -22,22 +22,35 @@ import (
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/manifest"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/monitor"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/sigintTrap"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/validate"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/walk"
 )
 
-// AddressMonitorMap carries arrays of appearances that have not yet been written to the monitor file
-type AddressMonitorMap map[base.Address]*monitor.Monitor
-
 // MonitorUpdate stores the original 'chifra list' command line options plus
 type MonitorUpdate struct {
-	MaxTasks   int
-	MonitorMap AddressMonitorMap
-	Options    *ListOptions
-	FirstBlock uint64
+	MaxTasks      int
+	MonitorMap    map[base.Address]*Monitor
+	Chain         string
+	PublisherAddr base.Address
+	TestMode      bool
+	Silent        bool
+	FirstBlock    uint64
+	Addrs         []string
+}
+
+func NewUpdater(chain string, testMode, silent bool, addrs []string) MonitorUpdate {
+	return MonitorUpdate{
+		MaxTasks:      12,
+		FirstBlock:    utils.NOPOS,
+		Chain:         chain,
+		PublisherAddr: base.Address{},
+		TestMode:      testMode,
+		Silent:        silent,
+		Addrs:         addrs,
+		MonitorMap:    make(map[base.Address]*Monitor, len(addrs)),
+	}
 }
 
 const maxTestingBlock = 17000000
@@ -70,15 +83,14 @@ func unlockForAddress(address string) {
 	mutex.Unlock()
 }
 
-func (opts *ListOptions) HandleFreshenMonitors(monitorArray *[]monitor.Monitor) (bool, error) {
-	chain := opts.Globals.Chain
+func (updater *MonitorUpdate) FreshenMonitors(monitorArray *[]Monitor) (bool, error) {
 
 	// TODO: There are special case addresses for the sender of mining rewards and
 	// TODO: prefund allocations that get ignored here because they are baddresses.
 	// TODO: We could, if we wished, create special cases here to (for example) report
 	// TODO: the balance sheet for "Ethereum" as a whole. How much has been spent on mining?
 	// TODO: How much was the original allocation worth?
-	for _, address := range opts.Addrs {
+	for _, address := range updater.Addrs {
 		lockForAddress(address)
 		defer unlockForAddress(address) // reminder: this defers until the function returns, not this loop
 	}
@@ -94,22 +106,15 @@ func (opts *ListOptions) HandleFreshenMonitors(monitorArray *[]monitor.Monitor)
 	trapChannel := sigintTrap.Enable(ctx, cancel, cleanOnQuit)
 	defer sigintTrap.Disable(trapChannel)
 
-	var updater = MonitorUpdate{
-		MaxTasks:   12,
-		MonitorMap: make(AddressMonitorMap, len(opts.Addrs)),
-		Options:    opts,
-		FirstBlock: utils.NOPOS,
-	}
-
 	// This removes dups and keeps a map from address to a pointer to the monitors
-	for _, addr := range opts.Addrs {
+	for _, addr := range updater.Addrs {
 		err := needsMigration(addr)
 		if err != nil {
 			return canceled, err
 		}
 
 		if updater.MonitorMap[base.HexToAddress(addr)] == nil {
-			mon, _ := monitor.NewMonitorStaged(chain, addr)
+			mon, _ := NewMonitorStaged(updater.Chain, addr)
 			_ = mon.ReadMonitorHeader()
 			if uint64(mon.LastScanned) < updater.FirstBlock {
 				updater.FirstBlock = uint64(mon.LastScanned)
@@ -120,7 +125,7 @@ func (opts *ListOptions) HandleFreshenMonitors(monitorArray *[]monitor.Monitor)
 		}
 	}
 
-	bloomPath := filepath.Join(config.PathToIndex(chain), "blooms/")
+	bloomPath := filepath.Join(config.PathToIndex(updater.Chain), "blooms/")
 	files, err := os.ReadDir(bloomPath)
 	if err != nil {
 		return canceled, err
@@ -155,7 +160,7 @@ func (opts *ListOptions) HandleFreshenMonitors(monitorArray *[]monitor.Monitor)
 				}
 			}
 
-			if opts.Globals.TestMode && fileRange.Last > maxTestingBlock {
+			if updater.TestMode && fileRange.Last > maxTestingBlock {
 				continue
 			}
 
@@ -187,9 +192,9 @@ func (opts *ListOptions) HandleFreshenMonitors(monitorArray *[]monitor.Monitor)
 		}
 	}
 
-	if !opts.Globals.TestMode {
+	if !updater.TestMode {
 		// TODO: Note we could actually test this if we had the concept of a FAKE_HEAD block
-		stagePath := index.ToStagingPath(config.PathToIndex(chain) + "staging")
+		stagePath := index.ToStagingPath(config.PathToIndex(updater.Chain) + "staging")
 		stageFn, _ := file.LatestFileInFolder(stagePath)
 		rng := base.RangeFromFilename(stageFn)
 		lines := []string{}
@@ -271,9 +276,9 @@ func (updater *MonitorUpdate) visitChunkToFreshenFinal(fileName string, resultCh
 
 	indexFilename := index.ToIndexPath(fileName)
 	if !file.FileExists(indexFilename) {
-		chain := updater.Options.Globals.Chain
+		chain := updater.Chain
 		var man *manifest.Manifest
-		man, err = manifest.ReadManifest(chain, updater.Options.PublisherAddr, manifest.LocalCache)
+		man, err = manifest.ReadManifest(chain, updater.PublisherAddr, manifest.LocalCache)
 		if err != nil {
 			results = append(results, index.AppearanceResult{Range: bl.Range, Err: err})
 			return
@@ -341,7 +346,7 @@ func (updater *MonitorUpdate) updateMonitors(result *index.AppearanceResult) {
 				_, err := mon.WriteAppearances(*result.AppRecords, true /* append */)
 				if err != nil {
 					logger.Error(err)
-				} else if !updater.Options.Globals.TestMode && !updater.Options.Silent {
+				} else if !updater.TestMode && !updater.Silent {
 					msg := fmt.Sprintf("%s appended %d apps at %s", mon.Address.Hex(), nWritten, result.Range)
 					logger.Info(msg)
 				}
@@ -351,7 +356,7 @@ func (updater *MonitorUpdate) updateMonitors(result *index.AppearanceResult) {
 }
 
 func needsMigration(addr string) error {
-	mon := monitor.Monitor{Address: base.HexToAddress(addr)}
+	mon := Monitor{Address: base.HexToAddress(addr)}
 	path := strings.Replace(mon.Path(), ".mon.bin", ".acct.bin", -1)
 	if file.FileExists(path) {
 		path = strings.Replace(path, config.PathToCache(mon.Chain), "./", -1)
@@ -391,3 +396,15 @@ func getAppearances(addrStr string, lines []string, lastVisited uint32, found in
 	}
 	return &results
 }
+
+// moveAllToProduction completes the update by moving the monitor files from ./cache/<chain>/monitors/staging to
+// ./cache/<chain>/monitors.
+func (updater *MonitorUpdate) moveAllToProduction() error {
+	for _, mon := range updater.MonitorMap {
+		err := mon.MoveToProduction()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

src/apps/chifra/internal/list/list_integration_test.go → src/apps/chifra/pkg/monitor/monitor_freshen_integration_test.go

@@ -5,31 +5,24 @@
 // Use of this source code is governed by a license that can
 // be found in the LICENSE file.
 
-package listPkg
+package monitor
 
 import (
 	"os"
 	"testing"
 
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/internal/globals"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
 	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
-	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/monitor"
+	"github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/utils"
 )
 
-func Test_HandleFreshenMonitors(t *testing.T) {
+func Test_FreshenMonitors(t *testing.T) {
 	_ = config.GetRootConfig()
-	opts := globals.GlobalOptions{}
-	opts.Chain = "mainnet"
-	listOpts := ListOptions{
-		Addrs:   []string{"0x846a9cb5593483b59bb386f5a878fbb2a0d1d8dc"},
-		Silent:  true,
-		Globals: opts,
-	}
+	addrs := []string{"0x846a9cb5593483b59bb386f5a878fbb2a0d1d8dc"}
 
 	// This is an address that we use for testing...early transactor but not for long so unlikely to be used for real
-	mon, _ := monitor.NewMonitor("mainnet", base.HexToAddress(listOpts.Addrs[0]), true)
+	mon, _ := NewMonitor("mainnet", base.HexToAddress(addrs[0]), true)
 	file.Remove(mon.Path())
 
 	got := mon.String()
@@ -39,8 +32,9 @@ func Test_HandleFreshenMonitors(t *testing.T) {
 	}
 
 	os.Setenv("FAKE_FINAL_BLOCK", "2500000")
-	monitorArray := make([]monitor.Monitor, 0, len(listOpts.Addrs))
-	_, err := listOpts.HandleFreshenMonitors(&monitorArray)
+	monitorArray := make([]Monitor, 0, len(addrs))
+	var updater = NewUpdater(utils.GetTestChain(), true, true, addrs)
+	_, err := updater.FreshenMonitors(&monitorArray)
 	if err != nil {
 		t.Error(err)
 	}
@@ -53,7 +47,9 @@ func Test_HandleFreshenMonitors(t *testing.T) {
 	// }
 
 	os.Setenv("FAKE_FINAL_BLOCK", "")
-	_, err = listOpts.HandleFreshenMonitors(&monitorArray)
+	// Must reset this or use a different one. We'll just reset it.
+	updater = NewUpdater(utils.GetTestChain(), true, true, addrs)
+	_, err = updater.FreshenMonitors(&monitorArray)
 	if err != nil {
 		t.Error(err)
 	}

src/apps/chifra/pkg/names/names.go

@@ -26,10 +26,11 @@ const (
 	Regular   Parts = 0x1
 	Custom    Parts = 0x2
 	Prefund   Parts = 0x4
-	Testing   Parts = 0x8
-	MatchCase Parts = 0x10
-	Expanded  Parts = 0x20
-	Tags      Parts = 0x40
+	Baddress  Parts = 0x8
+	Testing   Parts = 0x10
+	MatchCase Parts = 0x20
+	Expanded  Parts = 0x40
+	Tags      Parts = 0x80
 )
 
 type SortBy int

src/apps/chifra/pkg/names/regular.go

@@ -54,5 +54,40 @@ func loadRegularMap(chain string, thePath string, terms []string, parts Parts, r
 		}
 	}
 
+	if parts&Baddress != 0 {
+		loadKnownBadresses(terms, parts, ret)
+	}
+
 	return nil
 }
+
+// loadKnownBadresses loads the known bad addresses from the cache
+func loadKnownBadresses(terms []string, parts Parts, ret *map[base.Address]types.SimpleName) {
+	knownBadAddresses := []types.SimpleName{
+		{
+			Address: base.PrefundSender,
+			Name:    "PrefundSender",
+			Tags:    "75-Baddress",
+		},
+		{
+			Address: base.BlockRewardSender,
+			Name:    "BlockRewardSender",
+			Tags:    "75-Baddress",
+		},
+		{
+			Address: base.UncleRewardSender,
+			Name:    "UncleRewardSender",
+			Tags:    "75-Baddress",
+		},
+		{
+			Address: base.WithdrawalSender,
+			Name:    "WithdrawalSender",
+			Tags:    "75-Baddress",
+		},
+	}
+	for _, n := range knownBadAddresses {
+		if doSearch(&n, terms, parts) {
+			(*ret)[n.Address] = n
+		}
+	}
+}