go/runtime: Remove tag indexer

.changelog/3888.breaking.md

@@ -0,0 +1,5 @@
+go/runtime: Remove tag indexer
+
+The tag indexer is slow, buggy and unused by the SDK. Runtime transaction
+indexing and querying capabilities should be provided as part of the new
+event streaming API.

.changelog/3888.cfg.md

@@ -0,0 +1,4 @@
+go/runtime: Remove tag indexer
+
+The `runtime.history.tag_indexer.backend` configuration option has been
+removed.

go/go.mod

@@ -19,17 +19,19 @@ replace (
 )
 
 require (
-	github.com/blevesearch/bleve v1.0.14
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/cenkalti/backoff/v4 v4.1.1
 	github.com/dgraph-io/badger/v2 v2.2007.2
 	github.com/dgraph-io/badger/v3 v3.2103.1
 	github.com/eapache/channels v1.1.0
+	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
+	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/fxamacker/cbor/v2 v2.2.1-0.20200820021930-bafca87fa6db
 	github.com/go-kit/log v0.1.0
 	github.com/golang/protobuf v1.5.2
 	github.com/golang/snappy v0.0.4
 	github.com/google/btree v1.0.1
+	github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 // indirect
 	github.com/hashicorp/go-hclog v0.16.2
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-plugin v1.4.2

go/oasis-node/cmd/debug/txsource/workload/queries.go

@@ -697,67 +697,6 @@ func (q *queries) doRuntimeQueries(ctx context.Context, rng *rand.Rand) error {
 		)
 		return fmt.Errorf("runtimeClient.GetBlock, round: %d: %w", round, err)
 	}
-	// GetBlockByHash requires that the block was actually indexed, so wait for it.
-	err = q.runtime.WaitBlockIndexed(ctx, &runtimeClient.WaitBlockIndexedRequest{
-		RuntimeID: q.runtimeID,
-		Round:     round,
-	})
-	if err != nil {
-		q.logger.Error("runtime WaitBlockIndexed failure",
-			"round", round,
-			"err", err,
-		)
-		return fmt.Errorf("runtimeClient.WaitBlockIndexed, round: %d: %w", round, err)
-	}
-	block2, err := q.runtime.GetBlockByHash(ctx, &runtimeClient.GetBlockByHashRequest{
-		RuntimeID: q.runtimeID,
-		BlockHash: block.Header.EncodedHash(),
-	})
-	if err != nil {
-		q.logger.Error("runtime GetBlockByHash failure",
-			"hash", block.Header.EncodedHash(),
-			"latest_round", latestRound,
-			"err", err,
-		)
-		return fmt.Errorf("runtimeClient.GetBlockByHash, hash: %s: %w", block.Header.EncodedHash(), err)
-	}
-	if block.Header.EncodedHash() != block2.Header.EncodedHash() {
-		q.logger.Error("runtime block header hash mismatch",
-			"round", round,
-			"latest_round", latestRound,
-			"round_hash", block.Header.EncodedHash(),
-			"hash", block2.Header.EncodedHash(),
-		)
-		return fmt.Errorf("expected equal blocks, got: byRound: %s byHash: %s", block.Header.EncodedHash(), block2.Header.EncodedHash())
-	}
-
-	_, err = q.runtime.QueryTxs(ctx, &runtimeClient.QueryTxsRequest{
-		RuntimeID: q.runtimeID,
-		Query: runtimeClient.Query{
-			RoundMin: q.runtimeGenesisRound,
-			RoundMax: round,
-		},
-	})
-	if err != nil {
-		q.logger.Error("runtime QueryTxs failure",
-			"round", round,
-			"latest_round", latestRound,
-			"err", err,
-		)
-
-		switch round == latestRound {
-		case false:
-			q.queryTxsHistoricalFailures++
-			// Historical queries are allowed to fail few times.
-			if q.queryTxsHistoricalFailures <= queriesNumAllowedQueryTxsHistoricalFailures {
-				break
-			}
-			fallthrough
-		case true:
-			// Query for latest round should never fail.
-			return fmt.Errorf("runtimeClient.QueryTxs: %w", err)
-		}
-	}
 
 	q.logger.Debug("done runtime queries",
 		"latest_round", latestRound,

go/oasis-node/node_test.go

@@ -176,7 +176,6 @@ func newTestNode(t *testing.T) *testNode {
 	viper.Set("datadir", dataDir)
 	viper.Set("log.file", filepath.Join(dataDir, "test-node.log"))
 	viper.Set(runtimeRegistry.CfgSupported, testRuntimeID.String())
-	viper.Set(runtimeRegistry.CfgTagIndexerBackend, "bleve")
 	viper.Set(runtimeRegistry.CfgRuntimePaths, map[string]string{
 		testRuntimeID.String(): "mock-runtime",
 	})

go/oasis-test-runner/oasis/args.go

@@ -310,14 +310,6 @@ func (args *argBuilder) tendermintSupplementarySanity(interval uint64) *argBuild
 	return args
 }
 
-func (args *argBuilder) runtimeTagIndexerBackend(backend string) *argBuilder {
-	args.vec = append(args.vec, Argument{
-		Name:   runtimeRegistry.CfgTagIndexerBackend,
-		Values: []string{backend},
-	})
-	return args
-}
-
 func (args *argBuilder) runtimeClientMaxTransactionAge(maxTxAge int64) *argBuilder {
 	args.vec = append(args.vec, Argument{
 		Name:   runtimeClient.CfgMaxTransactionAge,

go/oasis-test-runner/oasis/client.go

@@ -49,9 +49,6 @@ func (client *Client) AddArgs(args *argBuilder) error {
 		args.runtimeClientMaxTransactionAge(client.maxTransactionAge)
 	}
 
-	if len(client.runtimes) > 0 {
-		args.runtimeTagIndexerBackend("bleve")
-	}
 	for _, idx := range client.runtimes {
 		v := client.net.runtimes[idx]
 		// XXX: could support configurable binary idx if ever needed.

go/oasis-test-runner/scenario/e2e/runtime/runtime_message.go

@@ -64,30 +64,34 @@ func (sc *runtimeMessageImpl) Run(childEnv *env.Env) error {
 	ctx := context.Background()
 	c := sc.Net.ClientController().RuntimeClient
 
-	// We should be at round 2 after the epoch transitions, plus 1 for the genesis block.
-	waitCtx, cancel := context.WithTimeout(ctx, waitTimeout)
-	defer cancel()
-	if err = c.WaitBlockIndexed(waitCtx, &api.WaitBlockIndexedRequest{RuntimeID: runtimeID, Round: 3}); err != nil {
+	blkCh, sub, err := c.WatchBlocks(ctx, runtimeID)
+	if err != nil {
 		return err
 	}
+	defer sub.Close()
 
-	// Save latest round.
+	// We should be at round 2 after the epoch transitions, plus 1 for the genesis block.
 	sc.Logger.Debug("querying latest round")
-	round, err := c.GetBlock(ctx, &api.GetBlockRequest{RuntimeID: runtimeID, Round: api.RoundLatest})
-	if err != nil {
-		return err
-	}
-	latestRound := round.Header.Round
-	sc.Logger.Debug("latest runtime round", "round", latestRound)
-	if latestRound != 3 {
-		return fmt.Errorf("unexpected latest round, got: %d, expected: %d", latestRound, 3)
-	}
+	const expectedRound uint64 = 2 + 1
+WaitLatestRound:
+	for {
+		select {
+		case blk := <-blkCh:
+			round := blk.Block.Header.Round
+			sc.Logger.Debug("seen runtime round", "round", round)
 
-	blkCh, sub, err := c.WatchBlocks(ctx, runtimeID)
-	if err != nil {
-		return err
+			if round < expectedRound {
+				continue
+			}
+			if round > expectedRound {
+				return fmt.Errorf("unexpected latest round, got: %d, expected: %d", round, expectedRound)
+			}
+			break WaitLatestRound
+		case <-time.After(waitTimeout):
+			return fmt.Errorf("timed out waiting for runtime rounds")
+		}
 	}
-	defer sub.Close()
+	latestRound := expectedRound
 
 	// Submit a consensus transfer transaction. This should result in two runtime
 	// rounds:

go/runtime/client/api/api.go

@@ -67,16 +67,6 @@ type RuntimeClient interface {
 	// GetBlock fetches the given runtime block.
 	GetBlock(ctx context.Context, request *GetBlockRequest) (*block.Block, error)
 
-	// GetBlockByHash fetches the given runtime block by its block hash.
-	GetBlockByHash(ctx context.Context, request *GetBlockByHashRequest) (*block.Block, error)
-
-	// GetTx fetches the given runtime transaction.
-	GetTx(ctx context.Context, request *GetTxRequest) (*TxResult, error)
-
-	// GetTxByBlockHash fetches the given rutnime transaction where the
-	// block is identified by its hash instead of its round number.
-	GetTxByBlockHash(ctx context.Context, request *GetTxByBlockHashRequest) (*TxResult, error)
-
 	// GetTransactions fetches all runtime transactions in a given block.
 	GetTransactions(ctx context.Context, request *GetTransactionsRequest) ([][]byte, error)
 
@@ -86,17 +76,8 @@ type RuntimeClient interface {
 	// Query makes a runtime-specific query.
 	Query(ctx context.Context, request *QueryRequest) (*QueryResponse, error)
 
-	// QueryTx queries the indexer for a specific runtime transaction.
-	QueryTx(ctx context.Context, request *QueryTxRequest) (*TxResult, error)
-
-	// QueryTxs queries the indexer for specific runtime transactions.
-	QueryTxs(ctx context.Context, request *QueryTxsRequest) ([]*TxResult, error)
-
 	// WatchBlocks subscribes to blocks for a specific runtimes.
 	WatchBlocks(ctx context.Context, runtimeID common.Namespace) (<-chan *roothash.AnnotatedBlock, pubsub.ClosableSubscription, error)
-
-	// WaitBlockIndexed waits for a runtime block to be indexed by the indexer.
-	WaitBlockIndexed(ctx context.Context, request *WaitBlockIndexedRequest) error
 }
 
 // RuntimeClientService is the runtime client service interface.
@@ -136,34 +117,6 @@ type GetBlockRequest struct {
 	Round     uint64           `json:"round"`
 }
 
-// GetBlockByHashRequest is a GetBlockByHash request.
-type GetBlockByHashRequest struct {
-	RuntimeID common.Namespace `json:"runtime_id"`
-	BlockHash hash.Hash        `json:"block_hash"`
-}
-
-// TxResult is the transaction query result.
-type TxResult struct {
-	Block  *block.Block `json:"block"`
-	Index  uint32       `json:"index"`
-	Input  []byte       `json:"input"`
-	Output []byte       `json:"output"`
-}
-
-// GetTxRequest is a GetTx request.
-type GetTxRequest struct {
-	RuntimeID common.Namespace `json:"runtime_id"`
-	Round     uint64           `json:"round"`
-	Index     uint32           `json:"index"`
-}
-
-// GetTxByBlockHashRequest is a GetTxByBlockHash request.
-type GetTxByBlockHashRequest struct {
-	RuntimeID common.Namespace `json:"runtime_id"`
-	BlockHash hash.Hash        `json:"block_hash"`
-	Index     uint32           `json:"index"`
-}
-
 // GetTransactionsRequest is a GetTransactions request.
 type GetTransactionsRequest struct {
 	RuntimeID common.Namespace `json:"runtime_id"`
@@ -197,53 +150,3 @@ type QueryRequest struct {
 type QueryResponse struct {
 	Data cbor.RawMessage `json:"data"`
 }
-
-// QueryTxRequest is a QueryTx request.
-type QueryTxRequest struct {
-	RuntimeID common.Namespace `json:"runtime_id"`
-	Key       []byte           `json:"key"`
-	Value     []byte           `json:"value"`
-}
-
-// QueryCondition is a query condition.
-type QueryCondition struct {
-	// Key is the tag key that should be matched.
-	Key []byte `json:"key"`
-	// Values are a list of tag values that the given tag key should
-	// have. They are combined using an OR query which means that any
-	// of the values will match.
-	Values [][]byte `json:"values"`
-}
-
-// Query is a complex query against the index.
-type Query struct {
-	// RoundMin is an optional minimum round (inclusive).
-	RoundMin uint64 `json:"round_min"`
-	// RoundMax is an optional maximum round (inclusive).
-	//
-	// A zero value means that there is no upper limit.
-	RoundMax uint64 `json:"round_max"`
-
-	// Conditions are the query conditions.
-	//
-	// They are combined using an AND query which means that all of
-	// the conditions must be satisfied for an item to match.
-	Conditions []QueryCondition `json:"conditions"`
-
-	// Limit is the maximum number of results to return.
-	//
-	// A zero value means that the `maxQueryLimit` limit is used.
-	Limit uint64 `json:"limit"`
-}
-
-// QueryTxsRequest is a QueryTxs request.
-type QueryTxsRequest struct {
-	RuntimeID common.Namespace `json:"runtime_id"`
-	Query     Query            `json:"query"`
-}
-
-// WaitBlockIndexedRequest is a WaitBlockIndexed request.
-type WaitBlockIndexedRequest struct {
-	RuntimeID common.Namespace `json:"runtime_id"`
-	Round     uint64           `json:"round"`
-}