Insight API UTXO+ [10] [11] [12] [3] [4] (decred#479)

Insight endpoints listed in decred#400: [10] [11] [12] [3] [4]

api/insight/apimiddleware.go

@@ -12,8 +12,10 @@ import (
 	"net/http"
 	"strconv"
 
+	"github.com/decred/dcrd/chaincfg/chainhash"
 	apitypes "github.com/decred/dcrdata/api/types"
 	m "github.com/decred/dcrdata/middleware"
+	"github.com/go-chi/chi"
 )
 
 type contextKey int
@@ -25,6 +27,8 @@ const (
 	ctxNoAsm
 	ctxNoScriptSig
 	ctxNoSpent
+	ctxBlockHash
+	ctxBlockIndex
 )
 
 // BlockHashPathAndIndexCtx is a middleware that embeds the value at the url
@@ -76,13 +80,11 @@ func (c *insightApiContext) PostBroadcastTxCtx(next http.Handler) http.Handler {
 		body, err := ioutil.ReadAll(r.Body)
 		r.Body.Close()
 		if err != nil {
-			apiLog.Errorf("error reading JSON message: %v", err)
-			writeInsightError(w, fmt.Sprintf("error reading JSON message: %v", err))
+			writeInsightError(w, fmt.Sprintf("Error reading JSON message: %v", err))
 			return
 		}
 		err = json.Unmarshal(body, &req)
 		if err != nil {
-			apiLog.Errorf("Failed to parse request: %v", err)
 			writeInsightError(w, fmt.Sprintf("Failed to parse request: %v", err))
 			return
 		}
@@ -225,7 +227,7 @@ func (c *insightApiContext) PostAddrsTxsCtx(next http.Handler) http.Handler {
 			}
 		}
 		if req.NoScriptSig != "" {
-			noScriptSig, err = req.To.Int64()
+			noScriptSig, err = req.NoScriptSig.Int64()
 			if err == nil && noScriptSig != 0 {
 				ctx = context.WithValue(ctx, ctxNoScriptSig, true)
 			}
@@ -237,6 +239,71 @@ func (c *insightApiContext) PostAddrsTxsCtx(next http.Handler) http.Handler {
 				ctx = context.WithValue(ctx, ctxNoSpent, true)
 			}
 		}
+
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+// Process params given in post body for an addrs Utxo endpoint
+func (c *insightApiContext) PostAddrsUtxoCtx(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		req := apitypes.InsightAddr{}
+		body, err := ioutil.ReadAll(r.Body)
+		r.Body.Close()
+		if err != nil {
+			writeInsightError(w, fmt.Sprintf("error reading JSON message: %v", err))
+			return
+		}
+		err = json.Unmarshal(body, &req)
+		if err != nil {
+			writeInsightError(w, fmt.Sprintf("Failed to parse request: %v", err))
+			return
+		}
+		// Successful extraction of Body JSON
+		ctx := context.WithValue(r.Context(), m.CtxAddress, req.Addrs)
+
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+// BlockIndexOrHashPathCtx returns a http.HandlerFunc that embeds the value at
+// the url part {idxorhash} into the request context.
+func (c *insightApiContext) BlockIndexOrHashPathCtx(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var ctx context.Context
+		pathIdxOrHashStr := chi.URLParam(r, "idxorhash")
+		if len(pathIdxOrHashStr) == 2*chainhash.HashSize {
+			ctx = context.WithValue(r.Context(), ctxBlockHash, pathIdxOrHashStr)
+		} else {
+			idx, err := strconv.Atoi(pathIdxOrHashStr)
+			if err != nil {
+				writeInsightError(w, "Valid hash or index not found")
+				return
+			}
+			ctx = context.WithValue(r.Context(), ctxBlockIndex, idx)
+		}
 		next.ServeHTTP(w, r.WithContext(ctx))
 	})
 }
+
+// GetInsightBlockIndexCtx retrieves the ctxBlockIndex data from the request context. If not
+// set, the return value is an 0 and false
+func (c *insightApiContext) GetInsightBlockIndexCtx(r *http.Request) (int, bool) {
+	idx, ok := r.Context().Value(ctxBlockIndex).(int)
+	if !ok {
+		apiLog.Trace("Rawtx hex transaction not set")
+		return 0, false
+	}
+	return idx, true
+}
+
+// GetInsightBlockHashCtx retrieves the ctxBlockHash data from the request context. If not
+// set, the return value is an empty string and false
+func (c *insightApiContext) GetInsightBlockHashCtx(r *http.Request) (string, bool) {
+	hash, ok := r.Context().Value(ctxBlockHash).(string)
+	if !ok {
+		apiLog.Trace("Rawtx hex transaction not set")
+		return "", false
+	}
+	return hash, true
+}

api/insight/apirouter.go

@@ -47,7 +47,7 @@ func NewInsightApiRouter(app *insightApiContext, userRealIP bool) ApiMux {
 	mux.With(m.BlockDateQueryCtx).Get("/blocks", app.getBlockSummaryByTime)
 	mux.With(app.BlockHashPathAndIndexCtx).Get("/block/{blockhash}", app.getBlockSummary)
 	mux.With(m.BlockIndexPathCtx).Get("/block-index/{idx}", app.getBlockHash)
-	mux.With(m.BlockIndexOrHashPathCtx).Get("/rawblock/{idx}", app.getRawBlock)
+	mux.With(app.BlockIndexOrHashPathCtx).Get("/rawblock/{idxorhash}", app.getRawBlock)
 
 	// Transaction endpoints
 	mux.With(middleware.AllowContentType("application/json"),
@@ -69,14 +69,15 @@ func NewInsightApiRouter(app *insightApiContext, userRealIP bool) ApiMux {
 		// POST methods
 		rd.With(middleware.AllowContentType("application/json"),
 			app.ValidatePostCtx, app.PostAddrsTxsCtx).Post("/txs", app.getAddressesTxn)
-		rd.With(m.AddressPostCtx).Post("/utxo", app.getAddressesTxnOutput)
+		rd.With(middleware.AllowContentType("application/json"),
+			app.ValidatePostCtx, app.PostAddrsUtxoCtx).Post("/utxo", app.getAddressesTxnOutput)
 	})
 
 	// Address endpoints
 	mux.Route("/addr/{address}", func(rd chi.Router) {
 		rd.Use(m.AddressPathCtx)
 		rd.With(m.PaginationCtx).Get("/", app.getAddressInfo)
-		rd.Get("/utxo", app.getAddressTxnOutput)
+		rd.Get("/utxo", app.getAddressesTxnOutput)
 		rd.Get("/balance", app.getAddressBalance)
 		rd.Get("/totalReceived", app.getAddressTotalReceived)
 		// TODO Missing unconfirmed balance implementation

api/insight/apiroutes.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -188,17 +189,36 @@ func (c *insightApiContext) getBlockChainHashCtx(r *http.Request) *chainhash.Has
 }
 
 func (c *insightApiContext) getRawBlock(w http.ResponseWriter, r *http.Request) {
-	hash := c.getBlockChainHashCtx(r)
-	blockMsg, err := c.nodeClient.GetBlock(hash)
+
+	hash, ok := c.GetInsightBlockHashCtx(r)
+	if !ok {
+		idx, ok := c.GetInsightBlockIndexCtx(r)
+		if !ok {
+			writeInsightError(w, "Must provide an index or block hash")
+			return
+		}
+		var err error
+		hash, err = c.BlockData.ChainDB.GetBlockHash(int64(idx))
+		if err != nil {
+			writeInsightError(w, "Unable to get block hash from index")
+			return
+		}
+	}
+	chainHash, err := chainhash.NewHashFromStr(hash)
 	if err != nil {
-		apiLog.Errorf("Failed to retrieve block %s: %v", hash.String(), err)
-		http.Error(w, http.StatusText(422), 422)
+		writeInsightError(w, fmt.Sprintf("Failed to parse block hash: %v", err))
+		return
+	}
+
+	blockMsg, err := c.nodeClient.GetBlock(chainHash)
+	if err != nil {
+		writeInsightNotFound(w, fmt.Sprintf("Failed to retrieve block %s: %v", chainHash.String(), err))
 		return
 	}
 	var blockHex bytes.Buffer
 	if err = blockMsg.Serialize(&blockHex); err != nil {
 		apiLog.Errorf("Failed to serialize block: %v", err)
-		http.Error(w, http.StatusText(422), 422)
+		writeInsightError(w, fmt.Sprintf("Failed to serialize block"))
 		return
 	}
 
@@ -220,7 +240,6 @@ func (c *insightApiContext) broadcastTransactionRaw(w http.ResponseWriter, r *ht
 
 	// Check maximum transaction size
 	if len(rawHexTx)/2 > c.params.MaxTxSize {
-		apiLog.Errorf("Rawtx length exceeds maximum allowable characters (%d bytes received)", len(rawHexTx)/2)
 		writeInsightError(w, fmt.Sprintf("Rawtx length exceeds maximum allowable characters (%d bytes received)", len(rawHexTx)/2))
 		return
 	}
@@ -242,29 +261,95 @@ func (c *insightApiContext) broadcastTransactionRaw(w http.ResponseWriter, r *ht
 	writeJSON(w, txidJSON, c.getIndentQuery(r))
 }
 
-func (c *insightApiContext) getAddressTxnOutput(w http.ResponseWriter, r *http.Request) {
-	address := m.GetAddressCtx(r)
+func (c *insightApiContext) getAddressesTxnOutput(w http.ResponseWriter, r *http.Request) {
+	address := m.GetAddressCtx(r) // Required
 	if address == "" {
-		http.Error(w, http.StatusText(422), 422)
+		writeInsightError(w, "Address cannot be empty")
 		return
 	}
-	txnOutputs := c.BlockData.ChainDB.GetAddressUTXO(address)
-	writeJSON(w, txnOutputs, c.getIndentQuery(r))
-}
 
-func (c *insightApiContext) getAddressesTxnOutput(w http.ResponseWriter, r *http.Request) {
-	addresses := strings.Split(m.GetAddressCtx(r), ",")
+	// Allow Addresses to be single or multiple separated by a comma.
+	addresses := strings.Split(address, ",")
 
-	var txnOutputs []apitypes.AddressTxnOutput
+	// Initialize Output Structure
+	txnOutputs := make([]apitypes.AddressTxnOutput, 0)
 
 	for _, address := range addresses {
-		if address == "" {
-			http.Error(w, http.StatusText(422), 422)
-			return
+
+		confirmedTxnOutputs := c.BlockData.ChainDB.GetAddressUTXO(address)
+
+		addressOuts, _, err := c.MemPool.UnconfirmedTxnsForAddress(address)
+		if err != nil {
+			apiLog.Errorf("Error in getting unconfirmed transactions")
+		}
+
+		if addressOuts != nil {
+			// If there is any mempool add to the utxo set
+		FUNDING_TX_DUPLICATE_CHECK:
+			for _, f := range addressOuts.Outpoints {
+				fundingTx, ok := addressOuts.TxnsStore[f.Hash]
+				if !ok {
+					apiLog.Errorf("An outpoint's transaction is not available in TxnStore.")
+					continue
+				}
+				if fundingTx.Confirmed() {
+					apiLog.Errorf("An outpoint's transaction is unexpectedly confirmed.")
+					continue
+				}
+				// TODO: Confirmed() not always return true for txs that have
+				// already been confirmed in a block.  The mempool cache update
+				// process should correctly update these.  Until we sort out why we
+				// need to do one more search on utxo and do not add if this is
+				// already in the list as a confirmed tx.
+				for _, utxo := range confirmedTxnOutputs {
+					if utxo.Vout == f.Index && utxo.TxnID == f.Hash.String() {
+						continue FUNDING_TX_DUPLICATE_CHECK
+					}
+				}
+
+				txnOutput := apitypes.AddressTxnOutput{
+					Address:       address,
+					TxnID:         fundingTx.Hash().String(),
+					Vout:          f.Index,
+					ScriptPubKey:  hex.EncodeToString(fundingTx.Tx.TxOut[f.Index].PkScript),
+					Amount:        dcrutil.Amount(fundingTx.Tx.TxOut[f.Index].Value).ToCoin(),
+					Satoshis:      fundingTx.Tx.TxOut[f.Index].Value,
+					Confirmations: 0,
+					BlockTime:     fundingTx.MemPoolTime,
+				}
+				txnOutputs = append(txnOutputs, txnOutput)
+			}
+		}
+		txnOutputs = append(txnOutputs, confirmedTxnOutputs...)
+
+		// Search for items in mempool that spend utxo (matching hash and index)
+		// and remove those from the set
+		for _, f := range addressOuts.PrevOuts {
+			spendingTx, ok := addressOuts.TxnsStore[f.TxSpending]
+			if !ok {
+				apiLog.Errorf("An outpoint's transaction is not available in TxnStore.")
+				continue
+			}
+			if spendingTx.Confirmed() {
+				apiLog.Errorf("A transaction spending the outpoint of an unconfirmed transaction is unexpectedly confirmed.")
+				continue
+			}
+			for g, utxo := range txnOutputs {
+				if utxo.Vout == f.PreviousOutpoint.Index && utxo.TxnID == f.PreviousOutpoint.Hash.String() {
+					// Found a utxo that is unconfirmed spent.  Remove from slice
+					txnOutputs = append(txnOutputs[:g], txnOutputs[g+1:]...)
+				}
+			}
 		}
-		utxo := c.BlockData.ChainDB.GetAddressUTXO(address)
-		txnOutputs = append(txnOutputs, utxo...)
 	}
+	// Final sort by timestamp desc if unconfirmed and by confirmations
+	// ascending if confirmed
+	sort.Slice(txnOutputs, func(i, j int) bool {
+		if txnOutputs[i].Confirmations == 0 && txnOutputs[j].Confirmations == 0 {
+			return txnOutputs[i].BlockTime > txnOutputs[j].BlockTime
+		}
+		return txnOutputs[i].Confirmations < txnOutputs[j].Confirmations
+	})
 
 	writeJSON(w, txnOutputs, c.getIndentQuery(r))
 }

api/types/insightapitypes.go

@@ -58,6 +58,11 @@ type InsightMultiAddrsTxOutput struct {
 	Items      []InsightTx `json:"items"`
 }
 
+// InsightAddr models the multi address post data structure
+type InsightAddr struct {
+	Addrs string `json:"addrs"`
+}
+
 // InsightPagination models basic pagination output
 // for a result
 type InsightPagination struct {
@@ -70,12 +75,14 @@ type InsightPagination struct {
 type AddressTxnOutput struct {
 	Address       string  `json:"address"`
 	TxnID         string  `json:"txid"`
-	Vout          uint32  `json:"vout,omitempty"`
-	ScriptPubKey  string  `json:"scriptPubKey,omitempty"`
+	Vout          uint32  `json:"vout"`
+	BlockTime     int64   `json:"ts,omitempty"`
+	ScriptPubKey  string  `json:"scriptPubKey"`
 	Height        int64   `json:"height,omitempty"`
 	BlockHash     string  `json:"block_hash,omitempty"`
-	Amount        float64 `json:"amount"`
-	Atoms         float64 `json:"atoms"`
+	Amount        float64 `json:"amount,omitempty"`
+	Atoms         int64   `json:"atoms,omitempty"` // Not Required per Insight spec
+	Satoshis      int64   `json:"satoshis,omitempty"`
 	Confirmations int64   `json:"confirmations"`
 }
 

db/dcrpg/insightapi.go

@@ -198,26 +198,6 @@ func (pgb *ChainDB) GetBlockSummaryTimeRange(min, max int64, limit int) []dbtype
 	return blockSummary
 }
 
-func makeAddressTxOutput(data *dcrjson.SearchRawTransactionsResult, address string) *apitypes.AddressTxnOutput {
-	tx := new(apitypes.AddressTxnOutput)
-	tx.Address = address
-	tx.TxnID = data.Txid
-	tx.Height = 0
-
-	for i := range data.Vout {
-		if len(data.Vout[i].ScriptPubKey.Addresses) != 0 {
-			if data.Vout[i].ScriptPubKey.Addresses[0] == address {
-				tx.ScriptPubKey = data.Vout[i].ScriptPubKey.Hex
-				tx.Vout = data.Vout[i].N
-				tx.Atoms += data.Vout[i].Value
-			}
-		}
-	}
-
-	tx.Amount = tx.Atoms * 100000000
-	return tx
-}
-
 // GetAddressUTXO returns the unspent transaction outputs (UTXOs) paying to the
 // specified address in a []apitypes.AddressTxnOutput.
 func (pgb *ChainDB) GetAddressUTXO(address string) []apitypes.AddressTxnOutput {

db/dcrpg/internal/addrstmts.go

@@ -68,14 +68,17 @@ const (
 									addresses.funding_tx_hash,
 									addresses.value,
 									transactions.block_height,
-									transactions.block_hash
+									block_time,
+									funding_tx_vout_index,
+									pkscript
 									FROM addresses 
 									JOIN transactions ON 
 									addresses.funding_tx_hash = transactions.tx_hash 
+									JOIN vouts on addresses.funding_tx_hash = vouts.tx_hash and addresses.funding_tx_vout_index=vouts.tx_index 
 									WHERE 
 									addresses.address=$1 
 									AND 
-									addresses.spending_tx_row_id IS NULL`
+									addresses.spending_tx_row_id IS NULL order by block_height desc`
 
 	SelectAddressLimitNByAddress = `SELECT * FROM addresses WHERE address=$1 order by id desc limit $2 offset $3;`