Merge pull request #741 from jbenet/feat/blocklist

add blocklist to gateway executable

Author: Brian Tiger Chow

cmd/ipfs-gateway-fs/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bufio"
 	"errors"
 	"flag"
 	"log"
@@ -17,13 +18,15 @@ import (
 )
 
 var (
-	writable               = flag.Bool("writable", false, "enable writing objects (with POST, PUT and DELETE)")
-	refreshAssetsInterval  = flag.Duration("refresh-assets-interval", 30*time.Second, "refresh assets")
-	garbageCollectInterval = flag.Duration("gc-interval", 24*time.Hour, "frequency of repo garbage collection")
-	assetsPath             = flag.String("assets-path", "", "if provided, periodically adds contents of path to IPFS")
-	host                   = flag.String("host", "/ip4/0.0.0.0/tcp/8080", "override the HTTP host listening address")
-	performGC              = flag.Bool("gc", false, "perform garbage collection")
-	nBitsForKeypair        = flag.Int("b", 1024, "number of bits for keypair (if repo is uninitialized)")
+	blocklistFilepath        = flag.String("blocklist", "", "keys that should not be served by the gateway")
+	writable                 = flag.Bool("writable", false, "enable writing objects (with POST, PUT and DELETE)")
+	refreshBlockListInterval = flag.Duration("refresh-blocklist-interval", 30*time.Second, "refresh blocklist")
+	refreshAssetsInterval    = flag.Duration("refresh-assets-interval", 30*time.Second, "refresh assets")
+	garbageCollectInterval   = flag.Duration("gc-interval", 24*time.Hour, "frequency of repo garbage collection")
+	assetsPath               = flag.String("assets-path", "", "if provided, periodically adds contents of path to IPFS")
+	host                     = flag.String("host", "/ip4/0.0.0.0/tcp/8080", "override the HTTP host listening address")
+	performGC                = flag.Bool("gc", false, "perform garbage collection")
+	nBitsForKeypair          = flag.Int("b", 1024, "number of bits for keypair (if repo is uninitialized)")
 )
 
 func main() {
@@ -77,8 +80,18 @@ func run() error {
 		}
 	}
 
+	blocklist := &corehttp.BlockList{}
+	gateway := corehttp.NewGateway(corehttp.GatewayConfig{
+		Writable:  *writable,
+		BlockList: blocklist,
+	})
+
+	if err := runBlockListWorker(blocklist, *blocklistFilepath); err != nil {
+		return err
+	}
+
 	opts := []corehttp.ServeOption{
-		corehttp.GatewayOption(*writable),
+		gateway.ServeOption(),
 	}
 	return corehttp.ListenAndServe(node, *host, opts...)
 }
@@ -112,3 +125,41 @@ func runFileServerWorker(ctx context.Context, node *core.IpfsNode) error {
 	}()
 	return nil
 }
+
+func runBlockListWorker(blocklist *corehttp.BlockList, filepath string) error {
+	if filepath == "" {
+		return nil
+	}
+	go func() {
+		for _ = range time.Tick(*refreshBlockListInterval) {
+			log.Println("updating the blocklist...")
+			func() { // in a func to allow defer f.Close()
+				f, err := os.Open(filepath)
+				if err != nil {
+					log.Println(err)
+				}
+				defer f.Close()
+				scanner := bufio.NewScanner(f)
+				blocked := make(map[string]struct{}) // Implement using Bloom Filter hybrid if blocklist gets large
+				for scanner.Scan() {
+					t := scanner.Text()
+					blocked[t] = struct{}{}
+				}
+
+				// If an error occurred, do not change the existing decider. This
+				// is to avoid accidentally clearing the list if the deploy is
+				// botched.
+				if err := scanner.Err(); err != nil {
+					log.Println(err)
+				} else {
+					blocklist.SetDecider(func(s string) bool {
+						_, ok := blocked[s]
+						return !ok
+					})
+					log.Printf("updated the blocklist (%d entries)", len(blocked))
+				}
+			}()
+		}
+	}()
+	return nil
+}

core/corehttp/gateway.go

@@ -2,13 +2,30 @@ package corehttp
 
 import (
 	"net/http"
+	"sync"
 
 	core "github.com/jbenet/go-ipfs/core"
 )
 
-func GatewayOption(writable bool) ServeOption {
+// Gateway should be instantiated using NewGateway
+type Gateway struct {
+	Config GatewayConfig
+}
+
+type GatewayConfig struct {
+	BlockList *BlockList
+	Writable  bool
+}
+
+func NewGateway(conf GatewayConfig) *Gateway {
+	return &Gateway{
+		Config: conf,
+	}
+}
+
+func (g *Gateway) ServeOption() ServeOption {
 	return func(n *core.IpfsNode, mux *http.ServeMux) error {
-		gateway, err := newGatewayHandler(n, writable)
+		gateway, err := newGatewayHandler(n, g.Config)
 		if err != nil {
 			return err
 		}
@@ -17,3 +34,41 @@ func GatewayOption(writable bool) ServeOption {
 		return nil
 	}
 }
+
+func GatewayOption(writable bool) ServeOption {
+	g := NewGateway(GatewayConfig{
+		Writable:  writable,
+		BlockList: &BlockList{},
+	})
+	return g.ServeOption()
+}
+
+// Decider decides whether to Allow string
+type Decider func(string) bool
+
+type BlockList struct {
+
+	mu sync.RWMutex
+	d  Decider
+}
+
+func (b *BlockList) ShouldAllow(s string) bool {
+	b.mu.RLock()
+	d := b.d
+	b.mu.RUnlock()
+	if d == nil {
+		return true
+	}
+	return d(s)
+}
+
+// SetDecider atomically swaps the blocklist's decider
+func (b *BlockList) SetDecider(d Decider) {
+	b.mu.Lock()
+	b.d = d
+	b.mu.Unlock()
+}
+
+func (b *BlockList) ShouldBlock(s string) bool {
+	return !b.ShouldAllow(s)
+}

core/corehttp/gateway_handler.go

@@ -50,13 +50,13 @@ type directoryItem struct {
 type gatewayHandler struct {
 	node     *core.IpfsNode
 	dirList  *template.Template
-	writable bool
+	config   GatewayConfig
 }
 
-func newGatewayHandler(node *core.IpfsNode, writable bool) (*gatewayHandler, error) {
+func newGatewayHandler(node *core.IpfsNode, conf GatewayConfig) (*gatewayHandler, error) {
 	i := &gatewayHandler{
 		node:     node,
-		writable: writable,
+		config:   conf,
 	}
 	err := i.loadTemplate()
 	if err != nil {
@@ -125,18 +125,20 @@ func (i *gatewayHandler) NewDagReader(nd *dag.Node) (uio.ReadSeekCloser, error)
 	return uio.NewDagReader(i.node.Context(), nd, i.node.DAG)
 }
 
+// TODO(btc): break this apart into separate handlers using a more expressive
+// muxer
 func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if i.writable && r.Method == "POST" {
+	if i.config.Writable && r.Method == "POST" {
 		i.postHandler(w, r)
 		return
 	}
 
-	if i.writable && r.Method == "PUT" {
+	if i.config.Writable && r.Method == "PUT" {
 		i.putHandler(w, r)
 		return
 	}
 
-	if i.writable && r.Method == "DELETE" {
+	if i.config.Writable && r.Method == "DELETE" {
 		i.deleteHandler(w, r)
 		return
 	}
@@ -147,7 +149,7 @@ func (i *gatewayHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	errmsg := "Method " + r.Method + " not allowed: "
-	if !i.writable {
+	if !i.config.Writable {
 		w.WriteHeader(http.StatusMethodNotAllowed)
 		errmsg = errmsg + "read only access"
 	} else {
@@ -164,6 +166,11 @@ func (i *gatewayHandler) getHandler(w http.ResponseWriter, r *http.Request) {
 
 	urlPath := r.URL.Path
 
+	if i.config.BlockList != nil && i.config.BlockList.ShouldBlock(urlPath) {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
 	nd, p, err := i.ResolvePath(ctx, urlPath)
 	if err != nil {
 		if err == routing.ErrNotFound {