Update dependencies and remove juju/errors (#79)

* *: replaced juju/errors by pingcap/errors and use builtin context package

* vendor: update tidb to v2.1.0-rc.3 and update its dependencies

* vendor: update the rest of the direct dependencies

* license: added commercial license

* vendor: update TiDB and PD to v2.1.0-rc.4

Removed the goyacc step since the parser has been moved out. For now we
rely on `git checkout` to recover the parser until v3.0.0-alpha is tagged

* tests: reduce the number of max-open-files used by tikv-server

Jenkin's open file limit is somehow reduced to 65536, and the default tikv
requirement is 40960*2+1000 = 82920, causing CI failing to start the
integration tests. Here we reduce the number to 4096*2+1000 = 9192 via
explicit config.

* license: fix the year range

LICENSE.txt

@@ -0,0 +1,15 @@
+PingCAP CONFIDENTIAL
+____________________
+
+ Copyright (C) 2017 - 2018 PingCAP Incorporated
+ All Rights Reserved.
+
+NOTICE:  All information contained herein is, and remains
+the property of PingCAP Incorporated and its suppliers,
+if any.  The intellectual and technical concepts contained
+herein are proprietary to PingCAP Incorporated
+and its suppliers and may be covered by P.R.China and Foreign Patents,
+patents in process, and are protected by trade secret or copyright law.
+Dissemination of this information or reproduction of this material
+is strictly forbidden unless prior written permission is obtained
+from PingCAP Incorporated.

Makefile

@@ -54,31 +54,6 @@ checksuccess:
 		echo "Lightning build successfully :-) !" ; \
 	fi
 
-# FIXME: move this to pingcap/goyacc so we could `go get` it instead?
-$(TIDBDIR)/bin/goyacc:
-	mkdir -p $(TIDBDIR)/bin
-	curl -f -L -o $(TIDBDIR)/bin/main.go https://raw.githubusercontent.com/pingcap/tidb/master/parser/goyacc/main.go
-	$(GOBUILD) -o $(TIDBDIR)/bin/goyacc $(TIDBDIR)/bin/main.go
-
-goyacc: $(TIDBDIR)/bin/goyacc
-
-parser: goyacc
-	$(TIDBDIR)/bin/goyacc -o /dev/null $(TIDBDIR)/parser/parser.y
-	$(TIDBDIR)/bin/goyacc -o $(TIDBDIR)/parser/parser.go $(TIDBDIR)/parser/parser.y 2>&1 | egrep "(shift|reduce)/reduce" | awk '{print} END {if (NR > 0) {print "Find conflict in parser.y. Please check y.output for more information."; exit 1;}}'
-	rm -f y.output
-
-	@if [ $(ARCH) = $(LINUX) ]; \
-	then \
-		sed -i -e 's|//line.*||' -e 's/yyEofCode/yyEOFCode/' $(TIDBDIR)/parser/parser.go; \
-	elif [ $(ARCH) = $(MAC) ]; \
-	then \
-		/usr/bin/sed -i "" 's|//line.*||' $(TIDBDIR)/parser/parser.go; \
-		/usr/bin/sed -i "" 's/yyEofCode/yyEOFCode/' $(TIDBDIR)/parser/parser.go; \
-	fi
-
-	@awk 'BEGIN{print "// Code generated by goyacc"} {print $0}' $(TIDBDIR)/parser/parser.go > tmp_parser.go && mv tmp_parser.go $(TIDBDIR)/parser/parser.go;
-
-
 lightning:
 	$(GOBUILD) $(RACE_FLAG) -ldflags '$(LDFLAGS)' -o $(LIGHTNING_BIN) cmd/main.go
 
@@ -108,7 +83,7 @@ coverage:
 	go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html"
 	grep -F '<option' "$(TEST_DIR)/all_cov.html"
 
-update: update_vendor parser clean_vendor
+update: update_vendor clean_vendor
 update_vendor:
 	rm -rf vendor/
 	GO111MODULE=on go mod verify

cmd/main.go

@@ -7,12 +7,11 @@ import (
 	"os/signal"
 	"syscall"
 
-	"github.com/juju/errors"
-
+	"github.com/pkg/errors"
 	"github.com/pingcap/tidb-lightning/lightning"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
-	"github.com/pingcap/tidb/plan"
+	plan "github.com/pingcap/tidb/planner/core"
 )
 
 func setGlobalVars() {

cmd/tidb-lightning-ctl/main.go

@@ -7,11 +7,11 @@ import (
 	"os"
 	"path"
 
-	"github.com/juju/errors"
 	"github.com/pingcap/kvproto/pkg/import_sstpb"
 	"github.com/pingcap/tidb-lightning/lightning/config"
 	"github.com/pingcap/tidb-lightning/lightning/kv"
 	"github.com/pingcap/tidb-lightning/lightning/restore"
+	"github.com/pkg/errors"
 )
 
 func main() {
@@ -91,7 +91,7 @@ func switchMode(ctx context.Context, cfg *config.Config, mode string) error {
 	case config.NormalMode:
 		m = import_sstpb.SwitchMode_Normal
 	default:
-		return errors.NotValidf("invalid mode %s, must use %s or %s", mode, config.ImportMode, config.NormalMode)
+		return errors.Errorf("invalid mode %s, must use %s or %s", mode, config.ImportMode, config.NormalMode)
 	}
 
 	importer, err := kv.NewImporter(ctx, cfg.TikvImporter.Addr, cfg.TiDB.PdAddr)

go.mod

@@ -1,7 +1,7 @@
 module github.com/pingcap/tidb-lightning
 
 require (
-	github.com/BurntSushi/toml v0.3.0
+	github.com/BurntSushi/toml v0.3.1
 	github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 // indirect
 	github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a // indirect
 	github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect
@@ -14,40 +14,41 @@ require (
 	github.com/cznic/strutil v0.0.0-20150430124730-1eb03e3cc9d3
 	github.com/cznic/y v0.0.0-20160420101755-9fdf92d4aac0
 	github.com/go-sql-driver/mysql v1.4.0
-	github.com/golang/protobuf v1.1.0 // indirect
+	github.com/gogo/protobuf v1.1.1 // indirect
 	github.com/golang/snappy v0.0.0-20150730031844-723cc1e459b8 // indirect
 	github.com/google/btree v0.0.0-20161217183710-316fb6d3f031 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20171020063731-82921fcf811d // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20160910222444-6b7015e65d36 // indirect
 	github.com/joho/sqltocsv v0.0.0-20180904231936-b24deec2b806
-	github.com/juju/errors v0.0.0-20160809030848-6f54ff631840
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 // indirect
 	github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef // indirect
+	github.com/opentracing/basictracer-go v1.0.0 // indirect
 	github.com/opentracing/opentracing-go v1.0.2 // indirect
 	github.com/pingcap/check v0.0.0-20161122095354-9b266636177e
 	github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e // indirect
-	github.com/pingcap/kvproto v0.0.0-20180624112519-716bb0d730ba
-	github.com/pingcap/pd v1.1.0-alpha.0.20171213134044-eed819b69991 // indirect
-	github.com/pingcap/tidb v2.1.0-beta.0.20180823101743-1fa5669fb48a+incompatible
-	github.com/pingcap/tidb-tools v0.0.0-20180806015954-b605dbc412c4
-	github.com/pingcap/tipb v0.0.0-20180614062814-015341a07dc3 // indirect
-	github.com/prometheus/client_golang v0.8.0
+	github.com/pingcap/kvproto v0.0.0-20181010074705-0ba3ca8a6e37
+	github.com/pingcap/pd v2.1.0-rc.4+incompatible // indirect
+	github.com/pingcap/tidb v2.1.0-rc.4+incompatible
+	github.com/pingcap/tidb-tools v0.0.0-20180806015954-b605dbc412c4 // indirect
+	github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 // indirect
+	github.com/pkg/errors v0.8.0
+	github.com/prometheus/client_golang v0.9.0
 	github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335 // indirect
 	github.com/prometheus/common v0.0.0-20160623151427-4402f4e5ea79 // indirect
 	github.com/prometheus/procfs v0.0.0-20160411190841-abf152e5f3e9 // indirect
 	github.com/satori/go.uuid v1.2.0
-	github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0
-	github.com/sirupsen/logrus v0.11.6-0.20170323161349-3bcb09397d6d
+	github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0 // indirect
+	github.com/sirupsen/logrus v1.1.1
 	github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 // indirect
 	github.com/twinj/uuid v0.1.0 // indirect
 	github.com/uber/jaeger-client-go v2.8.0+incompatible // indirect
 	github.com/uber/jaeger-lib v1.3.2-0.20180112221534-34d9cc24e47a // indirect
-	golang.org/x/net v0.0.0-20180724234803-3673e40ba225
-	golang.org/x/sys v0.0.0-20161006025142-8d1157a43547
+	golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33
 	golang.org/x/text v0.3.0
 	google.golang.org/appengine v1.1.1-0.20180731164958-4216e58b9158 // indirect
-	google.golang.org/genproto v0.0.0-20170711235230-b0a3dcfcd1a9 // indirect
-	google.golang.org/grpc v1.12.0
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20180817145747-7d6a1875575e
+	google.golang.org/grpc v1.15.0
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
+
+replace github.com/pkg/errors v0.8.0 => github.com/pingcap/errors v0.10.1

lightning/common/log.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"sync/atomic"
 
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/natefinch/lumberjack.v2"
 

lightning/common/util.go

@@ -16,7 +16,7 @@ import (
 	"path/filepath"
 
 	"github.com/go-sql-driver/mysql"
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	tmysql "github.com/pingcap/tidb/mysql"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"

lightning/config/config.go

@@ -8,7 +8,7 @@ import (
 	"runtime"
 
 	"github.com/BurntSushi/toml"
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 )
 

lightning/kv/importer.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	"github.com/satori/go.uuid"
 	"google.golang.org/grpc"
 

lightning/kv/sql2kv.go

@@ -1,7 +1,7 @@
 package kv
 
 import (
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/metric"
 	sqltool "github.com/pingcap/tidb-lightning/lightning/sql"

lightning/lightning.go

@@ -1,15 +1,15 @@
 package lightning
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"runtime"
 	"sync"
 
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	sstpb "github.com/pingcap/kvproto/pkg/import_sstpb"
 	"github.com/prometheus/client_golang/prometheus"
-	"golang.org/x/net/context"
 
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"

lightning/mydump/loader.go

@@ -5,9 +5,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/juju/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
+	"github.com/pkg/errors"
 )
 
 var (

lightning/mydump/reader.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"unicode/utf8"
 
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"golang.org/x/text/encoding"
 	"golang.org/x/text/encoding/simplifiedchinese"

lightning/mydump/region.go

@@ -7,8 +7,8 @@ import (
 	"sort"
 	"sync"
 
-	"github.com/juju/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
+	"github.com/pkg/errors"
 )
 
 type TableRegion struct {

lightning/restore/checkpoints.go

@@ -1,16 +1,16 @@
 package restore
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"io"
 	"strings"
 	"time"
 
 	"github.com/joho/sqltocsv"
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	"github.com/satori/go.uuid"
-	"golang.org/x/net/context"
 
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	verify "github.com/pingcap/tidb-lightning/lightning/verification"

lightning/restore/restore.go

@@ -1,6 +1,7 @@
 package restore
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"io"
@@ -12,7 +13,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-semver/semver"
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	sstpb "github.com/pingcap/kvproto/pkg/import_sstpb"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
@@ -22,11 +23,6 @@ import (
 	verify "github.com/pingcap/tidb-lightning/lightning/verification"
 	tidbcfg "github.com/pingcap/tidb/config"
 	"github.com/pingcap/tidb/util/kvencoder"
-
-	// hack for glide update, delete later
-	_ "github.com/pingcap/tidb-tools/pkg/table-router"
-	_ "github.com/siddontang/go/sync2"
-	"golang.org/x/net/context"
 )
 
 const (

lightning/restore/tidb.go

@@ -1,20 +1,20 @@
 package restore
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"net/http"
 	"net/url"
 	"regexp"
 	"time"
 
-	"github.com/juju/errors"
+	"github.com/pkg/errors"
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"
 	"github.com/pingcap/tidb-lightning/lightning/metric"
 	"github.com/pingcap/tidb-lightning/lightning/mydump"
 	"github.com/pingcap/tidb/model"
-	"golang.org/x/net/context"
 )
 
 type TiDBManager struct {

lightning/sql/parser_test.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/juju/errors"
 	. "github.com/pingcap/check"
+	"github.com/pkg/errors"
 
 	"github.com/pingcap/tidb-lightning/lightning/common"
 	"github.com/pingcap/tidb-lightning/lightning/config"

tests/run.sh

@@ -28,11 +28,20 @@ start_services() {
         sleep 1
     done
 
+    # Tries to limit the max number of open files under the system limit
+    cat - > "$TEST_DIR/tikv-config.toml" <<EOF
+[rocksdb]
+max-open-files = 4096
+[raftdb]
+max-open-files = 4096
+EOF
+
     echo "Starting TiKV..."
     bin/tikv-server \
         --pd 127.0.0.1:2379 \
         -A 127.0.0.1:20160 \
         --log-file "$TEST_DIR/tikv.log" \
+        -C "$TEST_DIR/tikv-config.toml" \
         -s "$TEST_DIR/tikv" &
     sleep 1