diff --git a/.dockerignore b/.dockerignore index bf4ffe706c1..4cabd9531f7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,6 +2,4 @@ !.git/HEAD !.git/refs/ !.git/packed-refs -cmd/ipfs/ipfs -vendor/gx/ -test/ +test/sharness/lib/sharness/ diff --git a/Makefile b/Makefile index a746944f5a7..2ac4d094960 100644 --- a/Makefile +++ b/Makefile @@ -1,195 +1,10 @@ -# Minimum version numbers for software required to build IPFS -IPFS_MIN_GO_VERSION = 1.7 -IPFS_MIN_GX_VERSION = 0.6 -IPFS_MIN_GX_GO_VERSION = 1.1 +# General tools -GOTAGS = -GOTAGS += "" # we have to have always at least one tag, empty tag works well +SHELL=PATH=$(PATH) /bin/sh -GOFLAGS = -GOTFLAGS = +PROTOC = protoc --gogo_out=. --proto_path=.:/usr/local/opt/protobuf/include:$(dir $@) $< -export IPFS_REUSEPORT=false -export GOFLAGS -export GOTFLAGS +# enable second expansion +.SECONDEXPANSION: -GOFLAGS += -tags $(call join-with,$(comma),$(GOTAGS)) - -ifeq ($(TEST_NO_FUSE),1) - GOTAGS += nofuse -endif - -ifeq ($(OS),Windows_NT) - GOPATH_DELIMITER = ; -else - GOPATH_DELIMITER = : -endif - -dist_root=/ipfs/QmNZL8wNsvAGdVYr8uGeUE9aGfHjFpHegAWywQFEdSaJbp -gx_bin=bin/gx-v0.9.0 -gx-go_bin=bin/gx-go-v1.3.0 - - -# util functions - -space = -space += -comma =, -join-with = $(subst $(space),$1,$(strip $2)) -# use things in our bin before any other system binaries -export PATH := bin:$(PATH) -export IPFS_API ?= v04x.ipfs.io - -all: help - -godep: - go get github.com/tools/godep - -go_check: - @bin/check_go_version $(IPFS_MIN_GO_VERSION) - -bin/gx-v%: - @echo "installing gx $(@:bin/gx-%=%)" - @bin/dist_get ${dist_root} gx $@ $(@:bin/gx-%=%) - rm -f bin/gx - ln -s $(@:bin/%=%) bin/gx - -bin/gx-go-v%: - @echo "installing gx-go $(@:bin/gx-go-%=%)" - @bin/dist_get ${dist_root} gx-go $@ $(@:bin/gx-go-%=%) - rm -f bin/gx-go - ln -s $(@:bin/%=%) bin/gx-go - -gx_check: ${gx_bin} ${gx-go_bin} - -path_check: - @bin/check_go_path $(realpath $(shell pwd)) $(realpath $(addsuffix /src/github.com/ipfs/go-ipfs,$(subst $(GOPATH_DELIMITER), ,$(GOPATH)))) - -deps: go_check gx_check path_check $(covertools_rule) - ${gx_bin} --verbose install --global - -deps_covertools: - go get -u github.com/wadey/gocovmerge - go get -u golang.org/x/tools/cmd/cover - -# saves/vendors third-party dependencies to Godeps/_workspace -# -r flag rewrites import paths to use the vendored path -# ./... performs operation on all packages in tree -vendor: godep - godep save -r ./... - -nofuse: GOTAGS += nofuse -nofuse: deps - $(MAKE) -C cmd/ipfs install - -install build: deps - $(MAKE) -C cmd/ipfs $@ - -clean: - $(MAKE) -C cmd/ipfs clean - $(MAKE) -C test clean - -uninstall: - $(MAKE) -C cmd/ipfs uninstall - -PHONY += all help godep gx_check covertools -PHONY += go_check deps vendor install build nofuse clean uninstall - -############################################################## -# tests targets - -test: test_expensive - -test_short: test_go_fmt build test_go_short test_sharness_short - -test_expensive: test_go_fmt build test_go_expensive test_sharness_expensive windows_build_check - -test_3node: - $(MAKE) -C test/3nodetest - -test_go_fmt: - bin/test-go-fmt - -test_go_short: GOTFLAGS += -test.short -test_go_race: GOTFLAGS += -race -test_go_expensive test_go_short test_go_race: - go test $(GOFLAGS) $(GOTFLAGS) ./... - -coverage: deps_covertools - @echo Running coverage - $(eval PKGS := $(shell go list -f '{{if (len .GoFiles)}}{{.ImportPath}}{{end}}' ./... | grep -v /vendor/ | grep -v /Godeps/)) -#$(eval PKGS_DELIM := $(call join-with,$(comma),$(PKGS))) - @go list -f '{{if or (len .TestGoFiles) (len .XTestGoFiles)}}go test $(GOFLAGS) $(GOTFLAGS) -covermode=atomic -coverprofile={{.Name}}_{{len .Imports}}_{{len .Deps}}.coverprofile {{.ImportPath}}{{end}}' $(GOFLAGS) $(PKGS) | xargs -I {} bash -c {} 2>&1 | grep -v 'warning: no packages being tested depend on' - gocovmerge `ls *.coverprofile` > coverage.txt - rm *.coverprofile - bash -c 'bash <(curl -s https://codecov.io/bash)' - -test_sharness_short: - $(MAKE) -j1 -C test/sharness/ - -test_sharness_expensive: - TEST_EXPENSIVE=1 $(MAKE) -j1 -C test/sharness/ - -test_all_commits: - @echo "testing all commits between origin/master..HEAD" - @echo "WARNING: this will 'git rebase --exec'." - @test/bin/continueyn - GIT_EDITOR=true git rebase -i --exec "$(MAKE) test" origin/master - -test_all_commits_travis: - # these are needed because travis. - # we don't use this yet because it takes way too long. - git config --global user.email "nemo@ipfs.io" - git config --global user.name "IPFS BOT" - git fetch origin master:master - GIT_EDITOR=true git rebase -i --exec "$(MAKE) test" master - -# since we have CI for osx and linux but not windows, this should help -windows_build_check: - GOOS=windows GOARCH=amd64 go build -o .test.ipfs.exe ./cmd/ipfs - rm -f .test.ipfs.exe - -PHONY += test test_short test_expensive - -############################################################## -# A semi-helpful help message - -help: - @echo 'DEPENDENCY TARGETS:' - @echo '' - @echo ' gx_check - Installs or upgrades gx and gx-go' - @echo ' deps - Download dependencies using gx' - @echo ' vendor - Create a Godep workspace of 3rd party dependencies' - @echo '' - @echo 'BUILD TARGETS:' - @echo '' - @echo ' all - print this help message' - @echo ' build - Build binary at ./cmd/ipfs/ipfs' - @echo ' nofuse - Build binary with no fuse support' - @echo ' install - Build binary and install into $$GOPATH/bin' -# @echo ' dist_install - TODO: c.f. ./cmd/ipfs/dist/README.md' - @echo '' - @echo 'CLEANING TARGETS:' - @echo '' - @echo ' clean - Remove binary from build directory' - @echo ' uninstall - Remove binary from $$GOPATH/bin' - @echo '' - @echo 'TESTING TARGETS:' - @echo '' - @echo ' test - Run expensive tests and Window$$ check' - @echo ' test_short - Run short tests and sharness tests' - @echo ' test_expensive - Run a few extras' - @echo ' test_3node' - @echo ' test_go_short' - @echo ' test_go_expensive' - @echo ' test_go_race' - @echo ' test_sharness_short' - @echo ' test_sharness_expensive' - @echo ' test_all_commits' - @echo " test_all_commits_travis - DON'T USE: takes way too long" - @echo ' windows_build_check' - @echo '' - -PHONY += help - -.PHONY: $(PHONY) +include Rules.mk diff --git a/Rules.mk b/Rules.mk new file mode 100644 index 00000000000..72f8cf5ea9b --- /dev/null +++ b/Rules.mk @@ -0,0 +1,141 @@ +TGT_BIN := +CLEAN := +COVERAGE := +DISTCLEAN := +TEST := +TEST_SHORT := + +all: help # all has to be first defined target +.PHONY: all + +include mk/util.mk +include mk/golang.mk +include mk/gx.mk + +# -------------------- # +# extra properties # +# -------------------- # + +ifeq ($(TEST_NO_FUSE),1) + GOTAGS += nofuse +endif +export IPFS_REUSEPORT=false + +# -------------------- # +# sub-files # +# -------------------- # +dir := bin +include $(dir)/Rules.mk + +dir := test +include $(dir)/Rules.mk + +dir := cmd/ipfs +include $(dir)/Rules.mk + +# include this file only if coverage target is executed +# it is quite expensive +ifneq ($(filter coverage% clean distclean,$(MAKECMDGOALS)),) + # has to be after cmd/ipfs due to PATH + dir := coverage + include $(dir)/Rules.mk +endif + +dir := namesys/pb +include $(dir)/Rules.mk + +dir := unixfs/pb +include $(dir)/Rules.mk + +dir := merkledag/pb +include $(dir)/Rules.mk + +dir := exchange/bitswap/message/pb +include $(dir)/Rules.mk + +dir := diagnostics/pb +include $(dir)/Rules.mk + +dir := pin/internal/pb +include $(dir)/Rules.mk + +# -------------------- # +# universal rules # +# -------------------- # + +%.pb.go: %.proto + $(PROTOC) + +# -------------------- # +# core targets # +# -------------------- # + + +build: $(TGT_BIN) +.PHONY: build + +clean: + rm -rf $(CLEAN) +.PHONY: clean + +coverage: $(COVERAGE) +.PHONY: coverage + +distclean: clean + rm -rf $(DISTCLEAN) +.PHONY: distclean + +test: $(TEST) +.PHONY: test + +test_short: $(TEST_SHORT) +.PHONY: test_short + +deps: gx-deps +.PHONY: deps + +nofuse: GOTAGS += nofuse +nofuse: build +.PHONY: nofuse + +install: $$(DEPS_GO) + go install $(go-flags-with-tags) ./cmd/ipfs +.PHONY: install + +uninstall: + go clean -i ./cmd/ipfs +.PHONY: uninstall + +help: + @echo 'DEPENDENCY TARGETS:' + @echo '' + @echo ' deps - Download dependencies using bundled gx' + @echo ' test_sharness_deps - Download and build dependencies for sharness' + @echo '' + @echo 'BUILD TARGETS:' + @echo '' + @echo ' all - print this help message' + @echo ' build - Build binary at ./cmd/ipfs/ipfs' + @echo ' nofuse - Build binary with no fuse support' + @echo ' install - Build binary and install into $$GOPATH/bin' +# @echo ' dist_install - TODO: c.f. ./cmd/ipfs/dist/README.md' + @echo '' + @echo 'CLEANING TARGETS:' + @echo '' + @echo ' clean - Remove files generated by build' + @echo ' distclean - Remove files that are no part of a repository' + @echo ' uninstall - Remove binary from $$GOPATH/bin' + @echo '' + @echo 'TESTING TARGETS:' + @echo '' + @echo ' test - Run expensive tests' + @echo ' test_short - Run short tests and short sharness tests' + @echo ' test_go_short' + @echo ' test_go_expensive' + @echo ' test_go_race' + @echo ' test_sharness_short' + @echo ' test_sharness_expensive' + @echo ' test_sharness_race' + @echo ' coverage - Collects coverage info from unit tests and sharness' + @echo +.PHONY: help diff --git a/bin/Rules.mk b/bin/Rules.mk new file mode 100644 index 00000000000..347aec3f3c5 --- /dev/null +++ b/bin/Rules.mk @@ -0,0 +1,26 @@ +include mk/header.mk + +dist_root_$(d)=/ipfs/QmTazYLCem8B7pQGLsCj1Q4MxkqGMmD9XugvStLJSQ8uxA + +$(d)/gx: $(d)/gx-v0.10.0 +$(d)/gx-go: $(d)/gx-go-v1.4.0 + +TGTS_$(d) := $(d)/gx $(d)/gx-go +DISTCLEAN += $(wildcard $(d)/gx-v*) $(wildcard $(d)/gx-go-v*) $(d)/tmp + +PATH := $(realpath $(d)):$(PATH) + +$(TGTS_$(d)): + rm -f $@ + ln -s $(notdir $^) $@ + +bin/gx-v%: + @echo "installing gx $(@:bin/gx-%=%)" + bin/dist_get $(dist_root_bin) gx $@ $(@:bin/gx-%=%) + +bin/gx-go-v%: + @echo "installing gx-go $(@:bin/gx-go-%=%)" + @bin/dist_get $(dist_root_bin) gx-go $@ $(@:bin/gx-go-%=%) + +CLEAN += $(TGTS_$(d)) +include mk/footer.mk diff --git a/bin/circle.sh b/bin/circle.sh new file mode 100755 index 00000000000..d7f13970031 --- /dev/null +++ b/bin/circle.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# the CircleCI build line got a bit out of hands +# thus we have sparate file for it + +curl -s https://codecov.io/bash > codecov + +case $CIRCLE_NODE_INDEX in + 0) make -j 1 coverage/unit_tests.coverprofile && + bash codecov -cF unittests -X search -f coverage/unit_tests.coverprofile + ;; + 1) make -j 1 coverage/sharness_tests.coverprofile && + bash codecov -cF sharness -X search -f coverage/sharness_tests.coverprofile + ;; +esac diff --git a/ci/Dockerfile.buildenv b/ci/Dockerfile.buildenv new file mode 100644 index 00000000000..d124dd92dc8 --- /dev/null +++ b/ci/Dockerfile.buildenv @@ -0,0 +1,45 @@ +FROM golang:1.7 +MAINTAINER Jakub Sztandera + + +RUN apt-get update && apt-get install -y --no-install-recommends \ + netcat-openbsd bash curl \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +ENV GOBIN $GOPATH/bin +ENV SRC_PATH /go/src/github.com/ipfs/go-ipfs + +RUN curl -s https://codecov.io/bash > /usr/bin/codecov && chmod +x /usr/bin/codecov \ + && go get -u github.com/Kubuxu/gocovmerge && go get -u golang.org/x/tools/cmd/cover +ENV IPFS_SKIP_COVER_BINS 1 + + +RUN useradd user +RUN chown -R user $GOPATH + +WORKDIR $SRC_PATH + +COPY ./bin $SRC_PATH/bin/ +COPY ./mk $SRC_PATH/mk/ +RUN chown -R user $GOPATH + +USER user +# install gx and gx-go +RUN make -j 4 -f bin/Rules.mk d=bin bin/gx bin/gx-go && cp bin/gx bin/gx-go $GOBIN +USER root +ENV IPFS_GX_USE_GLOBAL 1 + +COPY package.json $SRC_PATH/ +ENV PATH $SRC_PATH/bin:$PATH + +USER user +RUN make -f mk/gx.mk gx-deps +USER root + +COPY . $SRC_PATH/ +RUN chown -R user $GOPATH +USER user +RUN make cmd/ipfs/ipfs #populate go cache + +CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait"] diff --git a/ci/jenkins b/ci/jenkins new file mode 100644 index 00000000000..34e52dfe452 --- /dev/null +++ b/ci/jenkins @@ -0,0 +1 @@ +go-ipfs-jenkinsfile diff --git a/circle.yml b/circle.yml index acf3aae44b2..ccc888b06d1 100644 --- a/circle.yml +++ b/circle.yml @@ -35,6 +35,6 @@ dependencies: test: override: - - case $CIRCLE_NODE_INDEX in 0) make coverage ;; 1) make test_sharness_expensive ;; esac: + - bin/circle.sh: pwd: "../.go_workspace/src/$IMPORT_PATH" parallel: true diff --git a/cmd/ipfs/.gitignore b/cmd/ipfs/.gitignore index 264486b528d..9393c5fb564 100644 --- a/cmd/ipfs/.gitignore +++ b/cmd/ipfs/.gitignore @@ -1,2 +1,3 @@ ipfs +ipfs-test-cover ipfs.exe diff --git a/cmd/ipfs/Makefile b/cmd/ipfs/Makefile deleted file mode 100644 index 59746c62e74..00000000000 --- a/cmd/ipfs/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -COMMIT := $(shell git rev-parse --short HEAD) -GOFLAGS += -ldflags="-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(COMMIT)" - -all: install - -install: - go install $(GOFLAGS) - -build: - go build -i $(GOFLAGS) - -clean: - go clean $(GOFLAGS) - -uninstall: - go clean -i $(GOFLAGS) diff --git a/cmd/ipfs/Rules.mk b/cmd/ipfs/Rules.mk new file mode 100644 index 00000000000..38cc676c538 --- /dev/null +++ b/cmd/ipfs/Rules.mk @@ -0,0 +1,31 @@ +include mk/header.mk +IPFS_BIN_$(d) := $(call go-curr-pkg-tgt) + +TGT_BIN += $(IPFS_BIN_$(d)) +CLEAN += $(IPFS_BIN_$(d)) + +PATH := $(realpath $(d)):$(PATH) + +# disabled for now +# depend on *.pb.go files in the repo as Order Only (as they shouldn't be rebuilt if exist) +# DPES_OO_$(d) := diagnostics/pb/diagnostics.pb.go exchange/bitswap/message/pb/message.pb.go +# DEPS_OO_$(d) += merkledag/pb/merkledag.pb.go namesys/pb/namesys.pb.go +# DEPS_OO_$(d) += pin/internal/pb/header.pb.go unixfs/pb/unixfs.pb.go + +$(IPFS_BIN_$(d)): GOFLAGS += -ldflags="-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(shell git rev-parse --short HEAD)" + +# uses second expansion to collect all $(DEPS_GO) +$(IPFS_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS #| $(DEPS_OO_$(d)) + $(go-build) + +COVER_BIN_$(d) := $(d)/ipfs-test-cover +CLEAN += $(COVER_BIN_$(d)) + +$(COVER_BIN_$(d)): GOTAGS += testrunmain +$(COVER_BIN_$(d)): $(d) $$(DEPS_GO) ALWAYS + $(eval TMP_PKGS := $(shell go list -f '{{range .Deps}}{{.}} {{end}}' $(go-flags-with-tags) ./cmd/ipfs | sed 's/ /\n/g' | grep ipfs/go-ipfs | grep -v ipfs/go-ipfs/Godeps) $(call go-pkg-name,$<)) + $(eval TMP_LIST := $(call join-with,$(comma),$(TMP_PKGS))) + @echo go test $@ -c -covermode atomic -coverpkg ... $(go-flags-with-tags) ./$(@D) # for info + @go test -o $@ -c -covermode atomic -coverpkg $(TMP_LIST) $(go-flags-with-tags) ./$(@D) 2>&1 | (grep -v 'warning: no packages being tested' || true) + +include mk/footer.mk diff --git a/cmd/ipfs/main.go b/cmd/ipfs/main.go index 9159ce7c5ab..bc48a5bf5e9 100644 --- a/cmd/ipfs/main.go +++ b/cmd/ipfs/main.go @@ -64,6 +64,10 @@ type cmdInvocation struct { // - output the response // - if anything fails, print error, maybe with help func main() { + os.Exit(mainRet()) +} + +func mainRet() int { rand.Seed(time.Now().UnixNano()) ctx := logging.ContextWithLoggable(context.Background(), loggables.Uuid("session")) var err error @@ -79,7 +83,7 @@ func main() { stopFunc, err := profileIfEnabled() if err != nil { printErr(err) - os.Exit(1) + return 1 } defer stopFunc() // to be executed as late as possible @@ -104,7 +108,7 @@ func main() { if len(os.Args) == 2 { if os.Args[1] == "help" { printHelp(false, os.Stdout) - os.Exit(0) + return 0 } else if os.Args[1] == "--version" { os.Args[1] = "version" } @@ -119,11 +123,11 @@ func main() { longH, shortH, err := invoc.requestedHelp() if err != nil { printErr(err) - os.Exit(1) + return 1 } if longH || shortH { printHelp(longH, os.Stdout) - os.Exit(0) + return 0 } } @@ -138,7 +142,7 @@ func main() { fmt.Fprintf(os.Stderr, "\n") printHelp(false, os.Stderr) } - os.Exit(1) + return 1 } // here we handle the cases where @@ -146,7 +150,7 @@ func main() { // - the main command is invoked. if invoc.cmd == nil || invoc.cmd.Run == nil { printHelp(false, os.Stdout) - os.Exit(0) + return 0 } // ok, finally, run the command invocation. @@ -161,16 +165,16 @@ func main() { if isClientError(err) { printMetaHelp(os.Stderr) } - os.Exit(1) + return 1 } // everything went better than expected :) _, err = io.Copy(os.Stdout, output) if err != nil { printErr(err) - - os.Exit(1) + return 1 } + return 0 } func (i *cmdInvocation) Run(ctx context.Context) (output io.Reader, err error) { diff --git a/cmd/ipfs/runmain_test.go b/cmd/ipfs/runmain_test.go new file mode 100644 index 00000000000..8eecce3347f --- /dev/null +++ b/cmd/ipfs/runmain_test.go @@ -0,0 +1,30 @@ +// +build testrunmain + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "testing" +) + +// this abuses go so much that I felt dirty writing this code +// but it is the only way to do it without writing custom compiler that would +// be a clone of go-build with go-test +func TestRunMain(t *testing.T) { + args := flag.Args() + os.Args = append([]string{os.Args[0]}, args...) + ret := mainRet() + + p := os.Getenv("IPFS_COVER_RET_FILE") + if len(p) != 0 { + ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", ret)), 0777) + } + + // close outputs so go testing doesn't print anything + null, _ := os.Open(os.DevNull) + os.Stderr = null + os.Stdout = null +} diff --git a/coverage/.gitignore b/coverage/.gitignore new file mode 100644 index 00000000000..bfe701a4184 --- /dev/null +++ b/coverage/.gitignore @@ -0,0 +1,4 @@ +unitcover +sharnesscover +ipfs + diff --git a/coverage/Rules.mk b/coverage/Rules.mk new file mode 100644 index 00000000000..bf433621c0a --- /dev/null +++ b/coverage/Rules.mk @@ -0,0 +1,55 @@ +include mk/header.mk + +$(d)/coverage_deps: + rm -rf $(@D)/unitcover && mkdir $(@D)/unitcover + rm -rf $(@D)/sharnesscover && mkdir $(@D)/sharnesscover +ifneq ($(IPFS_SKIP_COVER_BINS),1) + go get -u github.com/Kubuxu/gocovmerge + go get -u golang.org/x/tools/cmd/cover +endif +.PHONY: $(d)/coverage_deps + +# unit tests coverage +UTESTS_$(d) := $(shell go list -f '{{if (len .TestGoFiles)}}{{.ImportPath}}{{end}}' $(go-flags-with-tags) ./... | grep -v go-ipfs/vendor | grep -v go-ipfs/Godeps) + +UCOVER_$(d) := $(addsuffix .coverprofile,$(addprefix $(d)/unitcover/, $(subst /,_,$(UTESTS_$(d))))) + +$(UCOVER_$(d)): $(d)/coverage_deps ALWAYS + $(eval TMP_PKG := $(subst _,/,$(basename $(@F)))) + $(eval TMP_DEPS := $(shell go list -f '{{range .Deps}}{{.}} {{end}}' $(go-flags-with-tags) $(TMP_PKG) | sed 's/ /\n/g' | grep ipfs/go-ipfs | grep -v ipfs/go-ipfs/Godeps) $(TMP_PKG)) + $(eval TMP_DEPS_LIST := $(call join-with,$(comma),$(TMP_DEPS))) + go test $(go-flags-with-tags) $(GOTFLAGS) -covermode=atomic -coverpkg=$(TMP_DEPS_LIST) -coverprofile=$@ $(TMP_PKG) + + +$(d)/unit_tests.coverprofile: $(UCOVER_$(d)) + gocovmerge $^ > $@ + +TGTS_$(d) := $(d)/unit_tests.coverprofile + + +# sharness tests coverage +$(d)/ipfs: GOTAGS += testrunmain +$(d)/ipfs: $(d)/main + $(go-build) + +CLEAN += $(d)/ipfs + +ifneq ($(filter coverage%,$(MAKECMDGOALS)),) + # this is quite hacky but it is best way I could fiture out + DEPS_test/sharness += cmd/ipfs/ipfs-test-cover $(d)/coverage_deps $(d)/ipfs +endif + +export IPFS_COVER_DIR:= $(realpath $(d))/sharnesscover/ + +$(d)/sharness_tests.coverprofile: $(d)/ipfs cmd/ipfs/ipfs-test-cover $(d)/coverage_deps test_sharness_short + (cd $(@D)/sharnesscover && find . -type f | gocovmerge -list -) > $@ + + +PATH := $(realpath $(d)):$(PATH) + +TGTS_$(d) += $(d)/sharness_tests.coverprofile + +CLEAN += $(TGTS_$(d)) +COVERAGE += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/coverage/main/main.go b/coverage/main/main.go new file mode 100644 index 00000000000..ba5d473e342 --- /dev/null +++ b/coverage/main/main.go @@ -0,0 +1,72 @@ +// +build testrunmain + +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "os/signal" + "strconv" + "syscall" +) + +func main() { + coverDir := os.Getenv("IPFS_COVER_DIR") + if len(coverDir) == 0 { + fmt.Println("IPFS_COVER_DIR not defined") + os.Exit(1) + } + coverFile, err := ioutil.TempFile(coverDir, "coverage-") + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + retFile, err := ioutil.TempFile("", "cover-ret-file") + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + args := []string{"-test.run", "^TestRunMain$", "-test.coverprofile=" + coverFile.Name(), "--"} + args = append(args, os.Args[1:]...) + + p := exec.Command("ipfs-test-cover", args...) + p.Stdin = os.Stdin + p.Stdout = os.Stdout + p.Stderr = os.Stderr + p.Env = append(os.Environ(), "IPFS_COVER_RET_FILE="+retFile.Name()) + + p.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, + } + + sig := make(chan os.Signal, 1) + go func() { + for { + p.Process.Signal(<-sig) + } + }() + signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM) + + err = p.Run() + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + + b, err := ioutil.ReadAll(retFile) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + b = b[:len(b)-1] + d, err := strconv.Atoi(string(b)) + if err != nil { + fmt.Println(err.Error()) + os.Exit(1) + } + os.Exit(d) +} diff --git a/diagnostics/pb/Makefile b/diagnostics/pb/Makefile deleted file mode 100644 index 334feee7411..00000000000 --- a/diagnostics/pb/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm *.pb.go diff --git a/diagnostics/pb/Rules.mk b/diagnostics/pb/Rules.mk new file mode 100644 index 00000000000..505f70e7541 --- /dev/null +++ b/diagnostics/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/exchange/bitswap/message/pb/Makefile b/exchange/bitswap/message/pb/Makefile deleted file mode 100644 index 5bbebea075a..00000000000 --- a/exchange/bitswap/message/pb/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -# TODO(brian): add proto tasks -all: message.pb.go - -message.pb.go: message.proto - protoc --gogo_out=. --proto_path=../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm message.pb.go diff --git a/exchange/bitswap/message/pb/Rules.mk b/exchange/bitswap/message/pb/Rules.mk new file mode 100644 index 00000000000..505f70e7541 --- /dev/null +++ b/exchange/bitswap/message/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/merkledag/pb/Makefile b/merkledag/pb/Makefile deleted file mode 100644 index 08ac883d0d0..00000000000 --- a/merkledag/pb/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm -f *.pb.go - rm -f *.go diff --git a/merkledag/pb/Rules.mk b/merkledag/pb/Rules.mk new file mode 100644 index 00000000000..505f70e7541 --- /dev/null +++ b/merkledag/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/mk/footer.mk b/mk/footer.mk new file mode 100644 index 00000000000..08e3d57e509 --- /dev/null +++ b/mk/footer.mk @@ -0,0 +1,3 @@ +# standard NR-make boilerplate, to be included at the end of a file +d := $(dirstack_$(sp)) +sp := $(basename $(sp)) diff --git a/mk/golang.mk b/mk/golang.mk new file mode 100644 index 00000000000..5d3b87db23f --- /dev/null +++ b/mk/golang.mk @@ -0,0 +1,50 @@ +# golang utilities +GO_MIN_VERSION = 1.7 + +# pre-definitions +GOTAGS ?= +GOFLAGS ?= +GOTFLAGS ?= + +DEPS_GO := +TEST_GO := +CHECK_GO := + +go-pkg-name=$(shell go list $(go-tags) ./$(1)) +go-main-name=$(notdir $(call go-pkg-name,$(1)))$(?exe) +go-curr-pkg-tgt=$(d)/$(call go-main-name,$(d)) + +go-tags=$(if $(GOTAGS), -tags="$(call join-with,$(space),$(GOTAGS))") +go-flags-with-tags=$(GOFLAGS)$(go-tags) + +define go-build +go build -i $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$<)" +endef + +test_go_short: GOTFLAGS += -test.short +test_go_short: test_go_expensive +.PHONY: test_go_short + +test_go_race: GOTFLAGS += -race +test_go_race: test_go_expensive +.PHONY: test_go_race + +test_go_expensive: $$(DEPS_GO) + go test $(go-flags-with-tags) $(GOTFLAGS) ./... +.PHONY: test_go_expensive +TEST_GO += test_go_expensive + +test_go_fmt: + bin/test-go-fmt +.PHONY: test_go_fmt +TEST_GO += test_go_fmt + +test_go: $(TEST_GO) + +check_go_version: + bin/check_go_version $(GO_MIN_VERSION) +.PHONY: check_go_version +DEPS_GO += check_go_version + +TEST += $(TEST_GO) +TEST_SHORT += test_go_fmt test_go_short diff --git a/mk/gx.mk b/mk/gx.mk new file mode 100644 index 00000000000..cd38c78484f --- /dev/null +++ b/mk/gx.mk @@ -0,0 +1,11 @@ +gx-path = gx/ipfs/$(shell gx deps find $(1))/$(1) + +gx-deps: + gx install --global +.PHONY: gx-deps + +ifneq ($(IPFS_GX_USE_GLOBAL),1) +gx-deps: bin/gx bin/gx-go +endif + +DEPS_GO += gx-deps diff --git a/mk/header.mk b/mk/header.mk new file mode 100644 index 00000000000..9ec77c7e182 --- /dev/null +++ b/mk/header.mk @@ -0,0 +1,5 @@ +# keep track of dirs +# standard NR-make boilerplate, to be included at the beginning of a file +p := $(sp).x +dirstack_$(sp) := $(d) +d := $(dir) diff --git a/mk/util.mk b/mk/util.mk new file mode 100644 index 00000000000..8f76013df39 --- /dev/null +++ b/mk/util.mk @@ -0,0 +1,20 @@ +# util functions +ifeq ($(OS),Windows_NT) + WINDOWS :=1 + ?exe :=.exe # windows compat +else + ?exe := +endif + +space:= +space+= +comma:=, +join-with=$(subst $(space),$1,$(strip $2)) + +# debug target, prints varaible. Example: `make print-GOFLAGS` +print-%: + @echo $*=$($*) + +# phony target that will mean that recipe is always exectued +ALWAYS: +.PHONY: ALWAYS diff --git a/namesys/pb/Makefile b/namesys/pb/Makefile deleted file mode 100644 index 334feee7411..00000000000 --- a/namesys/pb/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm *.pb.go diff --git a/namesys/pb/Rules.mk b/namesys/pb/Rules.mk new file mode 100644 index 00000000000..505f70e7541 --- /dev/null +++ b/namesys/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/pin/internal/pb/Rules.mk b/pin/internal/pb/Rules.mk new file mode 100644 index 00000000000..505f70e7541 --- /dev/null +++ b/pin/internal/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk diff --git a/test/Makefile b/test/Makefile deleted file mode 100644 index d7a320e58a6..00000000000 --- a/test/Makefile +++ /dev/null @@ -1,115 +0,0 @@ - -BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint bin/iptb bin/go-sleep -BINS += bin/go-timeout -IPFS_ROOT = ../ -IPFS_CMD = ../cmd/ipfs -RANDOM_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random -RANDOM_FILES_SRC = ../Godeps/_workspace/src/github.com/jbenet/go-random-files -POLLENDPOINT_SRC= ../thirdparty/pollEndpoint -GOSLEEP_SRC = ./dependencies/go-sleep -GOTIMEOUT_SRC = ./dependencies/go-timeout - -export PATH := ../bin:${PATH} - -# User might want to override those on the command line -GOFLAGS = - -all: deps - -global-deps: - $(MAKE) -C .. deps - -deps: global-deps bins - -clean: - $(MAKE) -C sharness clean - rm -rf $(BINS) - -bins: $(BINS) - -find_go_files = $(shell find $(1) -name "*.go") - -# Non gx dependencies - -bin/random: $(call find_go_files, $(RANDOM_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/random $(RANDOM_SRC)/random - -bin/random-files: - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/random-files $(RANDOM_FILES_SRC)/random-files - -bin/ipfs: $(call find_go_files, $(IPFS_ROOT)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/ipfs $(IPFS_CMD) - -bin/pollEndpoint: $(call find_go_files, $(POLLENDPOINT_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/pollEndpoint $(POLLENDPOINT_SRC) - -bin/go-sleep: $(call find_go_files, $(GOSLEEP_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/go-sleep $(GOSLEEP_SRC) - -bin/go-timeout: $(call find_go_files, $(GOTIMEOUT_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/go-timeout $(GOTIMEOUT_SRC) - -# gx dependencies - -multihash_src: - $(eval MULTIHASH_HASH := $(shell cd .. && bin/gx deps find go-multihash)) - $(eval MULTIHASH_SRC := gx/ipfs/$(MULTIHASH_HASH)/go-multihash) - -hang-fds_src: - $(eval HANG_FDS_HASH := $(shell cd .. && bin/gx deps find hang-fds)) - $(eval HANG_FDS_SRC := gx/ipfs/$(HANG_FDS_HASH)/hang-fds) - -bin/multihash: multihash_src $(call find_go_files, $(MULTIHASH_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/multihash $(MULTIHASH_SRC)/multihash - -bin/hang-fds: hang-fds_src $(call find_go_files, $(HANG_FDS_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/hang-fds $(HANG_FDS_SRC) - -iptb_src: - $(eval IPTB_HASH := $(shell cd .. && bin/gx deps find iptb)) - $(eval IPTB_SRC := gx/ipfs/$(IPTB_HASH)/iptb) - -bin/iptb: iptb_src $(call find_go_files, $(IPTB_SRC)) IPFS-BUILD-OPTIONS - @echo "*** installing $@ ***" - go build $(GOFLAGS) -o bin/iptb $(IPTB_SRC) - -# Tests - -test: test_expensive - -test_expensive: verify_gofmt - $(MAKE) -C sharness TEST_EXPENSIVE=1 - $(MAKE) -C 3nodetest - $(MAKE) -C dependencies - -test_cheap: verify_gofmt - $(MAKE) -C sharness - $(MAKE) -C 3nodetest - -test_race: verify_gofmt - $(MAKE) -C sharness GOFLAGS=-race TEST_EXPENSIVE=1 - $(MAKE) -C 3nodetest GOFLAGS=-race - $(MAKE) -C dependencies GOFLAGS=-race - -# Misc - -coverage: coverage_sharness - -coverage_sharness: - ./sharness_test_coverage_helper.sh - -IPFS-BUILD-OPTIONS: FORCE - @bin/checkflags '$@' '$(GOFLAGS)' '*** new Go flags ***' - -verify_gofmt: - bin/verify-go-fmt.sh - -.PHONY: all clean FORCE diff --git a/test/Rules.mk b/test/Rules.mk new file mode 100644 index 00000000000..e596dfd55a2 --- /dev/null +++ b/test/Rules.mk @@ -0,0 +1,9 @@ +include mk/header.mk + +dir := $(d)/bin +include $(dir)/Rules.mk + +dir := $(d)/sharness +include $(dir)/Rules.mk + +include mk/footer.mk diff --git a/test/bin/.gitignore b/test/bin/.gitignore index c032badeef4..cf2d847386d 100644 --- a/test/bin/.gitignore +++ b/test/bin/.gitignore @@ -8,3 +8,4 @@ !checkflags !continueyn !verify-go-fmt.sh +!Rules.mk diff --git a/test/bin/Rules.mk b/test/bin/Rules.mk new file mode 100644 index 00000000000..08f1a295cf6 --- /dev/null +++ b/test/bin/Rules.mk @@ -0,0 +1,44 @@ +include mk/header.mk + +TGTS_$(d) := + +$(d)/random: Godeps/_workspace/src/github.com/jbenet/go-random/random + $(go-build) +TGTS_$(d) += $(d)/random + +$(d)/random-files: Godeps/_workspace/src/github.com/jbenet/go-random-files/random-files + $(go-build) +TGTS_$(d) += $(d)/random-files + +$(d)/pollEndpoint: thirdparty/pollEndpoint + $(go-build) +TGTS_$(d) += $(d)/pollEndpoint + +$(d)/go-sleep: test/dependencies/go-sleep + $(go-build) +TGTS_$(d) += $(d)/go-sleep + +$(d)/go-timeout: test/dependencies/go-timeout + $(go-build) +TGTS_$(d) += $(d)/go-timeout + +TGTS_GX_$(d) := hang-fds iptb +TGTS_GX_$(d) := $(addprefix $(d)/,$(TGTS_GX_$(d))) + +$(TGTS_GX_$(d)): + go build -i $(go-flags-with-tags) -o "$@" "$(call gx-path,$(notdir $@))" + +TGTS_$(d) += $(TGTS_GX_$(d)) + +# multihash is special +$(d)/multihash: + go build -i $(go-flags-with-tags) -o "$@" "gx/ipfs/$(shell gx deps find go-multihash)/go-multihash/multihash" +TGTS_$(d) += $(d)/multihash + +$(TGTS_$(d)): $$(DEPS_GO) + +CLEAN += $(TGTS_$(d)) + +PATH := $(realpath $(d)):$(PATH) + +include mk/footer.mk diff --git a/test/sharness/Makefile b/test/sharness/Makefile index 922e1ecbb67..a96c5ea3fdf 100644 --- a/test/sharness/Makefile +++ b/test/sharness/Makefile @@ -1,57 +1,10 @@ -# Run tests -# -# Copyright (c) 2014 Christian Couder -# MIT Licensed; see the LICENSE file in this repository. -# - -# NOTE: Run with TEST_VERBOSE=1 for verbose sharness tests. - -T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) -BINS = bin/random bin/multihash bin/ipfs bin/pollEndpoint \ - bin/iptb bin/go-sleep bin/random-files bin/go-timeout bin/hang-fds -SHARNESS = lib/sharness/sharness.sh -IPFS_ROOT = ../.. - -# User might want to override those on the command line -GOFLAGS = - +# default target is to run all tests all: aggregate -clean: clean-test-results - @echo "*** $@ ***" - rm -rf $(BINS) - -clean-test-results: - @echo "*** $@ ***" - rm -rf test-results - -$(T): clean-test-results deps - @echo "*** $@ ***" - ./$@ - -aggregate: clean-test-results $(T) - @echo "*** $@ ***" - lib/test-aggregate-results.sh - -global-deps: - $(MAKE) -C ../.. deps - -deps: global-deps $(SHARNESS) $(BINS) curl - -$(SHARNESS): FORCE - @echo "*** checking $@ ***" - lib/install-sharness.sh - -bin/%: FORCE - $(MAKE) -C .. GOFLAGS=$(GOFLAGS) $@ - -race: - $(MAKE) GOFLAGS=-race all +SH := $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) -.PHONY: all clean $(T) aggregate FORCE +.DEFAULT $(SH): ALWAYS + $(MAKE) -C ../.. test/sharness/$@ -# will fail if curl is not installed. -# TODO: get rid of this and install curl with git or ipfs. -install_curl_pls = "curl is required to run tests. please install it" -curl: - @which curl >/dev/null || (echo "$(install_curl_pls)" && false) +ALWAYS: +.PHONY: ALWAYS diff --git a/test/sharness/README.md b/test/sharness/README.md index 910fa6e0d0d..39291d0c538 100644 --- a/test/sharness/README.md +++ b/test/sharness/README.md @@ -37,7 +37,8 @@ $ ./t0010-basic-commands.sh -v -i ## Sharness -When running "make" in this directory for the first time, sharness +When running sharness tests from main Makefile or when `test_sharness_deps` +target is run dependencies for sharness will be downloaded from its github repo and installed in a "lib/sharness" directory. diff --git a/test/sharness/Rules.mk b/test/sharness/Rules.mk new file mode 100644 index 00000000000..5832fb9b8a3 --- /dev/null +++ b/test/sharness/Rules.mk @@ -0,0 +1,57 @@ +include mk/header.mk + + +SHARNESS_$(d) = $(d)/lib/sharness/sharness.sh + +T_$(d) = $(sort $(wildcard $(d)/t[0-9][0-9][0-9][0-9]-*.sh)) + +DEPS_$(d) := test/bin/random test/bin/multihash test/bin/pollEndpoint \ + test/bin/iptb test/bin/go-sleep test/bin/random-files \ + test/bin/go-timeout test/bin/hang-fds +DEPS_$(d) += cmd/ipfs/ipfs +DEPS_$(d) += $(d)/clean-test-results +DEPS_$(d) += $(SHARNESS_$(d)) + +export MAKE_SKIP_PATH=1 + +$(T_$(d)): $$(DEPS_$(d)) # use second expansion so coverage can inject dependency + @echo "*** $@ ***" + @(cd $(@D) && ./$(@F)) 2>&1 +.PHONY: $(T_$(d)) + +$(d)/aggregate: $(T_$(d)) + @echo "*** $@ ***" + @(cd $(@D) && ./lib/test-aggregate-results.sh) +.PHONY: $(d)/aggregate + +$(d)/clean-test-results: + rm -rf $(@D)/test-results +.PHONY: $(d)/clean-test-results + +CLEAN += $(wildcard $(d)/test-results/*) + +$(SHARNESS_$(d)): $(d) ALWAYS + @clonedir=$(dir $(@D)) $&2 "Cannot find the tests' local ipfs tool." + echo >&2 "Please check test and ipfs tool installation." + exit 1 + fi +fi # set sharness verbosity. we set the env var directly as # it's too late to pass in --verbose, and --verbose is harder # to pass through in some cases. test "$TEST_VERBOSE" = 1 && verbose=t - -# assert the `ipfs` we're using is the right one. -if test `which ipfs` != ${BIN}/ipfs; then - echo >&2 "Cannot find the tests' local ipfs tool." - echo >&2 "Please check test and ipfs tool installation." - exit 1 -fi - - # source the common hashes first. . lib/test-lib-hashes.sh diff --git a/test/sharness/t0600-issues-and-regressions-online.sh b/test/sharness/t0600-issues-and-regressions-online.sh index 7499b0e1d15..f5e5b894075 100755 --- a/test/sharness/t0600-issues-and-regressions-online.sh +++ b/test/sharness/t0600-issues-and-regressions-online.sh @@ -37,7 +37,8 @@ test_kill_ipfs_daemon test_expect_success "ipfs daemon --offline --mount fails - #2995" ' test_expect_code 1 ipfs daemon --offline --mount 2>daemon_err && - grep "mount is not currently supported in offline mode" daemon_err + grep "mount is not currently supported in offline mode" daemon_err || + test_fsh cat daemon_err ' test_done diff --git a/unixfs/pb/Makefile b/unixfs/pb/Makefile deleted file mode 100644 index 334feee7411..00000000000 --- a/unixfs/pb/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -PB = $(wildcard *.proto) -GO = $(PB:.proto=.pb.go) - -all: $(GO) - -%.pb.go: %.proto - protoc --gogo_out=. --proto_path=../../../../../../:/usr/local/opt/protobuf/include:. $< - -clean: - rm *.pb.go diff --git a/unixfs/pb/Rules.mk b/unixfs/pb/Rules.mk new file mode 100644 index 00000000000..505f70e7541 --- /dev/null +++ b/unixfs/pb/Rules.mk @@ -0,0 +1,8 @@ +include mk/header.mk + +PB_$(d) = $(wildcard $(d)/*.proto) +TGTS_$(d) = $(PB_$(d):.proto=.pb.go) + +#DEPS_GO += $(TGTS_$(d)) + +include mk/footer.mk