Merge branch 'next'

trickstercache · Jan 17, 2020 · d6b87a5 · d6b87a5
2 parents eabd273 + 13431f8
commit d6b87a5
Show file tree

Hide file tree

Showing 1,227 changed files with 425,028 additions and 4,645 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,13 +6,20 @@
 
 # Test binary, build with `go test -c`
 *.test
+.coverprofile
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
 # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
 .glide/
 
+# IDE Files
+.vscode/
+
+# Test Coverage Profile Output
+.coverprofile
+
 # output path
 OPATH
 
@@ -24,11 +31,13 @@ OPATH
 .Trashes
 ehthumbs.db
 Thumbs.db
-vendor
-
-# The binary
-trickster
 
 # dep
 cacheKey.data
 cacheKey.expiration
+
+#Goland IDE
+.idea/
+
+#log testing
+out.log
diff --git a/.travis.yml b/.travis.yml
@@ -1,17 +1,21 @@
+env:
+  GO111MODULE: "on"
+
 matrix:
   include:
   - language: python
     install:
     - pip install codespell
     script:
-    - codespell --skip=".git,*.png"
+    - codespell --skip="vendor,*.git,*.png,*.pdf,*.tiff,*.plist,*.pem,rangesim*.go,*.gz"
 
   - language: go
     go:
-    - "1.11.x"
+    - "1.13.x"
     - master
     before_install:
     - go get github.com/mattn/goveralls
     script:
     - make style test build
-    - $GOPATH/bin/goveralls -service=travis-ci
+    - sed -i -e '/^.*_gen\.go:.*$/d' .coverprofile
+    - $GOPATH/bin/goveralls  -coverprofile=.coverprofile -service=travis-ci
diff --git a/Makefile b/Makefile
@@ -1,45 +1,70 @@
-DEFAULT: build
+# Copyright 2018 Comcast Cable Communications Management, LLC
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-GO           ?= go
-GOFMT        ?= $(GO)fmt
-FIRST_GOPATH := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
-DEP          := $(FIRST_GOPATH)/bin/dep
-TRICKSTER    := $(FIRST_GOPATH)/bin/trickster
+DEFAULT: build
 
-PROGVER = $(shell grep 'applicationVersion = ' main.go | awk '{print $$3}' | sed -e 's/\"//g')
+GO             ?= go
+GOFMT          ?= $(GO)fmt
+FIRST_GOPATH   := $(firstword $(subst :, ,$(shell $(GO) env GOPATH)))
+TRICKSTER_MAIN := cmd/trickster
+TRICKSTER      := $(FIRST_GOPATH)/bin/trickster
+PROGVER        := $(shell grep 'applicationVersion = ' $(TRICKSTER_MAIN)/main.go | awk '{print $$3}' | sed -e 's/\"//g')
+BUILD_TIME     := $(shell date -u +%FT%T%z)
+GIT_LATEST_COMMIT_ID     := $(shell git rev-parse HEAD)
+GO_VER         := $(shell go version | awk '{print $$3}')
+LDFLAGS=-ldflags "-s -X main.applicationBuildTime=$(BUILD_TIME) -X main.applicationGitCommitID=$(GIT_LATEST_COMMIT_ID) -X main.applicationGoVersion=$(GO_VER) -X main.applicationGoArch=$(GOARCH)"
+GO111MODULE    ?= on
+export GO111MODULE
 
 .PHONY: go-mod-vendor
 go-mod-vendor:
-	GO111MODULE=on $(GO) mod vendor
+	$(GO) mod vendor
+
+.PHONY: go-mod-tidy
+go-mod-tidy:
+	$(GO) mod tidy
+
+.PHONY: test-go-mod
+test-go-mod:
+	@git diff --quiet --exit-code go.mod go.sum || echo "There are changes to go.mod and go.sum which needs to be committed"
 
 .PHONY: build
-build: go-mod-vendor
-	GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO_ENABLED) go build -a -v
+build:
+	GOOS=$(GOOS) GOARCH=$(GOARCH) CGO_ENABLED=$(CGO_ENABLED) $(GO) build $(LDFLAGS) -o ./OPATH/trickster -a -v $(TRICKSTER_MAIN)/main.go 
 
 rpm: build
 	mkdir -p ./OPATH/SOURCES
-	cp -p trickster ./OPATH/SOURCES/
-	cp conf/trickster.service ./OPATH/SOURCES/
+	cp -p ./OPATH/trickster ./OPATH/SOURCES/
+	cp $(TRICKSTER_MAIN)/conf/trickster.service ./OPATH/SOURCES/
 	sed -e 's%^# log_file =.*$$%log_file = "/var/log/trickster/trickster.log"%' \
 		-e 's%prometheus:9090%localhost:9090%' \
-		< conf/example.conf > ./OPATH/SOURCES/trickster.conf
+		< $(TRICKSTER_MAIN)/conf/example.conf > ./OPATH/SOURCES/trickster.conf
 	rpmbuild --define "_topdir $(CURDIR)/OPATH" \
 		--define "_version $(PROGVER)" \
 		--define "_release 1" \
 		-ba deploy/packaging/trickster.spec
 
 .PHONY: install
-install: go-mod-vendor
-	echo go build -o $(TRICKSTER) $(PROGVER)
+install:
+	$(GO) install -o $(TRICKSTER) $(PROGVER)
 
 .PHONY: release
 release: build release-artifacts docker docker-release
 
 .PHONY: release-artifacts
 release-artifacts:
-	GOOS=darwin GOARCH=amd64 go build -o ./OPATH/trickster-$(PROGVER).darwin-amd64 && tar cvfz ./OPATH/trickster-$(PROGVER).darwin-amd64.tar.gz ./OPATH/trickster-$(PROGVER).darwin-amd64
-	GOOS=linux  GOARCH=amd64 go build -o ./OPATH/trickster-$(PROGVER).linux-amd64  && tar cvfz ./OPATH/trickster-$(PROGVER).linux-amd64.tar.gz ./OPATH/trickster-$(PROGVER).linux-amd64
+	GOOS=darwin GOARCH=amd64 $(GO) build -o ./OPATH/trickster-$(PROGVER).darwin-amd64 $(TRICKSTER_MAIN)/main.go && gzip -f ./OPATH/trickster-$(PROGVER).darwin-amd64
+	GOOS=linux  GOARCH=amd64 $(GO) build -o ./OPATH/trickster-$(PROGVER).linux-amd64  $(TRICKSTER_MAIN)/main.go && gzip -f ./OPATH/trickster-$(PROGVER).linux-amd64
 
+# Minikube and helm bootstrapping are done via deploy/helm/Makefile
 .PHONY: helm-local
 helm-local:
 	kubectl config use-context minikube --namespace=trickster
@@ -49,6 +74,7 @@ helm-local:
 	kubectl set image deployment/dev-trickster trickster=trickster:dev -n trickster
 	kubectl scale --replicas=1 deployment/dev-trickster -n trickster
 
+# Minikube and helm bootstrapping are done via deploy/kube/Makefile
 .PHONY: kube-local
 kube-local:
 	kubectl config use-context minikube
@@ -72,13 +98,16 @@ style:
 	! gofmt -d $$(find . -path ./vendor -prune -o -name '*.go' -print) | grep '^'
 
 .PHONY: test
-test: go-mod-vendor
-	go test -o $(TRICKSTER) -v ./...
+test:
+	@go test -v -coverprofile=.coverprofile ./... 
+
+.PHONY: bench
+bench:
+	$(GO) test -v -coverprofile=.coverprofile ./... -run=nonthingplease -bench=. | grep -v ' app=trickster '
 
 .PHONY: test-cover
-test-cover: go-mod-vendor
-	go test -o $(TRICKSTER) -coverprofile=cover.out ./...
-	go tool cover -html=cover.out
+test-cover: test
+	$(GO) tool cover -html=.coverprofile
 
 .PHONY: clean
 clean:

diff --git a/README.md b/README.md
@@ -1,28 +1,64 @@
-# <img src="./docs/images/logos/trickster-horizontal-sm.png" width=320 />
+# <img src="./docs/images/logos/trickster-logo.svg" width=90 />&nbsp;&nbsp;&nbsp;&nbsp;<img src="./docs/images/logos/trickster-text.svg" width=420 />
+
 [![Build Status](https://travis-ci.org/Comcast/trickster.svg?branch=master)](https://travis-ci.org/Comcast/trickster/)
 [![Go Report Card](https://goreportcard.com/badge/github.com/Comcast/trickster)](https://goreportcard.com/report/github.com/Comcast/trickster)
-[![Coverage Status](https://coveralls.io/repos/github/Comcast/trickster/badge.svg?branch=master)](https://coveralls.io/github/Comcast/trickster?branch=master)
-[![CLI Best Practices](https://bestpractices.coreinfrastructure.org/projects/2518/badge)](https://bestpractices.coreinfrastructure.org/en/projects/2518)
+[![Coverage Status](https://coveralls.io/repos/github/Comcast/trickster/badge.svg?branch=next)](https://coveralls.io/github/Comcast/trickster?branch=next)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2518/badge)](https://bestpractices.coreinfrastructure.org/en/projects/2518)
+[![Docker Pulls](https://img.shields.io/docker/pulls/tricksterio/trickster.svg?maxAge=86400)](https://hub.docker.com/r/tricksterio/trickster)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/tricksterio.svg?style=social&logo=twitter)](https://twitter.com/tricksterio)
+
+Trickster is an HTTP reverse proxy/cache for http applications and a dashboard query accelerator for time series databases.
+
+<img src="./docs/images/high-level-1.png" width=512/>
+
+## HTTP Reverse Proxy Cache
+
+Trickster is a fully-featured HTTP Reverse Proxy Cache for HTTP applications like static file servers and web API's.
+
+### Trickster Feature Highlights
+
+* [Supports TLS](./docs/tls.md) frontend termination and backend origination
+* Offers several options for a [caching layer](./docs/caches.md), including in-memory, filesystem, Redis and bbolt
+* [Highly customizable](./docs/configuring.md), using simple configuration settings, [down to the HTTP Path](./docs/paths.md)
+* Built-in Prometheus [metrics](./docs/metrics.md) and customizable [Health Check](./docs/health.md) Endpoints for end-to-end monitoring
+* [Negative Caching](./docs/negative-caching.md) to prevent domino effect outages
+* High-performance [Collapsed Forwarding](./docs/collapsed-forwarding.md)
+* Best-in-class [Range Request caching and acceleration](./docs/range_request.md).
+
+## Dashboard Acceleration
+
+Trickster dramatically improves dashboard chart rendering times for end users, while eliminating redundant computations on the TSDBs it fronts. In short, Trickster makes read-heavy Dashboard/TSDB environments, as well as those with highly-cardinalized datasets, significantly more performant and scalable.
 
-Trickster is a reverse proxy cache for the [Prometheus](https://github.com/prometheus/prometheus) [HTTP APIv1](https://prometheus.io/docs/prometheus/latest/querying/api/) that dramatically accelerates dashboard rendering times for any series queried from Prometheus.
+## Compatibility
 
-#### NOTE: Trickster is currently actively developed under the [next](https://github.com/Comcast/trickster/tree/next) Branch for our milestone 1.0 Release.
+Trickster works with virtually any Dashboard application that makes queries to any of these TSDB's:
 
-<img src="./docs/images/high-level.png" width=512 />
+<img src="./docs/images/external/prom_logo_60.png" width=16 /> Prometheus
 
-## How it works
+<img src="./docs/images/external/clickhouse_logo.png" width=16 /> ClickHouse
+
+<img src="./docs/images/external/influx_logo_60.png" width=16 /> InfluxDB
+
+<img src="./docs/images/external/irondb_logo_60.png" width=16 /> Circonus IRONdb
+
+See the [Supported Origin Types](./docs/supported-origin-types.md) document for full details
+
+## How Trickster Accelerates Time Series
 
 ### 1. Delta Proxy
-Most dashboards request the entire time range of data from the time series database, every time a dashboard loads or reloads. Trickster's Delta Proxy inspects the time range of a client query to determine what data points are already cached, and requests from Prometheus only the data points still needed to service the client request. This results in dramatically faster chart load times for everyone, since Prometheus is queried only for tiny incremental changes on each dashboard load, rather than several hundred data points of duplicative data.
+
+Most dashboards request from a time series database the entire time range of data they wish to present, every time a user's dashboard loads, as well as on every auto-refresh. Trickster's Delta Proxy inspects the time range of a client query to determine what data points are already cached, and requests from the tsdb only the data points still needed to service the client request. This results in dramatically faster chart load times for everyone, since the tsdb is queried only for tiny incremental changes on each dashboard load, rather than several hundred data points of duplicative data.
 
 <img src="./docs/images/partial-cache-hit.png" width=1024 />
 
 ### 2. Step Boundary Normalization
-When Trickster requests data from Prometheus, it adjusts the clients's requested time range slightly to ensure that all data points returned by Prometheus are aligned to normalized step boundaries. For example, if the step is 300s, all data points will fall on the clock 0's and 5's. This ensures that the data is highly cacheable, is conveyed visually to users in a more familiar way, and that all dashboard users see identical data on their screens.
+
+When Trickster requests data from a tsdb, it adjusts the clients's requested time range slightly to ensure that all data points returned are aligned to normalized step boundaries. For example, if the step is 300s, all data points will fall on the clock 0's and 5's. This ensures that the data is highly cacheable, is conveyed visually to users in a more familiar way, and that all dashboard users see identical data on their screens.
 
 <img src="./docs/images/step-boundary-normalization.png" width=640 />
 
 ### 3. Fast Forward
+
 Trickster's Fast Forward feature ensures that even with step boundary normalization, real-time graphs still always show the most recent data, regardless of how far away the next step boundary is. For example, if your chart step is 300s, and the time is currently 1:21p, you would normally be waiting another four minutes for a new data point at 1:25p. Trickster will break the step interval for the most recent data point and always include it in the response to clients requesting real-time data.
 
 <img src="./docs/images/fast-forward.png" width=640 />
@@ -52,7 +88,6 @@ binary into your `GOPATH`:
     $ go get github.com/Comcast/trickster
     $ trickster -origin http://prometheus.example.com:9090
 
-
 You can also clone the repository yourself and build using `make`:
 
     $ mkdir -p $GOPATH/src/github.com/Comcast
@@ -62,25 +97,26 @@ You can also clone the repository yourself and build using `make`:
     $ make build
     $ ./trickster -origin http://prometheus.example.com:9090
 
-The Makefile provides several targets:
+The Makefile provides several targets, including:
 
-  * *build*: build the `trickster` binary
-  * *docker*: build a docker container for the current `HEAD`
-  * *clean*: delete previously-built binaries and object files
+* *build*: build the `trickster` binary
+* *docker*: build a docker container for the current `HEAD`
+* *clean*: delete previously-built binaries and object files
+* *test*: runs unit tests
 
 ## More information
 
-  * Refer to the docs directory for additional info.
+* Refer to the [docs](./docs/) directory for additional info.
 
 ## Contributing
 
 Refer to [CONTRIBUTING.md](CONTRIBUTING.md)
 
-## Who uses Trickster?
+## Who Is Using Trickster
 
 As the Trickster community grows, we'd like to keep track of who is using it in their stack. We invite you to submit a PR with your company name and @githubhandle to be included on the list.
 
 1. [Comcast](https://comcast.github.io) [[@jranson](https://github.com/jranson)]
 2. [Selfnet e.V.](https://www.selfnet.de/) [[@ThoreKr](https://github.com/ThoreKr)]
 3. [swarmstack](https://github.com/swarmstack) [[@mh720](https://github.com/mh720)]
-4. [Hostinger](https://www.hostinger.com/) [[@ton31337](https://github.com/ton31337)]
+4. [Hostinger](https://www.hostinger.com/) [[@ton31337](https://github.com/ton31337)]