diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a0d1dc4f3183..5e1a8bb638c24 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,11 +5,13 @@ workflows: test-build-deploy: jobs: - test + - test-helm - build - lint - publish: requires: - test + - test-helm - build - lint filters: @@ -18,6 +20,7 @@ workflows: - publish-master: requires: - test + - test-helm - build - lint filters: @@ -26,6 +29,7 @@ workflows: - publish-helm: requires: - test + - test-helm - build - lint filters: @@ -41,7 +45,7 @@ workflows: # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/ defaults: &defaults docker: - - image: grafana/loki-build-image:0.1.0 + - image: grafana/loki-build-image:0.3.0 working_directory: /go/src/github.com/grafana/loki jobs: @@ -53,8 +57,7 @@ jobs: - run: name: Run Unit Tests command: | - touch loki-build-image/.uptodate && - make BUILD_IN_CONTAINER=false test + make test lint: <<: *defaults @@ -64,13 +67,11 @@ jobs: - run: name: Lint command: | - touch loki-build-image/.uptodate && - make BUILD_IN_CONTAINER=false lint + make lint - run: name: Check Generated Fies command: | - touch loki-build-image/.uptodate && make BUILD_IN_CONTAINER=false check-generated-files build: @@ -79,23 +80,33 @@ jobs: - checkout - setup_remote_docker + - run: + name: Promtail cross platform + command: | + make GOOS=linux BUILD_IN_CONTAINER=false promtail + rm cmd/promtail/promtail + make GOOS=windows BUILD_IN_CONTAINER=false promtail + - run: name: Build Images command: | - touch loki-build-image/.uptodate && - make BUILD_IN_CONTAINER=false + make images - run: name: Save Images command: | - touch loki-build-image/.uptodate && - make BUILD_IN_CONTAINER=false save-images + make save-images - save_cache: key: v1-loki-{{ .Branch }}-{{ .Revision }} paths: - images/ + - save_cache: + key: v1-loki-plugin-{{ .Branch }}-{{ .Revision }} + paths: + - cmd/docker-driver/docker-driver + publish: <<: *defaults steps: @@ -104,12 +115,13 @@ jobs: - restore_cache: key: v1-loki-{{ .Branch }}-{{ .Revision }} + - restore_cache: + key: v1-loki-plugin-{{ .Branch }}-{{ .Revision }} - run: name: Load Images command: | - touch loki-build-image/.uptodate && - make BUILD_IN_CONTAINER=false load-images + make load-images - run: name: Push Images @@ -119,6 +131,14 @@ jobs: make push-images fi + - run: + name: Push Docker Plugin + command: | + if [ -n "$DOCKER_USER" ]; then + docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" && + make BUILD_IN_CONTAINER=false docker-driver-push + fi + publish-master: <<: *defaults steps: @@ -127,12 +147,13 @@ jobs: - restore_cache: key: v1-loki-{{ .Branch }}-{{ .Revision }} + - restore_cache: + key: v1-loki-plugin-{{ .Branch }}-{{ .Revision }} - run: name: Load Images command: | - touch loki-build-image/.uptodate && - make BUILD_IN_CONTAINER=false load-images + make load-images - run: name: Push Images @@ -140,6 +161,47 @@ jobs: docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" && make push-latest + - run: + name: Push Docker Plugin + command: | + if [ -n "$DOCKER_USER" ]; then + docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" && + PLUGIN_TAG=master make BUILD_IN_CONTAINER=false docker-driver-push && PLUGIN_TAG=latest make BUILD_IN_CONTAINER=false docker-driver-push + fi + + test-helm: + environment: + CT_VERSION: 2.3.3 + machine: + image: ubuntu-1604:201903-01 + steps: + - checkout + - run: + name: Install k3s + command: | + curl -sfL https://get.k3s.io | sh - + sudo chmod 755 /etc/rancher/k3s/k3s.yaml + mkdir -p ~/.kube + cp /etc/rancher/k3s/k3s.yaml ~/.kube/config + - run: + name: Install Helm + command: | + curl -L https://git.io/get_helm.sh | bash + kubectl apply -f tools/helm.yaml + helm init --service-account helm --wait + - run: + name: Install Chart Testing tool + command: | + pip install yamale yamllint + curl -Lo ct.tgz https://github.com/helm/chart-testing/releases/download/v${CT_VERSION}/chart-testing_${CT_VERSION}_linux_amd64.tar.gz + sudo tar -C /usr/local/bin -xvf ct.tgz + sudo mv /usr/local/bin/etc /etc/ct/ + - run: + name: Run Chart Tests + command: | + ct lint --chart-dirs=production/helm --check-version-increment=false --validate-maintainers=false + ct install --build-id=${CIRCLE_BUILD_NUM} --charts production/helm/loki-stack + publish-helm: <<: *defaults steps: @@ -159,3 +221,24 @@ jobs: --data "{\"build_parameters\": {\"CIRCLE_JOB\": \"deploy\", \"IMAGE_NAMES\": \"$(make images)\"}}" \ --request POST \ https://circleci.com/api/v1.1/project/github/raintank/deployment_tools/tree/master?circle-token=$CIRCLE_TOKEN + + release: + <<: *defaults + steps: + - checkout + - setup_remote_docker + - restore_cache: + key: v1-loki-{{ .Branch }}-{{ .Revision }} + - run: + name: Load Images + command: | + touch loki-build-image/.uptodate && + make BUILD_IN_CONTAINER=false load-images + - run: + name: "Print Tag" + command: echo ${CIRCLE_TAG} + - run: + name: "Release" + command: | + docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" && + make VERSION=${CIRCLE_TAG} release-perform diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000000..e8e08c0eb805a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,19 @@ + + +**What this PR does / why we need it**: + +**Which issue(s) this PR fixes**: +Fixes # + +**Special notes for your reviewer**: + +**Checklist** +- [ ] Documentation added +- [ ] Tests updated + diff --git a/.gitignore b/.gitignore index dec47fd3a9606..a9a40e0fb4fe0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .uptodate +.uptodate-debug .pkg .cache *.output @@ -7,6 +8,13 @@ requirements.lock mixin/vendor/ cmd/loki/loki cmd/promtail/promtail +cmd/loki/loki-debug +cmd/promtail/promtail-debug +cmd/docker-driver/docker-driver +cmd/loki-canary/loki-canary /loki /promtail /logcli +/loki-canary +dlv +rootfs/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000000..2583383fdbcd2 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,66 @@ +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 2 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 5m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # list of build tags, all linters use it. Default is empty list. + build-tags: + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + +linters: + enable: + - deadcode + - errcheck + - goconst + - gofmt + - goimports + - golint + - gosimple + - ineffassign + - megacheck + - misspell + - structcheck + - unconvert + - unparam + - varcheck + - govet + - unused # new from here. + - interfacer + - typecheck + +issues: + exclude: + - Error return value of .*log\.Logger\)\.Log\x60 is not checked \ No newline at end of file diff --git a/.gometalinter.json b/.gometalinter.json deleted file mode 100644 index ed3fda665df43..0000000000000 --- a/.gometalinter.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "Vendor": true, - "Deadline": "5m", - "Concurrency": 2, - "Linters": { - "gofmt": {"Command": "gofmt -l -s -w"}, - "goimports": {"Command": "goimports -l -w"} - }, - "Exclude": [ - "\\.pb\\.go", - "method Seek.*should have signature", - "error return value not checked \\(level\\.", - "\"err\" shadows declaration" - ], - - "Enable": [ - "deadcode", - "errcheck", - "goconst", - "gofmt", - "goimports", - "golint", - "gosimple", - "gotypex", - "ineffassign", - "megacheck", - "misspell", - "structcheck", - "unconvert", - "unparam", - "varcheck", - "vet", - "vetshadow" - ] -} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000000..c6bffe79b060f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at contact@grafana.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 929b3bc7c5dce..9728341d22b1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,3 +18,8 @@ $ git remote add ``` Notice: `go get` return `package github.com/grafana/loki: no Go files in /go/src/github.com/grafana/loki` is normal. + + +## Contribute to helm + +Please follow [doc](./production/helm/README.md). diff --git a/Gopkg.lock b/Gopkg.lock index 250b4fa3606a3..67c3f93e03787 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -43,6 +43,17 @@ revision = "a1a2da0aba294fe51ba47119e652ff2f08a78afb" version = "v26.3.0" +[[projects]] + branch = "master" + digest = "1:6da51e5ec493ad2b44cb04129e2d0a068c8fb9bd6cb5739d199573558696bb94" + name = "github.com/Azure/go-ansiterm" + packages = [ + ".", + "winterm", + ] + pruneopts = "UT" + revision = "d6e3b3328b783f23731bc4d058875b0371ff8109" + [[projects]] digest = "1:ae0afbb4ba38fc78943a738a1d853294a9591d9c7dd69594d641b7649535d518" name = "github.com/Azure/go-autorest" @@ -60,6 +71,14 @@ revision = "134ac34d5a96fc72758c0f157f38f87cbdb3f532" version = "v11.5.1" +[[projects]] + digest = "1:f9ae348e1f793dcf9ed930ed47136a67343dbd6809c5c91391322267f4476892" + name = "github.com/Microsoft/go-winio" + packages = ["."] + pruneopts = "UT" + revision = "1a8911d1ed007260465c3bfbbc785ac6915a0bb8" + version = "v0.4.12" + [[projects]] branch = "master" digest = "1:315c5f2f60c76d89b871c73f9bd5fe689cad96597afd50fb9992228ef80bdd34" @@ -187,7 +206,34 @@ [[projects]] branch = "master" - digest = "1:5750dfa5a8160b51a01c02d6d7841b50fe7d2d97344c200fc4cdc3c46b3953f8" + digest = "1:9ab2182297ebe5a1433c9804ba65a382e4f41e3084485d1b4d31996e4c992e38" + name = "github.com/containerd/fifo" + packages = ["."] + pruneopts = "UT" + revision = "a9fb20d87448d386e6d50b1f2e1fa70dcf0de43c" + +[[projects]] + digest = "1:bc38b83376aa09bdc1e889c00ce73cb748b2140d535bb5c76cb9823da6c7a98a" + name = "github.com/coreos/go-systemd" + packages = [ + "activation", + "sdjournal", + ] + pruneopts = "UT" + revision = "95778dfbb74eb7e4dbaf43bf7d71809650ef8076" + version = "v19" + +[[projects]] + digest = "1:6e2ff82d2fe11ee35ec8dceb4346b8144a761f1c8655592c4ebe99a92fcec327" + name = "github.com/coreos/pkg" + packages = ["dlopen"] + pruneopts = "UT" + revision = "97fdf19511ea361ae1c100dd393cc47f8dcfa1e1" + version = "v4" + +[[projects]] + branch = "master" + digest = "1:5a07b5363e4c2aa127a3afd1e8e323d3a288ba1d90d37793d2e14843f5b5b82e" name = "github.com/cortexproject/cortex" packages = [ "pkg/chunk", @@ -211,10 +257,9 @@ "pkg/util/middleware", "pkg/util/spanlogger", "pkg/util/validation", - "pkg/util/wire", ] pruneopts = "UT" - revision = "ff51bd3c7267184042ea4cf347e6d1fa24934c91" + revision = "ef492f6bbafb185bbe61ae7a6955b7a4af5f3d9a" [[projects]] digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" @@ -232,6 +277,93 @@ revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e" version = "v3.2.0" +[[projects]] + digest = "1:699aca47d5af672db37b96102bb088edbf26f84f73d3dafff6fa917026eb6971" + name = "github.com/docker/distribution" + packages = ["registry/api/errcode"] + pruneopts = "UT" + revision = "2461543d988979529609e8cb6fca9ca190dc48da" + version = "v2.7.1" + +[[projects]] + branch = "master" + digest = "1:15c3bb44c339867455b8388edc16fbace9f3e5834cfa911389733b4ffd375496" + name = "github.com/docker/docker" + packages = [ + "api/types", + "api/types/backend", + "api/types/blkiodev", + "api/types/container", + "api/types/filters", + "api/types/mount", + "api/types/network", + "api/types/plugins/logdriver", + "api/types/registry", + "api/types/strslice", + "api/types/swarm", + "api/types/swarm/runtime", + "api/types/versions", + "daemon/logger", + "daemon/logger/jsonfilelog", + "daemon/logger/jsonfilelog/jsonlog", + "daemon/logger/loggerutils", + "daemon/logger/templates", + "errdefs", + "pkg/filenotify", + "pkg/ioutils", + "pkg/jsonmessage", + "pkg/longpath", + "pkg/plugingetter", + "pkg/plugins", + "pkg/plugins/transport", + "pkg/pools", + "pkg/progress", + "pkg/pubsub", + "pkg/streamformatter", + "pkg/stringid", + "pkg/tailfile", + "pkg/term", + "pkg/term/windows", + ] + pruneopts = "UT" + revision = "238f8eaa31aa74be843c81703fabf774863ec30c" + +[[projects]] + digest = "1:811c86996b1ca46729bad2724d4499014c4b9effd05ef8c71b852aad90deb0ce" + name = "github.com/docker/go-connections" + packages = [ + "nat", + "sockets", + "tlsconfig", + ] + pruneopts = "UT" + revision = "7395e3f8aa162843a74ed6d48e79627d9792ac55" + version = "v0.4.0" + +[[projects]] + branch = "master" + digest = "1:2b126e77be4ab4b92cdb3924c87894dd76bf365ba282f358a13133e848aa0059" + name = "github.com/docker/go-metrics" + packages = ["."] + pruneopts = "UT" + revision = "b84716841b82eab644a0c64fc8b42d480e49add5" + +[[projects]] + branch = "master" + digest = "1:64aa7651bec9ea1eb3535fda3590661da0e6c225b746745976fa94c4126d652b" + name = "github.com/docker/go-plugins-helpers" + packages = ["sdk"] + pruneopts = "UT" + revision = "1e6269c305b8c75cfda1c8aa91349c38d7335814" + +[[projects]] + digest = "1:e95ef557dc3120984bb66b385ae01b4bb8ff56bcde28e7b0d1beed0cccc4d69f" + name = "github.com/docker/go-units" + packages = ["."] + pruneopts = "UT" + revision = "519db1ee28dcc9fd2474ae59fca29a810482bfb1" + version = "v0.4.0" + [[projects]] digest = "1:3762d59edaa6e5c71d5e594c020c8391f274ff283e9c30fb43c518ec59a3f9b3" name = "github.com/etcd-io/bbolt" @@ -248,6 +380,14 @@ revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" version = "v1.7.0" +[[projects]] + digest = "1:abeb38ade3f32a92943e5be54f55ed6d6e3b6602761d74b4aab4c9dd45c18abd" + name = "github.com/fsnotify/fsnotify" + packages = ["."] + pruneopts = "UT" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" + [[projects]] branch = "master" digest = "1:4728aed43b5989e89bd5d88a00734adb2ec597e7823712cd05934c2d9fac89df" @@ -300,10 +440,11 @@ version = "v1.1.0" [[projects]] - digest = "1:452644cddfec8736ad410d9a5a4eacce21810873ceaa938ae373edf2f2c0fe84" + digest = "1:dfb2d8632b5a704b3b6c3eee885f18c23b0c2f5e942822f91ff8dbb58247ab07" name = "github.com/gogo/protobuf" packages = [ "gogoproto", + "io", "proto", "protoc-gen-gogo/descriptor", "sortkeys", @@ -386,8 +527,7 @@ version = "v0.2.0" [[projects]] - branch = "master" - digest = "1:848741e579410d2475b4f15411713e344320a3e766ce6e92700595159a62db08" + digest = "1:e57d9b8bf3a0e83d813416814937c4789ff89b06bccc65356912115af9977e7c" name = "github.com/gophercloud/gophercloud" packages = [ ".", @@ -404,7 +544,8 @@ "pagination", ] pruneopts = "UT" - revision = "07de4ce3c8b94e997c75205cfee74c33ffdaada7" + revision = "c2d73b246b48e239d3f03c455905e06fe26e33c3" + version = "v0.1.0" [[projects]] digest = "1:c79fb010be38a59d657c48c6ba1d003a8aa651fa56b579d959d74573b7dff8e1" @@ -430,17 +571,6 @@ revision = "66b9c49e59c6c48f0ffce28c2d8b8a5678502c6d" version = "v1.4.0" -[[projects]] - branch = "master" - digest = "1:86c1210529e69d69860f2bb3ee9ccce0b595aa3f9165e7dd1388e5c612915888" - name = "github.com/gregjones/httpcache" - packages = [ - ".", - "diskcache", - ] - pruneopts = "UT" - revision = "c63ab54fda8f77302f8d414e19933f2b6026a089" - [[projects]] digest = "1:1168584a5881d371e96cb0e66ef6db71d7cef0856cc7f311490bc856627f8328" name = "github.com/grpc-ecosystem/go-grpc-middleware" @@ -618,6 +748,14 @@ revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd" version = "1.0.1" +[[projects]] + branch = "master" + digest = "1:906eb1ca3c8455e447b99a45237b2b9615b665608fd07ad12cce847dd9a1ec43" + name = "github.com/morikuni/aec" + packages = ["."] + pruneopts = "UT" + revision = "39771216ff4c63d11f5e604076f9c45e8be1067b" + [[projects]] branch = "master" digest = "1:9f07f801988b225662081432361c430cad8f5293b134e80bdf1998d14969d7a6" @@ -642,6 +780,25 @@ revision = "02a8604050d8466dd915307496174adb9be4593a" version = "v1.3.1" +[[projects]] + digest = "1:ee4d4af67d93cc7644157882329023ce9a7bcfce956a079069a9405521c7cc8d" + name = "github.com/opencontainers/go-digest" + packages = ["."] + pruneopts = "UT" + revision = "279bed98673dd5bef374d3b6e4b09e2af76183bf" + version = "v1.0.0-rc1" + +[[projects]] + digest = "1:11db38d694c130c800d0aefb502fb02519e514dc53d9804ce51d1ad25ec27db6" + name = "github.com/opencontainers/image-spec" + packages = [ + "specs-go", + "specs-go/v1", + ] + pruneopts = "UT" + revision = "d60099175f88c47cd379c4738d158884749ed235" + version = "v1.0.1" + [[projects]] branch = "master" digest = "1:3173a0e98abdc5021301c4d34e6ac5ff8e6c792f3d22849c879ace70d5828978" @@ -678,14 +835,6 @@ pruneopts = "UT" revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" -[[projects]] - digest = "1:0e7775ebbcf00d8dd28ac663614af924411c868dca3d5aa762af0fae3808d852" - name = "github.com/peterbourgon/diskv" - packages = ["."] - pruneopts = "UT" - revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" - version = "v2.0.1" - [[projects]] digest = "1:cf31692c14422fa27c83a05292eb5cbe0fb2775972e8f1f8446a71549bd8980b" name = "github.com/pkg/errors" @@ -704,7 +853,7 @@ [[projects]] branch = "master" - digest = "1:8577a254ffb1b97e5c824fa32ccbc09bcbd327b2faa66b5abd83d7e6169a49b8" + digest = "1:dff66fce6bb8fa6111998bf3575e22aa3d0fd2560712a83de3390236d3cd1f6b" name = "github.com/prometheus/client_golang" packages = [ "api", @@ -712,6 +861,8 @@ "prometheus", "prometheus/internal", "prometheus/promauto", + "prometheus/promhttp", + "prometheus/testutil", ] pruneopts = "UT" revision = "c5e14697eaa7af985b2e326f1f5ed50bacb75c06" @@ -753,7 +904,7 @@ [[projects]] branch = "master" - digest = "1:c270fde3390cf824f004853135a8cb6347d013eab2d31e129bbae66219284284" + digest = "1:43edb14ccbe0d18f38bd55511250c39ccc3917c64a514a0e791408ac19a71558" name = "github.com/prometheus/prometheus" packages = [ "discovery", @@ -767,26 +918,29 @@ "discovery/kubernetes", "discovery/marathon", "discovery/openstack", + "discovery/refresh", "discovery/targetgroup", "discovery/triton", "discovery/zookeeper", "pkg/gate", "pkg/labels", + "pkg/modtimevfs", + "pkg/pool", "pkg/relabel", "pkg/textparse", "pkg/timestamp", "pkg/value", "promql", - "relabel", "storage", "storage/tsdb", + "template", "util/stats", "util/strutil", "util/testutil", "util/treecache", ] pruneopts = "UT" - revision = "0d1a69353e2864f9498e50fa51fa22bfb8346dcc" + revision = "e23fa22233cfbaa4fcea7abcd4e191ce5258870b" [[projects]] digest = "1:7aa5ae0ef5452ce26522f16677edee5016afeaa689f77e62e6b46374463b1884" @@ -821,6 +975,26 @@ revision = "f0a61d5e8ca1bcc7a607d6de3dfd51467791db88" version = "v2.1.0" +[[projects]] + branch = "master" + digest = "1:d716544630de6b0d07c24bd4690547b6db84a8f62f71939a2957aec1a7b824d2" + name = "github.com/shurcooL/httpfs" + packages = [ + "filter", + "union", + "vfsutil", + ] + pruneopts = "UT" + revision = "74dc9339e414ad069a8d04bba7e7aafd08043a25" + +[[projects]] + branch = "master" + digest = "1:98450c86949b8cdc4637b80c1c686ca955e503d3fbae9296d1f49c532895d281" + name = "github.com/shurcooL/vfsgen" + packages = ["."] + pruneopts = "UT" + revision = "6a9ea43bcacdf716a5c1b38efff722c07adf0069" + [[projects]] digest = "1:87c2e02fb01c27060ccc5ba7c5a407cc91147726f8f40b70cceeedbc52b1f3a8" name = "github.com/sirupsen/logrus" @@ -830,16 +1004,33 @@ version = "v1.3.0" [[projects]] - digest = "1:5da8ce674952566deae4dbc23d07c85caafc6cfa815b0b3e03e41979cedb8750" + digest = "1:ac83cf90d08b63ad5f7e020ef480d319ae890c208f8524622a2f3136e2686b02" + name = "github.com/stretchr/objx" + packages = ["."] + pruneopts = "UT" + revision = "477a77ecc69700c7cdeb1fa9e129548e1c1c393c" + version = "v0.1.1" + +[[projects]] + digest = "1:b762f96c1183763894e8dabbac5f8e8d5821754b6e2686a8c398a174b7a58ff5" name = "github.com/stretchr/testify" packages = [ "assert", + "mock", "require", ] pruneopts = "UT" revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" version = "v1.3.0" +[[projects]] + branch = "master" + digest = "1:9ab2182297ebe5a1433c9804ba65a382e4f41e3084485d1b4d31996e4c992e38" + name = "github.com/tonistiigi/fifo" + packages = ["."] + pruneopts = "UT" + revision = "a9fb20d87448d386e6d50b1f2e1fa70dcf0de43c" + [[projects]] branch = "master" digest = "1:c9fac76a6828c9dd4bf2a73804c35f28b9d15db602e11a7ffe87f44d52c8ab3b" @@ -878,8 +1069,8 @@ version = "v2.0.0" [[projects]] - branch = "master" - digest = "1:81403343bc9a102e3c924f87ad81e5a13bb7c36211714475c906853f4e887933" + branch = "server-listen-addr" + digest = "1:0184d699d4cbbbc3073fbbdd7cc0c8592d484c6f23914c89fb6d218e90de171a" name = "github.com/weaveworks/common" packages = [ "aws", @@ -896,7 +1087,8 @@ "user", ] pruneopts = "UT" - revision = "81a1a4d158e60de72dbead600ec011fb90344f8c" + revision = "5bf824591a6567784789cf9b2169f74f162bf80d" + source = "https://github.com/tomwilkie/weaveworks-common" [[projects]] digest = "1:bb40f7ff970145324f2a2acafdff3a23ed3f05db49cb5eb519b3d6bee86a5887" @@ -945,7 +1137,7 @@ [[projects]] branch = "master" - digest = "1:1d4b98f6810f39d93cba6ac580325528e02a6ebccafaa7628fb232cd8450b177" + digest = "1:1ffd895348a1d8f406811326cae5de5ba6de04e7a9ed03f26e8d39fc7ef27621" name = "golang.org/x/net" packages = [ "bpf", @@ -957,9 +1149,11 @@ "idna", "internal/iana", "internal/socket", + "internal/socks", "internal/timeseries", "ipv4", "ipv6", + "proxy", "trace", ] pruneopts = "UT" @@ -1034,7 +1228,7 @@ [[projects]] branch = "master" - digest = "1:d594bebb1e46761948a5162ce831bdee8bbf54204a9f87c5687143b4efe10cde" + digest = "1:961dd085b9c55912dc9c7d05e83bdad3a491b302fa7a844caa314d66a78734d9" name = "google.golang.org/api" packages = [ "cloudresourcemanager/v1", @@ -1054,7 +1248,7 @@ "transport/http/internal/propagation", ] pruneopts = "UT" - revision = "032faecc3e7e2c445ec37a1b1ec4e5a3016d96f2" + revision = "5213b809086156e6e2b262a41394993fcff97439" [[projects]] digest = "1:c4eaa5f79d36f76ef4bd0c4f96e36bc1b7b5a359528d1267f0cb7a5d58b7b5bb" @@ -1095,17 +1289,27 @@ revision = "6909d8a4a91b6d3fd1c4580b6e35816be4706fef" [[projects]] - digest = "1:10be9918abb568296ca66b6967197d0183e18d934e62f0f542e2e04a8b503270" + digest = "1:10488712ab5f5da69a7da557c169824c247b63219af7f072318919ba352bee7a" name = "google.golang.org/grpc" packages = [ ".", "balancer", "balancer/base", + "balancer/grpclb", + "balancer/grpclb/grpc_lb_v1", "balancer/roundrobin", "binarylog/grpc_binarylog_v1", "codes", "connectivity", "credentials", + "credentials/alts", + "credentials/alts/internal", + "credentials/alts/internal/authinfo", + "credentials/alts/internal/conn", + "credentials/alts/internal/handshaker", + "credentials/alts/internal/handshaker/service", + "credentials/alts/internal/proto/grpc_gcp", + "credentials/google", "credentials/internal", "credentials/oauth", "encoding", @@ -1115,6 +1319,7 @@ "health/grpc_health_v1", "internal", "internal/backoff", + "internal/balancerload", "internal/binarylog", "internal/channelz", "internal/envconfig", @@ -1134,8 +1339,8 @@ "tap", ] pruneopts = "UT" - revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8" - version = "v1.17.0" + revision = "501c41df7f472c740d0674ff27122f3f48c80ce7" + version = "v1.21.1" [[projects]] digest = "1:c06d9e11d955af78ac3bbb26bd02e01d2f61f689e1a3bce2ef6fb683ef8a7f2d" @@ -1187,10 +1392,10 @@ version = "v2.2.2" [[projects]] - digest = "1:0d299a04c6472e4458461d7034c76d014cc6f632a3262cbf21d123b19ce13e65" + branch = "master" + digest = "1:b2e959d92a38ffc25142da8e8bf4fefe0e5e687ecb871fabaac58db0497304df" name = "k8s.io/api" packages = [ - "admissionregistration/v1alpha1", "admissionregistration/v1beta1", "apps/v1", "apps/v1beta1", @@ -1207,15 +1412,20 @@ "batch/v1beta1", "batch/v2alpha1", "certificates/v1beta1", + "coordination/v1", "coordination/v1beta1", "core/v1", "events/v1beta1", "extensions/v1beta1", "networking/v1", + "networking/v1beta1", + "node/v1alpha1", + "node/v1beta1", "policy/v1beta1", "rbac/v1", "rbac/v1alpha1", "rbac/v1beta1", + "scheduling/v1", "scheduling/v1alpha1", "scheduling/v1beta1", "settings/v1alpha1", @@ -1224,7 +1434,7 @@ "storage/v1beta1", ] pruneopts = "UT" - revision = "05914d821849" + revision = "9b8cae951d65ea28a341b3262d38a058d0935f7c" [[projects]] digest = "1:846bcc1ec56accfcb542fa8834d3013b64e7930eb6e9789a7be1bb6448567265" @@ -1274,13 +1484,12 @@ revision = "2b1284ed4c93" [[projects]] - digest = "1:9edb9a30d4ec54889d1bb3137f0a24915a0b4306ee95f156ef6a111f755816b3" + digest = "1:d6168474843596c732e8e457ef6f66655a86a135b25a408d7f913b38c056819c" name = "k8s.io/client-go" packages = [ "discovery", "kubernetes", "kubernetes/scheme", - "kubernetes/typed/admissionregistration/v1alpha1", "kubernetes/typed/admissionregistration/v1beta1", "kubernetes/typed/apps/v1", "kubernetes/typed/apps/v1beta1", @@ -1297,15 +1506,20 @@ "kubernetes/typed/batch/v1beta1", "kubernetes/typed/batch/v2alpha1", "kubernetes/typed/certificates/v1beta1", + "kubernetes/typed/coordination/v1", "kubernetes/typed/coordination/v1beta1", "kubernetes/typed/core/v1", "kubernetes/typed/events/v1beta1", "kubernetes/typed/extensions/v1beta1", "kubernetes/typed/networking/v1", + "kubernetes/typed/networking/v1beta1", + "kubernetes/typed/node/v1alpha1", + "kubernetes/typed/node/v1beta1", "kubernetes/typed/policy/v1beta1", "kubernetes/typed/rbac/v1", "kubernetes/typed/rbac/v1alpha1", "kubernetes/typed/rbac/v1beta1", + "kubernetes/typed/scheduling/v1", "kubernetes/typed/scheduling/v1alpha1", "kubernetes/typed/scheduling/v1beta1", "kubernetes/typed/settings/v1alpha1", @@ -1325,16 +1539,15 @@ "tools/pager", "tools/reference", "transport", - "util/buffer", "util/cert", "util/connrotation", "util/flowcontrol", - "util/integer", + "util/keyutil", "util/retry", "util/workqueue", ] pruneopts = "UT" - revision = "a47917edff34" + revision = "1a26190bd76a9017e289958b9fba936430aa3704" [[projects]] digest = "1:e2999bf1bb6eddc2a6aa03fe5e6629120a53088926520ca3b4765f77d7ff7eab" @@ -1344,6 +1557,18 @@ revision = "a5bc97fbc634d635061f3146511332c7e313a55a" version = "v0.1.0" +[[projects]] + branch = "master" + digest = "1:03ce1a3e8094febc17dfaf3bfc7a445fb964e4fa96d3443822505dfc8567b648" + name = "k8s.io/utils" + packages = [ + "buffer", + "integer", + "trace", + ] + pruneopts = "UT" + revision = "6999998975a717e7f5fc2a7476497044cb111854" + [[projects]] digest = "1:7719608fe0b52a4ece56c2dde37bedd95b938677d1ab0f84b8a7852e4c59f849" name = "sigs.k8s.io/yaml" @@ -1357,8 +1582,10 @@ analyzer-version = 1 input-imports = [ "github.com/bmatcuk/doublestar", + "github.com/coreos/go-systemd/sdjournal", "github.com/cortexproject/cortex/pkg/chunk", "github.com/cortexproject/cortex/pkg/chunk/encoding", + "github.com/cortexproject/cortex/pkg/chunk/local", "github.com/cortexproject/cortex/pkg/chunk/storage", "github.com/cortexproject/cortex/pkg/ingester/client", "github.com/cortexproject/cortex/pkg/ingester/index", @@ -1366,11 +1593,17 @@ "github.com/cortexproject/cortex/pkg/util", "github.com/cortexproject/cortex/pkg/util/flagext", "github.com/cortexproject/cortex/pkg/util/validation", - "github.com/cortexproject/cortex/pkg/util/wire", + "github.com/docker/docker/api/types/plugins/logdriver", + "github.com/docker/docker/daemon/logger", + "github.com/docker/docker/daemon/logger/jsonfilelog", + "github.com/docker/docker/daemon/logger/templates", + "github.com/docker/docker/pkg/ioutils", + "github.com/docker/go-plugins-helpers/sdk", "github.com/fatih/color", "github.com/go-kit/kit/log", "github.com/go-kit/kit/log/level", "github.com/gogo/protobuf/gogoproto", + "github.com/gogo/protobuf/io", "github.com/gogo/protobuf/proto", "github.com/gogo/protobuf/types", "github.com/golang/snappy", @@ -1378,29 +1611,50 @@ "github.com/gorilla/websocket", "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc", "github.com/hpcloud/tail", + "github.com/jmespath/go-jmespath", + "github.com/mitchellh/mapstructure", "github.com/mwitkow/go-grpc-middleware", "github.com/opentracing/opentracing-go", "github.com/pkg/errors", "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/client_golang/prometheus/promauto", + "github.com/prometheus/client_golang/prometheus/promhttp", + "github.com/prometheus/client_golang/prometheus/testutil", + "github.com/prometheus/client_model/go", + "github.com/prometheus/common/config", + "github.com/prometheus/common/expfmt", "github.com/prometheus/common/model", "github.com/prometheus/common/version", "github.com/prometheus/prometheus/discovery", "github.com/prometheus/prometheus/discovery/config", "github.com/prometheus/prometheus/discovery/targetgroup", "github.com/prometheus/prometheus/pkg/labels", + "github.com/prometheus/prometheus/pkg/modtimevfs", + "github.com/prometheus/prometheus/pkg/pool", "github.com/prometheus/prometheus/pkg/relabel", - "github.com/prometheus/prometheus/relabel", + "github.com/prometheus/prometheus/pkg/textparse", + "github.com/prometheus/prometheus/promql", + "github.com/prometheus/prometheus/template", + "github.com/shurcooL/httpfs/filter", + "github.com/shurcooL/httpfs/union", + "github.com/shurcooL/vfsgen", "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/mock", "github.com/stretchr/testify/require", + "github.com/tonistiigi/fifo", "github.com/weaveworks/common/httpgrpc", + "github.com/weaveworks/common/httpgrpc/server", + "github.com/weaveworks/common/logging", "github.com/weaveworks/common/middleware", "github.com/weaveworks/common/server", "github.com/weaveworks/common/tracing", "github.com/weaveworks/common/user", "golang.org/x/net/context", "google.golang.org/grpc", + "google.golang.org/grpc/codes", "google.golang.org/grpc/health/grpc_health_v1", + "google.golang.org/grpc/metadata", + "google.golang.org/grpc/status", "gopkg.in/alecthomas/kingpin.v2", "gopkg.in/fsnotify.v1", "gopkg.in/yaml.v2", diff --git a/Gopkg.toml b/Gopkg.toml index e35786b0c7221..01e104fe0de51 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -30,7 +30,8 @@ [[constraint]] name = "github.com/weaveworks/common" - branch = "master" + source = "https://github.com/tomwilkie/weaveworks-common" + branch = "server-listen-addr" [[constraint]] name = "gopkg.in/fsnotify.v1" @@ -70,3 +71,10 @@ name = "github.com/hpcloud/tail" source = "github.com/grafana/tail" branch = "master" + +[[override]] + name = "k8s.io/client-go" + revision = "1a26190bd76a9017e289958b9fba936430aa3704" +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.3.0" diff --git a/Makefile b/Makefile index e7c27e0f41899..8c7e8cfcbf0cd 100644 --- a/Makefile +++ b/Makefile @@ -1,80 +1,64 @@ -.PHONY: all test clean images protos .DEFAULT_GOAL := all +.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos +.PHONY: helm helm-install helm-upgrade helm-publish helm-debug helm-clean +.PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push +.PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image +.PHONY: benchmark-store +############# +# Variables # +############# -CHARTS := production/helm/loki production/helm/promtail production/helm/loki-stack +DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) +IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) -# Boiler plate for bulding Docker containers. -# All this must go at top of file I'm afraid. -IMAGE_PREFIX ?= grafana/ +# Certain aspects of the build are done in containers for consistency (e.g. yacc/protobuf generation) +# If you have the correct tools installed and you want to speed up development you can run +# make BUILD_IN_CONTAINER=false target +# or you can override this with an environment variable +BUILD_IN_CONTAINER ?= true +BUILD_IMAGE_VERSION := "0.2.1" + +# Docker image info +IMAGE_PREFIX ?= grafana IMAGE_TAG := $(shell ./tools/image-tag) -UPTODATE := .uptodate + +# Version info for binaries GIT_REVISION := $(shell git rev-parse --short HEAD) GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) - -# Building Docker images is now automated. The convention is every directory -# with a Dockerfile in it builds an image calls quay.io/grafana/loki-. -# Dependencies (i.e. things that go in the image) still need to be explicitly -# declared. -%/$(UPTODATE): %/Dockerfile - $(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/ - $(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG) - touch $@ - # We don't want find to scan inside a bunch of directories, to accelerate the # 'make: Entering directory '/go/src/github.com/grafana/loki' phase. DONT_FIND := -name tools -prune -o -name vendor -prune -o -name .git -prune -o -name .cache -prune -o -name .pkg -prune -o -# Get a list of directories containing Dockerfiles -DOCKERFILES := $(shell find . $(DONT_FIND) -type f -name 'Dockerfile' -print) -UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES)) -DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) -IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir)))) -images: - $(info $(patsubst %,%:$(IMAGE_TAG),$(IMAGE_NAMES))) - @echo > /dev/null +# These are all the application files, they are included in the various binary rules as dependencies +# to make sure binaries are rebuilt if any source files change. +APP_GO_FILES := $(shell find . $(DONT_FIND) -name .y.go -prune -o -name .pb.go -prune -o -name cmd -prune -o -type f -name '*.go' -print) + +# Build flags +VPREFIX := github.com/grafana/loki/vendor/github.com/prometheus/common/version +GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo +# Per some websites I've seen to add `-gcflags "all=-N -l"`, the gcflags seem poorly if at all documented +# the best I could dig up is -N disables optimizations and -l disables inlining which should make debugging match source better. +# Also remove the -s and -w flags present in the normal build which strip the symbol table and the DWARF symbol table. +DEBUG_GO_FLAGS := -gcflags "all=-N -l" -ldflags "-extldflags \"-static\" -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo -# Generating proto code is automated. +NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ + rm $@; \ + echo "\nYour go standard library was built without the 'netgo' build tag."; \ + echo "To fix that, run"; \ + echo " sudo go clean -i net"; \ + echo " sudo go install -tags netgo std"; \ + false; \ +} + +# Protobuf files PROTO_DEFS := $(shell find . $(DONT_FIND) -type f -name '*.proto' -print) -PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) \ - vendor/github.com/cortexproject/cortex/pkg/ring/ring.pb.go \ - vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go \ - vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.pb.go +PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS)) -# Generating yacc code is automated. +# Yacc Files YACC_DEFS := $(shell find . $(DONT_FIND) -type f -name *.y -print) -YACC_GOS := $(patsubst %.y,%.go,$(YACC_DEFS)) - -# Building binaries is now automated. The convention is to build a binary -# for every directory with main.go in it, in the ./cmd directory. -MAIN_GO := $(shell find . $(DONT_FIND) -type f -name 'main.go' -print) -EXES := $(foreach exe, $(patsubst ./cmd/%/main.go, %, $(MAIN_GO)), ./cmd/$(exe)/$(exe)) -GO_FILES := $(shell find . $(DONT_FIND) -name cmd -prune -o -type f -name '*.go' -print) -define dep_exe -$(1): $(dir $(1))/main.go $(GO_FILES) $(PROTO_GOS) $(YACC_GOS) -$(dir $(1))$(UPTODATE): $(1) -endef -$(foreach exe, $(EXES), $(eval $(call dep_exe, $(exe)))) - -# Manually declared dependancies and what goes into each exe -pkg/logproto/logproto.pb.go: pkg/logproto/logproto.proto -vendor/github.com/cortexproject/cortex/pkg/ring/ring.pb.go: vendor/github.com/cortexproject/cortex/pkg/ring/ring.proto -vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.pb.go: vendor/github.com/cortexproject/cortex/pkg/ingester/client/cortex.proto -vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.pb.go: vendor/github.com/cortexproject/cortex/pkg/chunk/storage/caching_index_client.proto -pkg/parser/labels.go: pkg/parser/labels.y -pkg/parser/matchers.go: pkg/parser/matchers.y -all: $(UPTODATE_FILES) -test: $(PROTO_GOS) $(YACC_GOS) -yacc: $(YACC_GOS) -protos: $(PROTO_GOS) -yacc: $(YACC_GOS) - -# And now what goes into each image -loki-build-image/$(UPTODATE): loki-build-image/* +YACC_GOS := $(patsubst %.y,%.y.go,$(YACC_DEFS)) -# All the boiler plate for building golang follows: -SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E") -BUILD_IN_CONTAINER := true # RM is parameterized to allow CircleCI to run builds, as it # currently disallows `docker run --rm`. This value is overridden # in circle.yml @@ -84,122 +68,294 @@ RM := --rm # in any custom cloudbuild.yaml files TTY := --tty -VPREFIX := github.com/grafana/loki/vendor/github.com/prometheus/common/version -GO_FLAGS := -ldflags "-extldflags \"-static\" -s -w -X $(VPREFIX).Branch=$(GIT_BRANCH) -X $(VPREFIX).Version=$(IMAGE_TAG) -X $(VPREFIX).Revision=$(GIT_REVISION)" -tags netgo +################ +# Main Targets # +################ -NETGO_CHECK = @strings $@ | grep cgo_stub\\\.go >/dev/null || { \ - rm $@; \ - echo "\nYour go standard library was built without the 'netgo' build tag."; \ - echo "To fix that, run"; \ - echo " sudo go clean -i net"; \ - echo " sudo go install -tags netgo std"; \ - false; \ -} +all: promtail logcli loki loki-canary check-generated-files -ifeq ($(BUILD_IN_CONTAINER),true) -$(EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-files: loki-build-image/$(UPTODATE) +# This is really a check for the CI to make sure generated files are built and checked in manually +check-generated-files: yacc protos + @if ! (git diff --exit-code $(YACC_GOS) $(PROTO_GOS)); then \ + echo "\nChanges found in either generated protos or yaccs"; \ + echo "Run 'make all' and commit the changes to fix this error."; \ + echo "If you are actively developing these files you can ignore this error"; \ + echo "(Don't forget to check in the generated files when finished)\n"; \ + exit 1; \ + fi + + +########## +# Logcli # +########## + +logcli: yacc cmd/logcli/logcli + +cmd/logcli/logcli: $(APP_GO_FILES) cmd/logcli/main.go + CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +######## +# Loki # +######## + +loki: protos yacc cmd/loki/loki +loki-debug: protos yacc cmd/loki/loki-debug + +cmd/loki/loki: $(APP_GO_FILES) cmd/loki/main.go + CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +cmd/loki/loki-debug: $(APP_GO_FILES) cmd/loki/main.go + CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +############### +# Loki-Canary # +############### + +loki-canary: protos yacc cmd/loki-canary/loki-canary + +cmd/loki-canary/loki-canary: $(APP_GO_FILES) cmd/loki-canary/main.go + CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +############ +# Promtail # +############ + +promtail: yacc cmd/promtail/promtail +promtail-debug: yacc cmd/promtail/promtail-debug + +# Rule to generate promtail static assets file +pkg/promtail/server/ui/assets_vfsdata.go: + @echo ">> writing assets" + GOOS=$(shell go env GOHOSTOS) go generate -x -v ./pkg/promtail/server/ui + +cmd/promtail/promtail: $(APP_GO_FILES) pkg/promtail/server/ui/assets_vfsdata.go cmd/promtail/main.go + CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +cmd/promtail/promtail-debug: $(APP_GO_FILES) pkg/promtail/server/ui/assets_vfsdata.go cmd/promtail/main.go + CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D) + $(NETGO_CHECK) + +######## +# Lint # +######## + +lint: + GOGC=20 golangci-lint run + +######## +# Test # +######## + +test: all + go test -p=8 ./... + +######### +# Clean # +######### + +clean: + rm -rf cmd/promtail/promtail pkg/promtail/server/ui/assets_vfsdata.go + rm -rf cmd/loki/loki + rm -rf cmd/logcli/logcli + rm -rf cmd/loki-canary/loki-canary + rm -rf .cache + rm -rf cmd/docker-driver/rootfs + go clean ./... + +######### +# YACCs # +######### + +yacc: $(YACC_GOS) + +%.y.go: %.y +ifeq ($(BUILD_IN_CONTAINER),true) + # I wish we could make this a multiline variable however you can't pass more than simple arguments to them @mkdir -p $(shell pwd)/.pkg @mkdir -p $(shell pwd)/.cache $(SUDO) docker run $(RM) $(TTY) -i \ -v $(shell pwd)/.cache:/go/cache \ -v $(shell pwd)/.pkg:/go/pkg \ -v $(shell pwd):/go/src/github.com/grafana/loki \ - $(IMAGE_PREFIX)loki-build-image $@; + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; +else + goyacc -p $(basename $(notdir $<)) -o $@ $< +endif + +############# +# Protobufs # +############# +protos: $(PROTO_GOS) + +%.pb.go: $(PROTO_DEFS) +ifeq ($(BUILD_IN_CONTAINER),true) + @mkdir -p $(shell pwd)/.pkg + @mkdir -p $(shell pwd)/.cache + $(SUDO) docker run $(RM) $(TTY) -i \ + -v $(shell pwd)/.cache:/go/cache \ + -v $(shell pwd)/.pkg:/go/pkg \ + -v $(shell pwd):/go/src/github.com/grafana/loki \ + $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else + case "$@" in \ + vendor*) \ + protoc -I ./vendor:./$(@D) --gogoslick_out=plugins=grpc:./vendor ./$(patsubst %.pb.go,%.proto,$@); \ + ;; \ + *) \ + protoc -I ./vendor:./$(@D) --gogoslick_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,plugins=grpc:./$(@D) ./$(patsubst %.pb.go,%.proto,$@); \ + ;; \ + esac +endif + + +######## +# Helm # +######## -$(EXES): loki-build-image/$(UPTODATE) +CHARTS := production/helm/loki production/helm/promtail production/helm/loki-stack + +helm: + -rm -f production/helm/*/requirements.lock + @set -e; \ + helm init -c; \ + for chart in $(CHARTS); do \ + helm dependency build $$chart; \ + helm lint $$chart; \ + helm package $$chart; \ + done + rm -f production/helm/*/requirements.lock + +helm-install: + kubectl apply -f tools/helm.yaml + helm init --wait --service-account helm --upgrade + $(MAKE) helm-upgrade + +helm-upgrade: helm + helm upgrade --wait --install $(ARGS) loki-stack ./production/helm/loki-stack \ + --set promtail.image.tag=$(IMAGE_TAG) --set loki.image.tag=$(IMAGE_TAG) -f tools/dev.values.yaml + +helm-publish: helm + cp production/helm/README.md index.md + git config user.email "$CIRCLE_USERNAME@users.noreply.github.com" + git config user.name "${CIRCLE_USERNAME}" + git checkout gh-pages || (git checkout --orphan gh-pages && git rm -rf . > /dev/null) + mkdir -p charts + mv *.tgz index.md charts/ + helm repo index charts/ + git add charts/ + git commit -m "[skip ci] Publishing helm charts: ${CIRCLE_SHA1}" + git push origin gh-pages + +helm-debug: ARGS=--dry-run --debug +helm-debug: helm-upgrade + +helm-clean: + -helm delete --purge loki-stack + +################# +# Docker Driver # +################# + +PLUGIN_TAG ?= $(IMAGE_TAG) + +docker-driver: docker-driver-clean cmd/docker-driver/docker-driver + mkdir cmd/docker-driver/rootfs + docker build -t rootfsimage cmd/docker-driver + ID=$$(docker create rootfsimage true) && \ + (docker export $$ID | tar -x -C cmd/docker-driver/rootfs) && \ + docker rm -vf $$ID + docker rmi rootfsimage -f + docker plugin create grafana/loki-docker-driver:$(PLUGIN_TAG) cmd/docker-driver + +cmd/docker-driver/docker-driver: $(APP_GO_FILES) CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) $(NETGO_CHECK) -%.pb.go: loki-build-image/$(UPTODATE) - case "$@" in \ - vendor*) \ - protoc -I ./vendor:./$(@D) --gogoslick_out=plugins=grpc:./vendor ./$(patsubst %.pb.go,%.proto,$@); \ - ;; \ - *) \ - protoc -I ./vendor:./$(@D) --gogoslick_out=Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,plugins=grpc:./$(@D) ./$(patsubst %.pb.go,%.proto,$@); \ - ;; \ - esac - -%.go: %.y - goyacc -p $(basename $(notdir $<)) -o $@ $< +docker-driver-push: docker-driver + docker plugin push grafana/loki-docker-driver:$(PLUGIN_TAG) -lint: loki-build-image/$(UPTODATE) - gometalinter ./... +docker-driver-enable: + docker plugin enable grafana/loki-docker-driver:$(PLUGIN_TAG) -check-generated-files: loki-build-image/$(UPTODATE) yacc protos - @git diff-files || (echo "changed files; failing check" && exit 1) +docker-driver-clean: + -docker plugin disable grafana/loki-docker-driver:$(IMAGE_TAG) + -docker plugin rm grafana/loki-docker-driver:$(IMAGE_TAG) + rm -rf cmd/docker-driver/rootfs -test: loki-build-image/$(UPTODATE) - go test ./... -shell: loki-build-image/$(UPTODATE) - bash +########## +# Images # +########## -endif +images: promtail-image loki-image loki-canary-image docker-driver + +IMAGE_NAMES := grafana/loki grafana/promtail grafana/loki-canary save-images: @set -e; \ mkdir -p images; \ for image_name in $(IMAGE_NAMES); do \ - if ! echo $$image_name | grep build; then \ - docker save $$image_name:$(IMAGE_TAG) -o images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ - fi \ + echo ">> saving image $$image_name:$(IMAGE_TAG)"; \ + docker save $$image_name:$(IMAGE_TAG) -o images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ done load-images: @set -e; \ mkdir -p images; \ for image_name in $(IMAGE_NAMES); do \ - if ! echo $$image_name | grep build; then \ - docker load -i images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ - fi \ + docker load -i images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ done push-images: @set -e; \ for image_name in $(IMAGE_NAMES); do \ - if ! echo $$image_name | grep build; then \ - docker push $$image_name:$(IMAGE_TAG); \ - fi \ + docker push $$image_name:$(IMAGE_TAG); \ done push-latest: @set -e; \ for image_name in $(IMAGE_NAMES); do \ - if ! echo $$image_name | grep build; then \ - docker tag $$image_name:$(IMAGE_TAG) $$image_name:latest; \ - docker tag $$image_name:$(IMAGE_TAG) $$image_name:master; \ - docker push $$image_name:latest; \ - docker push $$image_name:master; \ - fi \ + docker tag $$image_name:$(IMAGE_TAG) $$image_name:latest; \ + docker tag $$image_name:$(IMAGE_TAG) $$image_name:master; \ + docker push $$image_name:latest; \ + docker push $$image_name:master; \ done -helm: - @set -e; \ - helm init -c; \ - for chart in $(CHARTS); do \ - helm lint $$chart; \ - helm dependency build $$chart; \ - helm package $$chart; \ - done - rm -f production/helm/*/requirements.lock -helm-publish: helm - cp production/helm/README.md index.md - git config user.email "$CIRCLE_USERNAME@users.noreply.github.com" - git config user.name "${CIRCLE_USERNAME}" - git checkout gh-pages || (git checkout --orphan gh-pages && git rm -rf . > /dev/null) - mkdir -p charts - mv *.tgz index.md charts/ - helm repo index charts/ - git add charts/ - git commit -m "[skip ci] Publishing helm charts: ${CIRCLE_SHA1}" - git push origin gh-pages +promtail-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/promtail -f cmd/promtail/Dockerfile . + $(SUDO) docker tag $(IMAGE_PREFIX)/promtail $(IMAGE_PREFIX)/promtail:$(IMAGE_TAG) +promtail-debug-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/promtail -f cmd/promtail/Dockerfile.debug . + $(SUDO) docker tag $(IMAGE_PREFIX)/promtail-debug $(IMAGE_PREFIX)/promtail-debug:$(IMAGE_TAG) -clean: - $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true - rm -rf $(UPTODATE_FILES) $(EXES) .cache - go clean ./... +loki-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki -f cmd/loki/Dockerfile . + $(SUDO) docker tag $(IMAGE_PREFIX)/loki $(IMAGE_PREFIX)/loki:$(IMAGE_TAG) +loki-debug-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki -f cmd/loki/Dockerfile.debug . + $(SUDO) docker tag $(IMAGE_PREFIX)/loki-debug $(IMAGE_PREFIX)/loki-debug:$(IMAGE_TAG) + +loki-canary-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-canary -f cmd/loki-canary/Dockerfile . + $(SUDO) docker tag $(IMAGE_PREFIX)/loki-canary $(IMAGE_PREFIX)/loki-canary:$(IMAGE_TAG) + +build-image: + $(SUDO) docker build -t $(IMAGE_PREFIX)/loki-build-image -f loki-build-image/Dockerfile . + $(SUDO) docker tag $(IMAGE_PREFIX)/loki-build-image $(IMAGE_PREFIX)/loki-build-image:$(IMAGE_TAG) + + +######## +# Misc # +######## + +benchmark-store: + go run ./pkg/storage/hack/main.go + go test ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out diff --git a/README.md b/README.md index fc4c72a9d242e..998eeb0b0908c 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,11 @@ Once you have promtail, Loki, and Grafana running, continue with [our usage docs - [API documentation](./docs/api.md) for alternative ways of getting logs into Loki. - [Operations](./docs/operations.md) for important aspects of running Loki. -- [Promtail](./docs/promtail-setup.md) on how to configure the agent that tails your logs. +- [Promtail](./docs/promtail.md) is an agent which can tail your log files and push them to Loki. +- [Processing Log Lines](./docs/logentry/processing-log-lines.md) for detailed log processing pipeline documentation +- [Docker Logging Driver](./cmd/docker-driver/README.md) is a docker plugin to send logs directly to Loki from Docker containers. - [Logcli](./docs/logcli.md) on how to query your logs without Grafana. +- [Loki Canary](./docs/canary/README.md) for monitoring your Loki installation for missing logs. - [Troubleshooting](./docs/troubleshooting.md) for help around frequent error messages. - [Usage](./docs/usage.md) for how to set up a Loki datasource in Grafana and query your logs. @@ -55,12 +58,16 @@ Your feedback is always welcome. ## Further Reading - The original [design doc](https://docs.google.com/document/d/11tjK_lvp1-SVsFZjgOTr1vV3-q6vBAsZYIQ5ZeYBkyM/view) for Loki is a good source for discussion of the motivation and design decisions. +- Callum Styan's March 2019 DevOpsDays Vancouver talk "[Grafana Loki: Log Aggregation for Incident Investigations][devopsdays19-talk]". +- Grafana Labs blog post "[How We Designed Loki to Work Easily Both as Microservices and as Monoliths][architecture-blog]". - Julien Garcia Gonzalez' March 2019 blog post "[Grafana Logging using Loki][giant-swarm-blog]". -- Tom Wilkie's early-2019 CNCF Paris/FODEM talk "[Grafana Loki: like Prometheus, but for logs][fosdem19-talk]" ([slides][fosdem19-slides], [video][fosdem19-video]). +- Tom Wilkie's early-2019 CNCF Paris/FOSDEM talk "[Grafana Loki: like Prometheus, but for logs][fosdem19-talk]" ([slides][fosdem19-slides], [video][fosdem19-video]). - David Kaltschmidt's KubeCon 2018 talk "[On the OSS Path to Full Observability with Grafana][kccna18-event]" ([slides][kccna18-slides], [video][kccna18-video]) on how Loki fits into a cloud-native environment. - Goutham Veeramachaneni's blog post "[Loki: Prometheus-inspired, open source logging for cloud natives](https://grafana.com/blog/2018/12/12/loki-prometheus-inspired-open-source-logging-for-cloud-natives/)" on details of the Loki architectire. - David Kaltschmidt's blog post "[Closer look at Grafana's user interface for Loki](https://grafana.com/blog/2019/01/02/closer-look-at-grafanas-user-interface-for-loki/)" on the ideas that went into the logging user interface. +[devopsdays19-talk]: https://grafana.com/blog/2019/05/06/how-loki-correlates-metrics-and-logs-and-saves-you-money/ +[architecture-blog]: https://grafana.com/blog/2019/04/15/how-we-designed-loki-to-work-easily-both-as-microservices-and-as-monoliths/ [giant-swarm-blog]: https://blog.giantswarm.io/grafana-logging-using-loki [fosdem19-talk]: https://fosdem.org/2019/schedule/event/loki_prometheus_for_logs/ [fosdem19-slides]: https://speakerdeck.com/grafana/grafana-loki-like-prometheus-but-for-logs diff --git a/build/Dockerfile b/build/Dockerfile new file mode 100644 index 0000000000000..72c6a9baf8ff7 --- /dev/null +++ b/build/Dockerfile @@ -0,0 +1,53 @@ +# These may be overwritten by --build-arg to build promtail or debug images +ARG APP=loki +ARG TYPE=production + +# ca-certificates +FROM alpine:3.9 as ssl +RUN apk add --update --no-cache ca-certificates + +# use grafana/loki-build-image to compile binaries +FROM grafana/loki-build-image as golang +ARG GOARCH="amd64" +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN touch loki-build-image/.uptodate &&\ + mkdir /build + +# production image +FROM golang as builder-production +ARG APP +RUN make BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\ + mv cmd/${APP}/${APP} /build/${APP} + +FROM scratch as production +COPY --from=ssl /etc/ssl /etc/ssl +COPY --from=builder-production /build/${APP} /usr/bin/${APP} + +# debug image (only arm64 supported, because of go-delve/delve#118) +FROM golang as builder-debug +ARG APP +RUN go get github.com/go-delve/delve/cmd/dlv &&\ + make BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\ + mv cmd/${APP}/${APP}-debug /build/app-debug &&\ + mv cmd/${APP}/dlv /build/dlv + +FROM alpine:3.9 as debug +COPY --from=ssl /etc/ssl /etc/ssl +COPY --from=builder-debug /build/app-debug /usr/bin/app-debug +COPY --from=builder-debug /build/dlv /usr/bin/dlv +RUN apk add --no-cache libc6-compat +EXPOSE 40000 +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/app-debug", "--"] + +# final image with configuration +FROM ${TYPE} as promtail +COPY cmd/promtail/promtail-local-config.yaml cmd/promtail/promtail-docker-config.yaml /etc/promtail/ +ENTRYPOINT ["/usr/bin/promtail"] + +FROM ${TYPE} as loki +COPY cmd/loki/loki-local-config.yaml /etc/loki/local-config.yaml +EXPOSE 80 +ENTRYPOINT ["/usr/bin/loki"] + +FROM ${APP} diff --git a/cmd/docker-driver/Dockerfile b/cmd/docker-driver/Dockerfile new file mode 100644 index 0000000000000..a44c52e38853d --- /dev/null +++ b/cmd/docker-driver/Dockerfile @@ -0,0 +1,5 @@ +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY docker-driver /bin/docker-driver +WORKDIR /bin/ +ENTRYPOINT [ "/bin/docker-driver" ] \ No newline at end of file diff --git a/cmd/docker-driver/README.md b/cmd/docker-driver/README.md new file mode 100644 index 0000000000000..983beabd2733c --- /dev/null +++ b/cmd/docker-driver/README.md @@ -0,0 +1,167 @@ +# Loki Docker Logging Driver + +## Overview + +Docker logging driver plugins extends Docker's logging capabilities. You can use Loki Docker logging driver plugin to send +Docker container logs directly to your Loki instance or [Grafana Cloud](https://grafana.com/loki). + +> Docker plugins are not yet supported on Windows; see Docker's logging driver plugin [documentation](https://docs.docker.com/engine/extend/) + +If you have any questions or issues using the Docker plugin feel free to open an issue in this [repository](https://github.com/grafana/loki/issues). + +## Plugin Installation + +You need to install the plugin on each Docker host with container from which you want to collect logs. + +You can install the plugin from our Docker hub repository by running on the Docker host the following command: + +```bash +docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions +``` + +To check the status of installed plugins, use the `docker plugin ls` command. Plugins that start successfully are listed as enabled in the output: + +```bash +docker plugin ls +ID NAME DESCRIPTION ENABLED +ac720b8fcfdb loki Loki Logging Driver true +``` + +You can now configure the plugin. + +## Plugin Configuration + +The Docker daemon on each Docker host has a default logging driver; each container on the Docker host uses the default driver, unless you configure it to use a different logging driver. + +### Configure the logging driver for a container + +When you start a container, you can configure it to use a different logging driver than the Docker daemon’s default, using the `--log-driver` flag. If the logging driver has configurable options, you can set them using one or more instances of the `--log-opt =` flag. Even if the container uses the default logging driver, it can use different configurable options. + +The following command configure the container `grafana` to start with the loki drivers which will send logs to `logs-us-west1.grafana.net` Loki instance, using a batch size of 400 entries and will retry maximum 5 times if it fails. + +```bash +docker run --log-driver=loki \ + --log-opt loki-url="https://:@logs-us-west1.grafana.net/api/prom/push" \ + --log-opt loki-retries=5 \ + --log-opt loki-batch-size=400 \ + grafana/grafana +``` + +### Configure the default logging driver + +To configure the Docker daemon to default to Loki logging driver, set the value of `log-driver` to `loki` logging driver in the `daemon.json` file, which is located in `/etc/docker/`. The following example explicitly sets the default logging driver to Loki: + +```json +{ + "debug" : true, + "log-driver": "loki" +} +``` + +The logging driver has configurable options, you can set them in the `daemon.json` file as a JSON array with the key log-opts. The following example sets the Loki push endpoint and batch size of the logging driver: + +```json +{ + "debug" : true, + "log-driver": "loki", + "log-opts": { + "loki-url": "https://:@logs-us-west1.grafana.net/api/prom/push", + "loki-batch-size": "400" + } +} +``` + +> **Note**: log-opt configuration options in the daemon.json configuration file must be provided as strings. Boolean and numeric values (such as the value for loki-batch-size in the example above) must therefore be enclosed in quotes ("). + +Restart the Docker daemon and it will be configured with Loki logging driver, all containers from that host will send logs to Loki instance. + +### Configure the logging driver for a Swarm service or Compose + +You can also configure the logging driver for a [swarm service](https://docs.docker.com/engine/swarm/how-swarm-mode-works/services/) directly in your compose file, this also work for a docker-compose deployment: + +```yaml +version: "3.7" +services: + logger: + image: grafana/grafana + logging: + driver: loki + options: + loki-url: "https://:@logs-us-west1.grafana.net/api/prom/push" +``` + +You can then deploy your stack using: + +```bash +docker stack deploy my_stack_name --compose-file docker-compose.yaml +``` + +Once deployed the Grafana service will be sending logs automatically to Loki. + +> **Note**: stack name and service name are automatically discovered and sent as Loki labels for each swarm service, this way you can filter by them in Grafana. + +## Labels + +Loki can received a set of labels along with log line. These labels are used to index log entries and query back logs using [LogQL stream selector](../../docs/usage.md#log-stream-selector). + +By default the Docker driver will add the `filename` where the log is written, the `host` where the log has been generated as well as the `container_name`. Additionally `swarm_stack` and `swarm_service` are added for Docker Swarm deployments. + +You can add more labels by using `loki-external-labels`,`loki-pipeline-stage-file`,`labels`,`env` and `env-regex` options as described below. + +## log-opt options + +To specify additional logging driver options, you can use the --log-opt NAME=VALUE flag. + +| Option | Required? | Default Value | Description +| ------------------------- | :-------: | :------------------: | -------------------------------------- | +| `loki-url` | Yes | | Loki HTTP push endpoint. +| `loki-external-labels` | No | `container_name={{.Name}}` | Additional label value pair separated by `,` to send with logs. The value is expanded with the [Docker tag template format](https://docs.docker.com/engine/admin/logging/log_tags/). (eg: `container_name={{.ID}}.{{.Name}},cluster=prod`) +| `loki-timeout` | No | `10s` | The timeout to use when sending logs to the Loki instance. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +| `loki-batch-wait` | No | `1s` | The amount of time to wait before sending a log batch complete or not. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +| `loki-batch-size` | No | `102400` | The maximum size of a log batch to send. +| `loki-min-backoff` | No | `100ms` | The minimum amount of time to wait before retrying a batch. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +| `loki-max-backoff` | No | `10s` | The maximum amount of time to wait before retrying a batch. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". +| `loki-retries` | No | `10` | The maximum amount of retries for a log batch. +| `loki-pipeline-stage-file` | No | | The location of a pipeline stage configuration file ([example](./pipeline-example.yaml)). Pipeline stages allows to parse log lines to extract more labels. [see documentation](../../docs/logentry/processing-log-lines.md) +| `loki-tls-ca-file` | No | | Set the path to a custom certificate authority. +| `loki-tls-cert-file` | No | | Set the path to a client certificate file. +| `loki-tls-key-file` | No | | Set the path to a client key. +| `loki-tls-server-name` | No | | Name used to validate the server certificate. +| `loki-tls-insecure-skip-verify` | No | `false` | Allow to skip tls verification. +| `loki-proxy-url` | No | | Proxy URL use to connect to Loki. +| `labels` | No | | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container. +| `env` | No | | Comma-separated list of keys of environment variables to be included in message if they specified for a container. +| `env-regex` | No | | A regular expression to match logging-related environment variables. Used for advanced log label options. If there is collision between the label and env keys, the value of the env takes precedence. Both options add additional fields to the labels of a logging message. + +## Uninstall the plugin + +To cleanly disable and remove the plugin, run: + +```bash +docker plugin disable loki +docker plugin rm loki +``` + +## Upgrade the plugin + +To upgrade the plugin to the last version, run: + +```bash +docker plugin disable loki +docker plugin upgrade loki grafana/loki-docker-driver:master +docker plugin enable loki +``` + +## Troubleshooting + +Plugin logs can be found as docker daemon log. To enable debug mode refer to the Docker daemon documentation: https://docs.docker.com/config/daemon/ + +Stdout of a plugin is redirected to Docker logs. Such entries have a plugin= suffix. + +To find out the plugin ID of Loki, use the command below and look for Loki plugin entry. + +```bash +docker plugin ls +``` + +Depending on your system, location of Docker daemon logging may vary. Refer to Docker documentation for Docker daemon log location for your specific platform. ([see](https://docs.docker.com/config/daemon/)) diff --git a/cmd/docker-driver/config.go b/cmd/docker-driver/config.go new file mode 100644 index 0000000000000..e8efaa8dab9ac --- /dev/null +++ b/cmd/docker-driver/config.go @@ -0,0 +1,296 @@ +package main + +import ( + "bytes" + "fmt" + "net/url" + "os" + "strconv" + "strings" + "time" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/templates" + "github.com/grafana/loki/pkg/helpers" + "github.com/grafana/loki/pkg/logentry/stages" + "github.com/grafana/loki/pkg/promtail/client" + "github.com/grafana/loki/pkg/promtail/targets" + "github.com/prometheus/common/model" +) + +const ( + driverName = "loki" + + cfgExternalLabelsKey = "loki-external-labels" + cfgURLKey = "loki-url" + cfgTLSCAFileKey = "loki-tls-ca-file" + cfgTLSCertFileKey = "loki-tls-cert-file" + cfgTLSKeyFileKey = "loki-tls-key-file" + cfgTLSServerNameKey = "loki-tls-server-name" + cfgTLSInsecure = "loki-tls-insecure-skip-verify" + cfgProxyURLKey = "loki-proxy-url" + cfgTimeoutKey = "loki-timeout" + cfgBatchWaitKey = "loki-batch-wait" + cfgBatchSizeKey = "loki-batch-size" + cfgMinBackoffKey = "loki-min-backoff" + cfgMaxBackoffKey = "loki-max-backoff" + cfgMaxRetriesKey = "loki-retries" + cfgPipelineStagesKey = "loki-pipeline-stage-file" + + swarmServiceLabelKey = "com.docker.swarm.service.name" + swarmStackLabelKey = "com.docker.stack.namespace" + + swarmServiceLabelName = "swarm_service" + swarmStackLabelName = "swarm_stack" + + defaultExternalLabels = "container_name={{.Name}}" + defaultHostLabelName = model.LabelName("host") +) + +var ( + defaultClientConfig = client.Config{ + BatchWait: 1 * time.Second, + BatchSize: 100 * 1024, + BackoffConfig: util.BackoffConfig{ + MinBackoff: 100 * time.Millisecond, + MaxBackoff: 10 * time.Second, + MaxRetries: 10, + }, + Timeout: 10 * time.Second, + } +) + +type config struct { + labels model.LabelSet + clientConfig client.Config + pipeline PipelineConfig +} + +type PipelineConfig struct { + PipelineStages stages.PipelineStages `yaml:"pipeline_stages,omitempty"` +} + +func validateDriverOpt(loggerInfo logger.Info) error { + config := loggerInfo.Config + + for opt := range config { + switch opt { + case cfgURLKey: + case cfgExternalLabelsKey: + case cfgTLSCAFileKey: + case cfgTLSCertFileKey: + case cfgTLSKeyFileKey: + case cfgTLSServerNameKey: + case cfgTLSInsecure: + case cfgTimeoutKey: + case cfgProxyURLKey: + case cfgBatchWaitKey: + case cfgBatchSizeKey: + case cfgMinBackoffKey: + case cfgMaxBackoffKey: + case cfgMaxRetriesKey: + case cfgPipelineStagesKey: + case "labels": + case "env": + case "env-regex": + default: + return fmt.Errorf("%s: wrong log-opt: '%s' - %s", driverName, opt, loggerInfo.ContainerID) + } + } + _, ok := config[cfgURLKey] + if !ok { + return fmt.Errorf("%s: %s is required in the config", driverName, cfgURLKey) + } + + return nil +} + +func parseConfig(logCtx logger.Info) (*config, error) { + if err := validateDriverOpt(logCtx); err != nil { + return nil, err + } + + clientConfig := defaultClientConfig + labels := model.LabelSet{} + + // parse URL + rawURL, ok := logCtx.Config[cfgURLKey] + if !ok { + return nil, fmt.Errorf("%s: option %s is required", driverName, cfgURLKey) + } + url, err := url.Parse(rawURL) + if err != nil { + return nil, fmt.Errorf("%s: option %s is invalid %s", driverName, cfgURLKey, err) + } + clientConfig.URL = flagext.URLValue{URL: url} + + // parse timeout + if err := parseDuration(cfgTimeoutKey, logCtx, func(d time.Duration) { clientConfig.Timeout = d }); err != nil { + return nil, err + } + + // parse batch wait and batch size + if err := parseDuration(cfgBatchWaitKey, logCtx, func(d time.Duration) { clientConfig.BatchWait = d }); err != nil { + return nil, err + } + if err := parseInt(cfgBatchSizeKey, logCtx, func(i int) { clientConfig.BatchSize = i }); err != nil { + return nil, err + } + + // parse backoff + if err := parseDuration(cfgMinBackoffKey, logCtx, func(d time.Duration) { clientConfig.BackoffConfig.MinBackoff = d }); err != nil { + return nil, err + } + if err := parseDuration(cfgMaxBackoffKey, logCtx, func(d time.Duration) { clientConfig.BackoffConfig.MaxBackoff = d }); err != nil { + return nil, err + } + if err := parseInt(cfgMaxRetriesKey, logCtx, func(i int) { clientConfig.BackoffConfig.MaxRetries = i }); err != nil { + return nil, err + } + + // parse http & tls config + if tlsCAFile, ok := logCtx.Config[cfgTLSCAFileKey]; ok { + clientConfig.Client.TLSConfig.CAFile = tlsCAFile + } + if tlsCertFile, ok := logCtx.Config[cfgTLSCertFileKey]; ok { + clientConfig.Client.TLSConfig.CertFile = tlsCertFile + } + if tlsCertFile, ok := logCtx.Config[cfgTLSCertFileKey]; ok { + clientConfig.Client.TLSConfig.CertFile = tlsCertFile + } + if tlsKeyFile, ok := logCtx.Config[cfgTLSKeyFileKey]; ok { + clientConfig.Client.TLSConfig.KeyFile = tlsKeyFile + } + if tlsServerName, ok := logCtx.Config[cfgTLSServerNameKey]; ok { + clientConfig.Client.TLSConfig.ServerName = tlsServerName + } + if tlsInsecureSkipRaw, ok := logCtx.Config[cfgTLSInsecure]; ok { + tlsInsecureSkip, err := strconv.ParseBool(tlsInsecureSkipRaw) + if err != nil { + return nil, fmt.Errorf("%s: invalid external labels: %s", driverName, tlsInsecureSkipRaw) + } + clientConfig.Client.TLSConfig.InsecureSkipVerify = tlsInsecureSkip + } + if tlsProxyURL, ok := logCtx.Config[cfgProxyURLKey]; ok { + proxyURL, err := url.Parse(tlsProxyURL) + if err != nil { + return nil, fmt.Errorf("%s: option %s is invalid %s", driverName, cfgProxyURLKey, err) + } + clientConfig.Client.ProxyURL.URL = proxyURL + } + + // parse external labels + extlbs, ok := logCtx.Config[cfgExternalLabelsKey] + if !ok { + extlbs = defaultExternalLabels + } + lvs := strings.Split(extlbs, ",") + for _, lv := range lvs { + lvparts := strings.Split(lv, "=") + if len(lvparts) != 2 { + return nil, fmt.Errorf("%s: invalid external labels: %s", driverName, extlbs) + } + labelName := model.LabelName(lvparts[0]) + if !labelName.IsValid() { + return nil, fmt.Errorf("%s: invalid external label name: %s", driverName, labelName) + } + + // expand the value using docker template {{.Name}}.{{.ImageName}} + value, err := expandLabelValue(logCtx, lvparts[1]) + if err != nil { + return nil, fmt.Errorf("%s: could not expand label value: %s err : %s", driverName, lvparts[1], err) + } + labelValue := model.LabelValue(value) + if !labelValue.IsValid() { + return nil, fmt.Errorf("%s: invalid external label value: %s", driverName, value) + } + labels[labelName] = labelValue + } + + // other labels coming from docker labels or env selected by user labels, labels-regex, env, env-regex config. + attrs, err := logCtx.ExtraAttributes(nil) + if err != nil { + return nil, err + } + + // parse docker swarms labels and adds them automatically to attrs + swarmService := logCtx.ContainerLabels[swarmServiceLabelKey] + if swarmService != "" { + attrs[swarmServiceLabelName] = swarmService + } + swarmStack := logCtx.ContainerLabels[swarmStackLabelKey] + if swarmStack != "" { + attrs[swarmStackLabelName] = swarmStack + } + + for key, value := range attrs { + labelName := model.LabelName(key) + if !labelName.IsValid() { + return nil, fmt.Errorf("%s: invalid label name from attribute: %s", driverName, key) + } + labelValue := model.LabelValue(value) + if !labelValue.IsValid() { + return nil, fmt.Errorf("%s: invalid label value from attribute: %s", driverName, value) + } + labels[labelName] = labelValue + } + + // adds host label and filename + host, err := os.Hostname() + if err == nil { + labels[defaultHostLabelName] = model.LabelValue(host) + } + labels[targets.FilenameLabel] = model.LabelValue(logCtx.LogPath) + + // parse pipeline stages + var pipeline PipelineConfig + pipelineFile, ok := logCtx.Config[cfgPipelineStagesKey] + if ok { + if err := helpers.LoadConfig(pipelineFile, &pipeline); err != nil { + return nil, fmt.Errorf("%s: error loading config file %s: %s", driverName, pipelineFile, err) + } + } + + return &config{ + labels: labels, + clientConfig: clientConfig, + pipeline: pipeline, + }, nil +} + +func expandLabelValue(info logger.Info, defaultTemplate string) (string, error) { + tmpl, err := templates.NewParse("label_value", defaultTemplate) + if err != nil { + return "", err + } + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, &info); err != nil { + return "", err + } + + return buf.String(), nil +} + +func parseDuration(key string, logCtx logger.Info, set func(d time.Duration)) error { + if raw, ok := logCtx.Config[key]; ok { + val, err := time.ParseDuration(raw) + if err != nil { + return fmt.Errorf("%s: invalid option %s format: %s", driverName, key, raw) + } + set(val) + } + return nil +} + +func parseInt(key string, logCtx logger.Info, set func(i int)) error { + if raw, ok := logCtx.Config[key]; ok { + val, err := strconv.Atoi(raw) + if err != nil { + return fmt.Errorf("%s: invalid option %s format: %s", driverName, key, raw) + } + set(val) + } + return nil +} diff --git a/cmd/docker-driver/config.json b/cmd/docker-driver/config.json new file mode 100644 index 0000000000000..7216230404401 --- /dev/null +++ b/cmd/docker-driver/config.json @@ -0,0 +1,20 @@ +{ + "description": "Loki Logging Driver", + "documentation": "https://github.com/grafana/loki", + "entrypoint": ["/bin/docker-driver"], + "network": { + "type": "host" + }, + "interface": { + "types": ["docker.logdriver/1.0"], + "socket": "loki.sock" + }, + "env": [ + { + "name": "LOG_LEVEL", + "description": "Set log level to output for plugin logs", + "value": "info", + "settable": ["value"] + } + ] +} \ No newline at end of file diff --git a/cmd/docker-driver/driver.go b/cmd/docker-driver/driver.go new file mode 100644 index 0000000000000..f99e5c33412fa --- /dev/null +++ b/cmd/docker-driver/driver.go @@ -0,0 +1,200 @@ +package main + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + "time" + + "github.com/docker/docker/api/types/backend" + "github.com/docker/docker/api/types/plugins/logdriver" + "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/jsonfilelog" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + protoio "github.com/gogo/protobuf/io" + "github.com/pkg/errors" + "github.com/tonistiigi/fifo" +) + +type driver struct { + mu sync.Mutex + logs map[string]*logPair + idx map[string]*logPair + logger log.Logger +} + +type logPair struct { + jsonl logger.Logger + lokil logger.Logger + stream io.ReadCloser + info logger.Info + logger log.Logger +} + +func (l *logPair) Close() { + if err := l.stream.Close(); err != nil { + level.Error(l.logger).Log("msg", "error while closing fifo stream", "err", err) + } + if err := l.lokil.Close(); err != nil { + level.Error(l.logger).Log("msg", "error while closing loki logger", "err", err) + } + if err := l.jsonl.Close(); err != nil { + level.Error(l.logger).Log("msg", "error while closing json logger", "err", err) + } +} + +func newDriver(logger log.Logger) *driver { + return &driver{ + logs: make(map[string]*logPair), + idx: make(map[string]*logPair), + logger: logger, + } +} + +func (d *driver) StartLogging(file string, logCtx logger.Info) error { + d.mu.Lock() + if _, exists := d.logs[file]; exists { + d.mu.Unlock() + return fmt.Errorf("logger for %q already exists", file) + } + d.mu.Unlock() + + if logCtx.LogPath == "" { + logCtx.LogPath = filepath.Join("/var/log/docker", logCtx.ContainerID) + } + if err := os.MkdirAll(filepath.Dir(logCtx.LogPath), 0755); err != nil { + return errors.Wrap(err, "error setting up logger dir") + } + jsonl, err := jsonfilelog.New(logCtx) + if err != nil { + return errors.Wrap(err, "error creating jsonfile logger") + } + + lokil, err := New(logCtx, d.logger) + if err != nil { + return errors.Wrap(err, "error creating loki logger") + } + level.Debug(d.logger).Log("msg", "Start logging", "id", logCtx.ContainerID, "file", file, "logpath", logCtx.LogPath) + f, err := fifo.OpenFifo(context.Background(), file, syscall.O_RDONLY, 0700) + if err != nil { + return errors.Wrapf(err, "error opening logger fifo: %q", file) + } + + d.mu.Lock() + lf := &logPair{jsonl, lokil, f, logCtx, d.logger} + d.logs[file] = lf + d.idx[logCtx.ContainerID] = lf + d.mu.Unlock() + + go consumeLog(lf) + return nil +} + +func (d *driver) StopLogging(file string) { + level.Debug(d.logger).Log("msg", "Stop logging", "file", file) + d.mu.Lock() + lf, ok := d.logs[file] + if ok { + lf.Close() + delete(d.logs, file) + } + d.mu.Unlock() +} + +func consumeLog(lf *logPair) { + dec := protoio.NewUint32DelimitedReader(lf.stream, binary.BigEndian, 1e6) + defer dec.Close() + defer lf.Close() + var buf logdriver.LogEntry + for { + if err := dec.ReadMsg(&buf); err != nil { + if err == io.EOF || err == os.ErrClosed || strings.Contains(err.Error(), "file already closed") { + level.Debug(lf.logger).Log("msg", "shutting down log logger", "id", lf.info.ContainerID, "err", err) + return + } + dec = protoio.NewUint32DelimitedReader(lf.stream, binary.BigEndian, 1e6) + } + var msg logger.Message + msg.Line = buf.Line + msg.Source = buf.Source + if buf.PartialLogMetadata != nil { + if msg.PLogMetaData == nil { + msg.PLogMetaData = &backend.PartialLogMetaData{} + } + msg.PLogMetaData.ID = buf.PartialLogMetadata.Id + msg.PLogMetaData.Last = buf.PartialLogMetadata.Last + msg.PLogMetaData.Ordinal = int(buf.PartialLogMetadata.Ordinal) + } + msg.Timestamp = time.Unix(0, buf.TimeNano) + + // loki goes first as the json logger reset the message on completion. + if err := lf.lokil.Log(&msg); err != nil { + level.Error(lf.logger).Log("msg", "error pushing message to loki", "id", lf.info.ContainerID, "err", err, "message", msg) + } + + if err := lf.jsonl.Log(&msg); err != nil { + level.Error(lf.logger).Log("msg", "error writing log message", "id", lf.info.ContainerID, "err", err, "message", msg) + continue + } + + buf.Reset() + } +} + +func (d *driver) ReadLogs(info logger.Info, config logger.ReadConfig) (io.ReadCloser, error) { + d.mu.Lock() + lf, exists := d.idx[info.ContainerID] + d.mu.Unlock() + if !exists { + return nil, fmt.Errorf("logger does not exist for %s", info.ContainerID) + } + + r, w := io.Pipe() + lr, ok := lf.jsonl.(logger.LogReader) + if !ok { + return nil, fmt.Errorf("logger does not support reading") + } + + go func() { + watcher := lr.ReadLogs(config) + + enc := protoio.NewUint32DelimitedWriter(w, binary.BigEndian) + defer enc.Close() + defer watcher.ConsumerGone() + + var buf logdriver.LogEntry + for { + select { + case msg, ok := <-watcher.Msg: + if !ok { + w.Close() + return + } + + buf.Line = msg.Line + buf.Partial = msg.PLogMetaData != nil + buf.TimeNano = msg.Timestamp.UnixNano() + buf.Source = msg.Source + + if err := enc.WriteMsg(&buf); err != nil { + _ = w.CloseWithError(err) + return + } + case err := <-watcher.Err: + _ = w.CloseWithError(err) + return + } + + buf.Reset() + } + }() + + return r, nil +} diff --git a/cmd/docker-driver/http.go b/cmd/docker-driver/http.go new file mode 100644 index 0000000000000..5045e2c9d1c41 --- /dev/null +++ b/cmd/docker-driver/http.go @@ -0,0 +1,95 @@ +package main + +import ( + "encoding/json" + "errors" + "io" + "net/http" + + "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/go-plugins-helpers/sdk" +) + +type StartLoggingRequest struct { + File string + Info logger.Info +} + +type StopLoggingRequest struct { + File string +} + +type CapabilitiesResponse struct { + Err string + Cap logger.Capability +} + +type ReadLogsRequest struct { + Info logger.Info + Config logger.ReadConfig +} + +func handlers(h *sdk.Handler, d *driver) { + h.HandleFunc("/LogDriver.StartLogging", func(w http.ResponseWriter, r *http.Request) { + var req StartLoggingRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if req.Info.ContainerID == "" { + respond(errors.New("must provide container id in log context"), w) + return + } + + err := d.StartLogging(req.File, req.Info) + respond(err, w) + }) + + h.HandleFunc("/LogDriver.StopLogging", func(w http.ResponseWriter, r *http.Request) { + var req StopLoggingRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + d.StopLogging(req.File) + respond(nil, w) + }) + + h.HandleFunc("/LogDriver.Capabilities", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(&CapabilitiesResponse{ + Cap: logger.Capability{ReadLogs: true}, + }) + }) + + h.HandleFunc("/LogDriver.ReadLogs", func(w http.ResponseWriter, r *http.Request) { + var req ReadLogsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + stream, err := d.ReadLogs(req.Info, req.Config) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer stream.Close() + + w.Header().Set("Content-Type", "application/x-json-stream") + wf := ioutils.NewWriteFlusher(w) + _, _ = io.Copy(wf, stream) + }) +} + +type response struct { + Err string +} + +func respond(err error, w io.Writer) { + var res response + if err != nil { + res.Err = err.Error() + } + _ = json.NewEncoder(w).Encode(&res) +} diff --git a/cmd/docker-driver/loki.go b/cmd/docker-driver/loki.go new file mode 100644 index 0000000000000..e8d99cea5862f --- /dev/null +++ b/cmd/docker-driver/loki.go @@ -0,0 +1,70 @@ +package main + +import ( + "bytes" + + "github.com/docker/docker/daemon/logger" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/grafana/loki/pkg/logentry/stages" + "github.com/grafana/loki/pkg/promtail/api" + "github.com/grafana/loki/pkg/promtail/client" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" +) + +var jobName = "docker" + +type loki struct { + client client.Client + handler api.EntryHandler + labels model.LabelSet + logger log.Logger +} + +// New create a new Loki logger that forward logs to Loki instance +func New(logCtx logger.Info, logger log.Logger) (logger.Logger, error) { + logger = log.With(logger, "container_id", logCtx.ContainerID) + cfg, err := parseConfig(logCtx) + if err != nil { + return nil, err + } + c, err := client.New(cfg.clientConfig, logger) + if err != nil { + return nil, err + } + var handler api.EntryHandler = c + if len(cfg.pipeline.PipelineStages) != 0 { + pipeline, err := stages.NewPipeline(logger, cfg.pipeline.PipelineStages, &jobName, prometheus.DefaultRegisterer) + if err != nil { + return nil, err + } + handler = pipeline.Wrap(c) + } + return &loki{ + client: c, + labels: cfg.labels, + logger: logger, + handler: handler, + }, nil +} + +// Log implements `logger.Logger` +func (l *loki) Log(m *logger.Message) error { + if len(bytes.Fields(m.Line)) == 0 { + level.Info(l.logger).Log("msg", "ignoring empty line", "line", string(m.Line)) + return nil + } + return l.handler.Handle(l.labels.Clone(), m.Timestamp, string(m.Line)) +} + +// Log implements `logger.Logger` +func (l *loki) Name() string { + return driverName +} + +// Log implements `logger.Logger` +func (l *loki) Close() error { + l.client.Stop() + return nil +} diff --git a/cmd/docker-driver/main.go b/cmd/docker-driver/main.go new file mode 100644 index 0000000000000..1598998f7e8c0 --- /dev/null +++ b/cmd/docker-driver/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "fmt" + "os" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/docker/go-plugins-helpers/sdk" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/common/version" + "github.com/weaveworks/common/logging" +) + +const socketAddress = "/run/docker/plugins/loki.sock" + +var logLevel logging.Level + +func main() { + levelVal := os.Getenv("LOG_LEVEL") + if levelVal == "" { + levelVal = "info" + } + + if err := logLevel.Set(levelVal); err != nil { + fmt.Fprintln(os.Stdout, "invalid log level: ", levelVal) + os.Exit(1) + } + logger := newLogger(logLevel) + level.Info(util.Logger).Log("msg", "Starting docker-plugin", "version", version.Info()) + h := sdk.NewHandler(`{"Implements": ["LoggingDriver"]}`) + handlers(&h, newDriver(logger)) + if err := h.ServeUnix(socketAddress, 0); err != nil { + panic(err) + } + +} + +func newLogger(lvl logging.Level) log.Logger { + // plugin logs must be stdout to appear. + logger := log.NewLogfmtLogger(log.NewSyncWriter(os.Stdout)) + logger = level.NewFilter(logger, lvl.Gokit) + logger = log.With(logger, "ts", log.DefaultTimestampUTC) + logger = log.With(logger, "caller", log.Caller(3)) + return logger +} diff --git a/cmd/docker-driver/pipeline-example.yaml b/cmd/docker-driver/pipeline-example.yaml new file mode 100644 index 0000000000000..91310d7974826 --- /dev/null +++ b/cmd/docker-driver/pipeline-example.yaml @@ -0,0 +1,5 @@ +pipeline_stages: +- regex: + expression: '(level|lvl|severity)=(?P\\w+)' +- labels: + level: \ No newline at end of file diff --git a/cmd/logcli/client.go b/cmd/logcli/client.go index 3acaea6b65eea..0984ba4538993 100644 --- a/cmd/logcli/client.go +++ b/cmd/logcli/client.go @@ -12,6 +12,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/prometheus/common/config" "github.com/grafana/loki/pkg/logproto" ) @@ -20,12 +21,18 @@ const ( queryPath = "/api/prom/query?query=%s&limit=%d&start=%d&end=%d&direction=%s®exp=%s" labelsPath = "/api/prom/label" labelValuesPath = "/api/prom/label/%s/values" - tailPath = "/api/prom/tail?query=%s®exp=%s" + tailPath = "/api/prom/tail?query=%s®exp=%s&delay_for=%d&limit=%d&start=%d" ) func query(from, through time.Time, direction logproto.Direction) (*logproto.QueryResponse, error) { - path := fmt.Sprintf(queryPath, url.QueryEscape(*queryStr), *limit, from.UnixNano(), - through.UnixNano(), direction.String(), url.QueryEscape(*regexpStr)) + path := fmt.Sprintf(queryPath, + url.QueryEscape(*queryStr), // query + *limit, // limit + from.UnixNano(), // start + through.UnixNano(), // end + direction.String(), // direction + url.QueryEscape(*regexpStr), // regexp + ) var resp logproto.QueryResponse if err := doRequest(path, &resp); err != nil { @@ -53,22 +60,45 @@ func listLabelValues(name string) (*logproto.LabelResponse, error) { } func doRequest(path string, out interface{}) error { - url := *addr + path - log.Print(url) + us := *addr + path + if !*quiet { + log.Print(us) + } - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequest("GET", us, nil) if err != nil { return err } + req.SetBasicAuth(*username, *password) - resp, err := http.DefaultClient.Do(req) + // Parse the URL to extract the host + u, err := url.Parse(us) + if err != nil { + return err + } + clientConfig := config.HTTPClientConfig{ + TLSConfig: config.TLSConfig{ + CAFile: *tlsCACertPath, + CertFile: *tlsClientCertPath, + KeyFile: *tlsClientCertKeyPath, + ServerName: u.Host, + InsecureSkipVerify: *tlsSkipVerify, + }, + } + + client, err := config.NewClientFromConfig(clientConfig, "logcli") + if err != nil { + return err + } + + resp, err := client.Do(req) if err != nil { return err } defer func() { if err := resp.Body.Close(); err != nil { - fmt.Println("error closing body", err) + log.Println("error closing body", err) } }() @@ -81,21 +111,51 @@ func doRequest(path string, out interface{}) error { } func liveTailQueryConn() (*websocket.Conn, error) { - path := fmt.Sprintf(tailPath, url.QueryEscape(*queryStr), url.QueryEscape(*regexpStr)) + path := fmt.Sprintf(tailPath, + url.QueryEscape(*queryStr), // query + url.QueryEscape(*regexpStr), // regexp + *delayFor, // delay_for + *limit, // limit + getStart(time.Now()).UnixNano(), // start + ) return wsConnect(path) } func wsConnect(path string) (*websocket.Conn, error) { - url := *addr + path - if strings.HasPrefix(url, "https") { - url = strings.Replace(url, "https", "wss", 1) - } else if strings.HasPrefix(url, "http") { - url = strings.Replace(url, "http", "ws", 1) + us := *addr + path + + // Parse the URL to extract the host + u, err := url.Parse(us) + if err != nil { + return nil, err + } + tlsConfig, err := config.NewTLSConfig(&config.TLSConfig{ + CAFile: *tlsCACertPath, + CertFile: *tlsClientCertPath, + KeyFile: *tlsClientCertKeyPath, + ServerName: u.Host, + InsecureSkipVerify: *tlsSkipVerify, + }) + if err != nil { + return nil, err + } + + if strings.HasPrefix(us, "https") { + us = strings.Replace(us, "https", "wss", 1) + } else if strings.HasPrefix(us, "http") { + us = strings.Replace(us, "http", "ws", 1) + } + if !*quiet { + log.Println(us) } - fmt.Println(url) h := http.Header{"Authorization": {"Basic " + base64.StdEncoding.EncodeToString([]byte(*username+":"+*password))}} - c, resp, err := websocket.DefaultDialer.Dial(url, h) + + ws := websocket.Dialer{ + TLSClientConfig: tlsConfig, + } + + c, resp, err := ws.Dial(us, h) if err != nil { if resp == nil { diff --git a/cmd/logcli/main.go b/cmd/logcli/main.go index 596bdda6f0f0e..38ce7791f7938 100644 --- a/cmd/logcli/main.go +++ b/cmd/logcli/main.go @@ -4,37 +4,53 @@ import ( "log" "os" - kingpin "gopkg.in/alecthomas/kingpin.v2" + "gopkg.in/alecthomas/kingpin.v2" ) var ( - app = kingpin.New("logcli", "A command-line for loki.") + app = kingpin.New("logcli", "A command-line for loki.") + quiet = app.Flag("quiet", "suppress everything but log lines").Default("false").Short('q').Bool() + outputMode = app.Flag("output", "specify output mode [default, raw, jsonl]").Default("default").Short('o').Enum("default", "raw", "jsonl") + + addr = app.Flag("addr", "Server address.").Default("https://logs-us-west1.grafana.net").Envar("GRAFANA_ADDR").String() - addr = app.Flag("addr", "Server address.").Default("https://logs-us-west1.grafana.net").Envar("GRAFANA_ADDR").String() username = app.Flag("username", "Username for HTTP basic auth.").Default("").Envar("GRAFANA_USERNAME").String() password = app.Flag("password", "Password for HTTP basic auth.").Default("").Envar("GRAFANA_PASSWORD").String() - queryCmd = app.Command("query", "Run a LogQL query.") - queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String() - regexpStr = queryCmd.Arg("regex", "").String() - limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int() - since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration() - forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool() - tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool() - noLabels = queryCmd.Flag("no-labels", "Do not print labels").Default("false").Bool() - - labelsCmd = app.Command("labels", "Find values for a given label.") - labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(listLabels).String() + tlsCACertPath = app.Flag("ca-cert", "Path to the server Certificate Authority.").Default("").Envar("LOKI_CA_CERT_PATH").String() + tlsSkipVerify = app.Flag("tls-skip-verify", "Server certificate TLS skip verify.").Default("false").Bool() + tlsClientCertPath = app.Flag("cert", "Path to the client certificate.").Default("").Envar("LOKI_CLIENT_CERT_PATH").String() + tlsClientCertKeyPath = app.Flag("key", "Path to the client certificate key.").Default("").Envar("LOKI_CLIENT_KEY_PATH").String() + + queryCmd = app.Command("query", "Run a LogQL query.") + queryStr = queryCmd.Arg("query", "eg '{foo=\"bar\",baz=\"blip\"}'").Required().String() + regexpStr = queryCmd.Arg("regex", "").String() + limit = queryCmd.Flag("limit", "Limit on number of entries to print.").Default("30").Int() + since = queryCmd.Flag("since", "Lookback window.").Default("1h").Duration() + from = queryCmd.Flag("from", "Start looking for logs at this absolute time (inclusive)").String() + to = queryCmd.Flag("to", "Stop looking for logs at this absolute time (exclusive)").String() + forward = queryCmd.Flag("forward", "Scan forwards through logs.").Default("false").Bool() + tail = queryCmd.Flag("tail", "Tail the logs").Short('t').Default("false").Bool() + delayFor = queryCmd.Flag("delay-for", "Delay in tailing by number of seconds to accumulate logs for re-ordering").Default("0").Int() + noLabels = queryCmd.Flag("no-labels", "Do not print any labels").Default("false").Bool() + ignoreLabelsKey = queryCmd.Flag("exclude-label", "Exclude labels given the provided key during output.").Strings() + showLabelsKey = queryCmd.Flag("include-label", "Include labels given the provided key during output.").Strings() + fixedLabelsLen = queryCmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").Int() + labelsCmd = app.Command("labels", "Find values for a given label.") + labelName = labelsCmd.Arg("label", "The name of the label.").HintAction(listLabels).String() ) func main() { log.SetOutput(os.Stderr) - switch kingpin.MustParse(app.Parse(os.Args[1:])) { + cmd := kingpin.MustParse(app.Parse(os.Args[1:])) + + if *addr == "" { + log.Fatalln("Server address cannot be empty") + } + + switch cmd { case queryCmd.FullCommand(): - if *addr == "" { - log.Fatalln("Server address cannot be empty") - } doQuery() case labelsCmd.FullCommand(): doLabels() diff --git a/cmd/logcli/output.go b/cmd/logcli/output.go new file mode 100644 index 0000000000000..d822775058f21 --- /dev/null +++ b/cmd/logcli/output.go @@ -0,0 +1,73 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "strings" + "time" + + "github.com/fatih/color" + "github.com/prometheus/prometheus/pkg/labels" +) + +// Outputs is an enum with all possible output modes +var Outputs = map[string]LogOutput{ + "default": &DefaultOutput{}, + "jsonl": &JSONLOutput{}, + "raw": &RawOutput{}, +} + +// LogOutput is the interface any output mode must implement +type LogOutput interface { + Print(ts time.Time, lbls *labels.Labels, line string) +} + +// DefaultOutput provides logs and metadata in human readable format +type DefaultOutput struct { + MaxLabelsLen int + CommonLabels labels.Labels +} + +// Print a log entry in a human readable format +func (f DefaultOutput) Print(ts time.Time, lbls *labels.Labels, line string) { + ls := subtract(*lbls, f.CommonLabels) + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } + + labels := "" + if !*noLabels { + labels = padLabel(ls, f.MaxLabelsLen) + } + fmt.Println( + color.BlueString(ts.Format(time.RFC3339)), + color.RedString(labels), + strings.TrimSpace(line), + ) +} + +// JSONLOutput prints logs and metadata as JSON Lines, suitable for scripts +type JSONLOutput struct{} + +// Print a log entry as json line +func (f JSONLOutput) Print(ts time.Time, lbls *labels.Labels, line string) { + entry := map[string]interface{}{ + "timestamp": ts, + "labels": lbls, + "line": line, + } + out, err := json.Marshal(entry) + if err != nil { + log.Fatalf("error marshalling entry: %s", err) + } + fmt.Println(string(out)) +} + +// RawOutput prints logs in their original form, without any metadata +type RawOutput struct{} + +// Print a log entry as is +func (f RawOutput) Print(ts time.Time, lbls *labels.Labels, line string) { + fmt.Println(line) +} diff --git a/cmd/logcli/query.go b/cmd/logcli/query.go index 4a7b47f0d0346..735527ea40c97 100644 --- a/cmd/logcli/query.go +++ b/cmd/logcli/query.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "log" "strings" "time" @@ -11,9 +10,20 @@ import ( "github.com/grafana/loki/pkg/iter" "github.com/grafana/loki/pkg/logproto" - "github.com/grafana/loki/pkg/parser" ) +func getStart(end time.Time) time.Time { + start := end.Add(-*since) + if *from != "" { + var err error + start, err = time.Parse(time.RFC3339Nano, *from) + if err != nil { + log.Fatalf("error parsing date '%s': %s", *from, err) + } + } + return start +} + func doQuery() { if *tail { tailQuery() @@ -21,13 +31,21 @@ func doQuery() { } var ( - i iter.EntryIterator - common labels.Labels - maxLabelsLen = 100 + i iter.EntryIterator + common labels.Labels ) end := time.Now() - start := end.Add(-*since) + start := getStart(end) + + if *to != "" { + var err error + end, err = time.Parse(time.RFC3339Nano, *to) + if err != nil { + log.Fatalf("error parsing --to date '%s': %s", *to, err) + } + } + d := logproto.BACKWARD if *forward { d = logproto.FORWARD @@ -43,126 +61,48 @@ func doQuery() { labelsCache := func(labels string) labels.Labels { return cache[labels] } + common = commonLabels(lss) - i = iter.NewQueryResponseIterator(resp, d) - if len(common) > 0 { - fmt.Println("Common labels:", color.RedString(common.String())) + // Remove the labels we want to show from common + if len(*showLabelsKey) > 0 { + common = common.MatchLabels(false, *showLabelsKey...) + } + + if len(common) > 0 && !*quiet { + log.Println("Common labels:", color.RedString(common.String())) + } + + if len(*ignoreLabelsKey) > 0 && !*quiet { + log.Println("Ignoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ","))) } + // Get the max size of labels + maxLabelsLen := *fixedLabelsLen for _, ls := range cache { ls = subtract(common, ls) + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } len := len(ls.String()) if maxLabelsLen < len { maxLabelsLen = len } } - for i.Next() { - ls := labelsCache(i.Labels()) - ls = subtract(ls, common) + i = iter.NewQueryResponseIterator(resp, d) - labels := "" - if !*noLabels { - labels = padLabel(ls, maxLabelsLen) - } + Outputs["default"] = DefaultOutput{ + MaxLabelsLen: maxLabelsLen, + CommonLabels: common, + } - printLogEntry(i.Entry().Timestamp, labels, i.Entry().Line) + for i.Next() { + ls := labelsCache(i.Labels()) + Outputs[*outputMode].Print(i.Entry().Timestamp, &ls, i.Entry().Line) } if err := i.Error(); err != nil { log.Fatalf("Error from iterator: %v", err) } } - -func printLogEntry(ts time.Time, lbls string, line string) { - fmt.Println( - color.BlueString(ts.Format(time.RFC3339)), - color.RedString(lbls), - strings.TrimSpace(line), - ) -} - -func padLabel(ls labels.Labels, maxLabelsLen int) string { - labels := ls.String() - if len(labels) < maxLabelsLen { - labels += strings.Repeat(" ", maxLabelsLen-len(labels)) - } - return labels -} - -func mustParseLabels(labels string) labels.Labels { - ls, err := parser.Labels(labels) - if err != nil { - log.Fatalf("Failed to parse labels: %+v", err) - } - return ls -} - -func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) { - cache := make(map[string]labels.Labels, len(resp.Streams)) - lss := make([]labels.Labels, 0, len(resp.Streams)) - for _, stream := range resp.Streams { - ls := mustParseLabels(stream.Labels) - cache[stream.Labels] = ls - lss = append(lss, ls) - } - return cache, lss -} - -func commonLabels(lss []labels.Labels) labels.Labels { - if len(lss) == 0 { - return nil - } - - result := lss[0] - for i := 1; i < len(lss); i++ { - result = intersect(result, lss[i]) - } - return result -} - -func intersect(a, b labels.Labels) labels.Labels { - var result labels.Labels - for i, j := 0, 0; i < len(a) && j < len(b); { - k := strings.Compare(a[i].Name, b[j].Name) - switch { - case k == 0: - if a[i].Value == b[j].Value { - result = append(result, a[i]) - } - i++ - j++ - case k < 0: - i++ - case k > 0: - j++ - } - } - return result -} - -// subtract b from a -func subtract(a, b labels.Labels) labels.Labels { - var result labels.Labels - i, j := 0, 0 - for i < len(a) && j < len(b) { - k := strings.Compare(a[i].Name, b[j].Name) - if k != 0 || a[i].Value != b[j].Value { - result = append(result, a[i]) - } - switch { - case k == 0: - i++ - j++ - case k < 0: - i++ - case k > 0: - j++ - } - } - for ; i < len(a); i++ { - result = append(result, a[i]) - } - return result -} diff --git a/cmd/logcli/query_test.go b/cmd/logcli/query_test.go new file mode 100644 index 0000000000000..a8a3db231df97 --- /dev/null +++ b/cmd/logcli/query_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "reflect" + "testing" + + "github.com/prometheus/prometheus/pkg/labels" +) + +func Test_commonLabels(t *testing.T) { + type args struct { + lss []labels.Labels + } + tests := []struct { + name string + args args + want labels.Labels + }{ + { + "Extract common labels source > target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`)}, + }, + mustParseLabels(`{bar="foo"}`), + }, + { + "Extract common labels source > target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo", foo="bar", baz="baz"}`)}, + }, + mustParseLabels(`{foo="bar", bar="foo"}`), + }, + { + "Extract common labels source < target", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{bar="foo"}`)}, + }, + mustParseLabels(`{bar="foo"}`), + }, + { + "Extract common labels source < target no common", + args{ + []labels.Labels{mustParseLabels(`{foo="bar", bar="foo"}`), mustParseLabels(`{fo="bar"}`)}, + }, + labels.Labels{}, + }, + { + "Extract common labels source = target no common", + args{ + []labels.Labels{mustParseLabels(`{foo="bar"}`), mustParseLabels(`{fooo="bar"}`)}, + }, + labels.Labels{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := commonLabels(tt.args.lss); !reflect.DeepEqual(got, tt.want) { + t.Errorf("commonLabels() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_subtract(t *testing.T) { + type args struct { + a labels.Labels + b labels.Labels + } + tests := []struct { + name string + args args + want labels.Labels + }{ + { + "Subtract labels source > target", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{bar="foo", foo="foo", baz="baz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source < target", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{bar="foo"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source < target no sub", + args{ + mustParseLabels(`{foo="bar", bar="foo"}`), + mustParseLabels(`{fo="bar"}`), + }, + mustParseLabels(`{bar="foo", foo="bar"}`), + }, + { + "Subtract labels source = target no sub", + args{ + mustParseLabels(`{foo="bar"}`), + mustParseLabels(`{fiz="buz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source > target no sub", + args{ + mustParseLabels(`{foo="bar"}`), + mustParseLabels(`{fiz="buz", foo="baz"}`), + }, + mustParseLabels(`{foo="bar"}`), + }, + { + "Subtract labels source > target no sub", + args{ + mustParseLabels(`{a="b", foo="bar", baz="baz", fizz="fizz"}`), + mustParseLabels(`{foo="bar", baz="baz", buzz="buzz", fizz="fizz"}`), + }, + mustParseLabels(`{a="b"}`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("subtract() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/logcli/tail.go b/cmd/logcli/tail.go index 51e64bd221e0e..94ee75b3432ec 100644 --- a/cmd/logcli/tail.go +++ b/cmd/logcli/tail.go @@ -2,8 +2,11 @@ package main import ( "log" + "strings" - "github.com/grafana/loki/pkg/logproto" + "github.com/grafana/loki/pkg/querier" + + "github.com/fatih/color" ) func tailQuery() { @@ -12,21 +15,58 @@ func tailQuery() { log.Fatalf("Tailing logs failed: %+v", err) } - stream := new(logproto.Stream) + tailReponse := new(querier.TailResponse) + + if len(*ignoreLabelsKey) > 0 { + log.Println("Ingoring labels key:", color.RedString(strings.Join(*ignoreLabelsKey, ","))) + } + + if len(*showLabelsKey) > 0 { + log.Println("Print only labels key:", color.RedString(strings.Join(*showLabelsKey, ","))) + } for { - err := conn.ReadJSON(stream) + err := conn.ReadJSON(tailReponse) if err != nil { log.Println("Error reading stream:", err) return } labels := "" - if !*noLabels { - labels = stream.Labels + for _, stream := range tailReponse.Streams { + if !*noLabels { + + if len(*ignoreLabelsKey) > 0 || len(*showLabelsKey) > 0 { + + ls := mustParseLabels(stream.GetLabels()) + + if len(*showLabelsKey) > 0 { + ls = ls.MatchLabels(true, *showLabelsKey...) + } + + if len(*ignoreLabelsKey) > 0 { + ls = ls.MatchLabels(false, *ignoreLabelsKey...) + } + + labels = ls.String() + + } else { + + labels = stream.Labels + } + } + + for _, entry := range stream.Entries { + lbls := mustParseLabels(labels) + Outputs[*outputMode].Print(entry.Timestamp, &lbls, entry.Line) + } + } - for _, entry := range stream.Entries { - printLogEntry(entry.Timestamp, labels, entry.Line) + if len(tailReponse.DroppedEntries) != 0 { + log.Println("Server dropped following entries due to slow client") + for _, d := range tailReponse.DroppedEntries { + log.Println(d.Timestamp, d.Labels) + } } } } diff --git a/cmd/logcli/utils.go b/cmd/logcli/utils.go new file mode 100644 index 0000000000000..6054f86ff6e70 --- /dev/null +++ b/cmd/logcli/utils.go @@ -0,0 +1,97 @@ +package main + +import ( + "log" + "sort" + "strings" + + "github.com/grafana/loki/pkg/logproto" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/promql" +) + +// add some padding after labels +func padLabel(ls labels.Labels, maxLabelsLen int) string { + labels := ls.String() + if len(labels) < maxLabelsLen { + labels += strings.Repeat(" ", maxLabelsLen-len(labels)) + } + return labels +} + +// parse labels from string +func mustParseLabels(labels string) labels.Labels { + ls, err := promql.ParseMetric(labels) + if err != nil { + log.Fatalf("Failed to parse labels: %+v", err) + } + return ls +} + +// parse labels from response stream +func parseLabels(resp *logproto.QueryResponse) (map[string]labels.Labels, []labels.Labels) { + cache := make(map[string]labels.Labels, len(resp.Streams)) + lss := make([]labels.Labels, 0, len(resp.Streams)) + for _, stream := range resp.Streams { + ls := mustParseLabels(stream.Labels) + cache[stream.Labels] = ls + lss = append(lss, ls) + } + return cache, lss +} + +// return commonLabels labels between given lavels set +func commonLabels(lss []labels.Labels) labels.Labels { + if len(lss) == 0 { + return nil + } + + result := lss[0] + for i := 1; i < len(lss); i++ { + result = intersect(result, lss[i]) + } + return result +} + +// intersect two labels set +func intersect(a, b labels.Labels) labels.Labels { + + set := labels.Labels{} + ma := a.Map() + mb := b.Map() + + for ka, va := range ma { + if vb, ok := mb[ka]; ok { + if vb == va { + set = append(set, labels.Label{ + Name: ka, + Value: va, + }) + } + } + } + sort.Sort(set) + return set +} + +// subtract labels set b from labels set a +func subtract(a, b labels.Labels) labels.Labels { + + set := labels.Labels{} + ma := a.Map() + mb := b.Map() + + for ka, va := range ma { + if vb, ok := mb[ka]; ok { + if vb == va { + continue + } + } + set = append(set, labels.Label{ + Name: ka, + Value: va, + }) + } + sort.Sort(set) + return set +} diff --git a/cmd/loki-canary/Dockerfile b/cmd/loki-canary/Dockerfile new file mode 100644 index 0000000000000..fbfb024a78061 --- /dev/null +++ b/cmd/loki-canary/Dockerfile @@ -0,0 +1,14 @@ +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . + +FROM grafana/loki-build-image:0.2.1 as build +ARG GOARCH="amd64" +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && make loki-canary + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki-canary/loki-canary /usr/bin/loki-canary +ENTRYPOINT [ "/usr/bin/loki-canary" ] diff --git a/cmd/loki-canary/main.go b/cmd/loki-canary/main.go new file mode 100644 index 0000000000000..a289a1631165e --- /dev/null +++ b/cmd/loki-canary/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/grafana/loki/pkg/canary/comparator" + "github.com/grafana/loki/pkg/canary/reader" + "github.com/grafana/loki/pkg/canary/writer" +) + +func main() { + + lName := flag.String("labelname", "name", "The label name for this instance of loki-canary to use in the log selector") + lVal := flag.String("labelvalue", "loki-canary", "The unique label value for this instance of loki-canary to use in the log selector") + port := flag.Int("port", 3500, "Port which loki-canary should expose metrics") + addr := flag.String("addr", "", "The Loki server URL:Port, e.g. loki:3100") + tls := flag.Bool("tls", false, "Does the loki connection use TLS?") + user := flag.String("user", "", "Loki username") + pass := flag.String("pass", "", "Loki password") + + interval := flag.Duration("interval", 1000*time.Millisecond, "Duration between log entries") + size := flag.Int("size", 100, "Size in bytes of each log line") + wait := flag.Duration("wait", 60*time.Second, "Duration to wait for log entries before reporting them lost") + pruneInterval := flag.Duration("pruneinterval", 60*time.Second, "Frequency to check sent vs received logs, also the frequency which queries for missing logs will be dispatched to loki") + buckets := flag.Int("buckets", 10, "Number of buckets in the response_latency histogram") + flag.Parse() + + if *addr == "" { + _, _ = fmt.Fprintf(os.Stderr, "Must specify a Loki address with -addr\n") + os.Exit(1) + } + + sentChan := make(chan time.Time) + receivedChan := make(chan time.Time) + + w := writer.NewWriter(os.Stdout, sentChan, *interval, *size) + r := reader.NewReader(os.Stderr, receivedChan, *tls, *addr, *user, *pass, *lName, *lVal) + c := comparator.NewComparator(os.Stderr, *wait, *pruneInterval, *buckets, sentChan, receivedChan, r) + + http.Handle("/metrics", promhttp.Handler()) + go func() { + err := http.ListenAndServe(":"+strconv.Itoa(*port), nil) + if err != nil { + panic(err) + } + }() + + interrupt := make(chan os.Signal, 1) + terminate := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + signal.Notify(terminate, syscall.SIGTERM) + + for { + select { + case <-interrupt: + _, _ = fmt.Fprintf(os.Stderr, "suspending indefinetely\n") + w.Stop() + r.Stop() + c.Stop() + case <-terminate: + _, _ = fmt.Fprintf(os.Stderr, "shutting down\n") + w.Stop() + r.Stop() + c.Stop() + return + } + } + +} diff --git a/cmd/loki/Dockerfile b/cmd/loki/Dockerfile index feb074da4a561..81111e192f0d4 100644 --- a/cmd/loki/Dockerfile +++ b/cmd/loki/Dockerfile @@ -1,6 +1,17 @@ +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/loki -f cmd/loki/Dockerfile . + +FROM grafana/loki-build-image:0.2.1 as build +ARG GOARCH="amd64" +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && make loki + FROM alpine:3.9 RUN apk add --update --no-cache ca-certificates -COPY loki /bin/loki -COPY loki-local-config.yaml /etc/loki/local-config.yaml +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki/loki /usr/bin/loki +COPY cmd/loki/loki-local-config.yaml /etc/loki/local-config.yaml EXPOSE 80 -ENTRYPOINT [ "/bin/loki" ] +ENTRYPOINT [ "/usr/bin/loki" ] +CMD ["-config.file=/etc/loki/local-config.yaml"] diff --git a/cmd/loki/Dockerfile.debug b/cmd/loki/Dockerfile.debug new file mode 100644 index 0000000000000..58ff97d27b898 --- /dev/null +++ b/cmd/loki/Dockerfile.debug @@ -0,0 +1,28 @@ +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/loki -f cmd/loki/Dockerfile.debug . + +FROM grafana/loki-build-image as build +ARG GOARCH="amd64" +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && make loki-debug + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates +COPY --from=build /go/src/github.com/grafana/loki/cmd/loki/loki-debug /usr/bin/loki-debug +COPY --from=build /go/bin/dlv /usr/bin/dlv +COPY cmd/loki/loki-local-config.yaml /etc/loki/local-config.yaml +EXPOSE 80 + +# Expose 40000 for delve +EXPOSE 40000 + +# Allow delve to run on Alpine based containers. +RUN apk add --no-cache libc6-compat + +# Run delve, ending with -- because we pass params via kubernetes, per the docs: +# Pass flags to the program you are debugging using --, for example:` +# dlv exec ./hello -- server --config conf/config.toml` +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/loki-debug", "--"] +CMD ["-config.file=/etc/loki/local-config.yaml"] diff --git a/cmd/loki/loki-local-config.yaml b/cmd/loki/loki-local-config.yaml index 24d7a8344a528..3320a99ebdde2 100644 --- a/cmd/loki/loki-local-config.yaml +++ b/cmd/loki/loki-local-config.yaml @@ -7,13 +7,16 @@ ingester: lifecycler: address: 127.0.0.1 ring: - store: inmemory + kvstore: + store: inmemory replication_factor: 1 - chunk_idle_period: 15m + final_sleep: 0s + chunk_idle_period: 5m + chunk_retain_period: 30s schema_config: configs: - - from: 0 + - from: 2018-04-15 store: boltdb object_store: filesystem schema: v9 @@ -30,3 +33,23 @@ storage_config: limits_config: enforce_metric_name: false + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0 + +table_manager: + chunk_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + index_tables_provisioning: + inactive_read_throughput: 0 + inactive_write_throughput: 0 + provisioned_read_throughput: 0 + provisioned_write_throughput: 0 + retention_deletes_enabled: false + retention_period: 0 + diff --git a/cmd/loki/main.go b/cmd/loki/main.go index 073e8a0bd88ed..60676cf3603ac 100644 --- a/cmd/loki/main.go +++ b/cmd/loki/main.go @@ -4,16 +4,19 @@ import ( "flag" "fmt" "os" + "reflect" "github.com/go-kit/kit/log/level" "github.com/grafana/loki/pkg/helpers" "github.com/grafana/loki/pkg/loki" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" + "github.com/weaveworks/common/logging" "github.com/weaveworks/common/tracing" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" + "github.com/cortexproject/cortex/pkg/util/validation" ) func init() { @@ -29,6 +32,15 @@ func main() { flagext.RegisterFlags(&cfg) flag.Parse() + // LimitsConfig has a customer UnmarshalYAML that will set the defaults to a global. + // This global is set to the config passed into the last call to `NewOverrides`. If we don't + // call it atleast once, the defaults are set to an empty struct. + // We call it with the flag values so that the config file unmarshalling only overrides the values set in the config. + if _, err := validation.NewOverrides(cfg.LimitsConfig); err != nil { + level.Error(util.Logger).Log("msg", "error loading limits", "err", err) + os.Exit(1) + } + util.InitLogger(&cfg.Server) if configFile != "" { @@ -38,6 +50,13 @@ func main() { } } + // Re-init the logger which will now honor a different log level set in cfg.Server + if reflect.DeepEqual(&cfg.Server.LogLevel, &logging.Level{}) { + level.Error(util.Logger).Log("msg", "invalid log level") + os.Exit(1) + } + util.InitLogger(&cfg.Server) + // Setting the environment variable JAEGER_AGENT_HOST enables tracing trace := tracing.NewFromEnv(fmt.Sprintf("loki-%s", cfg.Target)) defer func() { diff --git a/cmd/promtail/Dockerfile b/cmd/promtail/Dockerfile index 136672534b742..58ba144f46210 100644 --- a/cmd/promtail/Dockerfile +++ b/cmd/promtail/Dockerfile @@ -1,6 +1,16 @@ +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile . + +FROM grafana/loki-build-image:0.2.1 as build +ARG GOARCH="amd64" +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && make promtail + FROM alpine:3.9 -RUN apk add --update --no-cache ca-certificates -ADD promtail /usr/bin -COPY promtail-local-config.yaml /etc/promtail/local-config.yaml -COPY promtail-docker-config.yaml /etc/promtail/docker-config.yaml +RUN apk add --update --no-cache ca-certificates tzdata +COPY --from=build /go/src/github.com/grafana/loki/cmd/promtail/promtail /usr/bin/promtail +COPY cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml +COPY cmd/promtail/promtail-docker-config.yaml /etc/promtail/docker-config.yaml ENTRYPOINT ["/usr/bin/promtail"] diff --git a/cmd/promtail/Dockerfile.debug b/cmd/promtail/Dockerfile.debug new file mode 100644 index 0000000000000..fc4d3fe35d778 --- /dev/null +++ b/cmd/promtail/Dockerfile.debug @@ -0,0 +1,28 @@ +# Directories in this file are referenced from the root of the project not this folder +# This file is intented to be called from the root like so: +# docker build -t grafana/promtail -f cmd/promtail/Dockerfile.debug . + +FROM grafana/loki-build-image as build +ARG GOARCH="amd64" +COPY . /go/src/github.com/grafana/loki +WORKDIR /go/src/github.com/grafana/loki +RUN make clean && make promtail-debug + + +FROM alpine:3.9 +RUN apk add --update --no-cache ca-certificates tzdata +COPY --from=build /go/src/github.com/grafana/loki/cmd/promtail/promtail-debug /usr/bin/promtail-debug +COPY --from=build /go/bin/dlv /usr/bin/dlv +COPY cmd/promtail/promtail-local-config.yaml /etc/promtail/local-config.yaml +COPY cmd/promtail/promtail-docker-config.yaml /etc/promtail/docker-config.yaml + +# Expose 40000 for delve +EXPOSE 40000 + +# Allow delve to run on Alpine based containers. +RUN apk add --no-cache libc6-compat + +# Run delve, ending with -- because we pass params via kubernetes, per the docs: +# Pass flags to the program you are debugging using --, for example:` +# dlv exec ./hello -- server --config conf/config.toml` +ENTRYPOINT ["/usr/bin/dlv", "--listen=:40000", "--headless=true", "--api-version=2", "exec", "/usr/bin/promtail-debug", "--"] diff --git a/cmd/promtail/main.go b/cmd/promtail/main.go index 77ece24615fef..fac17045e319d 100644 --- a/cmd/promtail/main.go +++ b/cmd/promtail/main.go @@ -3,12 +3,14 @@ package main import ( "flag" "os" + "reflect" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/version" + "github.com/weaveworks/common/logging" "github.com/grafana/loki/pkg/helpers" "github.com/grafana/loki/pkg/promtail" @@ -28,7 +30,7 @@ func main() { flagext.RegisterFlags(&config) flag.Parse() - util.InitLogger(&config.ServerConfig) + util.InitLogger(&config.ServerConfig.Config) if configFile != "" { if err := helpers.LoadConfig(configFile, &config); err != nil { @@ -37,6 +39,13 @@ func main() { } } + // Re-init the logger which will now honor a different log level set in ServerConfig.Config + if reflect.DeepEqual(&config.ServerConfig.Config.LogLevel, &logging.Level{}) { + level.Error(util.Logger).Log("msg", "invalid log level") + os.Exit(1) + } + util.InitLogger(&config.ServerConfig.Config) + p, err := promtail.New(config) if err != nil { level.Error(util.Logger).Log("msg", "error creating promtail", "error", err) diff --git a/cmd/promtail/promtail-docker-config.yaml b/cmd/promtail/promtail-docker-config.yaml index 8dc8f27307c72..9dca263256802 100644 --- a/cmd/promtail/promtail-docker-config.yaml +++ b/cmd/promtail/promtail-docker-config.yaml @@ -5,12 +5,11 @@ server: positions: filename: /tmp/positions.yaml -client: - url: http://loki:3100/api/prom/push +clients: + - url: http://loki:3100/api/prom/push scrape_configs: - job_name: system - entry_parser: raw static_configs: - targets: - localhost diff --git a/cmd/promtail/promtail-local-config.yaml b/cmd/promtail/promtail-local-config.yaml index d5b13c5e0c23a..9744741c57381 100644 --- a/cmd/promtail/promtail-local-config.yaml +++ b/cmd/promtail/promtail-local-config.yaml @@ -5,12 +5,11 @@ server: positions: filename: /tmp/positions.yaml -client: - url: http://localhost:3100/api/prom/push +clients: + - url: http://localhost:3100/api/prom/push scrape_configs: - job_name: system - entry_parser: raw static_configs: - targets: - localhost diff --git a/debug/README.md b/debug/README.md new file mode 100644 index 0000000000000..b4e571325a2ab --- /dev/null +++ b/debug/README.md @@ -0,0 +1,57 @@ +## Debug images + +To build debug images run + +```shell +make debug +``` + +You can use the `docker-compose.yaml` in this directory to launch the debug versions of the image in docker + + +## Promtail in kubernetes + +If you want to debug promtail in kubernetes, I have done so with the ksonnet setup: + +```shell +ks init promtail +cd promtail +ks env add promtail +jb init +jb install github.com/grafana/loki/production/ksonnet/promtail +vi environments/promtail/main.jsonnet +``` + +Replace the contents with: + +```jsonnet +local promtail = import 'promtail/promtail.libsonnet'; + + +promtail + { + _images+:: { + promtail: 'grafana/promtail-debug:latest', + }, + _config+:: { + namespace: 'default', + + promtail_config+: { + external_labels+: { + cluster: 'some_cluster_name', + }, + scheme: 'https', + hostname: 'hostname', + username: 'username', + password: 'password', + }, + }, +} +``` + +change the `some_cluster_name` to anything meaningful to help find your logs in loki + +also update the `hostname`, `username`, and `password` for your loki instance. + +## Loki in kubernetes + +Haven't tried this yet, it works from docker-compose so it should run in kubernetes just fine also. \ No newline at end of file diff --git a/debug/docker-compose.yaml b/debug/docker-compose.yaml new file mode 100644 index 0000000000000..88c0811ac1256 --- /dev/null +++ b/debug/docker-compose.yaml @@ -0,0 +1,37 @@ +version: "3" + +networks: + loki: + +services: + loki: + # this is required according to https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code#linuxdocker + security_opt: + - seccomp:unconfined + image: grafana/loki-debug:latest + ports: + - "40000:40000" + - "3100:3100" + command: -config.file=/etc/loki/local-config.yaml + networks: + - loki + + promtail: + # this is required according to https://github.com/Microsoft/vscode-go/wiki/Debugging-Go-code-using-VS-Code#linuxdocker + security_opt: + - seccomp:unconfined + image: grafana/promtail-debug:latest + ports: + - "40100:40000" + volumes: + - /var/log:/var/log + command: -config.file=/etc/promtail/docker-config.yaml + networks: + - loki + + grafana: + image: grafana/grafana:master + ports: + - "3000:3000" + networks: + - loki diff --git a/docs/api.md b/docs/api.md index 9980878ffec59..2432f307dfe09 100644 --- a/docs/api.md +++ b/docs/api.md @@ -26,12 +26,12 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o For doing queries, accepts the following parameters in the query-string: - - `query`: a logQL query + - `query`: a [logQL query](./usage.md) (eg: `{name=~"mysql.+"}` or `{name=~"mysql.+"} |= "error"`) - `limit`: max number of entries to return - - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970). Default is always one hour ago. - - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970). Default is current time. + - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970) or as RFC3339Nano (eg: "2006-01-02T15:04:05.999999999-07:00"). Default is always one hour ago. + - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970) or as RFC3339Nano (eg: "2006-01-02T15:04:05.999999999-07:00"). Default is current time. - `direction`: `forward` or `backward`, useful when specifying a limit. Default is backward. - - `regexp`: a regex to filter the returned results, will eventually be rolled into the query language + - `regexp`: a regex to filter the returned results Loki needs to query the index store in order to find log streams for particular labels and the store is spread out by time, so you need to specify the start and end labels accordingly. Querying a long time into the history will cause additional @@ -39,14 +39,14 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o Responses looks like this: - ``` + ```json { "streams": [ { "labels": "{instance=\"...\", job=\"...\", namespace=\"...\"}", "entries": [ { - "timestamp": "2018-06-27T05:20:28.699492635Z", + "ts": "2018-06-27T05:20:28.699492635Z", "line": "..." }, ... @@ -59,11 +59,14 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o - `GET /api/prom/label` - For retrieving the names of the labels one can query on. + For doing label name queries, accepts the following parameters in the query-string: + + - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970). Default is always 6 hour ago. + - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970). Default is current time. Responses looks like this: - ``` + ```json { "values": [ "instance", @@ -74,11 +77,15 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o ``` - `GET /api/prom/label//values` - For retrieving the label values one can query on. + + For doing label values queries, accepts the following parameters in the query-string: + + - `start`: the start time for the query, as a nanosecond Unix epoch (nanoseconds since 1970). Default is always 6 hour ago. + - `end`: the end time for the query, as a nanosecond Unix epoch (nanoseconds since 1970). Default is current time. Responses looks like this: - ``` + ```json { "values": [ "default", @@ -88,6 +95,20 @@ The Loki server has the following API endpoints (_Note:_ Authentication is out o } ``` -## Example of using the API in a third-party client library +- `GET /ready` + + This endpoint returns 200 when Loki ingester is ready to accept traffic. If you're running Loki on Kubernetes, this endpoint can be used as readiness probe. + +- `GET /flush` + + This endpoint triggers a flush of all in memory chunks in the ingester. Mainly used for local testing. + +- `GET /metrics` + + This endpoint returns Loki metrics for Prometheus. See "[Operations > Observability > Metrics](./operations.md)" to have a list of exported metrics. + + +## Examples of using the API in a third-party client library -Take a look at this [client](https://github.com/afiskon/promtail-client), but be aware that the API is not stable yet. +1) Take a look at this [client](https://github.com/afiskon/promtail-client), but be aware that the API is not stable yet (Golang). +2) Example on [Python3](https://github.com/sleleko/devops-kb/blob/master/python/push-to-loki.py) diff --git a/docs/canary/README.md b/docs/canary/README.md new file mode 100644 index 0000000000000..45144399b6c8c --- /dev/null +++ b/docs/canary/README.md @@ -0,0 +1,108 @@ + +# loki-canary + +A standalone app to audit the log capturing performance of Loki. + +## how it works + +![block_diagram](block.png) + +loki-canary writes a log to a file and stores the timestamp in an internal array, the contents look something like this: + +```nohighlight +1557935669096040040 ppppppppppppppppppppppppppppppppppppppppppppppppppppppppppp +``` + +The relevant part is the timestamp, the `p`'s are just filler bytes to make the size of the log configurable. + +Promtail (or another agent) then reads the log file and ships it to Loki. + +Meanwhile loki-canary opens a websocket connection to loki and listens for logs it creates + +When a log is received on the websocket, the timestamp in the log message is compared to the internal array. + +If the received log is: + + * The next in the array to be received, it is removed from the array and the (current time - log timestamp) is recorded in the `response_latency` histogram, this is the expected behavior for well behaving logs + * Not the next in the array received, is is removed from the array, the response time is recorded in the `response_latency` histogram, and the `out_of_order_entries` counter is incremented + * Not in the array at all, it is checked against a separate list of received logs to either increment the `duplicate_entries` counter or the `unexpected_entries` counter. + +In the background, loki-canary also runs a timer which iterates through all the entries in the internal array, if any are older than the duration specified by the `-wait` flag (default 60s), they are removed from the array and the `websocket_missing_entries` counter is incremented. Then an additional query is made directly to loki for these missing entries to determine if they were actually missing or just didn't make it down the websocket. If they are not found in the followup query the `missing_entries` counter is incremented. + +## building and running + +`make` will run tests and build a docker image + +`make build` will create a binary `loki-canary` alongside the makefile + +To run the image, you can do something simple like: + +`kubectl run loki-canary --generator=run-pod/v1 --image=grafana/loki-canary:latest --restart=Never --image-pull-policy=Never --labels=name=loki-canary -- -addr=loki:3100` + +Or you can do something more complex like deploy it as a daemonset, there is a ksonnet setup for this in the `production` folder, you can import it using jsonnet-bundler: + +```shell +jb install github.com/grafana/loki-canary/production/ksonnet/loki-canary +``` + +Then in your ksonnet environments `main.jsonnet` you'll want something like this: + +```nohighlight +local loki_canary = import 'loki-canary/loki-canary.libsonnet'; + +loki_canary { + loki_canary_args+:: { + addr: "loki:3100", + port: 80, + labelname: "instance", + interval: "100ms", + size: 1024, + wait: "3m", + }, + _config+:: { + namespace: "default", + } +} + +``` + +## config + +It is required to pass in the Loki address with the `-addr` flag, if your server uses TLS, also pass `-tls=true` (this will create a wss:// instead of ws:// connection) + +You should also pass the `-labelname` and `-labelvalue` flags, these are used by loki-canary to filter the log stream to only process logs for this instance of loki-canary, so they must be unique per each of your loki-canary instances. The ksonnet config in this project accomplishes this by passing in the pod name as the labelvalue + +If you get a high number of `unexpected_entries` you may not be waiting long enough and should increase `-wait` from 60s to something larger. + +__Be cognizant__ of the relationship between `pruneinterval` and the `interval`. For example, with an interval of 10ms (100 logs per second) and a prune interval of 60s, you will write 6000 logs per minute, if those logs were not received over the websocket, the canary will attempt to query loki directly to see if they are completely lost. __However__ the query return is limited to 1000 results so you will not be able to return all the logs even if they did make it to Loki. + +__Likewise__, if you lower the `pruneinterval` you risk causing a denial of service attack as all your canaries attempt to query for missing logs at whatever your `pruneinterval` is defined at. + +All options: + +```nohighlight + -addr string + The Loki server URL:Port, e.g. loki:3100 + -buckets int + Number of buckets in the response_latency histogram (default 10) + -interval duration + Duration between log entries (default 1s) + -labelname string + The label name for this instance of loki-canary to use in the log selector (default "name") + -labelvalue string + The unique label value for this instance of loki-canary to use in the log selector (default "loki-canary") + -pass string + Loki password + -port int + Port which loki-canary should expose metrics (default 3500) + -pruneinterval duration + Frequency to check sent vs received logs, also the frequency which queries for missing logs will be dispatched to loki (default 1m0s) + -size int + Size in bytes of each log line (default 100) + -tls + Does the loki connection use TLS? + -user string + Loki username + -wait duration + Duration to wait for log entries before reporting them lost (default 1m0s) +``` diff --git a/docs/canary/block.png b/docs/canary/block.png new file mode 100644 index 0000000000000..f7dd39047bed7 Binary files /dev/null and b/docs/canary/block.png differ diff --git a/docs/logcli.md b/docs/logcli.md index d7330a2826bbb..116d3475ee1fe 100644 --- a/docs/logcli.md +++ b/docs/logcli.md @@ -16,7 +16,7 @@ $ go get github.com/grafana/loki/cmd/logcli ``` $ go get github.com/grafana/loki -$ cd $GOPATH/github.com/grafana/loki +$ cd $GOPATH/src/github.com/grafana/loki $ go build ./cmd/logcli ``` @@ -44,8 +44,8 @@ Common labels: {job="cortex-ops/consul", namespace="cortex-ops"} ### Configuration - Configuration values are considered in the following order (lowest to highest): + - environment value - command line @@ -53,17 +53,24 @@ The URLs of the requests are printed to help with integration work. ### Details -``` +```console $ logcli help usage: logcli [] [ ...] A command-line for loki. Flags: - --help Show context-sensitive help (also try --help-long and --help-man). - --addr="" Server address, need to specify. - --username="" Username for HTTP basic auth. - --password="" Password for HTTP basic auth. + --help Show context-sensitive help (also try --help-long and --help-man). + -q, --quiet suppress everything but log lines + -o, --output=default specify output mode [default, raw, jsonl] + --addr="https://logs-us-west1.grafana.net" + Server address. + --username="" Username for HTTP basic auth. + --password="" Password for HTTP basic auth. + --ca-cert="" Path to the server Certificate Authority. + --tls-skip-verify Server certificate TLS skip verify. + --cert="" Path to the client certificate. + --key="" Path to the client certificate key. Commands: help [...] @@ -72,7 +79,7 @@ Commands: query [] [] Run a LogQL query. - labels