From 2618a368abd60724981839d0ee6dd9627c7843c0 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:54:42 -0800 Subject: [PATCH] remove cruft and update workflows --- .github/ISSUE_TEMPLATE/config.yml | 5 - .../ISSUE_TEMPLATE/documentation_request.yml | 30 - .github/ISSUE_TEMPLATE/enhancement.yml | 53 - .github/ISSUE_TEMPLATE/question.yml | 16 - .../ci-aqua-security-trivy-tests.yml | 21 +- .github/workflows/ci-dgraph-code-coverage.yml | 21 +- .github/workflows/ci-dgraph-fuzz.yml | 3 + .../ci-dgraph-integration2-tests.yml | 3 + .github/workflows/ci-dgraph-ldbc-tests.yml | 21 +- .github/workflows/ci-dgraph-load-tests.yml | 21 +- .github/workflows/ci-dgraph-oss-build.yml | 21 +- .github/workflows/ci-dgraph-tests-arm64.yml | 33 +- .github/workflows/ci-dgraph-tests.yml | 33 +- .github/workflows/ci-dgraph-upgrade-tests.yml | 3 + .github/workflows/ci-golang-lint.yml | 24 +- CHANGELOG_CLOUD.md | 683 ------- contrib/config/datadog/docker-compose.yml | 67 - contrib/config/docker/docker-compose-ha.yml | 159 -- .../config/docker/docker-compose-multi.yml | 83 - contrib/config/docker/docker-compose.yml | 31 - contrib/config/kubernetes/dgraph-ha/README.md | 64 - .../config/kubernetes/dgraph-ha/cluster.yaml | 19 - .../kubernetes/dgraph-ha/dgraph-ha.yaml | 360 ---- .../kubernetes/dgraph-ha/kustomization.yaml | 2 - .../dgraph-single/dgraph-single.yaml | 108 -- .../dgraph-single/kustomization.yaml | 2 - contrib/config/marketplace/aws/README.md | 65 - .../aws/build_ubuntu_ami_mappings.rb | 104 - contrib/config/marketplace/aws/deploy.sh | 29 - contrib/config/marketplace/aws/dgraph.json | 1725 ----------------- .../config/marketplace/aws/tests/.gitignore | 6 - .../marketplace/aws/tests/.python-version | 1 - .../config/marketplace/aws/tests/.taskcat.yml | 24 - .../config/marketplace/aws/tests/README.md | 37 - .../marketplace/aws/tests/requirements.txt | 41 - .../marketplace/aws/tests/seed_keypairs.sh | 73 - contrib/gh-runner/README.md | 17 - contrib/gh-runner/gh-runner.sh | 62 - 38 files changed, 46 insertions(+), 4024 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/config.yml delete mode 100644 .github/ISSUE_TEMPLATE/documentation_request.yml delete mode 100644 .github/ISSUE_TEMPLATE/enhancement.yml delete mode 100644 .github/ISSUE_TEMPLATE/question.yml delete mode 100644 CHANGELOG_CLOUD.md delete mode 100644 contrib/config/datadog/docker-compose.yml delete mode 100644 contrib/config/docker/docker-compose-ha.yml delete mode 100644 contrib/config/docker/docker-compose-multi.yml delete mode 100644 contrib/config/docker/docker-compose.yml delete mode 100644 contrib/config/kubernetes/dgraph-ha/README.md delete mode 100644 contrib/config/kubernetes/dgraph-ha/cluster.yaml delete mode 100644 contrib/config/kubernetes/dgraph-ha/dgraph-ha.yaml delete mode 100644 contrib/config/kubernetes/dgraph-ha/kustomization.yaml delete mode 100644 contrib/config/kubernetes/dgraph-single/dgraph-single.yaml delete mode 100644 contrib/config/kubernetes/dgraph-single/kustomization.yaml delete mode 100644 contrib/config/marketplace/aws/README.md delete mode 100755 contrib/config/marketplace/aws/build_ubuntu_ami_mappings.rb delete mode 100755 contrib/config/marketplace/aws/deploy.sh delete mode 100644 contrib/config/marketplace/aws/dgraph.json delete mode 100644 contrib/config/marketplace/aws/tests/.gitignore delete mode 100644 contrib/config/marketplace/aws/tests/.python-version delete mode 100644 contrib/config/marketplace/aws/tests/.taskcat.yml delete mode 100644 contrib/config/marketplace/aws/tests/README.md delete mode 100644 contrib/config/marketplace/aws/tests/requirements.txt delete mode 100755 contrib/config/marketplace/aws/tests/seed_keypairs.sh delete mode 100644 contrib/gh-runner/README.md delete mode 100644 contrib/gh-runner/gh-runner.sh diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 5a2fd507fde..00000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,5 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Dgraph Community Support. - url: https://discuss.dgraph.io/ - about: Please ask and answer questions here. diff --git a/.github/ISSUE_TEMPLATE/documentation_request.yml b/.github/ISSUE_TEMPLATE/documentation_request.yml deleted file mode 100644 index 2d26d8502d3..00000000000 --- a/.github/ISSUE_TEMPLATE/documentation_request.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: "📃 Documentation Request" -description: Suggest improvements, additions, or revisions to Dgraph documentation -title: "[Documentation]: " -labels: ["area/documentation", "status/triage"] -body: - - type: markdown - attributes: - value: | - Thank you for reporting an issue. - If you think Dgraph's documentation at https://dgraph.io/docs/ is lacking, please explain it here. - - type: textarea - attributes: - label: What version of Dgraph is the target? - description: Is it latest? old? - placeholder: | - Type here. - validations: - required: false - - type: textarea - attributes: - label: Documentation. - description: Explain which part of the documents is lacking. - placeholder: | - Type here. - validations: - required: true - - type: textarea - attributes: - label: Additional information. - description: Tell us anything else you think we should know. diff --git a/.github/ISSUE_TEMPLATE/enhancement.yml b/.github/ISSUE_TEMPLATE/enhancement.yml deleted file mode 100644 index 10544f588c6..00000000000 --- a/.github/ISSUE_TEMPLATE/enhancement.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: "âš¡ Enhancement Request" -description: Something could be better. -title: "[ENHANCEMENT]: <Title>" -labels: ["kind/enhancement","status/triage"] -body: - - type: markdown - attributes: - value: | - Something could be better? - If your request is about a net new feature please use the Feature Request template. - Enhancements are tagged `kind/enhancement`. - - type: textarea - attributes: - label: Use case and current behavior - description: The context in which the feature is used and what is achieved. - placeholder: | - Type here. - validations: - required: true - - type: textarea - attributes: - label: Enhancement - description: Which enhancement is required and what is the new output? - validations: - required: true - - type: textarea - attributes: - label: Solution proposal - description: Any idea on the how? - validations: - required: false - - type: textarea - attributes: - label: Links to Discuss, RFC or previous Issues and PRs - description: Reference of any prior work, post on Discuss or other. - placeholder: | - Type here. - validations: - required: false - - type: textarea - attributes: - label: Links to examples and research - description: Is there a software that has this feature? Any relevant standards or conventions? - placeholder: | - Type here. - validations: - required: false - - type: textarea - attributes: - label: Additional Information - description: Any useful additional info? - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml deleted file mode 100644 index 13dad91b2fa..00000000000 --- a/.github/ISSUE_TEMPLATE/question.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: "🙋 Question" -description: Question related to Dgraph -title: "[QUESTION]: <Title>" -labels: ["kind/question ", "status/triage"] -body: - - type: markdown - attributes: - value: | - The issue tracker is not for questions. - - If you have a question, please try asking it on https://discuss.dgraph.io - - Our Docs: https://dgraph.io/docs/ - - If it is code related, you could open it here - - type: textarea - attributes: - label: Question. - description: Ask your Question. diff --git a/.github/workflows/ci-aqua-security-trivy-tests.yml b/.github/workflows/ci-aqua-security-trivy-tests.yml index 5aee2227c7f..f17abe6337e 100644 --- a/.github/workflows/ci-aqua-security-trivy-tests.yml +++ b/.github/workflows/ci-aqua-security-trivy-tests.yml @@ -1,24 +1,9 @@ name: ci-aqua-security-trivy-tests on: pull_request: - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened diff --git a/.github/workflows/ci-dgraph-code-coverage.yml b/.github/workflows/ci-dgraph-code-coverage.yml index f9795b65eaf..e2300fffbe9 100644 --- a/.github/workflows/ci-dgraph-code-coverage.yml +++ b/.github/workflows/ci-dgraph-code-coverage.yml @@ -1,24 +1,9 @@ name: ci-dgraph-code-coverage on: push: - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' + paths: + - '**/*.go' + - '**/go.mod' branches: - main - 'release/**' diff --git a/.github/workflows/ci-dgraph-fuzz.yml b/.github/workflows/ci-dgraph-fuzz.yml index a02b83dd712..b6afd0ddab3 100644 --- a/.github/workflows/ci-dgraph-fuzz.yml +++ b/.github/workflows/ci-dgraph-fuzz.yml @@ -1,6 +1,9 @@ name: ci-dgraph-fuzz on: pull_request: + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened diff --git a/.github/workflows/ci-dgraph-integration2-tests.yml b/.github/workflows/ci-dgraph-integration2-tests.yml index 9af8f364eb5..13c01ee6e54 100644 --- a/.github/workflows/ci-dgraph-integration2-tests.yml +++ b/.github/workflows/ci-dgraph-integration2-tests.yml @@ -1,6 +1,9 @@ name: ci-dgraph-integration2-tests on: pull_request: + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened diff --git a/.github/workflows/ci-dgraph-ldbc-tests.yml b/.github/workflows/ci-dgraph-ldbc-tests.yml index fda1a17a80b..3e0d822e3ae 100644 --- a/.github/workflows/ci-dgraph-ldbc-tests.yml +++ b/.github/workflows/ci-dgraph-ldbc-tests.yml @@ -1,24 +1,9 @@ name: ci-dgraph-ldbc-tests on: pull_request: - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened diff --git a/.github/workflows/ci-dgraph-load-tests.yml b/.github/workflows/ci-dgraph-load-tests.yml index 7eda3385a64..4ab729f01f1 100644 --- a/.github/workflows/ci-dgraph-load-tests.yml +++ b/.github/workflows/ci-dgraph-load-tests.yml @@ -1,29 +1,14 @@ name: ci-dgraph-load-tests on: pull_request: + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened - synchronize - ready_for_review - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' branches: - main - 'release/**' diff --git a/.github/workflows/ci-dgraph-oss-build.yml b/.github/workflows/ci-dgraph-oss-build.yml index 43ea8044c76..5dc4ac21e9d 100644 --- a/.github/workflows/ci-dgraph-oss-build.yml +++ b/.github/workflows/ci-dgraph-oss-build.yml @@ -1,24 +1,9 @@ name: ci-dgraph-oss-build on: pull_request: - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened diff --git a/.github/workflows/ci-dgraph-tests-arm64.yml b/.github/workflows/ci-dgraph-tests-arm64.yml index 0a36bb09514..c389b323747 100644 --- a/.github/workflows/ci-dgraph-tests-arm64.yml +++ b/.github/workflows/ci-dgraph-tests-arm64.yml @@ -1,32 +1,17 @@ name: ci-dgraph-tests-arm64 on: pull_request: - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' + paths: + - '**/*.go' + - '**/go.mod' types: - - opened - - reopened - - synchronize - - ready_for_review + - opened + - reopened + - synchronize + - ready_for_review branches: - - main - - 'release/**' + - main + - 'release/**' jobs: dgraph-tests: if: github.event.pull_request.draft == false diff --git a/.github/workflows/ci-dgraph-tests.yml b/.github/workflows/ci-dgraph-tests.yml index d171fa2a471..139cd309357 100644 --- a/.github/workflows/ci-dgraph-tests.yml +++ b/.github/workflows/ci-dgraph-tests.yml @@ -1,32 +1,17 @@ name: ci-dgraph-tests on: pull_request: - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' + paths: + - '**/*.go' + - '**/go.mod' types: - - opened - - reopened - - synchronize - - ready_for_review + - opened + - reopened + - synchronize + - ready_for_review branches: - - main - - 'release/**' + - main + - 'release/**' jobs: dgraph-tests: if: github.event.pull_request.draft == false diff --git a/.github/workflows/ci-dgraph-upgrade-tests.yml b/.github/workflows/ci-dgraph-upgrade-tests.yml index 7b6612dfc9a..9c1550cb336 100644 --- a/.github/workflows/ci-dgraph-upgrade-tests.yml +++ b/.github/workflows/ci-dgraph-upgrade-tests.yml @@ -1,6 +1,9 @@ name: ci-dgraph-upgrade-tests on: pull_request: + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened diff --git a/.github/workflows/ci-golang-lint.yml b/.github/workflows/ci-golang-lint.yml index d0e9368f14f..86673b1fdab 100644 --- a/.github/workflows/ci-golang-lint.yml +++ b/.github/workflows/ci-golang-lint.yml @@ -1,29 +1,14 @@ name: ci-golang-lint on: pull_request: + paths: + - '**/*.go' + - '**/go.mod' types: - opened - reopened - synchronize - ready_for_review - paths-ignore: - - '.github/CODEOWNERS' - - '.vscode/**' - - 'compose/**' - - 'contrib/systemd/**' - - 'licenses/**' - - 'paper/**' - - 'present/**' - - 'RFC/**' - - 'static/**' - - 'wiki/**' - - '**/**.dockerignore' - - '**/**.gitignore' - - '**/**.md' - - '**/**.png' - - '**/**.jpg' - - '**/**.gif' - - '**/**.ini' branches: - main - 'release/**' @@ -50,5 +35,4 @@ jobs: version: latest only-new-issues: true args: --timeout=10m - skip-pkg-cache: true - skip-build-cache: true + skip-cache: true \ No newline at end of file diff --git a/CHANGELOG_CLOUD.md b/CHANGELOG_CLOUD.md deleted file mode 100644 index 5c31c32298d..00000000000 --- a/CHANGELOG_CLOUD.md +++ /dev/null @@ -1,683 +0,0 @@ -# Changelog -All notable changes to this project for *Dgraph's cloud releases* will be documented in this file. -The v23.0.0 release combines the open source and cloud releases and we won't have any -separate cloud releases going forward. - -The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - - -## [v23.0.0-beta1] - 2021-10-30 -[v23.0.0-beta1]: https://github.com/dgraph-io/dgraph/compare/v21.03.0-96-g65fff46c4-slash...v23.0.0-beta1 - -### Added - -- **Core Dgraph** - - chore(deps): update prometheus dependency, adds new metrics (https://github.com/dgraph-io/dgraph/pull/8655) - - chore(deps): bump badger up to v4 (https://github.com/dgraph-io/dgraph/pull/8709) - -- **ARM Support** - - Dgraph now supports ARM64 Architecture for development (https://github.com/dgraph-io/dgraph/pull/8543 https://github.com/dgraph-io/dgraph/pull/8520 https://github.com/dgraph-io/dgraph/pull/8503 https://github.com/dgraph-io/dgraph/pull/8436 https://github.com/dgraph-io/dgraph/pull/8405 https://github.com/dgraph-io/dgraph/pull/8395) - -### Fixed - -- **GraphQL** - - Fix(GraphQL): Make mutation rewriting tests more robust (https://github.com/dgraph-io/dgraph/pull/8449) - - fix(GraphQL): add validation of null values with correct order of graphql rule validation (https://github.com/dgraph-io/dgraph/pull/8333) - - fix(GraphQL): fix auth query rewriting with ID filter (https://github.com/dgraph-io/dgraph/pull/8157) - - fix(GraphQL): Handle extend keyword for Queries and Mutations (https://github.com/dgraph-io/dgraph/pull/7923) - - Fix(Graphql): Fix error message of lambdaOnMutate directive (https://github.com/dgraph-io/dgraph/pull/7751) (https://github.com/dgraph-io/dgraph/pull/7754) - -- **Core Dgraph** - - Additional logging and trace tags for debugging (https://github.com/dgraph-io/dgraph/pull/8490) - - fix(rollups): Fix splits in roll-up (https://github.com/dgraph-io/dgraph/pull/8297) - - Fix(badger): Upgrade badger version to fix manifest corruption (https://github.com/dgraph-io/dgraph/pull/8365) - - fix(pagination): Fix after for regexp, match functions (https://github.com/dgraph-io/dgraph/pull/8471) - - fix(admin): make config changes to pass through gog middlewares (https://github.com/dgraph-io/dgraph/pull/8442) - - fix(DQL): optimize query for has function with offset (https://github.com/dgraph-io/dgraph/pull/8431) - - fix(core): fixed infinite loop in CommitToDisk (https://github.com/dgraph-io/dgraph/pull/8614) - - fix(zero): fix waiting for random time while rate limiting (https://github.com/dgraph-io/dgraph/pull/8656) - - chore(deps): upgrade badger (https://github.com/dgraph-io/dgraph/pull/8654, https://github.com/dgraph-io/dgraph/pull/8658) - - fix(backup): create directory before writing backup (https://github.com/dgraph-io/dgraph/pull/8638) - - fix(query): Prevent multiple entries for same predicate in mutations (https://github.com/dgraph-io/dgraph/pull/8332) - - fix(Chunker): don't delete node with empty facet in mutation (https://github.com/dgraph-io/dgraph/pull/7737) (https://github.com/dgraph-io/dgraph/pull/7745) - - fix(bulk): throw the error instead of crashing (https://github.com/dgraph-io/dgraph/pull/7722) (https://github.com/dgraph-io/dgraph/pull/7749) - - fix(raftwal): take snapshot after restore (https://github.com/dgraph-io/dgraph/pull/7719) (https://github.com/dgraph-io/dgraph/pull/7750) - - fix(drop): attach galaxy namespace to drop attr done on 20.11 backup (https://github.com/dgraph-io/dgraph/pull/7827) - - fix(metrics): Expose dgraph_num_backups_failed_total metric view. (https://github.com/dgraph-io/dgraph/pull/7900) (https://github.com/dgraph-io/dgraph/pull/7904) - -- **Test** - - fix(test): avoid host volume mount in minio container (https://github.com/dgraph-io/dgraph/pull/8569) - - chore(test): add tests for lex/iri.go,chunker/chunk.go (https://github.com/dgraph-io/dgraph/pull/8515) - - chore(test): add Backup/Restore test for NFS (https://github.com/dgraph-io/dgraph/pull/8551) - - chore(test): add test that after snapshot is applied, GraphQL schema is refreshed (https://github.com/dgraph-io/dgraph/pull/8619) - - chore(test): upgrade graphql tests to use go 1.19 (https://github.com/dgraph-io/dgraph/pull/8662) - - chore(test): add automated test to test multitenant --limit flag (https://github.com/dgraph-io/dgraph/pull/8646) - - chore(test): add restore test for more than 127 namespaces (https://github.com/dgraph-io/dgraph/pull/8643) - - fix(test): fix the corner case for raft entries test (https://github.com/dgraph-io/dgraph/pull/8617) - - Added more unit tests (https://github.com/dgraph-io/dgraph/pull/8470 https://github.com/dgraph-io/dgraph/pull/8489 https://github.com/dgraph-io/dgraph/pull/8479 https://github.com/dgraph-io/dgraph/pull/8488 https://github.com/dgraph-io/dgraph/pull/8433) - -- **Security** - - chore(deps): bump certifi from 2020.4.5.1 to 2022.12.7 in /contrib/config/marketplace/aws/tests (https://github.com/dgraph-io/dgraph/pull/8496) - - chore(deps): bump github.com/docker/distribution from 2.7.1+incompatible to 2.8.0+incompatible (https://github.com/dgraph-io/dgraph/pull/8575) - - chore(deps): bump werkzeug from 0.16.1 to 2.2.3 in /contrib/embargo (https://github.com/dgraph-io/dgraph/pull/8676) - - fix(sec): upgrade networkx to (https://github.com/dgraph-io/dgraph/pull/8613) - - fix(sec): CVE-2022-41721 (https://github.com/dgraph-io/dgraph/pull/8633) - - fix(sec): CVE & OS Patching (https://github.com/dgraph-io/dgraph/pull/8634) - - fix(sec): fixing HIGH CVEs (https://github.com/dgraph-io/dgraph/pull/8289) - - fix(sec): CVE High Vulnerability (https://github.com/dgraph-io/dgraph/pull/8277) - - fix(sec): Fixing CVE-2021-31525 (https://github.com/dgraph-io/dgraph/pull/8274) - - fix(sec): CVE-2019-11254 (https://github.com/dgraph-io/dgraph/pull/8270) - - <details> - <summary>CVE Fixes (35 total)</summary> - - #### CVE Fixes (35 total) - - CVE-2013-4235 - - CVE-2016-20013 - - CVE-2016-2781 - - CVE-2017-11164 - - CVE-2018-16886 - - CVE-2019-0205 - - CVE-2019-0210 - - CVE-2019-11254 - - CVE-2019-16167 - - CVE-2020-29652 - - CVE-2021-31525 - - CVE-2021-33194 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-38561 - - CVE-2021-39537 - - CVE-2021-43565 - - CVE-2021-44716 - - CVE-2021-44758 - - CVE-2022-21698 - - CVE-2022-27191 - - CVE-2022-27664 - - CVE-2022-29458 - - CVE-2022-29526 - - CVE-2022-3219 - - CVE-2022-32221 - - CVE-2022-3437 - - CVE-2022-35737 - - CVE-2022-3715 - - CVE-2022-3821 - - CVE-2022-39377 - - CVE-2022-41916 - - CVE-2022-42800 - - CVE-2022-42898 - - CVE-2022-44640 - - <details> - <summary>GHSA Fixes (2 total)</summary> - - #### GHSE Fixes (2 total) - - GHSA-69ch-w2m2-3vjp - - GHSA-m332-53r6-2w93 - - <details> - <summary>CVE Fixes (417 total)</summary> - - #### CVE Fixes (417 total) - - CVE-2019-0210 - - CVE-2019-0205 - - CVE-2021-43565 - - CVE-2022-27664 - - CVE-2021-38561 - - CVE-2021-44716 - - CVE-2021-33194 - - CVE-2022-27191 - - CVE-2020-29652 - - CVE-2018-16886 - - CVE-2022-21698 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-3116 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2022-37434 - - CVE-2020-16156 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2021-37750 - - CVE-2021-36222 - - CVE-2020-35525 - - CVE-2020-35527 - - CVE-2021-20223 - - CVE-2020-9794 - - CVE-2022-29526 - - CVE-2021-31525 - - CVE-2019-11254 - - CVE-2022-3219 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2017-11164 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2016-2781 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2022-3219 - - CVE-2016-2781 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2017-11164 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2022-3219 - - CVE-2016-2781 - - CVE-2021-3671 - - CVE-2022-3219 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2021-3671 - - CVE-2022-3219 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2022-3219 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2019-16167 - - CVE-2013-4235 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2013-4235 - - CVE-2021-3671 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2017-11164 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2022-29458 - - CVE-2021-39537 - - CVE-2021-3671 - - CVE-2021-43618 - - CVE-2016-20013 - - CVE-2021-3671 - - CVE-2016-2781 - - CVE-2021-3671 - - CVE-2022-1587 - - CVE-2022-1586 - - CVE-2021-3671 - - CVE-2020-9991 - - CVE-2020-9849 - - <details> - <summary>GHSA Fixes (5 total)</summary> - - #### GHSA Fixes (5 total) - - GHSA-jq7p-26h5-w78r - - GHSA-8c26-wmh5-6g9v - - GHSA-h6xx-pmxh-3wgp - - GHSA-cg3q-j54f-5p7p - - GHSA-wxc4-f4m6-wwqv - - -### Changed - -- **Core Dgraph** - - *REVERTED* fix(rollups): Write rolled-up keys at ts+1 (https://github.com/dgraph-io/dgraph/pull/7957) - - *REVERTED* add update_manifest tool (https://github.com/dgraph-io/dgraph/pull/7815) - - *REVERTED* opt(dropPrefix): allow logical drop for deleting predicates and indexing (https://github.com/dgraph-io/dgraph/pull/7779) - - fix(query): handle bad timezone correctly (https://github.com/dgraph-io/dgraph/pull/8657) - - fix(mutation): validate mutation before applying it (https://github.com/dgraph-io/dgraph/pull/8623) - -- **CI Enhancements** - - [**LDBC Benchmarking**](https://ldbcouncil.org) in enabled on [CI](https://github.com/dgraph-io/dgraph/actions/workflows/ci-dgraph-ldbc-tests.yml) - - fix(ci): unpin curl (https://github.com/dgraph-io/dgraph/pull/8577) - - fix(ci): adjust cron schedules (https://github.com/dgraph-io/dgraph/pull/8592) - - chore(ci): Capture coverage from bulk load and LDBC tests (https://github.com/dgraph-io/dgraph/pull/8478) - - chore(linter): enable gosec linter (https://github.com/dgraph-io/dgraph/pull/8678) - - chore: apply go vet improvements (https://github.com/dgraph-io/dgraph/pull/8620) - - chore(linter): fix some of the warnings from gas linter (https://github.com/dgraph-io/dgraph/pull/8664) - - chore(linter): fix golangci config and some issues in tests (https://github.com/dgraph-io/dgraph/pull/8669) - - fix(linter): address gosimple linter reports & errors (https://github.com/dgraph-io/dgraph/pull/8628) - - Configured to run with [Github Actions](https://github.com/dgraph-io/dgraph/tree/main/.github/workflows) - - Stability Improvements to test harness - - Enabled [Unit/Integration Tests](https://github.com/dgraph-io/dgraph/actions/workflows/ci-dgraph-tests.yml) - - Enabled [Load Tests](https://github.com/dgraph-io/dgraph/actions/workflows/ci-dgraph-load-tests.yml) - - Enabled [Linters](https://github.com/dgraph-io/dgraph/actions/workflows/ci-golang-lint.yml) - - Enabled [Code Coverage](https://coveralls.io/github/dgraph-io/dgraph?branch=main) - - Configured to run with [Github Actions](https://github.com/dgraph-io/dgraph/blob/main/.github/workflows/ci-aqua-security-trivy-tests.yml) - - Enabled [Trivy Scans](https://github.com/dgraph-io/dgraph/actions/workflows/ci-aqua-security-trivy-tests.yml) - - Enabled dependabot scans - - Configured to run with [Github Actions](https://github.com/dgraph-io/dgraph/blob/main/.github/workflows/ci-aqua-security-trivy-tests.yml) - -- **CD Enhancements** - - Badger Binary fetch steps added to the release CD pipeline (https://github.com/dgraph-io/dgraph/pull/8425) - - fix(build): update dockerfile to use cache busting and reduce image size (https://github.com/dgraph-io/dgraph/pull/8652) - - chore(deps): update min go build version (https://github.com/dgraph-io/dgraph/pull/8423) - - Enhanced our [CD Pipeline](https://github.com/dgraph-io/dgraph/actions/workflows/cd-dgraph.yml) to support ARM64 binaries and docker-images (https://github.com/dgraph-io/dgraph/pull/8520) - - Enhanced [dgraph-lambda](https://github.com/dgraph-io/dgraph-lambda) to support arm64 (https://github.com/dgraph-io/dgraph-lambda/pull/39 https://github.com/dgraph-io/dgraph-lambda/pull/38 https://github.com/dgraph-io/dgraph-lambda/pull/37) - - Enhanced [badger](https://github.com/dgraph-io/badger) to support arm64 (https://github.com/dgraph-io/badger/pull/1838) - - Automated [Release Pipeline](https://github.com/dgraph-io/dgraph/blob/main/.github/workflows/cd-dgraph.yml) to facilitate building of dgraph-binary & corresponding docker-images. The built artifacts are published to repositories through the same pipeline. - - -## [v21.03.0-96-g65fff46c4-slash] - 2023-03-02 -[v21.03.0-96-g65fff46c4-slash]: https://github.com/dgraph-io/dgraph/compare/v21.03.0-92-g0c9f60156...v21.03.0-96-g65fff46c4-slash - - - fix(ACL): Prevents permissions overrride and merges acl cache to persist permissions across different namespaces (https://github.com/dgraph-io/dgraph/pull/8506) - - fix(chore): Add more logging for cloud instances (https://github.com/dgraph-io/dgraph/pull/8507) - - -## [v21.03.0-92-g0c9f60156] - 2021-10-30 -[v21.03.0-92-g0c9f60156]: https://github.com/dgraph-io/dgraph/compare/v21.03.0...v21.03.0-92-g0c9f60156 - -### Added - -- **GraphQL** - - fix(GraphQL): pass on HTTP request headers for subscriptions (https://github.com/dgraph-io/dgraph/pull/8574) - -- **Core Dgraph** - - feat(acl): allow access to all the predicates using wildcard (https://github.com/dgraph-io/dgraph/pull/7993) - - feat(cdc): add superflag `tls` to enable TLS without CA or certs (https://github.com/dgraph-io/dgraph/pull/8564) - - feat(Multi-tenancy): Add namespaces field to state. (https://github.com/dgraph-io/dgraph/pull/7936) - - chore(debug): add `only-summary` flag in `dgraph debug` to show LSM tree and namespace size (https://github.com/dgraph-io/dgraph/pull/8516) - - feat(schema): do schema versioning and make backup non-blocking for indexing (https://github.com/dgraph-io/dgraph/pull/7856) (https://github.com/dgraph-io/dgraph/pull/7873) - - add update_manifest tool (https://github.com/dgraph-io/dgraph/pull/7815) - - feat(multitenancy): namespace aware drop data (https://github.com/dgraph-io/dgraph/pull/8511) - - feat(cloud): add `shared-instance` flag in limit superflag in alpha (https://github.com/dgraph-io/dgraph/pull/8625) - - feat(cdc): Add support for SCRAM SASL mechanism (https://github.com/dgraph-io/dgraph/pull/7765) (https://github.com/dgraph-io/dgraph/pull/7767) - -### Fixed - -- **GraphQL** - - fix(GraphQL): nested Auth Rules not working properly (https://github.com/dgraph-io/dgraph/pull/8571) - - fix(GraphQL): optimize eq filter queries (https://github.com/dgraph-io/dgraph/pull/7895) - - chore(graphql): fixing query timeouts for graphql queries too (https://github.com/dgraph-io/dgraph/pull/7796) - - Fix(GraphQL): Add filter in DQL query in case of reverse predicate (https://github.com/dgraph-io/dgraph/pull/7728) (https://github.com/dgraph-io/dgraph/pull/7733) - - Fix(GraphQL): Fix GraphQL encoding in case of empty list (https://github.com/dgraph-io/dgraph/pull/7726) (https://github.com/dgraph-io/dgraph/pull/7730) - - fix(GraphQL): fix @cascade with Pagination for @auth queries (https://github.com/dgraph-io/dgraph/pull/7695) - -- **Core Dgraph** - - fix(zero): fix update membership to make bulk tablet proposal instead of multiple small (https://github.com/dgraph-io/dgraph/pull/8573) - - adding bulk call for alpha to inform zero about the tablets (https://github.com/dgraph-io/dgraph/pull/8088) - - fix(fragment): merge the nested fragments fields (https://github.com/dgraph-io/dgraph/pull/8435) - - fix(sort): Only filter out nodes with positive offsets (https://github.com/dgraph-io/dgraph/pull/8441) - - opt(schema): optimize populateSchema() (https://github.com/dgraph-io/dgraph/pull/8565) - - fix(probe): do not contend for lock in lazy load (https://github.com/dgraph-io/dgraph/pull/8566) - - fix(audit): fixing audit logs for websocket connections (https://github.com/dgraph-io/dgraph/pull/8627) - - chore(logs): add logs to track dropped proposals (https://github.com/dgraph-io/dgraph/pull/8568) - - fix(proposals): incremental proposal key for zero proposals (https://github.com/dgraph-io/dgraph/pull/8567) - - fix(live): quote the xid when doing upsert (https://github.com/dgraph-io/dgraph/pull/7999) - - fix(acl): filter out the results based on type (https://github.com/dgraph-io/dgraph/pull/7981) - - fix(export): Write temporary files for export to the t directory. (https://github.com/dgraph-io/dgraph/pull/7998) - - fix(pool): use write lock when getting health info (https://github.com/dgraph-io/dgraph/pull/7967) - - fix(query): Do not execute filters if there are no source uids(https://github.com/dgraph-io/dgraph/pull/8452) - - fix(rollups): Write rolled-up keys at ts+1 (https://github.com/dgraph-io/dgraph/pull/7957) - - fix(conn): JoinCluster loop should use latest conn (https://github.com/dgraph-io/dgraph/pull/7952) - - fix(acl): The Acl cache should be updated on restart and restore. (https://github.com/dgraph-io/dgraph/pull/7964) - - opt(schema): load schema and types using Stream framework (https://github.com/dgraph-io/dgraph/pull/8562) - - fix(backup): Fix full backup request (https://github.com/dgraph-io/dgraph/pull/7934) - - fix(restore): set kv version to restoreTs for all keys (https://github.com/dgraph-io/dgraph/pull/8563) - - fix(groot): do not upsert groot for all namespaces on restart (https://github.com/dgraph-io/dgraph/pull/8561) - - fix(Raft): Reconnect via a redial in case of disconnection. (https://github.com/dgraph-io/dgraph/pull/7921) - - fix(Raft): Detect network partition when streaming (https://github.com/dgraph-io/dgraph/pull/7908) - - fix(debug): check length of wal entry before parsing (https://github.com/dgraph-io/dgraph/pull/8560) - - fix(DQL): revert changes related to cascade pagination with sort (https://github.com/dgraph-io/dgraph/pull/7885) (https://github.com/dgraph-io/dgraph/pull/7888) - - fix(restore): append galaxy namespace to type name (https://github.com/dgraph-io/dgraph/pull/7881) - - chore(tracing): Add ns attribute to doQuery (https://github.com/dgraph-io/dgraph/pull/7850) (https://github.com/dgraph-io/dgraph/pull/7853) - - fix(schema-update): Start opIndexing only when index creation is required. (https://github.com/dgraph-io/dgraph/pull/7845) (https://github.com/dgraph-io/dgraph/pull/7847) - - fix(admin): remove exportedFiles field (https://github.com/dgraph-io/dgraph/pull/7835) (https://github.com/dgraph-io/dgraph/pull/7836) - - fix(restore): consider the banned namespaces while bumping (https://github.com/dgraph-io/dgraph/pull/8559) - - fix(auth): preserve the status code while returning error (https://github.com/dgraph-io/dgraph/pull/7832) (https://github.com/dgraph-io/dgraph/pull/7834) - - fix: Prevent proposal from being dropped accidentally (https://github.com/dgraph-io/dgraph/pull/7741) (https://github.com/dgraph-io/dgraph/pull/7811) - - bug fix to permit audit streaming to stdout writer(https://github.com/dgraph-io/dgraph/pull/7803) (https://github.com/dgraph-io/dgraph/pull/7804) - - fix(lease): prevent ID lease overflow (https://github.com/dgraph-io/dgraph/pull/7802) - - fix(ee): GetKeys should return an error (https://github.com/dgraph-io/dgraph/pull/7713) (https://github.com/dgraph-io/dgraph/pull/7797) - - fix(lease): don't do rate limiting when not limit is not specified (https://github.com/dgraph-io/dgraph/pull/7787) - - opt(dropPrefix): allow logical drop for deleting predicates and indexing (https://github.com/dgraph-io/dgraph/pull/7779) - - fix(backup): make the /admin/backup and /admin/export API asynchronous (https://github.com/dgraph-io/dgraph/pull/8554) - - Add asynchronous task API (https://github.com/dgraph-io/dgraph/pull/7781) - - fix(txn): ensure that txn hash is set (https://github.com/dgraph-io/dgraph/pull/7782) (https://github.com/dgraph-io/dgraph/pull/7784) - - fix(bulk): upsert guardian/groot for all existing namespaces (https://github.com/dgraph-io/dgraph/pull/7759) (https://github.com/dgraph-io/dgraph/pull/7769) - - fix(export): Fix facet export of reference type postings to JSON format (https://github.com/dgraph-io/dgraph/pull/7744) (https://github.com/dgraph-io/dgraph/pull/7756) - - Fix(lsbackup): Fix profiler in lsBackup (https://github.com/dgraph-io/dgraph/pull/8432) - - fix(backup): use StreamWriter instead of KVLoader during backup restore (https://github.com/dgraph-io/dgraph/pull/8510) - - fix(vault): Hide ACL flags when not required (https://github.com/dgraph-io/dgraph/pull/7701) - -### Changed - -- **GraphQL** - - opt(GraphQL): filter existence queries on GraphQL side instead of using @filter(type) (https://github.com/dgraph-io/dgraph/pull/7757) (https://github.com/dgraph-io/dgraph/pull/7760) - -- **Core Dgraph** - - protobuf: upgrade golang/protobuf library v1.4.1 -> v1.5.2 (https://github.com/dgraph-io/dgraph/pull/7949) - - chore(raft): Log packets message less frequently. (https://github.com/dgraph-io/dgraph/pull/7913) - - fix(multitenancy) store namespace in predicate as a hex separated by a hyphen to prevent json marshal issues (https://github.com/dgraph-io/dgraph/pull/8601) - - -## [v21.03.0] - 2021-04-07 - -see [CHANGELOG.md](https://github.com/dgraph-io/dgraph/blob/main/CHANGELOG.md) diff --git a/contrib/config/datadog/docker-compose.yml b/contrib/config/datadog/docker-compose.yml deleted file mode 100644 index 180121f812f..00000000000 --- a/contrib/config/datadog/docker-compose.yml +++ /dev/null @@ -1,67 +0,0 @@ -version: "3.5" -services: - alpha1: - image: dgraph/dgraph:latest - container_name: alpha1 - working_dir: /data/alpha1 - labels: - cluster: test - ports: - - 8180:8180 - - 9180:9180 - volumes: - - type: bind - source: $GOPATH/bin - target: /gobin - read_only: true - command: /gobin/dgraph ${COVERAGE_OUTPUT} alpha -o 100 --my=alpha1:7180 --zero=zero1:5080 --logtostderr -v=2 - --trace "jaeger=http://jaeger:14268; datadog=datadog:8126;" - zero1: - image: dgraph/dgraph:latest - container_name: zero1 - working_dir: /data/zero1 - labels: - cluster: test - ports: - - 5080:5080 - - 6080:6080 - volumes: - - type: bind - source: $GOPATH/bin - target: /gobin - read_only: true - command: /gobin/dgraph ${COVERAGE_OUTPUT} zero -o 0 --raft "idx=1;" --my=zero1:5080 --replicas=3 --logtostderr -v=2 --bindall - --trace "jaeger=http://jaeger:14268; datadog=datadog:8126;" - datadog: - image: datadog/agent:latest - container_name: datadog - working_dir: /working/datadog - volumes: - - type: bind - source: /var/run/docker.sock - target: /var/run/docker.sock - read_only: true - - type: bind - source: /proc/ - target: /proc/ - read_only: true - - type: bind - source: /sys/fs/cgroup/ - target: /host/sys/fs/cgroup - read_only: true - environment: - - DD_API_KEY - - DD_APM_ENABLED=true - - DD_APM_NON_LOCAL_TRAFFIC=true - ports: - - 8126:8126 - jaeger: - image: jaegertracing/all-in-one:latest - container_name: jaeger - working_dir: /working/jaeger - environment: - - COLLECTOR_ZIPKIN_HTTP_PORT=9411 - ports: - - 16686:16686 - command: --memory.max-traces=1000000 -volumes: {} diff --git a/contrib/config/docker/docker-compose-ha.yml b/contrib/config/docker/docker-compose-ha.yml deleted file mode 100644 index 0e18cc0202c..00000000000 --- a/contrib/config/docker/docker-compose-ha.yml +++ /dev/null @@ -1,159 +0,0 @@ -# This file can be used to setup a Dgraph cluster with 6 Dgraph Alphas and 3 Dgraph Zero nodes on a -# Docker Swarm with replication. This setup ensures high availability for both Zero and Alpha. - -# It expects six virtual machines with hostnames host1, host2, host3, host4, host5 and host6 to -# be part of the swarm. There is a constraint to make sure that each Dgraph Alpha runs on a -# particular host. Dgraph Zero nodes run on host1, host2 and host3. - -# Data would be persisted to a docker volume called data-volume on the virtual machines which are -# part of the swarm. -# Run `docker stack deploy -c docker-compose-ha.yml` on the Swarm leader to start the cluster. - -version: "3.2" -networks: - dgraph: -services: - zero1: - image: dgraph/dgraph:latest - volumes: - - data-volume:/dgraph - ports: - - 5080:5080 - - 6080:6080 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws01 - command: dgraph zero --my=zero1:5080 --replicas 3 --raft="idx=1" - zero2: - image: dgraph/dgraph:latest - volumes: - - data-volume:/dgraph - ports: - - 5081:5081 - - 6081:6081 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws02 - command: dgraph zero -o 1 --my=zero2:5081 --replicas 3 --peer zero1:5080 --raft="idx=2" - zero3: - image: dgraph/dgraph:latest - volumes: - - data-volume:/dgraph - ports: - - 5082:5082 - - 6082:6082 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws03 - command: dgraph zero -o 2 --my=zero3:5082 --replicas 3 --peer zero1:5080 --raft="idx=3" - alpha1: - image: dgraph/dgraph:latest - hostname: "alpha1" - volumes: - - data-volume:/dgraph - ports: - - 8080:8080 - - 9080:9080 - networks: - - dgraph - deploy: - replicas: 1 - placement: - constraints: - - node.hostname == aws01 - command: dgraph alpha --my=alpha1:7080 --zero=zero1:5080,zero2:5081,zero3:5082 - alpha2: - image: dgraph/dgraph:latest - hostname: "alpha2" - volumes: - - data-volume:/dgraph - ports: - - 8081:8081 - - 9081:9081 - networks: - - dgraph - deploy: - replicas: 1 - placement: - constraints: - - node.hostname == aws02 - command: dgraph alpha --my=alpha2:7081 --zero=zero1:5080,zero2:5081,zero3:5082 -o 1 - alpha3: - image: dgraph/dgraph:latest - hostname: "alpha3" - volumes: - - data-volume:/dgraph - ports: - - 8082:8082 - - 9082:9082 - networks: - - dgraph - deploy: - replicas: 1 - placement: - constraints: - - node.hostname == aws03 - command: dgraph alpha --my=alpha3:7082 --zero=zero1:5080,zero2:5081,zero3:5082 -o 2 - alpha4: - image: dgraph/dgraph:latest - hostname: "alpha4" - volumes: - - data-volume:/dgraph - ports: - - 8083:8083 - - 9083:9083 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws04 - command: dgraph alpha --my=alpha4:7083 --zero=zero1:5080,zero2:5081,zero3:5082 -o 3 - alpha5: - image: dgraph/dgraph:latest - hostname: "alpha5" - volumes: - - data-volume:/dgraph - ports: - - 8084:8084 - - 9084:9084 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws05 - command: dgraph alpha --my=alpha5:7084 --zero=zero1:5080,zero2:5081,zero3:5082 -o 4 - alpha6: - image: dgraph/dgraph:latest - hostname: "alpha6" - volumes: - - data-volume:/dgraph - ports: - - 8085:8085 - - 9085:9085 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws06 - command: dgraph alpha --my=alpha6:7085 --zero=zero1:5080,zero2:5081,zero3:5082 -o 5 - ratel: - image: dgraph/ratel:latest - hostname: "ratel" - ports: - - 8000:8000 - networks: - - dgraph -volumes: - data-volume: diff --git a/contrib/config/docker/docker-compose-multi.yml b/contrib/config/docker/docker-compose-multi.yml deleted file mode 100644 index 95b83728a6c..00000000000 --- a/contrib/config/docker/docker-compose-multi.yml +++ /dev/null @@ -1,83 +0,0 @@ -# This file can be used to setup a Dgraph cluster with 3 Dgraph Alphas and 1 Dgraph Zero node on a -# Docker Swarm with replication.to -# It expects three virtual machines with hostnames aws01, aws02, and aws03 to be part of the swarm. -# There is a constraint to make sure that each Dgraph Alpha runs on a particular host. - -# Data would be persisted to a Docker volume called data-volume on the virtual machines which are -# part of the swarm. -# Run `docker stack deploy -c docker-compose-multi.yml` on the Swarm leader to start the cluster. - -version: "3.2" -networks: - dgraph: -services: - zero: - image: dgraph/dgraph:latest - volumes: - - data-volume:/dgraph - ports: - - 5080:5080 - - 6080:6080 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws01 - command: dgraph zero --my=zero:5080 --replicas 3 - alpha1: - image: dgraph/dgraph:latest - hostname: "alpha1" - volumes: - - data-volume:/dgraph - ports: - - 8080:8080 - - 9080:9080 - networks: - - dgraph - deploy: - placement: - constraints: - - node.hostname == aws01 - command: dgraph alpha --my=alpha1:7080 --zero=zero:5080 - alpha2: - image: dgraph/dgraph:latest - hostname: "alpha2" - volumes: - - data-volume:/dgraph - ports: - - 8081:8081 - - 9081:9081 - networks: - - dgraph - deploy: - replicas: 1 - placement: - constraints: - - node.hostname == aws02 - command: dgraph alpha --my=alpha2:7081 --zero=zero:5080 -o 1 - alpha3: - image: dgraph/dgraph:latest - hostname: "alpha3" - volumes: - - data-volume:/dgraph - ports: - - 8082:8082 - - 9082:9082 - networks: - - dgraph - deploy: - replicas: 1 - placement: - constraints: - - node.hostname == aws03 - command: dgraph alpha --my=alpha3:7082 --zero=zero:5080 -o 2 - ratel: - image: dgraph/ratel:latest - hostname: "ratel" - ports: - - 8000:8000 - networks: - - dgraph -volumes: - data-volume: diff --git a/contrib/config/docker/docker-compose.yml b/contrib/config/docker/docker-compose.yml deleted file mode 100644 index 614eee675c4..00000000000 --- a/contrib/config/docker/docker-compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -# This Docker Compose file can be used to quickly bootup Dgraph Zero -# and Alpha in different Docker containers. - -# It mounts /tmp/data on the host machine to /dgraph within the -# container. You will need to change /tmp/data to a more appropriate location. -# Run `docker-compose up` to start Dgraph. - -version: "3.2" -services: - zero: - image: dgraph/dgraph:latest - volumes: - - /tmp/data:/dgraph - ports: - - 5080:5080 - - 6080:6080 - restart: on-failure - command: dgraph zero --my=zero:5080 - alpha: - image: dgraph/dgraph:latest - volumes: - - /tmp/data:/dgraph - ports: - - 8080:8080 - - 9080:9080 - restart: on-failure - command: dgraph alpha --my=alpha:7080 --zero=zero:5080 - ratel: - image: dgraph/ratel:latest - ports: - - 8000:8000 diff --git a/contrib/config/kubernetes/dgraph-ha/README.md b/contrib/config/kubernetes/dgraph-ha/README.md deleted file mode 100644 index 0c0e9eb30c1..00000000000 --- a/contrib/config/kubernetes/dgraph-ha/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# Dgraph High Availability - -`dgraph-ha.yaml` is an example manifest to deploy Dgraph cluster on Kubernetes: - -* 3 zero nodes -* 3 alpha nodes -* 1 ratel UI node - -You can deploy the manifest with `kubectl`: - -```bash -kubectl apply --filename dgraph-ha.yaml -``` - -## Accessing the Services - -You can access services deploy from `dgraph-ha.yaml` running each of these commands in a separate terminal window or tab: - -```bash -# port-forward to ratel ui -kubectl port-forward svc/dgraph-ratel-public 8000:8000 -# port-forward to alpha -kubectl port-forward svc/dgraph-alpha-public 8080:8080 -``` - -## Public Services - -There are three services specified in the manifest that can be used to expose services outside the cluster. Highly recommend that when doing this, they are only accessible on a private subnet, and not exposed to the public Internet. - -* `dgraph-zero-public` - To load data using Live & Bulk Loaders -* `dgraph-alpha-public` - To connect clients and for HTTP APIs -* `dgraph-ratel-public` - For Dgraph UI - -For security best practices, these are set as `ClusterIP` service type, so they are only accessible from within the Kubernetes cluster. If you need to expose these to outside of the cluster, there are a few options: - -* Change the [service type](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) to `LoadBalancer` for the private intranet access. -* Install an [ingress controller](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) that can provide access to the service from outside the Kubernetes cluster for private intranet access. - -Configuring a service with `LoadBalancer` service type or an Ingress Controller to use a private subnet is very implementation specific. Here are some examples: - -|Provider | Documentation Reference | Annotation | -|------------|---------------------------|------------| -|AWS |[Amazon EKS: Load Balancing](https://docs.aws.amazon.com/eks/latest/userguide/load-balancing.html)|`service.beta.kubernetes.io/aws-load-balancer-internal: "true"`| -|Azure |[AKS: Internal Load Balancer](https://docs.microsoft.com/en-us/azure/aks/internal-lb)|`service.beta.kubernetes.io/azure-load-balancer-internal: "true"`| -|Google Cloud|[GKE: Internal Load Balancing](https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing)|`cloud.google.com/load-balancer-type: "Internal"`| - - - - -## Amazon EKS - -### Create K8S using eksctl (optional) - -An example cluster manifest for use with `eksctl` is provided quickly provision an Amazon EKS cluster. - -The `eksctl` tool can be installed following these instructions: - -* https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html - -Once installed, you can provision Amazon EKS with the following command (takes ~20 minutes): - -```bash -eksctl create cluster --config-file cluster.yaml -``` diff --git a/contrib/config/kubernetes/dgraph-ha/cluster.yaml b/contrib/config/kubernetes/dgraph-ha/cluster.yaml deleted file mode 100644 index 1df0b4a8caa..00000000000 --- a/contrib/config/kubernetes/dgraph-ha/cluster.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: eksctl.io/v1alpha5 -kind: ClusterConfig -metadata: - name: dgraph-ha-cluster - region: us-east-2 -managedNodeGroups: - - name: dgraph-ha-cluster-workers - minSize: 3 - maxSize: 6 - desiredCapacity: 3 - labels: {role: worker} - tags: - nodegroup-role: worker - iam: - withAddonPolicies: - # allow k8s update Route53 if external-dns installed - externalDNS: true - # access ACM (AWS Cert Mngr) for LoadBalancer or Ingress - certManager: true diff --git a/contrib/config/kubernetes/dgraph-ha/dgraph-ha.yaml b/contrib/config/kubernetes/dgraph-ha/dgraph-ha.yaml deleted file mode 100644 index 1e8834fb689..00000000000 --- a/contrib/config/kubernetes/dgraph-ha/dgraph-ha.yaml +++ /dev/null @@ -1,360 +0,0 @@ -# This highly available config creates 3 Dgraph Zeros, 3 Dgraph -# Alphas with 3 replicas, and 1 Ratel UI client. The Dgraph cluster -# will still be available to service requests even when one Zero -# and/or one Alpha are down. -# -# There are 3 services can can be used to expose outside the cluster as needed: -# dgraph-zero-public - To load data using Live & Bulk Loaders -# dgraph-alpha-public - To connect clients and for HTTP APIs -# dgraph-ratel-public - For Dgraph UI - -apiVersion: v1 -kind: Service -metadata: - name: dgraph-zero-public - labels: - app: dgraph-zero - monitor: zero-dgraph-io -spec: - type: ClusterIP - ports: - - port: 5080 - targetPort: 5080 - name: grpc-zero - - port: 6080 - targetPort: 6080 - name: http-zero - selector: - app: dgraph-zero ---- -apiVersion: v1 -kind: Service -metadata: - name: dgraph-alpha-public - labels: - app: dgraph-alpha - monitor: alpha-dgraph-io -spec: - type: ClusterIP - ports: - - port: 8080 - targetPort: 8080 - name: http-alpha - - port: 9080 - targetPort: 9080 - name: grpc-alpha - selector: - app: dgraph-alpha ---- -apiVersion: v1 -kind: Service -metadata: - name: dgraph-ratel-public - labels: - app: dgraph-ratel -spec: - type: ClusterIP - ports: - - port: 8000 - targetPort: 8000 - name: http-ratel - selector: - app: dgraph-ratel ---- -# This is a headless service which is necessary for discovery for a dgraph-zero StatefulSet. -# https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset -apiVersion: v1 -kind: Service -metadata: - name: dgraph-zero - labels: - app: dgraph-zero -spec: - ports: - - port: 5080 - targetPort: 5080 - name: grpc-zero - clusterIP: None - # We want all pods in the StatefulSet to have their addresses published for - # the sake of the other Dgraph Zero pods even before they're ready, since they - # have to be able to talk to each other in order to become ready. - publishNotReadyAddresses: true - selector: - app: dgraph-zero ---- -# This is a headless service which is necessary for discovery for a dgraph-alpha StatefulSet. -# https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#creating-a-statefulset -apiVersion: v1 -kind: Service -metadata: - name: dgraph-alpha - labels: - app: dgraph-alpha -spec: - ports: - - port: 7080 - targetPort: 7080 - name: grpc-alpha-int - clusterIP: None - # We want all pods in the StatefulSet to have their addresses published for - # the sake of the other Dgraph alpha pods even before they're ready, since they - # have to be able to talk to each other in order to become ready. - publishNotReadyAddresses: true - selector: - app: dgraph-alpha ---- -# This StatefulSet runs 3 Dgraph Zero. -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: dgraph-zero -spec: - serviceName: "dgraph-zero" - replicas: 3 - selector: - matchLabels: - app: dgraph-zero - template: - metadata: - labels: - app: dgraph-zero - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - dgraph-zero - topologyKey: kubernetes.io/hostname - containers: - - name: zero - image: dgraph/dgraph:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 5080 - name: grpc-zero - - containerPort: 6080 - name: http-zero - volumeMounts: - - name: datadir - mountPath: /dgraph - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - bash - - "-c" - - | - set -ex - [[ `hostname` =~ -([0-9]+)$ ]] || exit 1 - ordinal=${BASH_REMATCH[1]} - idx=$(($ordinal + 1)) - if [[ $ordinal -eq 0 ]]; then - exec dgraph zero --my=$(hostname -f):5080 --raft="idx=$idx" --replicas 3 - else - exec dgraph zero --my=$(hostname -f):5080 --peer dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080 --raft="idx=$idx" --replicas 3 - fi - livenessProbe: - httpGet: - path: /health - port: 6080 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - successThreshold: 1 - readinessProbe: - httpGet: - path: /health - port: 6080 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - successThreshold: 1 - terminationGracePeriodSeconds: 60 - volumes: - - name: datadir - persistentVolumeClaim: - claimName: datadir - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - name: datadir - annotations: - volume.alpha.kubernetes.io/storage-class: anything - spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 5Gi ---- -# This StatefulSet runs 3 replicas of Dgraph Alpha. -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: dgraph-alpha -spec: - serviceName: "dgraph-alpha" - replicas: 3 - selector: - matchLabels: - app: dgraph-alpha - template: - metadata: - labels: - app: dgraph-alpha - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - dgraph-alpha - topologyKey: kubernetes.io/hostname - # Initializing the Alphas: - # - # You may want to initialize the Alphas with data before starting, e.g. - # with data from the Dgraph Bulk Loader: https://dgraph.io/docs/deploy/#bulk-loader. - # You can accomplish by uncommenting this initContainers config. This - # starts a container with the same /dgraph volume used by Alpha and runs - # before Alpha starts. - # - # You can copy your local p directory to the pod's /dgraph/p directory - # with this command: - # - # kubectl cp path/to/p dgraph-alpha-0:/dgraph/ -c init-alpha - # (repeat for each alpha pod) - # - # When you're finished initializing each Alpha data directory, you can signal - # it to terminate successfully by creating a /dgraph/doneinit file: - # - # kubectl exec dgraph-alpha-0 -c init-alpha touch /dgraph/doneinit - # - # Note that pod restarts cause re-execution of Init Containers. Since - # /dgraph is persisted across pod restarts, the Init Container will exit - # automatically when /dgraph/doneinit is present and proceed with starting - # the Alpha process. - # - # Tip: StatefulSet pods can start in parallel by configuring - # .spec.podManagementPolicy to Parallel: - # - # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#deployment-and-scaling-guarantees - # - # initContainers: - # - name: init-alpha - # image: dgraph/dgraph:latest - # command: - # - bash - # - "-c" - # - | - # trap "exit" SIGINT SIGTERM - # echo "Write to /dgraph/doneinit when ready." - # until [ -f /dgraph/doneinit ]; do sleep 2; done - # volumeMounts: - # - name: datadir - # mountPath: /dgraph - containers: - - name: alpha - image: dgraph/dgraph:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 7080 - name: grpc-alpha-int - - containerPort: 8080 - name: http-alpha - - containerPort: 9080 - name: grpc-alpha - volumeMounts: - - name: datadir - mountPath: /dgraph - env: - # This should be the same namespace as the dgraph-zero - # StatefulSet to resolve a Dgraph Zero's DNS name for - # Alpha's --zero flag. - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - # dgraph versions earlier than v1.2.3 and v20.03.0 can only support one zero: - # `dgraph alpha --zero dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080` - # dgraph-alpha versions greater than or equal to v1.2.3 or v20.03.1 can support - # a comma-separated list of zeros. The value below supports 3 zeros - # (set according to number of replicas) - command: - - bash - - "-c" - - | - set -ex - dgraph alpha --my=$(hostname -f):7080 --zero dgraph-zero-0.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080,dgraph-zero-1.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080,dgraph-zero-2.dgraph-zero.${POD_NAMESPACE}.svc.cluster.local:5080 - livenessProbe: - httpGet: - path: /health?live=1 - port: 8080 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - successThreshold: 1 - readinessProbe: - httpGet: - path: /health - port: 8080 - initialDelaySeconds: 15 - periodSeconds: 10 - timeoutSeconds: 5 - failureThreshold: 6 - successThreshold: 1 - terminationGracePeriodSeconds: 600 - volumes: - - name: datadir - persistentVolumeClaim: - claimName: datadir - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - name: datadir - annotations: - volume.alpha.kubernetes.io/storage-class: anything - spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 5Gi ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dgraph-ratel - labels: - app: dgraph-ratel -spec: - selector: - matchLabels: - app: dgraph-ratel - template: - metadata: - labels: - app: dgraph-ratel - spec: - containers: - - name: ratel - image: dgraph/ratel:latest - ports: - - containerPort: 8000 diff --git a/contrib/config/kubernetes/dgraph-ha/kustomization.yaml b/contrib/config/kubernetes/dgraph-ha/kustomization.yaml deleted file mode 100644 index 9978f5ea870..00000000000 --- a/contrib/config/kubernetes/dgraph-ha/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- dgraph-ha.yaml diff --git a/contrib/config/kubernetes/dgraph-single/dgraph-single.yaml b/contrib/config/kubernetes/dgraph-single/dgraph-single.yaml deleted file mode 100644 index b24a7141e68..00000000000 --- a/contrib/config/kubernetes/dgraph-single/dgraph-single.yaml +++ /dev/null @@ -1,108 +0,0 @@ -# This is the service that should be used by the clients of Dgraph to talk to the cluster. -apiVersion: v1 -kind: Service -metadata: - name: dgraph-public - labels: - app: dgraph -spec: - type: ClusterIP - ports: - - port: 5080 - targetPort: 5080 - name: grpc-zero - - port: 6080 - targetPort: 6080 - name: http-zero - - port: 8080 - targetPort: 8080 - name: http-alpha - - port: 9080 - targetPort: 9080 - name: grpc-alpha - - port: 8000 - targetPort: 8000 - name: http-ratel - selector: - app: dgraph ---- -# This StatefulSet runs 1 pod with one Zero, one Alpha & one Ratel containers. -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: dgraph -spec: - serviceName: "dgraph" - replicas: 1 - selector: - matchLabels: - app: dgraph - template: - metadata: - labels: - app: dgraph - spec: - containers: - - name: ratel - image: dgraph/ratel:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8000 - name: http-ratel - - name: zero - image: dgraph/dgraph:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 5080 - name: grpc-zero - - containerPort: 6080 - name: http-zero - volumeMounts: - - name: datadir - mountPath: /dgraph - command: - - bash - - "-c" - - | - set -ex - dgraph zero --my=$(hostname -f):5080 - - name: alpha - image: dgraph/dgraph:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - name: http-alpha - - containerPort: 9080 - name: grpc-alpha - volumeMounts: - - name: datadir - mountPath: /dgraph - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - command: - - bash - - "-c" - - | - set -ex - dgraph alpha --my=$(hostname -f):7080 --zero dgraph-0.dgraph.${POD_NAMESPACE}.svc.cluster.local:5080 - terminationGracePeriodSeconds: 60 - volumes: - - name: datadir - persistentVolumeClaim: - claimName: datadir - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - name: datadir - annotations: - volume.alpha.kubernetes.io/storage-class: anything - spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: 5Gi diff --git a/contrib/config/kubernetes/dgraph-single/kustomization.yaml b/contrib/config/kubernetes/dgraph-single/kustomization.yaml deleted file mode 100644 index 0965599ee37..00000000000 --- a/contrib/config/kubernetes/dgraph-single/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- dgraph-single.yaml diff --git a/contrib/config/marketplace/aws/README.md b/contrib/config/marketplace/aws/README.md deleted file mode 100644 index d2bcac1cdf7..00000000000 --- a/contrib/config/marketplace/aws/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# **AWS CloudFormation template** - -> This is an AWS CloudFormation template to deploy a Dgraph cluster on AWS using EC2 instances in a separate VPC. - -To deploy the cluster using the CloudFormation template we need two things set up: - -1. SSH Keypair to assign to the created instances. -2. S3 bucket to store the applied CloudFormation templates. A S3 bucket will be created for you if one does not exist. - -Edit the `deploy.sh` file to change these variables to the configured values. - -```sh -./deploy.sh <name-of-cloudformation-stack> <name-of-ssh-key-pair> [target-region] [s3-bucket-name] -``` - -Parameters: - -* **name-of-cloudformation-stack**, e.g. `mydgraph-cluster` -* **name-of-key-pair**, e.g. `mydgraph-cluster-key` -* **target-region** (optional), e.g. `us-east-2`, region from default profile using `aws configure get region` will be used if not specified. -* **s3-bucket-name** (optional), e.g. `dgraph-marketplace-cf-stack-mydgraph-cluster-us-east-2`. This will be created if not specified. - - -## **Notes** - -### **Accessing Endpoint** - -The security groups created will allow access from the Load Balancer. If you wish to access the endpoints from your public IP, you will need to edit the security group attached to the Load Balancer. In the AWS web console, this can be found in the Description tab of the Load Balancer, from EC2 → Load Balancers → dgraph-load-balancer ( e.g. `xxxxx-Dgrap-XXXXXXXXXXXXX`). - - -You can also find the security group with this command: - -```bash -MY_STACK_NAME=<name-of-cloudformation-stack> -MY_STACK_REGION=<target-region> - -aws cloudformation describe-stack-resources \ ---stack-name ${MY_STACK_NAME} \ ---region ${MY_STACK_REGION} \ ---logical-resource-id 'DgraphALBSecurityGroup' \ ---query 'StackResources[0].PhysicalResourceId' \ ---output text -``` - -In the Security field, there the `sg-xxxxxxxxxxxxxxxxx`, which you can click this link to get sent Security Groups, then edit the inbound rules for the same SG. There should be existing inbound rules for ports `8000`, `8080`, `9080`. Add new entries from your public IP to access those ports. - -Also note the DNS information on Load Balancer description tab, like `xxxxx-Dgrap-XXXXXXXXXXXXX-1111111111.us-east-2.elb.amazonaws.com`, which you'll need to use to access the endpoint once access is enabled. - -Afterward, you can visit the website `http://xxxxx-Dgrap-XXXXXXXXXXXXX-1111111111.us-east-2.elb.amazonaws.com:8000`. Once in the Dgraph Ratel UI, configure the server connection as: `http://xxxxx-Dgrap-XXXXXXXXXXXXX-1111111111.us-east-2.elb.amazonaws.com:8080`. - -### **Accessing Systems with SSH** - -If you need to access the EC2 instances themselves through SSH, update the security group on those instances. On any EC2 instance, edit the security group that looks like this `mydgraph-cluster-DgraphSecurityGroup-XXXXXXXXXXXX` and open up port 22 to your public IP. Afterward, you can log into the system with something like this: - -```bash -ssh -i /path/to/my-dgraph-cluster-key.pem ubuntu@ec2-X-XX-XXX-XXX.us-east-2.compute.amazonaws.com -``` - -### **ALB vs gRPC** - -AWS ALBs (Application Load Balancers) configured with this template do not support gRPC load balancing. To get the best performance out of -the dgraph cluster, you can use an externally configured load balancer with gRPC capabilities like [HA Proxy](https://www.haproxy.com/blog/haproxy-1-9-2-adds-grpc-support/) -or [Nginx](https://www.nginx.com/blog/nginx-1-13-10-grpc/). - -To know more about gRPC issues with AWS application load balancer, you can give [this blog](https://rokt.com/engineering_blog/learnings-grpc-aws/) a read. diff --git a/contrib/config/marketplace/aws/build_ubuntu_ami_mappings.rb b/contrib/config/marketplace/aws/build_ubuntu_ami_mappings.rb deleted file mode 100755 index 909de5cd6d0..00000000000 --- a/contrib/config/marketplace/aws/build_ubuntu_ami_mappings.rb +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env ruby - -# Purpose: -# This builds a list of mappings of AMI images in either YAML or JSON -# for use with CFN (CloudFormation) scripts. -# Background: -# In AWS, each region has unique AMI id for the desired images, so -# you need to build a list of target AMI IDs for use with your scripts. -# Requirements: -# aws cli tools with profile (~/.aws/) configured -# -require 'yaml' -require 'json' - -# main -# main -def main - # get arguments - (mode, owner, filter) = parse_arguments - - # print results in JSON or YAML - print_mappings(mode, owner, filter) -end - -# parse_arguments -# process command line arguments -def parse_arguments - # get command line arguments - (mode, owner, filter) = ARGV[0, 2] - - # set to defaults if not set - mode = 'json' if mode.nil? || mode.empty? - owner = 'canonical' if owner.nil? || owner.empty? - # default filter for Ubuntu 18.04 Bionic - if filter.nil? || filter.empty? - filter = 'ubuntu/images/hvm-ssd/ubuntu-bionic-*amd64-server*' - end - - [mode, owner, filter] -end - -# print_mappings -# output final rendered result -def print_mappings(mode, owner, filter) - mappings = build_ami_mappings(owner, filter) - - if mode =~ /json/ - puts JSON.pretty_generate(mappings) - elsif mode =~ /yaml|yml/ - puts mappings.to_yaml - end -end - -# list_regions -# return a list of regions that are accessible given your profile -def list_regions - `aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text` -end - -# get_latest_image -# returns the most recent image given the owner and filter -def get_latest_image(owner, filter) - owners = { canonical: '099720109477' } - - images = `aws ec2 describe-images \ - --owners #{owners[owner]} \ - --filters "Name=name,Values=#{filter}" \ - --query 'sort_by(Images, &CreationDate)[].Name' \ - --output text` - - # return latest - images.split[-1] -end - -# get_region_ami_id -# returns ami for a given region given an example image name for the filter -def get_region_ami_id(owner, filter, region) - owners = { canonical: '099720109477' } - - ami_id = `aws ec2 describe-images \ - --region #{region} \ - --owners #{owners[owner]} \ - --filters "Name=name,Values=#{filter}" \ - --query Images[].ImageId \ - --output text`.chomp - - ami_id -end - -# build_ami_mappings -# returns a hash of AMI mappings -def build_ami_mappings(owner, filter) - ami_mappings = {} - image_name = get_latest_image(owner, filter) - list_regions.split.each do |region| - ami_id = get_region_ami_id(owner, image_name, region) - ami_mappings.merge!({ region => { '64' => ami_id } }) - end - - # return final structure - { 'Mappings' => { 'AWSRegionArch2AMI' => ami_mappings } } -end - -main diff --git a/contrib/config/marketplace/aws/deploy.sh b/contrib/config/marketplace/aws/deploy.sh deleted file mode 100755 index 605900952d4..00000000000 --- a/contrib/config/marketplace/aws/deploy.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -readonly stack_name="${1}" -readonly ssh_key_name="${2}" -readonly region="${3:-$(aws configure get region)}" -readonly s3_bucket_name="${4:-dgraph-marketplace-cf-stack-${stack_name}-${region}}" -readonly template="dgraph.json" - -# validate arguments -[[ $# -lt 2 ]] && \ - { echo "Usage $0 STACK_NAME SSH_KEY_NAME [REGION] [S3_BUCKET_NAME]." &> /dev/stderr; exit 1; } -[[ -z $stack_name ]] && \ - { echo "Stack name not specified. Exiting." &> /dev/stderr; exit 1; } -[[ -z $ssh_key_name ]] && \ - { echo "SSH Key Name not specified. Exiting." &> /dev/stderr; exit 1; } - -# create required bucket if it doesn't exist -aws s3 ls --region ${region} "s3://${s3_bucket_name}" &> /dev/null || \ - aws s3 mb --region ${region} "s3://${s3_bucket_name}" - -# create cfn stack -aws cloudformation deploy \ - --capabilities CAPABILITY_IAM \ - --template-file "${template}" \ - --s3-bucket "${s3_bucket_name}" \ - --stack-name "${stack_name}" \ - --region "${region}" \ - --parameter-overrides \ - KeyName="${ssh_key_name}" diff --git a/contrib/config/marketplace/aws/dgraph.json b/contrib/config/marketplace/aws/dgraph.json deleted file mode 100644 index 6f563f1d379..00000000000 --- a/contrib/config/marketplace/aws/dgraph.json +++ /dev/null @@ -1,1725 +0,0 @@ -{ - "AWSTemplateFormatVersion": "2010-09-09", - "Description": "Dgraph free deployment.", - "Metadata": { - "AWS::CloudFormation::Interface": { - "ParameterGroups": [ - { - "Label": { - "default": "Network Configuration" - }, - "Parameters": [ - "VpcClassB", - "KeyName", - "SSHLocationCIDR", - "AllowSSH" - ] - }, - { - "Label": { - "default": "Dgraph deployment Configuration" - }, - "Parameters": [ - "DgraphVersion", - "DgraphReplicaCount", - "AlphaInstanceCount", - "AlphaInstanceType", - "AlphaDiskSize", - "AlphaDiskIops", - "ZeroInstanceCount", - "ZeroInstanceType", - "ZeroDiskSize", - "ZeroDiskIops", - "RatelInstanceCount", - "RatelInstanceType", - "Tag" - ] - } - ], - "ParameterLabels": { - "KeyName": { - "default": "SSH Key Name to associate with the instances" - }, - "SSHLocationCIDR": { - "default": "IP CIDR to allow ssh from into the instances, defaults to 0.0.0.0/0" - }, - "AllowSSH": { - "default": "Allow ssh on all the instances, from the specified CIDR in SSHLocationCIDR." - }, - "DgraphReplicaCount": { - "default": "Replication factor for each group in Dgraph." - }, - "AlphaInstanceCount": { - "default": "Number of EC2 instances to launch as dgraph alpha nodes." - }, - "AlphaInstanceType": { - "default": "AWS instance types to use for dgraph alpha nodes" - }, - "AlphaDiskSize": { - "default": "Disk size to use for dgraph alpha nodes." - }, - "AlphaDiskIops": { - "default": "Alpha node Disk IOPS" - }, - "ZeroInstanceCount": { - "default": "Number of EC2 instances to launch as dgraph zero nodes." - }, - "ZeroInstanceType": { - "default": "AWS instance types to use for dgraph Zero nodes." - }, - "ZeroDiskSize": { - "default": "Disk size to use for dgraph Zero nodes." - }, - "ZeroDiskIops": { - "default": "Zero node Disk IOPS" - }, - "RatelInstanceCount": { - "default": "Number of EC2 instances to launch as dgraph ratel nodes." - }, - "RatelInstanceType": { - "default": "AWS instance types to use for dgraph Ratel nodes" - }, - "DgraphVersion": { - "default": "Dgraph version to deploy." - }, - "Tag": { - "default": "Common tag for all the deployed resources by template" - } - } - } - }, - "Mappings": { - "AWSRegionArch2AMI": { - "eu-north-1": { - "64": "ami-0d9bc23ca843cabc8" - }, - "ap-south-1": { - "64": "ami-0e9957f2fec33e8b0" - }, - "eu-west-3": { - "64": "ami-032a77ae350eac5a4" - }, - "eu-west-2": { - "64": "ami-0f333ab822d8abf05" - }, - "eu-west-1": { - "64": "ami-0980f143956f4c4a0" - }, - "ap-northeast-2": { - "64": "ami-0550660e405846d55" - }, - "ap-northeast-1": { - "64": "ami-00b90aa0c4c5188a4" - }, - "sa-east-1": { - "64": "ami-0ce05d16ddab2990b" - }, - "ca-central-1": { - "64": "ami-0ac0973ba9211ad6d" - }, - "ap-southeast-1": { - "64": "ami-04dfc6348dc03c931" - }, - "ap-southeast-2": { - "64": "ami-0d2d2286f0655e95e" - }, - "eu-central-1": { - "64": "ami-0f7c5b9619538a39d" - }, - "us-east-1": { - "64": "ami-0dc45e3d9be6ab7b5" - }, - "us-east-2": { - "64": "ami-0a118efdb3bf2c184" - }, - "us-west-1": { - "64": "ami-0ccd40218fe8440fc" - }, - "us-west-2": { - "64": "ami-04bb0cc469b2b81cc" - } - } - }, - "Parameters": { - "VpcClassB": { - "Type": "Number", - "Description": "Class B of Virtual Private Cloud's (VPC) CIDR, e.g. 10.XXX.0.0/16", - "Default": 0, - "ConstraintDescription": "Allowed values are 0 through 255", - "MinValue": 0, - "MaxValue": 255 - }, - "AllowPublicAccess": { - "Description": "Allow public access for dgraph ratel and dgraph alpha HTTP endpoints. *Avoid this in production environment*.", - "Type": "String", - "Default": "false" - }, - "SSHLocationCIDR": { - "Description": "The IP address range that can be used to SSH to the EC2 instances", - "Type": "String", - "Default": "0.0.0.0/0", - "MinLength": "9", - "MaxLength": "18", - "AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})", - "ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x." - }, - "AllowSSH": { - "Description": "Allow ssh on all the instances, from the specified CIDR in SSHLocationCIDR.", - "Type": "String", - "Default": "false" - }, - "DgraphReplicaCount": { - "Description": "Replication factor for each group in dgraph.", - "Type": "Number", - "Default": 3, - "MinValue": 1, - "MaxValue": 5 - }, - "AlphaMinInstanceCount": { - "Description": "Min alpha nodes to have.", - "Type": "Number", - "Default": 1 - }, - "AlphaMaxInstanceCount": { - "Description": "Maximum alpha nodes to have.", - "Type": "Number", - "Default": 5 - }, - "AlphaInstanceCount": { - "Description": "Number of EC2 instances to launch as dgraph alpha nodes.", - "Type": "Number", - "Default": 3, - "MinValue": 3, - "MaxValue": 5 - }, - "AlphaInstanceType": { - "Description": "Instance type for dgraph alpha nodes", - "Type": "String", - "Default": "m5a.large", - "AllowedValues": [ - "m5a.large", - "m5a.4xlarge", - "m4.4xlarge", - "r5.4xlarge", - "r4.4xlarge", - "c5.4xlarge", - "c4.4xlarge" - ], - "ConstraintDescription": "must be a valid EC2 instance type which supports autoscaling groups." - }, - "AlphaDiskSize": { - "Description": "Size in GB of the EBS io1 volume on each alpha node", - "Type": "Number", - "Default": 100, - "MinValue": 50 - }, - "AlphaDiskIops": { - "Description": "IOPS of the EBS io1 volume on each alpha node", - "Type": "Number", - "Default": 1000 - }, - "ZeroMinInstanceCount": { - "Description": "Min zero nodes to have.", - "Type": "Number", - "Default": 1 - }, - "ZeroMaxInstanceCount": { - "Description": "Maximum zero nodes to have.", - "Type": "Number", - "Default": 5 - }, - "ZeroInstanceCount": { - "Description": "Number of EC2 instances to launch as dgraph zero nodes.", - "Type": "Number", - "Default": 3, - "MinValue": 3, - "MaxValue": 5 - }, - "ZeroInstanceType": { - "Description": "Instance type for dgraph Zero nodes", - "Type": "String", - "Default": "m5a.large", - "AllowedValues": [ - "m5a.large", - "m4.large", - "r5.large", - "r4.large", - "c5.large", - "c4.large" - ], - "ConstraintDescription": "must be a valid EC2 instance type." - }, - "ZeroDiskSize": { - "Description": "Size in GB of the EBS io1 volume on each Zero node", - "Type": "Number", - "Default": 50, - "MinValue": 50 - }, - "ZeroDiskIops": { - "Description": "IOPS of the EBS io1 volume on each Zero node", - "Type": "Number", - "Default": 1000 - }, - "RatelMinInstanceCount": { - "Description": "Min alpha nodes to have.", - "Type": "Number", - "Default": 0 - }, - "RatelMaxInstanceCount": { - "Description": "Maximum alpha nodes to have.", - "Type": "Number", - "Default": 3 - }, - "RatelInstanceCount": { - "Description": "Number of EC2 instances to launch as dgraph ratel nodes.", - "Type": "Number", - "Default": 1, - "MinValue": 1, - "MaxValue": 3 - }, - "RatelInstanceType": { - "Description": "Instance type for dgraph Ratel nodes", - "Type": "String", - "Default": "t3a.small", - "AllowedValues": [ - "t3a.small", - "t3a.medium", - "t3a.large" - ], - "ConstraintDescription": "must be a valid EC2 instance type." - }, - "KeyName": { - "Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances", - "Type": "AWS::EC2::KeyPair::KeyName", - "ConstraintDescription": "must be the name of an existing EC2 KeyPair." - }, - "DgraphVersion": { - "Description": "Dgraph version to deploy.", - "Type": "String", - "Default": "v20.03.0", - "AllowedValues": [ - "v20.03.0", - "v1.2.2", - "v1.1.1" - ], - }, - "Tag": { - "Description": "Common tag to use for all the deployments.", - "Type": "String", - "Default": "dgraph.io" - } - }, - "Conditions": { - "AllowSSHToInstances": { - "Fn::Equals": [ - { - "Ref": "AllowSSH" - }, - "true" - ] - }, - "AllowPublicAccess": { - "Fn::Equals": [ - { - "Ref": "AllowPublicAccess" - }, - "true" - ] - }, - "AllowThirdSubnet": { - "Fn::Not" : [ - { - "Fn::Equals": [ - { - "Ref": "AWS::Region" - }, - "us-west-1" - ] - } - ] - } - }, - "Resources": { - "VPC": { - "Type": "AWS::EC2::VPC", - "Properties": { - "CidrBlock": { - "Fn::Sub": "10.${VpcClassB}.0.0/16" - }, - "EnableDnsSupport": true, - "EnableDnsHostnames": true, - "InstanceTenancy": "default", - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "InternetGateway": { - "Type": "AWS::EC2::InternetGateway", - "Properties": { - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "VPCGatewayAttachment": { - "Type": "AWS::EC2::VPCGatewayAttachment", - "Properties": { - "InternetGatewayId": { - "Ref": "InternetGateway" - }, - "VpcId": { - "Ref": "VPC" - } - } - }, - "RouteTable": { - "Type": "AWS::EC2::RouteTable", - "Properties": { - "VpcId": { - "Ref": "VPC" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "Route": { - "DependsOn": "VPCGatewayAttachment", - "Type": "AWS::EC2::Route", - "Properties": { - "RouteTableId": { - "Ref": "RouteTable" - }, - "DestinationCidrBlock": "0.0.0.0/0", - "GatewayId": { - "Ref": "InternetGateway" - } - } - }, - "SubnetA": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "VPC" - }, - "CidrBlock": { - "Fn::Sub": "10.${VpcClassB}.0.0/20" - }, - "AvailabilityZone": { - "Fn::Select": [ - "0", - { - "Fn::GetAZs": "" - } - ] - }, - "MapPublicIpOnLaunch": true, - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "SubnetARouteTableAssociation": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "SubnetA" - }, - "RouteTableId": { - "Ref": "RouteTable" - } - } - }, - "SubnetB": { - "Type": "AWS::EC2::Subnet", - "Properties": { - "VpcId": { - "Ref": "VPC" - }, - "CidrBlock": { - "Fn::Sub": "10.${VpcClassB}.16.0/20" - }, - "AvailabilityZone": { - "Fn::Select": [ - "1", - { - "Fn::GetAZs": "" - } - ] - }, - "MapPublicIpOnLaunch": true, - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "SubnetBRouteTableAssociation": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Properties": { - "SubnetId": { - "Ref": "SubnetB" - }, - "RouteTableId": { - "Ref": "RouteTable" - } - } - }, - "SubnetC": { - "Type": "AWS::EC2::Subnet", - "Condition": "AllowThirdSubnet", - "Properties": { - "VpcId": { - "Ref": "VPC" - }, - "CidrBlock": { - "Fn::Sub": "10.${VpcClassB}.32.0/20" - }, - "AvailabilityZone": { - "Fn::Select": [ - "2", - { - "Fn::GetAZs": "" - } - ] - }, - "MapPublicIpOnLaunch": true, - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "SubnetCRouteTableAssociation": { - "Type": "AWS::EC2::SubnetRouteTableAssociation", - "Condition": "AllowThirdSubnet", - "Properties": { - "SubnetId": { - "Ref": "SubnetC" - }, - "RouteTableId": { - "Ref": "RouteTable" - } - } - }, - "DgraphLoadBalancer": { - "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", - "Properties": { - "SecurityGroups": [ - { - "Ref": "DgraphALBSecurityGroup" - } - ], - "Subnets": - { - "Fn::If": [ - "AllowThirdSubnet", - [ - { - "Ref": "SubnetA" - }, - { - "Ref": "SubnetB" - }, - { - "Ref": "SubnetC" - } - ], - [ - { - "Ref": "SubnetA" - }, - { - "Ref": "SubnetB" - } - ] - ] - } - , - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "DgraphRatelLoadBalancerListener": { - "Type": "AWS::ElasticLoadBalancingV2::Listener", - "Properties": { - "DefaultActions": [ - { - "Type": "forward", - "TargetGroupArn": { - "Ref": "RatelTargetGroup" - } - } - ], - "LoadBalancerArn": { - "Ref": "DgraphLoadBalancer" - }, - "Port": 8000, - "Protocol": "HTTP" - } - }, - "DgraphAlphaHTTPLoadBalancerListener": { - "Type": "AWS::ElasticLoadBalancingV2::Listener", - "Properties": { - "DefaultActions": [ - { - "Type": "forward", - "TargetGroupArn": { - "Ref": "AlphaTargetGroup" - } - } - ], - "LoadBalancerArn": { - "Ref": "DgraphLoadBalancer" - }, - "Port": 8080, - "Protocol": "HTTP" - } - }, - "DgraphAlphaGRPCLoadBalancerListener": { - "Type": "AWS::ElasticLoadBalancingV2::Listener", - "Properties": { - "DefaultActions": [ - { - "Type": "forward", - "TargetGroupArn": { - "Ref": "AlphaTargetGroup" - } - } - ], - "LoadBalancerArn": { - "Ref": "DgraphLoadBalancer" - }, - "Port": 9080, - "Protocol": "HTTP" - } - }, - "RatelTargetGroup": { - "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", - "Properties": { - "HealthCheckIntervalSeconds": 30, - "HealthCheckPath": "/", - "HealthCheckPort": "8000", - "HealthCheckProtocol": "HTTP", - "HealthCheckTimeoutSeconds": 10, - "HealthyThresholdCount": 3, - "Matcher": { - "HttpCode": "200" - }, - "Port": 8000, - "Protocol": "HTTP", - "UnhealthyThresholdCount": 3, - "VpcId": { - "Ref": "VPC" - } - } - }, - "AlphaTargetGroup": { - "Type": "AWS::ElasticLoadBalancingV2::TargetGroup", - "Properties": { - "HealthCheckIntervalSeconds": 30, - "HealthCheckPath": "/health", - "HealthCheckPort": "8080", - "HealthCheckProtocol": "HTTP", - "HealthCheckTimeoutSeconds": 10, - "HealthyThresholdCount": 3, - "Matcher": { - "HttpCode": "200" - }, - "Port": 8080, - "Protocol": "HTTP", - "UnhealthyThresholdCount": 3, - "VpcId": { - "Ref": "VPC" - } - } - }, - "DgraphClientSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "Security group for the client to connect to the dgraph cluster.", - "VpcId": { - "Ref": "VPC" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "client.", - { - "Ref": "Tag" - } - ] - ] - } - } - ] - } - }, - "DgraphALBSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "Allow traffic from public internet to go to port 8080 on Dgraph alpha nodes", - "VpcId": { - "Ref": "VPC" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "DgraphALBAlphaSGIngressHTTP": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphALBSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8080, - "ToPort": 8080, - "SourceSecurityGroupId": { - "Ref": "DgraphClientSecurityGroup" - } - } - }, - "DgraphALBAlphaPublicSGIngressHTTP": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Condition": "AllowPublicAccess", - "Properties": { - "GroupId": { - "Ref": "DgraphALBSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8080, - "ToPort": 8080, - "CidrIp": "0.0.0.0/0" - } - }, - "DgraphALBAlphaSGIngressGRPC": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphALBSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 9080, - "ToPort": 9080, - "SourceSecurityGroupId": { - "Ref": "DgraphClientSecurityGroup" - } - } - }, - "DgraphALBRatelSGIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphALBSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8000, - "ToPort": 8000, - "SourceSecurityGroupId": { - "Ref": "DgraphClientSecurityGroup" - } - } - }, - "DgraphALBRatelPublicSGIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Condition": "AllowPublicAccess", - "Properties": { - "GroupId": { - "Ref": "DgraphALBSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8000, - "ToPort": 8000, - "CidrIp": "0.0.0.0/0" - } - }, - "DgraphALBSGEgress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "DgraphALBSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 0, - "ToPort": 65535, - "CidrIp": "0.0.0.0/0" - } - }, - "DgraphAlphaSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "Allow traffic from ALB to go to port 8080 on Dgraph alpha nodes", - "VpcId": { - "Ref": "VPC" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "alpha.", - { - "Ref": "Tag" - } - ] - ] - } - } - ] - } - }, - "DgraphAlphaSGIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphAlphaSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8080, - "ToPort": 8080, - "SourceSecurityGroupId": { - "Ref": "DgraphALBSecurityGroup" - } - } - }, - "DgraphRatelSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "Allow traffic from ALB to go to port 8000 on Dgraph Ratel nodes", - "VpcId": { - "Ref": "VPC" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "ratel.", - { - "Ref": "Tag" - } - ] - ] - } - } - ] - } - }, - "DgraphRatelSGIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphRatelSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8000, - "ToPort": 8000, - "SourceSecurityGroupId": { - "Ref": "DgraphALBSecurityGroup" - } - } - }, - "DgraphSecurityGroup": { - "Type": "AWS::EC2::SecurityGroup", - "Properties": { - "GroupDescription": "Allow all traffic between Dgraph instances, on required ports.", - "VpcId": { - "Ref": "VPC" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Ref": "Tag" - } - } - ] - } - }, - "DgraphSecurityGroupSSH": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Condition": "AllowSSHToInstances", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 22, - "ToPort": 22, - "CidrIp": { - "Ref": "SSHLocationCIDR" - } - } - }, - "DgraphSGZeroInternalGRPCIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 5080, - "ToPort": 5080, - "SourceSecurityGroupId": { - "Ref": "DgraphSecurityGroup" - } - } - }, - "DgraphSGZeroExternalGRPCIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 6080, - "ToPort": 6080, - "SourceSecurityGroupId": { - "Ref": "DgraphSecurityGroup" - } - } - }, - "DgraphSGAlphaInternalGRPCIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 7080, - "ToPort": 7080, - "SourceSecurityGroupId": { - "Ref": "DgraphSecurityGroup" - } - } - }, - "DgraphSGRatelExternalIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8000, - "ToPort": 8000, - "SourceSecurityGroupId": { - "Ref": "DgraphSecurityGroup" - } - } - }, - "DgraphSGAlphaExternalHTTPIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 8080, - "ToPort": 8080, - "SourceSecurityGroupId": { - "Ref": "DgraphSecurityGroup" - } - } - }, - "DgraphSGAlphaExternalGRPCIngress": { - "Type": "AWS::EC2::SecurityGroupIngress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 9080, - "ToPort": 9080, - "SourceSecurityGroupId": { - "Ref": "DgraphSecurityGroup" - } - } - }, - "DgraphSGEgress": { - "Type": "AWS::EC2::SecurityGroupEgress", - "Properties": { - "GroupId": { - "Ref": "DgraphSecurityGroup" - }, - "IpProtocol": "tcp", - "FromPort": 0, - "ToPort": 65535, - "CidrIp": "0.0.0.0/0" - } - }, - "DgraphHealthyZero": { - "Type": "AWS::EC2::Instance", - "Properties": { - "ImageId": { - "Fn::FindInMap": [ - "AWSRegionArch2AMI", - { - "Ref": "AWS::Region" - }, - "64" - ] - }, - "KeyName": { - "Ref": "KeyName" - }, - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 20 - } - }, - { - "DeviceName": "/dev/sdk", - "Ebs": { - "VolumeType": "io1", - "Iops": { - "Ref": "ZeroDiskIops" - }, - "DeleteOnTermination": false, - "VolumeSize": { - "Ref": "ZeroDiskSize" - } - } - } - ], - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": "0", - "SubnetId": { - "Ref": "SubnetA" - }, - "GroupSet": [ - { - "Ref": "DgraphSecurityGroup" - } - ] - } - ], - "DisableApiTermination": false, - "InstanceType": { - "Ref": "ZeroInstanceType" - }, - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#!/usr/bin/env bash\n\n", - "set -euxo pipefail\n\n", - "groupadd --system dgraph\n", - "useradd --system -d /var/run/dgraph -s /bin/false -g dgraph dgraph\n", - "dev_name=\"/dev/$(lsblk | grep ", - { - "Ref": "ZeroDiskSize" - }, - "G | tr -s ' ' | cut -d ' ' -f 1)\"\n", - "until [ -b $(readlink -f $dev_name) ]; do sleep 1; done\n", - "if [[ \"$(lsblk -no FSTYPE $dev_name)\" != \"ext4\" ]]; then\n", - " mkfs -t ext4 $dev_name\n", - " sleep 5\n", - "fi\n", - "\n\n", - "mkdir -p /var/log/dgraph/ && mkdir -p /var/run/dgraph/\n", - "mount $dev_name /var/run/dgraph\n", - "resize2fs $dev_name\n", - "chown -R dgraph:dgraph /var/run/dgraph\n", - "chown -R dgraph:dgraph /var/log/dgraph\n", - "DEVICE_UUID=\"$(blkid -s UUID -o value $dev_name)\"\n", - "if grep -q \"$DEVICE_UUID\" /etc/fstab; then\n", - " echo \"fstab already set\"\n", - "else\n", - " cp /etc/fstab /etc/fstab.original\n", - " echo -e \"UUID=$DEVICE_UUID\t/dgraph\text4\tdefaults,nofail\t0\t2\" >> /etc/fstab\n", - "fi\n", - "\n\n", - "cat <<EOT > /etc/systemd/system/dgraph-zero.service\n", - "[Unit]\n", - "Description=dgraph.io data server\n", - "Wants=network.target\n", - "After=network.target\n", - "[Service]\n", - "Type=simple\n", - "ExecStart=/usr/local/bin/dgraph zero --my=$(hostname).", - { - "Ref": "AWS::Region" - }, - ".compute.internal:5080 -w /var/run/dgraph/w --raft='idx=1'\n", - "StandardOutput=journal\n", - "StandardError=journal\n", - "User=dgraph\n", - "Group=dgraph\n", - "[Install]\n", - "WantedBy=multi-user.target\n", - "EOT\n", - "\n\n", - "chmod +x /etc/systemd/system/dgraph-zero.service\n", - "curl -sSf https://get.dgraph.io | ACCEPT_LICENSE=y VERSION=", - { - "Ref": "DgraphVersion" - }, - " bash\n", - "systemctl daemon-reload\n", - "systemctl enable --now dgraph-zero.service\n" - ] - ] - } - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "zero.", - { - "Ref": "Tag" - } - ] - ] - } - } - ] - } - }, - "ZeroLaunchTemplate": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 20 - } - }, - { - "DeviceName": "/dev/sdk", - "Ebs": { - "VolumeType": "io1", - "DeleteOnTermination": false, - "Iops": { - "Ref": "ZeroDiskIops" - }, - "VolumeSize": { - "Ref": "ZeroDiskSize" - } - } - } - ], - "DisableApiTermination": false, - "ImageId": { - "Fn::FindInMap": [ - "AWSRegionArch2AMI", - { - "Ref": "AWS::Region" - }, - "64" - ] - }, - "InstanceType": { - "Ref": "ZeroInstanceType" - }, - "KeyName": { - "Ref": "KeyName" - }, - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeviceIndex": 0, - "DeleteOnTermination": true, - "Groups": [ - { - "Ref": "DgraphSecurityGroup" - } - ] - } - ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#!/usr/bin/env bash", - "\n\n", - "set -euxo pipefail", - "\n\n", - "groupadd --system dgraph\n", - "useradd --system -d /var/run/dgraph -s /bin/false -g dgraph dgraph\n", - "dev_name=\"/dev/$(lsblk | grep ", - { - "Ref": "ZeroDiskSize" - }, - "G | tr -s ' ' | cut -d ' ' -f 1)\"\n", - "until [ -b $(readlink -f $dev_name) ]; do sleep 1; done\n", - "if [[ \"$(lsblk -no FSTYPE $dev_name)\" != \"ext4\" ]]; then\n", - " mkfs -t ext4 $dev_name\n", - " sleep 5\n", - "fi\n", - "\n\n", - "mkdir -p /var/log/dgraph/ && mkdir -p /var/run/dgraph/\n", - "mount $dev_name /var/run/dgraph\n", - "resize2fs $dev_name\n", - "chown -R dgraph:dgraph /var/run/dgraph\n", - "chown -R dgraph:dgraph /var/log/dgraph\n", - "DEVICE_UUID=\"$(blkid -s UUID -o value $dev_name)\"\n", - "if grep -q \"$DEVICE_UUID\" /etc/fstab; then\n", - " echo \"fstab already set\"\n", - "else\n", - " cp /etc/fstab /etc/fstab.original\n", - " echo -e \"UUID=$DEVICE_UUID\t/dgraph\text4\tdefaults,nofail\t0\t2\" >> /etc/fstab\n", - "fi\n", - "\n\n", - "cat <<EOT > /etc/systemd/system/dgraph-zero.service\n", - "[Unit]\n", - "Description=dgraph.io data server\n", - "Wants=network.target\n", - "After=network.target\n", - "[Service]\n", - "Type=simple\n", - "ExecStart=/usr/local/bin/dgraph zero --my=$(hostname).", - { - "Ref": "AWS::Region" - }, - ".compute.internal:5080 --peer ", - { - "Fn::GetAtt": [ - "DgraphHealthyZero", - "PrivateDnsName" - ] - }, - ":5080 --replicas ", - { - "Ref": "DgraphReplicaCount" - }, - " -w /var/run/dgraph/w\n", - "StandardOutput=journal\n", - "StandardError=journal\n", - "User=dgraph\n", - "Group=dgraph\n", - "[Install]\n", - "WantedBy=multi-user.target\n", - "EOT\n", - "\n\n", - "chmod +x /etc/systemd/system/dgraph-zero.service\n", - "curl -sSf https://get.dgraph.io | ACCEPT_LICENSE=y VERSION=", - { - "Ref": "DgraphVersion" - }, - " bash\n", - "systemctl daemon-reload\n", - "systemctl enable --now dgraph-zero.service" - ] - ] - } - } - }, - "LaunchTemplateName": "zero.dgraph.io" - } - }, - "ZeroAutoscalingGroup": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "Properties": { - "VPCZoneIdentifier": { - "Fn::If": [ - "AllowThirdSubnet", - [ - { - "Ref": "SubnetA" - }, - { - "Ref": "SubnetB" - }, - { - "Ref": "SubnetC" - } - ], - [ - { - "Ref": "SubnetA" - }, - { - "Ref": "SubnetB" - } - ] - ] - }, - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "ZeroLaunchTemplate" - }, - "Version": { - "Fn::GetAtt": [ - "ZeroLaunchTemplate", - "LatestVersionNumber" - ] - } - }, - "DesiredCapacity": { - "Ref": "ZeroInstanceCount" - }, - "MinSize": { - "Ref": "ZeroMinInstanceCount" - }, - "MaxSize": { - "Ref": "ZeroMaxInstanceCount" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "zero.", - { - "Ref": "Tag" - } - ] - ] - }, - "PropagateAtLaunch": true - } - ] - } - }, - "AlphaLaunchTemplate": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 20 - } - }, - { - "DeviceName": "/dev/sdk", - "Ebs": { - "VolumeType": "io1", - "DeleteOnTermination": false, - "Iops": { - "Ref": "AlphaDiskIops" - }, - "VolumeSize": { - "Ref": "AlphaDiskSize" - } - } - } - ], - "DisableApiTermination": false, - "ImageId": { - "Fn::FindInMap": [ - "AWSRegionArch2AMI", - { - "Ref": "AWS::Region" - }, - "64" - ] - }, - "InstanceType": { - "Ref": "AlphaInstanceType" - }, - "KeyName": { - "Ref": "KeyName" - }, - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeviceIndex": 0, - "DeleteOnTermination": true, - "Groups": [ - { - "Ref": "DgraphSecurityGroup" - }, - { - "Ref": "DgraphAlphaSecurityGroup" - } - ] - } - ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#!/usr/bin/env bash", - "\n\n", - "set -euxo pipefail", - "\n\n", - "groupadd --system dgraph\n", - "useradd --system -d /var/run/dgraph -s /bin/false -g dgraph dgraph\n", - "dev_name=\"/dev/$(lsblk | grep ", - { - "Ref": "AlphaDiskSize" - }, - "G | tr -s ' ' | cut -d ' ' -f 1)\"\n", - "until [ -b $(readlink -f $dev_name) ]; do sleep 1; done\n", - "if [[ \"$(lsblk -no FSTYPE $dev_name)\" != \"ext4\" ]]; then\n", - " mkfs -t ext4 $dev_name\n", - " sleep 5\n", - "fi\n", - "\n\n", - "mkdir -p /var/log/dgraph/ && mkdir -p /var/run/dgraph/\n", - "mount $dev_name /var/run/dgraph\n", - "resize2fs $dev_name\n", - "chown -R dgraph:dgraph /var/run/dgraph\n", - "chown -R dgraph:dgraph /var/log/dgraph\n", - "DEVICE_UUID=\"$(blkid -s UUID -o value $dev_name)\"\n", - "if grep -q \"$DEVICE_UUID\" /etc/fstab; then\n", - " echo \"fstab already set\"\n", - "else\n", - " cp /etc/fstab /etc/fstab.original\n", - " echo -e \"UUID=$DEVICE_UUID\t/dgraph\text4\tdefaults,nofail\t0\t2\" >> /etc/fstab\n", - "fi\n", - "\n\n", - "cat <<EOT > /etc/systemd/system/dgraph-alpha.service\n", - "[Unit]\n", - "Description=dgraph.io alpha server\n", - "Wants=network.target\n", - "After=network.target\n", - "[Service]\n", - "Type=simple\n", - "ExecStart=/usr/local/bin/dgraph alpha --my=$(hostname).", - { - "Ref": "AWS::Region" - }, - ".compute.internal:7080 --zero ", - { - "Fn::GetAtt": [ - "DgraphHealthyZero", - "PrivateDnsName" - ] - }, - ":5080 -p /var/run/dgraph/p -w /var/run/dgraph/w\n", - "StandardOutput=journal\n", - "StandardError=journal\n", - "User=dgraph\n", - "Group=dgraph\n", - "[Install]\n", - "WantedBy=multi-user.target\n", - "EOT\n", - "\n\n", - "chmod +x /etc/systemd/system/dgraph-alpha.service\n", - "curl -sSf https://get.dgraph.io | ACCEPT_LICENSE=y VERSION=", - { - "Ref": "DgraphVersion" - }, - " bash\n", - "systemctl daemon-reload\n", - "systemctl enable --now dgraph-alpha.service" - ] - ] - } - } - }, - "LaunchTemplateName": "alpha.dgraph.io" - } - }, - "AlphaAutoscalingGroup": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "DependsOn": [ - "ZeroAutoscalingGroup" - ], - "Properties": { - "VPCZoneIdentifier": { - "Fn::If": [ - "AllowThirdSubnet", - [ - { - "Ref": "SubnetA" - }, - { - "Ref": "SubnetB" - }, - { - "Ref": "SubnetC" - } - ], - [ - { - "Ref": "SubnetA" - }, - { - "Ref": "SubnetB" - } - ] - ] - }, - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "AlphaLaunchTemplate" - }, - "Version": { - "Fn::GetAtt": [ - "AlphaLaunchTemplate", - "LatestVersionNumber" - ] - } - }, - "DesiredCapacity": { - "Ref": "AlphaInstanceCount" - }, - "TargetGroupARNs": [ - { - "Ref": "AlphaTargetGroup" - } - ], - "MinSize": { - "Ref": "AlphaMinInstanceCount" - }, - "MaxSize": { - "Ref": "AlphaMaxInstanceCount" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "alpha.", - { - "Ref": "Tag" - } - ] - ] - }, - "PropagateAtLaunch": true - } - ] - } - }, - "RatelLaunchTemplate": { - "Type": "AWS::EC2::LaunchTemplate", - "Properties": { - "LaunchTemplateData": { - "BlockDeviceMappings": [ - { - "DeviceName": "/dev/sda1", - "Ebs": { - "VolumeSize": 20 - } - } - ], - "DisableApiTermination": false, - "ImageId": { - "Fn::FindInMap": [ - "AWSRegionArch2AMI", - { - "Ref": "AWS::Region" - }, - "64" - ] - }, - "InstanceType": { - "Ref": "RatelInstanceType" - }, - "KeyName": { - "Ref": "KeyName" - }, - "NetworkInterfaces": [ - { - "AssociatePublicIpAddress": true, - "DeleteOnTermination": true, - "DeviceIndex": 0, - "SubnetId": { - "Ref": "SubnetA" - }, - "Groups": [ - { - "Ref": "DgraphSecurityGroup" - }, - { - "Ref": "DgraphRatelSecurityGroup" - } - ] - } - ], - "UserData": { - "Fn::Base64": { - "Fn::Join": [ - "", - [ - "#!/usr/bin/env bash", - "\n\n", - "set -euxo pipefail", - "\n\n", - "groupadd --system dgraph\n", - "useradd --system -d /var/run/dgraph -s /bin/false -g dgraph dgraph\n", - "mkdir -p /var/log/dgraph/ && mkdir -p /var/run/dgraph/\n", - "chown -R dgraph:dgraph /var/run/dgraph\n", - "chown -R dgraph:dgraph /var/log/dgraph\n", - "\n\n", - "cat <<EOT > /etc/systemd/system/dgraph-ratel.service\n", - "[Unit]\n", - "Description=dgraph.io UI server\n", - "Wants=network.target\n", - "After=network.target\n", - "[Service]\n", - "Type=simple\n", - "ExecStart=/usr/local/bin/dgraph-ratel\n", - "StandardOutput=journal\n", - "StandardError=journal\n", - "User=dgraph\n", - "Group=dgraph\n", - "[Install]\n", - "WantedBy=multi-user.target\n", - "EOT\n", - "\n\n", - "chmod +x /etc/systemd/system/dgraph-ratel.service\n", - "curl -sSf https://get.dgraph.io | ACCEPT_LICENSE=y VERSION=", - { - "Ref": "DgraphVersion" - }, - " bash\n", - "systemctl daemon-reload\n", - "systemctl enable --now dgraph-ratel.service" - ] - ] - } - } - }, - "LaunchTemplateName": "ratel.dgraph.io" - } - }, - "RatelAutoscalingGroup": { - "Type": "AWS::AutoScaling::AutoScalingGroup", - "DependsOn": [ - "AlphaAutoscalingGroup" - ], - "Properties": { - "VPCZoneIdentifier": [ - { - "Ref": "SubnetA" - } - ], - "LaunchTemplate": { - "LaunchTemplateId": { - "Ref": "RatelLaunchTemplate" - }, - "Version": { - "Fn::GetAtt": [ - "RatelLaunchTemplate", - "LatestVersionNumber" - ] - } - }, - "DesiredCapacity": { - "Ref": "RatelInstanceCount" - }, - "TargetGroupARNs": [ - { - "Ref": "RatelTargetGroup" - } - ], - "MinSize": { - "Ref": "RatelMinInstanceCount" - }, - "MaxSize": { - "Ref": "RatelMaxInstanceCount" - }, - "Tags": [ - { - "Key": "context", - "Value": { - "Fn::Join": [ - "", - [ - "ratel.", - { - "Ref": "Tag" - } - ] - ] - }, - "PropagateAtLaunch": true - } - ] - } - } - }, - "Outputs": { - "VpcCidr": { - "Description": "VPC CIDR block created for the dgraph cluster deployment.", - "Value": { - "Fn::GetAtt": [ - "VPC", - "CidrBlock" - ] - } - }, - "ClientSecurityGroupId": { - "Description": "Client security group Id for access to dgraph endpoints via created ALB.", - "Value": { - "Fn::GetAtt": [ - "DgraphClientSecurityGroup", - "GroupId" - ] - } - }, - "LBPublicDNS": { - "Description": "DNS entry corresponding to the ALB associated with the dgraph cluster.", - "Value": { - "Fn::GetAtt": [ - "DgraphLoadBalancer", - "DNSName" - ] - } - } - } -} diff --git a/contrib/config/marketplace/aws/tests/.gitignore b/contrib/config/marketplace/aws/tests/.gitignore deleted file mode 100644 index 6f42515e941..00000000000 --- a/contrib/config/marketplace/aws/tests/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -# secret keys -*.pub -*.pem -taskcat_outputs/ -.taskcat_overrides.yml -dgraph.json diff --git a/contrib/config/marketplace/aws/tests/.python-version b/contrib/config/marketplace/aws/tests/.python-version deleted file mode 100644 index cd7b97de612..00000000000 --- a/contrib/config/marketplace/aws/tests/.python-version +++ /dev/null @@ -1 +0,0 @@ -aws diff --git a/contrib/config/marketplace/aws/tests/.taskcat.yml b/contrib/config/marketplace/aws/tests/.taskcat.yml deleted file mode 100644 index 56d120b1029..00000000000 --- a/contrib/config/marketplace/aws/tests/.taskcat.yml +++ /dev/null @@ -1,24 +0,0 @@ -project: - name: dgraph-aws-marketplace - regions: - - ap-northeast-1 - - ap-northeast-2 - - ap-south-1 - - ap-southeast-1 - - ap-southeast-2 - - ca-central-1 - - eu-central-1 - - eu-north-1 - - eu-west-1 - - eu-west-2 - - eu-west-3 - - sa-east-1 - - us-east-1 - - us-east-2 - - us-west-1 - - us-west-2 -tests: - default: - parameters: - KeyName: dgraph-marketplace-cf-stack-key - template: ./dgraph.json diff --git a/contrib/config/marketplace/aws/tests/README.md b/contrib/config/marketplace/aws/tests/README.md deleted file mode 100644 index c233719b621..00000000000 --- a/contrib/config/marketplace/aws/tests/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# **TESTING** - - -## **Installing Tests Tools** - -### **Python VirtualEnv** - -A recent version of python3 should be sufficient. I you have `pyenv` + `pyenv-virtualenv`, you setup a local virtual environment with this: - -```bash -pyenv install 3.8.0 -pyenv virtualenv 3.8.0 aws -cd . -``` - -### **Python Requirements** - -```bash -pip install -r requirements.txt -``` - -## **Generate Key Pairs** - -Generate keys and install them as keypairs into AWS regions. - -```bash -./seed_keypairs.sh -``` - -## **Run Tests** - -```bash -# copy template to current working directory -cp ../dgraph.json . -# run tests -taskcat test run -``` diff --git a/contrib/config/marketplace/aws/tests/requirements.txt b/contrib/config/marketplace/aws/tests/requirements.txt deleted file mode 100644 index aec22f7f08e..00000000000 --- a/contrib/config/marketplace/aws/tests/requirements.txt +++ /dev/null @@ -1,41 +0,0 @@ -attrs==19.3.0 -aws-sam-translator==1.22.0 -awscli==1.18.40 -backports.shutil-get-terminal-size==1.0.0 -boto3==1.12.40 -botocore==1.15.40 -certifi==2024.7.4 -cfn-lint==0.29.5 -chardet==3.0.4 -colorama==0.4.3 -dataclasses-jsonschema==2.12.0 -decorator==4.4.2 -docker==4.2.0 -docutils==0.15.2 -dulwich==0.19.15 -idna==3.7 -Jinja2==3.1.4 -jmespath==0.9.5 -jsonpatch==1.25 -jsonpointer==2.0 -jsonschema==3.2.0 -MarkupSafe==1.1.1 -mock==2.0.0 -mypy-extensions==0.4.3 -networkx==2.6 -pbr==5.4.5 -pyasn1==0.4.8 -pyrsistent==0.16.0 -python-dateutil==2.8.1 -PyYAML==5.4 -reprint==0.5.2 -requests==2.32.0 -rsa==4.7 -s3transfer==0.3.3 -six==1.14.0 -tabulate==0.8.7 -taskcat==0.9.17 -typing-extensions==3.7.4.2 -urllib3==1.26.19 -websocket-client==0.57.0 -yattag==1.13.2 diff --git a/contrib/config/marketplace/aws/tests/seed_keypairs.sh b/contrib/config/marketplace/aws/tests/seed_keypairs.sh deleted file mode 100755 index 624a9cfb1bd..00000000000 --- a/contrib/config/marketplace/aws/tests/seed_keypairs.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -# Purpose: -# This builds a keypair and installs them into all availavble regions -# Background: -# CloudFormation script requires keypair as parameter to be in existing region -# Requirements: -# aws cli tools with profile (~/.aws/) configured -# - -##### -# main -###### -main() { - parse_arguments - # generate local private/public key - generate_key - # install keys into target regions - seed_keys "$REGIONS" -} - -##### -# parse_arguments - set global vars from command line args -###### -parse_arguments() { - REGIONS=${1:-$(aws ec2 describe-regions \ - --query 'Regions[].{Name:RegionName}' \ - --output text - )} - - KEYPAIR=${2:-"dgraph-marketplace-cf-stack-key"} - KEYPATH=${3:-"."} - KEYNAME=${4:-"dgraph"} -} - -##### -# seed_keys - install public key with keypair name into target regions -###### -seed_keys() { - local REGIONS="$1" - - for REGION in $REGIONS; do - create_key_pair $REGION - done -} - -##### -# generate_key - generate private/public key pair if private key doesn't exist -###### -generate_key() { - if [[ ! -f $KEYPATH/$KEYNAME.pem ]]; then - # Generate Key Pair - openssl genrsa -out "$KEYPATH/$KEYNAME.pem" 4096 - openssl rsa -in "$KEYPATH/$KEYNAME.pem" -pubout > "$KEYPATH/$KEYNAME.pub" - chmod 400 "$KEYPATH/$KEYNAME.pem" - fi -} - -##### -# create_key_pair - upload public key with key_pair name -###### -create_key_pair() { - local REGION=${1:-$(aws configure get region)} - - # Install Keys into Metadata - echo "Creating KeyPair in $REGION" - aws ec2 import-key-pair \ - --region $REGION \ - --key-name $KEYPAIR \ - --public-key-material "$(grep -v PUBLIC $KEYPATH/$KEYNAME.pub | tr -d '\n')" -} - -main diff --git a/contrib/gh-runner/README.md b/contrib/gh-runner/README.md deleted file mode 100644 index afd2c3ab285..00000000000 --- a/contrib/gh-runner/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# GH Actions Runner - -How to bake an AWS Virtual Machine for GH Actions? - -1. Get a Github Actions Runner Token from the UI -``` -export TOKEN=<GITHUB ACTIONS RUNNER TOKEN> -``` -2. Download the setup script onto the machine -``` -wget https://raw.githubusercontent.com/dgraph-io/dgraph/main/contrib/gh-runner/gh-runner.sh -``` -3. Run the script to attach this machine to Github Actions Runner Pool -``` -sh gh-runner.sh -``` -NOTE: this will reboot the machine, once the machine is back up it will connect to Github diff --git a/contrib/gh-runner/gh-runner.sh b/contrib/gh-runner/gh-runner.sh deleted file mode 100644 index 2fd35382a8a..00000000000 --- a/contrib/gh-runner/gh-runner.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash -# Machine Setup -sudo apt-get update -sudo apt-get -y upgrade -sudo apt-get -y install \ - ca-certificates \ - curl \ - gnupg \ - lsb-release \ - build-essential #Libc Packages -# Docker Setup -sudo mkdir -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get -y update -sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin -sudo apt-get -y install docker-compose -#Add Docker to sudoers group -sudo usermod -aG docker ${USER} -newgrp docker & -# Install & Setup GH Actions Runner -mkdir actions-runner && cd actions-runner -if [ "$(uname -m)" = "aarch64" ]; then - echo "Detected arm64 architecture" - # Download the latest runner package - curl -o actions-runner-linux-arm64-2.298.2.tar.gz -L https://github.com/actions/runner/releases/download/v2.298.2/actions-runner-linux-arm64-2.298.2.tar.gz - # Optional: Validate the hash - echo "803e4aba36484ef4f126df184220946bc151ae1bbaf2b606b6e2f2280f5042e8 actions-runner-linux-arm64-2.298.2.tar.gz" | shasum -a 256 -c - # Extract the installer - tar xzf ./actions-runner-linux-arm64-2.298.2.tar.gz -elif [ "$(uname -m)" = "x86_64" ]; then - echo "Detected amd64 architecture" - # Download the latest runner package - curl -o actions-runner-linux-x64-2.298.2.tar.gz -L https://github.com/actions/runner/releases/download/v2.298.2/actions-runner-linux-x64-2.298.2.tar.gz - # Optional: Validate the hash - echo "0bfd792196ce0ec6f1c65d2a9ad00215b2926ef2c416b8d97615265194477117 actions-runner-linux-x64-2.298.2.tar.gz" | shasum -a 256 -c - # Extract the installer - tar xzf ./actions-runner-linux-x64-2.298.2.tar.gz -else - echo "Unrecognized architecture" - exit 1 -fi -# Create the runner and start the configuration experience -./config.sh --url https://github.com/dgraph-io/dgraph --token $TOKEN -# CI Permission Issue -sudo touch /etc/cron.d/ci_permissions_resetter -sudo chown $USER:$USER /etc/cron.d/ci_permissions_resetter -sudo echo "* * * * * root for i in {0..60}; do chown -R $USER:$USER /home/ubuntu/actions-runner/_work & sleep 1; done" > /etc/cron.d/ci_permissions_resetter -sudo chown root:root /etc/cron.d/ci_permissions_resetter -# Increase file descriptor limit -sudo sh -c 'echo "* hard nofile 2000000" >> /etc/security/limits.conf' -sudo sh -c 'echo "* soft nofile 2000000" >> /etc/security/limits.conf' -sudo sh -c 'echo "root hard nofile 2000000" >> /etc/security/limits.conf' -sudo sh -c 'echo "root soft nofile 2000000" >> /etc/security/limits.conf' -sudo sh -c 'echo "fs.nr_open = 2000000" >> /etc/sysctl.conf' -# Start GH Actions -sudo ./svc.sh install -sudo ./svc.sh start -# Reboot Machine -sudo reboot