From 77d0840f023dde702ec971b0ac8d7b6d194f7d96 Mon Sep 17 00:00:00 2001 From: paulfantom Date: Sat, 26 Jan 2019 22:00:54 +0100 Subject: [PATCH 1/7] synchronize dashboards by default; group tasks in blocks --- README.md | 4 +- defaults/main.yml | 2 +- tasks/dashboards.yml | 268 +++++++++++++++++++++---------------------- 3 files changed, 134 insertions(+), 140 deletions(-) diff --git a/README.md b/README.md index 805d218..6874338 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Provision and manage [grafana](https://github.com/grafana/grafana) - platform fo - Ansible >= 2.5 - libselinux-python on deployer host (only when deployer machine has SELinux) - grafana >= 5.1 (for older grafana versions use this role in version 0.10.1 or earlier) -- rsync if you plan to use grafana provisioning +- jmespath on deployer machine. If you are using Ansible from a Python virtualenv, install *jmespath* to the same virtualenv via pip. ## Role Variables @@ -24,7 +24,7 @@ All variables which can be overridden are stored in [defaults/main.yml](defaults | Name | Default Value | Description | | -------------- | ------------- | -----------------------------------| | `grafana_use_provisioning` | true | Use Grafana provisioning capalibity when possible (**grafana_version=latest will assume >= 5.0**). | -| `grafana_provisioning_synced` | false | Ensure no previously provisioned dashboards are kept if not referenced anymore. | +| `grafana_provisioning_synced` | true | Ensure no previously provisioned dashboards are kept if not referenced anymore. | | `grafana_system_user` | grafana | Grafana server system user | | `grafana_system_group` | grafana | Grafana server system group | | `grafana_version` | latest | Grafana package version | diff --git a/defaults/main.yml b/defaults/main.yml index 6ecfadc..655c4c7 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -6,7 +6,7 @@ grafana_yum_repo_template: etc/yum.repos.d/grafana.repo.j2 grafana_use_provisioning: true # Should the provisioning be kept synced. If true, previous provisioned objects will be removed if not referenced anymore. -grafana_provisioning_synced: false +grafana_provisioning_synced: true grafana_instance: "{{ ansible_fqdn | default(ansible_host) | default(inventory_hostname) }}" diff --git a/tasks/dashboards.yml b/tasks/dashboards.yml index dc1981c..df0f276 100644 --- a/tasks/dashboards.yml +++ b/tasks/dashboards.yml @@ -1,116 +1,89 @@ --- -- name: Empty local grafana dashboard directory +- name: Locally prepare doashboards for upload become: false - file: - path: /tmp/dashboards - state: absent delegate_to: localhost run_once: true - check_mode: false - changed_when: false - when: grafana_use_provisioning and grafana_provisioning_synced + block: + - name: Create local grafana dashboard directory + become: false + tempfile: + state: directory + register: _tmp_dashboards + changed_when: false + check_mode: false -- name: Create local grafana dashboard directories - become: false - file: - path: /tmp/dashboards - state: directory - mode: 0755 - delegate_to: localhost - run_once: true - check_mode: false - changed_when: false + # Use curl to solve issue #77 + - name: download grafana dashboard from grafana.net to local directory + become: false + command: > + "curl --fail --compressed + https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download + -o {{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json" + args: + creates: "{{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json" + warn: false + register: _download_dashboards + until: _download_dashboards is succeeded + retries: 5 + delay: 2 + with_items: "{{ grafana_dashboards }}" + when: grafana_dashboards | length > 0 + check_mode: false + tags: + - skip_ansible_lint -# - name: download grafana dashboard from grafana.net to local folder -# become: false -# get_url: -# url: "https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download" -# dest: "/tmp/dashboards/{{ item.dashboard_id }}.json" -# register: _download_dashboards -# until: _download_dashboards is succeeded -# retries: 5 -# delay: 2 -# delegate_to: localhost -# run_once: true -# changed_when: false -# with_items: "{{ grafana_dashboards }}" -# when: grafana_dashboards | length > 0 + # As noted in [1] an exported dashboard replaces the exporter's datasource + # name with a representative name, something like 'DS_GRAPHITE'. The name + # is different for each datasource plugin, but always begins with 'DS_'. + # In the rest of the data, the same name is used, but captured in braces, + # for example: '${DS_GRAPHITE}'. + # + # [1] http://docs.grafana.org/reference/export_import/#import-sharing-with-grafana-2-x-or-3-0 + # + # The data structure looks (massively abbreviated) something like: + # + # "name": "DS_GRAPHITE", + # "datasource": "${DS_GRAPHITE}", + # + # If we import the downloaded dashboard verbatim, it will not automatically + # be connected to the data source like we want it. The Grafana UI expects + # us to do the final connection by hand, which we do not want to do. + # So, in the below task we ensure that we replace instances of this string + # with the data source name we want. + # To make sure that we're not being too greedy with the regex replacement + # of the data source to use for each dashboard that's uploaded, we make the + # regex match very specific by using the following: + # + # 1. Literal boundaries for " on either side of the match. + # 2. Non-capturing optional group matches for the ${} bits which may, or + # or may not, be there.. + # 3. A case-sensitive literal match for DS . + # 4. A one-or-more case-sensitive match for the part that follows the + # underscore, with only A-Z, 0-9 and - or _ allowed. + # + # This regex can be tested and understood better by looking at the + # matches and non-matches in https://regex101.com/r/f4Gkvg/6 -# Use curl to solve issue #77 -- name: download grafana dashboard from grafana.net to local directory - become: false - command: "curl --fail --compressed https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download -o /tmp/dashboards/{{ item.dashboard_id }}.json" - args: - creates: "/tmp/dashboards/{{ item.dashboard_id }}.json" - warn: false - register: _download_dashboards - until: _download_dashboards is succeeded - retries: 5 - delay: 2 - delegate_to: localhost - run_once: true - with_items: "{{ grafana_dashboards }}" - when: grafana_dashboards | length > 0 - tags: - - skip_ansible_lint - check_mode: false - -# As noted in [1] an exported dashboard replaces the exporter's datasource -# name with a representative name, something like 'DS_GRAPHITE'. The name -# is different for each datasource plugin, but always begins with 'DS_'. -# In the rest of the data, the same name is used, but captured in braces, -# for example: '${DS_GRAPHITE}'. -# -# [1] http://docs.grafana.org/reference/export_import/#import-sharing-with-grafana-2-x-or-3-0 -# -# The data structure looks (massively abbreviated) something like: -# -# "name": "DS_GRAPHITE", -# "datasource": "${DS_GRAPHITE}", -# -# If we import the downloaded dashboard verbatim, it will not automatically -# be connected to the data source like we want it. The Grafana UI expects -# us to do the final connection by hand, which we do not want to do. -# So, in the below task we ensure that we replace instances of this string -# with the data source name we want. -# To make sure that we're not being too greedy with the regex replacement -# of the data source to use for each dashboard that's uploaded, we make the -# regex match very specific by using the following: -# -# 1. Literal boundaries for " on either side of the match. -# 2. Non-capturing optional group matches for the ${} bits which may, or -# or may not, be there.. -# 3. A case-sensitive literal match for DS . -# 4. A one-or-more case-sensitive match for the part that follows the -# underscore, with only A-Z, 0-9 and - or _ allowed. -# -# This regex can be tested and understood better by looking at the -# matches and non-matches in https://regex101.com/r/f4Gkvg/6 - -- name: Set the correct data source name in the dashboard - become: false - replace: - dest: "/tmp/dashboards/{{ item.dashboard_id }}.json" - regexp: '"(?:\${)?DS_[A-Z0-9_-]+(?:})?"' - replace: '"{{ item.datasource }}"' - delegate_to: localhost - run_once: true - changed_when: false - with_items: "{{ grafana_dashboards }}" - when: grafana_dashboards | length > 0 + - name: Set the correct data source name in the dashboard + become: false + replace: + dest: "{{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json" + regexp: '"(?:\${)?DS_[A-Z0-9_-]+(?:})?"' + replace: '"{{ item.datasource }}"' + changed_when: false + with_items: "{{ grafana_dashboards }}" + when: grafana_dashboards | length > 0 -- name: copy local grafana dashboards - become: false - copy: - src: "{{ item }}" - dest: "/tmp/dashboards/{{ item | basename }}" - with_fileglob: - - "{{ grafana_dashboards_dir }}/*.json" - delegate_to: localhost - run_once: true - changed_when: false + - name: Copy grafana dashboards to temporary directory + become: false + copy: + src: "{{ item }}" + dest: "{{ _tmp_dashboards.path }}/{{ item | basename }}" + changed_when: false + with_fileglob: + - "{{ grafana_dashboards_dir }}/*.json" -- name: import grafana dashboards through API +- name: Import grafana dashboards through API uri: url: "{{ grafana_api_url }}/api/dashboards/db" user: "{{ grafana_security.admin_user }}" @@ -118,10 +91,15 @@ force_basic_auth: true method: POST body_format: json - body: '{ "dashboard": {{ lookup("file", item) }}, "overwrite": true, "message": "Updated by ansible" }' + body: > + '{ + "dashboard": {{ lookup("file", item) }}, + "overwrite": true, + "message": "Updated by ansible" + }' no_log: true with_fileglob: - - "/tmp/dashboards/*" + - "{{ _tmp_dashboards.path }}/*" when: not grafana_use_provisioning # TODO: uncomment this when ansible 2.7 will be min supported version @@ -138,36 +116,52 @@ # with_fileglob: # - "/tmp/dashboards/*" -- name: Create/Update dashboards file (provisioning) - become: true - copy: - dest: "/etc/grafana/provisioning/dashboards/ansible.yml" - content: | - apiVersion: 1 - providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - options: - path: /var/lib/grafana/dashboards - backup: false - owner: root - group: grafana - mode: 0640 - notify: restart grafana +- name: Upload dashboards with Grafana provisioning system when: grafana_use_provisioning + block: + - name: Create/Update dashboards file (provisioning) + become: true + copy: + dest: "/etc/grafana/provisioning/dashboards/ansible.yml" + content: | + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + options: + path: /var/lib/grafana/dashboards + backup: false + owner: root + group: grafana + mode: 0640 + notify: restart grafana -- name: Import grafana dashboards through provisioning - become: true - synchronize: - src: "/tmp/dashboards/" - dest: "/var/lib/grafana/dashboards" - archive: false - checksum: true - recursive: true - delete: "{{ grafana_provisioning_synced }}" - rsync_opts: - - "--no-motd" - when: grafana_use_provisioning - notify: "provisioned dashboards changed" + - name: Register previously copied dashboards + find: + paths: "/var/lib/grafana/dashboards" + hidden: true + patterns: + - "*.json" + register: _dashboards_present + when: grafana_provisioning_synced + + - name: Import grafana dashboards + become: true + copy: + src: "{{ item }}" + dest: "/var/lib/grafana/dashboards/{{ item | basename }}" + with_fileglob: + - "{{ _tmp_dashboards.path }}/*" + register: _dashboards_copied + notify: "provisioned dashboards changed" + + - name: Remove dashbards not present on deployer machine (synchronize) + become: true + file: + path: "{{ item }}" + state: absent + with_items: "{{ _dashboards_present | json_query('files[*].path') + | difference( _dashboards_copied | json_query('results[*].dest')) }}" + when: grafana_provisioning_synced From 7437466164d1fe81f19d5d52e2bee08cdc68daac Mon Sep 17 00:00:00 2001 From: paulfantom Date: Sat, 26 Jan 2019 22:21:30 +0100 Subject: [PATCH 2/7] remove names from blocks --- tasks/dashboards.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tasks/dashboards.yml b/tasks/dashboards.yml index df0f276..c4cffa1 100644 --- a/tasks/dashboards.yml +++ b/tasks/dashboards.yml @@ -1,6 +1,5 @@ --- -- name: Locally prepare doashboards for upload - become: false +- become: false delegate_to: localhost run_once: true block: @@ -116,8 +115,7 @@ # with_fileglob: # - "/tmp/dashboards/*" -- name: Upload dashboards with Grafana provisioning system - when: grafana_use_provisioning +- when: grafana_use_provisioning block: - name: Create/Update dashboards file (provisioning) become: true From d789e3a28ba0f1ee4f316c402d2795f28280b0e2 Mon Sep 17 00:00:00 2001 From: paulfantom Date: Sat, 26 Jan 2019 22:23:43 +0100 Subject: [PATCH 3/7] remove quotes --- tasks/dashboards.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/dashboards.yml b/tasks/dashboards.yml index c4cffa1..585231f 100644 --- a/tasks/dashboards.yml +++ b/tasks/dashboards.yml @@ -15,9 +15,9 @@ - name: download grafana dashboard from grafana.net to local directory become: false command: > - "curl --fail --compressed + curl --fail --compressed https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download - -o {{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json" + -o {{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json args: creates: "{{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json" warn: false @@ -91,11 +91,11 @@ method: POST body_format: json body: > - '{ + { "dashboard": {{ lookup("file", item) }}, "overwrite": true, "message": "Updated by ansible" - }' + } no_log: true with_fileglob: - "{{ _tmp_dashboards.path }}/*" From f2286068f276aa5407bbf0a2a362c88e38e5f0b7 Mon Sep 17 00:00:00 2001 From: paulfantom Date: Fri, 22 Mar 2019 15:03:28 +0100 Subject: [PATCH 4/7] do not move dashboards around --- tasks/dashboards.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tasks/dashboards.yml b/tasks/dashboards.yml index 585231f..5e0974e 100644 --- a/tasks/dashboards.yml +++ b/tasks/dashboards.yml @@ -27,6 +27,7 @@ delay: 2 with_items: "{{ grafana_dashboards }}" when: grafana_dashboards | length > 0 + changed_when: false check_mode: false tags: - skip_ansible_lint @@ -73,15 +74,6 @@ with_items: "{{ grafana_dashboards }}" when: grafana_dashboards | length > 0 - - name: Copy grafana dashboards to temporary directory - become: false - copy: - src: "{{ item }}" - dest: "{{ _tmp_dashboards.path }}/{{ item | basename }}" - changed_when: false - with_fileglob: - - "{{ grafana_dashboards_dir }}/*.json" - - name: Import grafana dashboards through API uri: url: "{{ grafana_api_url }}/api/dashboards/db" @@ -99,6 +91,7 @@ no_log: true with_fileglob: - "{{ _tmp_dashboards.path }}/*" + - "{{ grafana_dashboards_dir }}/*.json" when: not grafana_use_provisioning # TODO: uncomment this when ansible 2.7 will be min supported version @@ -152,6 +145,7 @@ dest: "/var/lib/grafana/dashboards/{{ item | basename }}" with_fileglob: - "{{ _tmp_dashboards.path }}/*" + - "{{ grafana_dashboards_dir }}/*.json" register: _dashboards_copied notify: "provisioned dashboards changed" From 6dd8f36b50ff70289b1c7d9a6b563107f91ff920 Mon Sep 17 00:00:00 2001 From: paulfantom Date: Fri, 22 Mar 2019 15:10:15 +0100 Subject: [PATCH 5/7] simplify priviledge escalation --- tasks/dashboards.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/tasks/dashboards.yml b/tasks/dashboards.yml index 5e0974e..87a26b5 100644 --- a/tasks/dashboards.yml +++ b/tasks/dashboards.yml @@ -4,7 +4,6 @@ run_once: true block: - name: Create local grafana dashboard directory - become: false tempfile: state: directory register: _tmp_dashboards @@ -13,7 +12,6 @@ # Use curl to solve issue #77 - name: download grafana dashboard from grafana.net to local directory - become: false command: > curl --fail --compressed https://grafana.com/api/dashboards/{{ item.dashboard_id }}/revisions/{{ item.revision_id }}/download @@ -65,7 +63,6 @@ # matches and non-matches in https://regex101.com/r/f4Gkvg/6 - name: Set the correct data source name in the dashboard - become: false replace: dest: "{{ _tmp_dashboards.path }}/{{ item.dashboard_id }}.json" regexp: '"(?:\${)?DS_[A-Z0-9_-]+(?:})?"' From 70ca11423002ecaa11b3f092c0414d4b036e3e8a Mon Sep 17 00:00:00 2001 From: paulfantom Date: Fri, 22 Mar 2019 15:22:51 +0100 Subject: [PATCH 6/7] do not install rsync in test runs --- molecule/default/prepare.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml index f899721..5358b3b 100644 --- a/molecule/default/prepare.yml +++ b/molecule/default/prepare.yml @@ -2,11 +2,4 @@ - name: Prepare hosts: all gather_facts: false - tasks: - - name: Install rsync for grafana dashboard provisioning - package: - name: ["rsync"] - register: task_result - until: task_result is success - retries: 10 - delay: 2 + tasks: [] From c9691b10710e5b1fde014aa0aae01ca5c9771a35 Mon Sep 17 00:00:00 2001 From: paulfantom Date: Fri, 22 Mar 2019 17:04:19 +0100 Subject: [PATCH 7/7] move dashboard lists manipulation to separate task --- tasks/dashboards.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tasks/dashboards.yml b/tasks/dashboards.yml index 87a26b5..9dc0265 100644 --- a/tasks/dashboards.yml +++ b/tasks/dashboards.yml @@ -146,11 +146,16 @@ register: _dashboards_copied notify: "provisioned dashboards changed" + - name: Get dashboard lists + set_fact: + _dashboards_present_list: "{{ _dashboards_present | json_query('files[*].path') | default([]) }}" + _dashboards_copied_list: "{{ _dashboards_copied | json_query('results[*].dest') | default([]) }}" + when: grafana_provisioning_synced + - name: Remove dashbards not present on deployer machine (synchronize) become: true file: path: "{{ item }}" state: absent - with_items: "{{ _dashboards_present | json_query('files[*].path') - | difference( _dashboards_copied | json_query('results[*].dest')) }}" + with_items: "{{ _dashboards_present_list | difference( _dashboards_copied_list ) }}" when: grafana_provisioning_synced