diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 192175457..db5bd6bc4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -168,8 +168,8 @@ So without further ado, run the following: ## Start the build process $ make dev/up -Any missing images (i.e., postgresql, memcachd, rabbitmq) will be pulled. -Getting all the images downloaded may take a few minutes. +Any missing images (i.e. postgresql, rabbitmq, prometheus, influxdb, grafana) +will be pulled. Getting all the images downloaded may take a few minutes. Once all the images are available, the ontainers will launch. Aftr the above commands completes, you can take a look at the containers by @@ -179,12 +179,13 @@ running ``docker ps`` in your second terminal session: $ docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - fc06225cdfd5 galaxy-dev:latest "/bin/sh -c /galax..." 6 hours ago Up 5 hours 0.0.0.0:8000->8000/tcp galaxy_galaxy_1 - dc007355a69a elasticsearch:2.4.1 "/docker-entrypoin..." 6 hours ago Up 6 hours 9200/tcp, 9300/tcp galaxy_elastic_1 - fa48c619cc3d postgres:9.5.4 "/docker-entrypoin..." 6 hours ago Up 6 hours 5432/tcp galaxy_postgres_1 - b7f374cf7d56 rabbitmq:latest "docker-entrypoint..." 6 hours ago Up 6 hours 4369/tcp, 5671-5672/tcp, 25672/tcp galaxy_rabbitmq_1 - 9b8245eea91b memcached:latest "docker-entrypoint..." 6 hours ago Up 6 hours 11211/tcp galaxy_memcache_1 + CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES + b76488f94890 galaxy-dev:latest "/entrypoint.sh /g..." 2 minutes ago Up 2 minutes 0.0.0.0:8000->8000/tcp galaxy_galaxy_1 + dfe97d19197e centos/postgresql-95-centos7 "container-entrypo..." 22 hours ago Up 2 minutes 0.0.0.0:2345->5432/tcp galaxy_postgres_1 + fd3dd5f663f2 rabbitmq:latest "docker-entrypoint..." 22 hours ago Up 2 minutes 4369/tcp, 5671-5672/tcp, 25672/tcp galaxy_rabbitmq_1 + 9561d0cea1ec prom/prometheus:latest "/bin/prometheus -..." 2 minutes ago Up 2 minutes 0.0.0.0:9090->9090/tcp galaxy_prometheus_1 + 21e8b688f2ab influxdb:latest "/entrypoint.sh in..." 22 hours ago Up 2 minutes 0.0.0.0:8086->8086/tcp galaxy_influxdb_1 + 92186c792b4d grafana/grafana:latest "/run.sh" 2 minutes ago Up 2 minutes 0.0.0.0:3000->3000/tcp galaxy_grafana_1 Running detached ^^^^^^^^^^^^^^^^ @@ -323,6 +324,26 @@ OK, go for it! Your Galaxy web site is available at: `http://localhost:8000`_. Post build setup ---------------- +Metrics +^^^^^^^ + +From the root of the project tree, run ``make dev/setup-metrics`` to initialize +InfluxDB and import data sources and dashboards for Prometheus and InfluxDB into Grafana. + +.. code-block:: console + + $ make dev/setup-metrics + +Log into Grafana at `http://localhost:3000`_ using the admin user with +password ``admin``. Navigate to ``Galaxy Search Metrics - InfluxDB`` and +``Galaxy Search Metrics - Prometheus`` dashboards. + +Search metrics are exposed at `http://localhost:8000/api/metrics`_. From +there, the metrics are being scraped by Prometheus +(`http://localhost:9090/`_). Prometheus serves as a short-term storage. +For a long-term metrics storage, the metrics are being sent from +Prometheus to InfluxDB. + Create an admin user ^^^^^^^^^^^^^^^^^^^^ diff --git a/Makefile b/Makefile index 4e9ba6f84..9c51a14ef 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ celery: .PHONY: ng_server ng_server: - cd /galaxy/galaxyui; ng serve --host '0.0.0.0' --port '8000' --poll '5000' --watch --live-reload --progress=false ----proxy-config proxy.conf.js + cd /galaxy/galaxyui; ng serve --disable-host-check --host '0.0.0.0' --port '8000' --poll '5000' --watch --live-reload --progress=false ----proxy-config proxy.conf.js .PHONY: waitenv waitenv: @@ -261,5 +261,9 @@ dev/lint-rule-update: @$(DOCKER_COMPOSE) exec galaxy git config --global user.name "dev galaxy_1" @$(DOCKER_COMPOSE) exec galaxy bash -c "cd /galaxy-lint-rules && git pull --ff-only https://github.com/ansible/galaxy-lint-rules.git master" +.PHONY: dev/setup-metrics +dev/setup-metrics: + cd scripts/metrics-setup-playbook; ansible-playbook setup.yml + %: @: diff --git a/galaxy/api/urls.py b/galaxy/api/urls.py index d6188f289..0b7f6b4b5 100644 --- a/galaxy/api/urls.py +++ b/galaxy/api/urls.py @@ -270,4 +270,5 @@ urlpatterns = [ url(r'^$', views.ApiRootView.as_view(), name='api_root_view'), url(r'^v1/', include(v1_urls)), + url(r'', include('django_prometheus.urls')), ] diff --git a/galaxy/api/views/search.py b/galaxy/api/views/search.py index e94a89bc0..20538e358 100644 --- a/galaxy/api/views/search.py +++ b/galaxy/api/views/search.py @@ -33,6 +33,7 @@ from galaxy.api import filters from galaxy.api import serializers from galaxy.api.views import base_views as base +from galaxy.common import metrics from galaxy.main import models __all__ = [ @@ -79,6 +80,7 @@ def get_queryset(self): repository__provider_namespace__namespace__active=True)) # TODO(cutwater): Use serializer to parse request arguments + @metrics.send('search') def list(self, request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) diff --git a/galaxy/common/metrics.py b/galaxy/common/metrics.py new file mode 100644 index 000000000..aea1eb9ae --- /dev/null +++ b/galaxy/common/metrics.py @@ -0,0 +1,95 @@ +# (c) 2012-2018, Ansible by Red Hat +# +# This file is part of Ansible Galaxy +# +# Ansible Galaxy is free software: you can redistribute it and/or modify +# it under the terms of the Apache License as published by +# the Apache Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# Ansible Galaxy is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Apache License for more details. +# +# You should have received a copy of the Apache License +# along with Galaxy. If not, see . + +import logging +import sys + +from django.conf import settings + +from rest_framework.request import Request + + +__all__ = [ + 'send', +] + +logger = logging.getLogger(__name__) + + +def send(func_name): + def decorator(function): + def wrapper(*args, **kwargs): + # call the view + result = function(*args, **kwargs) + + if settings.METRICS_ENABLED: + # TODO do this async + _send(func_name, args[1]) + + return result + return wrapper + return decorator + + +def _send(func_name, request): + if not isinstance(request, Request): + raise AssertionError( + 'Could not send metrics. ' + 'Expected rest_framework.request.Request, got %s!' + % type(request) + ) + + func = getattr(sys.modules[__name__], func_name, None) + if not callable(func): + raise AssertionError( + 'Could not send metrics. % is not callable!' % func_name + ) + + try: + # FIXME POST? + func(request.GET) + except IOError as e: + logger.exception(e) + + +def search(data): + platforms = data.get('platforms', '').split() + cloud_platforms = data.get('cloud_platforms', '').split() + tags = data.get('tags', '').split() + keywords = data.get('keywords', '').split() + + if not any((platforms, cloud_platforms, tags, keywords)): + return + + search_criteria = { + 'keyword': keywords, + 'platform': platforms, + 'cloud_platform': cloud_platforms, + 'tag': tags, + } + + settings.PROM_CNTR_SEARCH.labels( + keywords=','.join(keywords), + platforms=','.join(platforms), + cloud_platforms=','.join(cloud_platforms), + tags=','.join(tags) + ).inc() + + for criteria_type, criteria_values in search_criteria.items(): + for criteria_value in criteria_values: + settings.PROM_CNTR_SEARCH_CRITERIA.labels( + ctype=criteria_type, cvalue=criteria_value).inc() diff --git a/galaxy/settings/default.py b/galaxy/settings/default.py index c484ed477..d771578fa 100644 --- a/galaxy/settings/default.py +++ b/galaxy/settings/default.py @@ -19,6 +19,7 @@ import os import djcelery +import prometheus_client djcelery.setup_loader() @@ -43,6 +44,8 @@ # Application definition # --------------------------------------------------------- +PROMETHEUS_EXPORT_MIGRATIONS = False + # TODO(cutwater): Review 3rd party apps usage INSTALLED_APPS = ( # Django apps @@ -65,6 +68,7 @@ # 3rd part apps 'bootstrapform', + 'django_prometheus', 'djcelery', 'rest_framework', 'rest_framework.authtoken', @@ -288,6 +292,26 @@ If set to `None`, system temporary directory is used. """ +# ========================================================= +# Metrics Settings +# ========================================================= + +METRICS_ENABLED = False + +PROM_CNTR_SEARCH = prometheus_client.Counter( + 'search', + '', + ['keywords', 'platforms', 'cloud_platforms', 'tags'], + registry=prometheus_client.REGISTRY +) + +PROM_CNTR_SEARCH_CRITERIA = prometheus_client.Counter( + 'search_criteria', + '', + ['ctype', 'cvalue'], + registry=prometheus_client.REGISTRY +) + # ========================================================= # Logging # ========================================================= @@ -368,6 +392,11 @@ 'level': 'DEBUG', 'propagate': True, }, + 'galaxy.common.metrics': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': True, + }, 'galaxy.main': { 'handlers': ['console'], 'level': 'DEBUG', diff --git a/galaxy/settings/development.py b/galaxy/settings/development.py index 4ad5bbed8..687e654d0 100644 --- a/galaxy/settings/development.py +++ b/galaxy/settings/development.py @@ -82,6 +82,15 @@ WAIT_FOR = [ {'host': 'postgres', 'port': 5432}, {'host': 'rabbitmq', 'port': 5672}, + {'host': 'influxdb', 'port': 8086}, + {'host': 'grafana', 'port': 3000}, + {'host': 'prometheus', 'port': 9090}, ] STATIC_ROOT = '' + +# ========================================================= +# Metrics Settings +# ========================================================= + +METRICS_ENABLED = True diff --git a/galaxy/settings/production.py b/galaxy/settings/production.py index 338292f11..5bea92f16 100644 --- a/galaxy/settings/production.py +++ b/galaxy/settings/production.py @@ -37,6 +37,7 @@ * GALAXY_RABBITMQ_USER * GALAXY_RABBITMQ_PASSWORD * GALAXY_ADMIN_PATH +* GALAXY_METRICS_ENABLED """ import os @@ -203,6 +204,12 @@ def _set_logging(): LOGGING = _set_logging() +# ========================================================= +# Metrics Settings +# ========================================================= + +METRICS_ENABLED = True + # ========================================================= # System Settings # ========================================================= diff --git a/requirements.txt b/requirements.txt index 60d7e2d2a..80367b54c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,3 +57,7 @@ Sphinx django-log-request-id # format log output as json jog + +# Web analytics/metrics +prometheus_client +django-prometheus diff --git a/scripts/docker/dev/compose.yml b/scripts/docker/dev/compose.yml index 6016cafbc..9e209a3e7 100644 --- a/scripts/docker/dev/compose.yml +++ b/scripts/docker/dev/compose.yml @@ -32,3 +32,32 @@ services: - RABBITMQ_DEFAULT_USER=galaxy - RABBITMQ_DEFAULT_PASS=galaxy - RABBITMQ_DEFAULT_VHOST=galaxy + + influxdb: + image: influxdb:latest + ports: + - '8086:8086' + environment: + - INFLUXDB_REPORTING_DISABLED=true + - INFLUXDB_DB=galaxy + - INFLUXDB_ADMIN_USER=admin + - INFLUXDB_ADMIN_PASSWORD=admin + - INFLUXDB_WRITE_USER=galaxy + - INFLUXDB_WRITE_USER_PASSWORD=galaxy + + grafana: + image: grafana/grafana:latest + ports: + - '3000:3000' + links: + - influxdb + - prometheus + + prometheus: + image: prom/prometheus:latest + ports: + - '9090:9090' + volumes: + - './prometheus.yml:/etc/prometheus/prometheus.yml' + links: + - galaxy:galaxy.svc diff --git a/scripts/docker/dev/prometheus.yml b/scripts/docker/dev/prometheus.yml new file mode 100644 index 000000000..d170318f9 --- /dev/null +++ b/scripts/docker/dev/prometheus.yml @@ -0,0 +1,11 @@ +scrape_configs: + - job_name: 'galaxy' + scrape_interval: '60s' + honor_labels: true + metrics_path: '/api/metrics' + static_configs: + - targets: + - 'galaxy.svc:8000' + +remote_write: + - url: "http://influxdb:8086/api/v1/prom/write?u=galaxy&p=galaxy&db=galaxy" diff --git a/scripts/metrics-setup-playbook/influxdb_init.py b/scripts/metrics-setup-playbook/influxdb_init.py new file mode 100644 index 000000000..bb5205adc --- /dev/null +++ b/scripts/metrics-setup-playbook/influxdb_init.py @@ -0,0 +1,44 @@ +from influxdb import InfluxDBClient + + +INFLUXDB_HOST = 'localhost' +INFLUXDB_PORT = 8086 +INFLUXDB_USER = 'galaxy' +INFLUXDB_PASS = 'galaxy' +INFLUXDB_DB = 'galaxy' + + +client = InfluxDBClient( + host=INFLUXDB_HOST, + port=INFLUXDB_PORT, + username=INFLUXDB_USER, + password=INFLUXDB_PASS +) + +client.create_database(INFLUXDB_DB) +client.switch_database(INFLUXDB_DB) + +# Retention policies +#client.query( +# "CREATE RETENTION POLICY a_month ON %s DURATION 30d REPLICATION 1" +# % INFLUXDB_DB +#) +client.query( + "CREATE RETENTION POLICY a_year ON %s DURATION 365d REPLICATION 1 DEFAULT" + % INFLUXDB_DB +) + +# Continuous queries +#client.query( +# "DROP CONTINUOUS QUERY cq_search_criteria_per_day ON %s" % INFLUXDB_DB +#) +# +#client.query( +# "CREATE CONTINUOUS QUERY cq_search_criteria_per_day ON %s " +# "RESAMPLE EVERY 30m " +# "BEGIN " +# "SELECT count(value) AS value INTO search_criteria_per_day FROM a_month.search_criteria " +# "GROUP BY time(1d), * " +# "END" +# % INFLUXDB_DB +#) diff --git a/scripts/metrics-setup-playbook/search-metrics-dashboard-influxdb.json b/scripts/metrics-setup-playbook/search-metrics-dashboard-influxdb.json new file mode 100644 index 000000000..5a78b1f3e --- /dev/null +++ b/scripts/metrics-setup-playbook/search-metrics-dashboard-influxdb.json @@ -0,0 +1,588 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "influxdb", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT TOP(value, cvalue, 10) FROM search_criteria WHERE ctype = 'keyword'", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Top 10 keywords for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "influxdb", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT TOP(value, cvalue, 10) FROM search_criteria WHERE ctype = 'tag'", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Top 10 tags for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "influxdb", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT TOP(value, cvalue, 10) FROM search_criteria WHERE ctype = 'platform'", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Top 10 platforms for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "influxdb", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 7, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT TOP(value, cvalue,10) FROM search_criteria WHERE ctype = 'cloud_platform'", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Top 10 cloud platforms for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "influxdb", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 4, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT * FROM search", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Raw search data - 'search' measurement", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "influxdb", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT * FROM search_criteria", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "Raw search data - 'search_criteria' measurement", + "transform": "table", + "type": "table" + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now/y", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Galaxy Search Metrics - InfluxDB", + "uid": "vAIJB6Kmz", + "version": 3 + }, + "meta": { + "canAdmin": true, + "canEdit": true, + "canSave": true, + "canStar": true, + "created": "2018-08-20T12:32:07Z", + "createdBy": "admin", + "expires": "0001-01-01T00:00:00Z", + "folderId": 0, + "folderTitle": "General", + "folderUrl": "", + "hasAcl": false, + "isFolder": false, + "provisioned": false, + "slug": "galaxy-search-metrics-influxdb", + "type": "db", + "updated": "2018-08-20T13:04:07Z", + "updatedBy": "admin", + "url": "/d/vAIJB6Kmz/galaxy-search-metrics-influxdb", + "version": 3 + } +} diff --git a/scripts/metrics-setup-playbook/search-metrics-dashboard-prometheus.json b/scripts/metrics-setup-playbook/search-metrics-dashboard-prometheus.json new file mode 100644 index 000000000..4142ce51a --- /dev/null +++ b/scripts/metrics-setup-playbook/search-metrics-dashboard-prometheus.json @@ -0,0 +1,407 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "prometheus", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 6, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10, search_criteria{ctype=\"keyword\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Top 10 keywords for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "prometheus", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 6, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 6, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10, search_criteria{ctype=\"tag\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Top 10 tags for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "prometheus", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 7, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 6, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10, search_criteria{ctype=\"platform\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Top 10 platforms for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "prometheus", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 6, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk(10, search_criteria{ctype=\"cloud_platform\"})", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Top 10 cloud platforms for the past week", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "prometheus", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "search", + "format": "table", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Raw search data - 'search' metric", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "prometheus", + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 3, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "search_criteria", + "format": "table", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Raw search data - 'search_criteria' metric", + "transform": "table", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now/w", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Galaxy Search Metrics - Prometheus", + "uid": "al-z02tik", + "version": 6 + }, + "meta": { + "canAdmin": true, + "canEdit": true, + "canSave": true, + "canStar": true, + "created": "2018-08-22T08:50:23Z", + "createdBy": "admin", + "expires": "0001-01-01T00:00:00Z", + "folderId": 0, + "folderTitle": "General", + "folderUrl": "", + "hasAcl": false, + "isFolder": false, + "provisioned": false, + "slug": "galaxy-search-metrics-prometheus", + "type": "db", + "updated": "2018-08-22T08:56:26Z", + "updatedBy": "admin", + "url": "/d/al-z02tik/galaxy-search-metrics-prometheus", + "version": 6 + } +} diff --git a/scripts/metrics-setup-playbook/setup.yml b/scripts/metrics-setup-playbook/setup.yml new file mode 100644 index 000000000..0881df1e0 --- /dev/null +++ b/scripts/metrics-setup-playbook/setup.yml @@ -0,0 +1,47 @@ +- hosts: localhost + gather_facts: no + tasks: + - name: Check python-influxdb is installed + command: python -c 'import influxdb' + register: influxdb_installed + ignore_errors: yes + + - fail: + msg: "The InfluxDB client seems to be not installed. Run 'pip install influxdb' to install it." + when: influxdb_installed.rc != 0 + + - name: InfluxDB init + command: python influxdb_init.py + + - name: Add Prometheus datasource + grafana_datasource: + name: prometheus + grafana_url: http://localhost:3000 + ds_type: prometheus + url: http://prometheus:9090 + time_interval: ">10s" + + - name: Add InfluxDB datasource + grafana_datasource: + name: influxdb + grafana_url: http://localhost:3000 + ds_type: influxdb + url: http://influxdb:8086 + database: galaxy + user: galaxy + password: galaxy + time_interval: ">10s" + + - name: Import Prometheus dashboard + grafana_dashboard: + grafana_url: http://localhost:3000 + state: present + message: Imported by Ansible + path: search-metrics-dashboard-prometheus.json + + - name: Import InfluxDB dashboard + grafana_dashboard: + grafana_url: http://localhost:3000 + state: present + message: Imported by Ansible + path: search-metrics-dashboard-influxdb.json