Merge pull request #20 from pipech/dev-monitor_tool

Added server monitor tool

docs/_includes/ec2.md

@@ -2,4 +2,6 @@ Guide to setup EC2 instance for running erpnext-docker-debian image.
 
 * [Setup AWS EC2](/erpnext-docker-debian/setup_ec2)
 
+* [Setup Node Exporter on AWS EC2](/erpnext-docker-debian/setup_node_exporter_ec2)
+
 * [Backup AWS EC2 to AMI](https://github.com/pipech/aws-lambda-ami-backup)

docs/setup/production_setup.md

@@ -216,10 +216,10 @@ and domain pointing to those IP address, otherwise it won't works.**
 
     `mysql -u "root" "-p<your_password>" < "/home/init.sql"`
 
-### Install Custom-app
+### Misc
 
 * [Install custom app](/erpnext-docker-debian/create_custom_app_image)
 
-### Backing up
+* [Backing up docker volume](/erpnext-docker-debian/backup)
 
-* [Backing up docker volume](/erpnext-docker-debian/backup)
+* [Setup up monitoring tools](/erpnext-docker-debian/setup_monitoring)

docs/setup_monitoring.md

@@ -0,0 +1,25 @@
+---
+layout: page
+title: Setting up monitoring tools
+permalink: /setup_monitoring/
+---
+
+**Setting up monitoring tools**
+
+* Setup prometheus.yml
+
+    setup aws key and secret key on ./production_setup/conf/prometheus-conf/prometheus.yml
+    if you're not using aws you colud setup node-exporter using node-exporter docker image
+
+* Setup frontend rules in monitor.yml
+
+    `- "traefik.frontend.rule=Host:graf.example1.com"`
+
+* Start docker stack
+
+    `docker stack deploy -c monitor.yml <stack_name>`
+
+* Login
+
+    Username: admin
+    Password: admin

docs/setup_node-exporter.md

@@ -0,0 +1,95 @@
+---
+layout: page
+title: Setup Node Exporter on AWS EC2
+permalink: /setup_node_exporter_ec2/
+---
+
+**Install go prerequisite**
+
+* Install prerequisite
+
+    `sudo yum install -y glibc-static`
+
+**Install go and dependencies**
+
+* Install go
+
+    `sudo amazon-linux-extras install -y golang1.9`
+
+* Update go to version 1.10.3
+
+    unfortunately we need go version > 1.9.4
+    https://github.com/prometheus/node_exporter/issues/880
+
+    ```
+    # check where go is
+    whereis go
+
+    # remove go
+    sudo rm -rf /usr/bin/go
+
+    # get go https://golang.org/dl/
+    wget -c https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz
+
+    # extract go
+    sudo tar -C /usr/bin -xvzf go1.10.3.linux-amd64.tar.gz
+
+    # set go path
+    export PATH=$PATH:/usr/bin/go/bin
+
+    # check go version
+    go version
+
+    # add go path to bash profile
+    nano .bash_profile
+    export PATH=$PATH:/usr/bin/go/bin
+    ```
+
+**Install node_exporter**
+
+* install node_exporter
+
+    ```
+    go get github.com/prometheus/node_exporter
+    cd ${GOPATH-$HOME/go}/src/github.com/prometheus/node_exporter
+    make
+    ```
+
+* test node_exporter
+
+    `/home/ec2-user/go/src/github.com/prometheus/node_exporter/node_exporter`
+
+* create systemd service file for Node Exporter
+
+    `sudo nano /etc/systemd/system/node_exporter.service`
+
+    ```
+    [Unit]
+    Description=Node Exporter
+
+    [Service]
+    User=ec2-user
+    ExecStart=/home/ec2-user/go/src/github.com/prometheus/node_exporter/node_exporter
+
+    [Install]
+    WantedBy=multi-user.target
+    ```
+
+* run node_exporter by systemctl
+
+    ```
+    sudo systemctl daemon-reload
+    sudo systemctl start node_exporter
+    sudo systemctl enable node_exporter
+    ```
+
+* restart server
+
+    `sudo reboot`
+
+* check node_exporter
+
+    ```
+    sudo systemctl status node_exporter
+    curl localhost:9100/metrics
+    ```

production_setup/conf/grafana-conf/datasource.yml

@@ -0,0 +1,7 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090

production_setup/conf/prometheus-conf/prometheus.yml

@@ -0,0 +1,27 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'cadvisor'
+    scrape_interval: 5s
+    static_configs:
+         - targets: ['cadvisor:8484']
+
+  - job_name: 'traefik'
+    scrape_interval: 5s
+    static_configs:
+         - targets: ['traefik:8080']
+
+  - job_name: 'node-exporter'
+    scrape_interval: 5s
+    ec2_sd_configs:
+      # acccess_key and secret_key contain the keys for a user created in IAM with AWS managed AmazonEC2ReadOnlyAccess policy.
+      - region: aws_region
+        access_key: aws_key
+        secret_key: secret_key
+        port: 9100

production_setup/monitor.yml

@@ -0,0 +1,63 @@
+version: '3.3'
+
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - prometheus-data-volumes:/prometheus
+    networks:
+      - monitor
+      - traefik_proxy
+    configs:
+      - source: prometheus-conf
+        target: /etc/prometheus/prometheus.yml
+
+  cadvisor:
+    # latest version is not the newest as 2018/07/25
+    # [0.29.1 and above] has error > "cAdvisor cannot reach containerd.sock" > https://github.com/google/cadvisor/issues/1910
+    image: google/cadvisor:v0.28.5
+    volumes:
+      - /dev/disk/:/dev/disk:ro
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      # Fixed for "Failed to start container manager: inotify_add_watch /sys/fs/cgroup/cpuset: no such file or directory"
+      # https://github.com/google/cadvisor/issues/1850
+      # - /cgroup:/sys/fs/cgroup:ro
+    command:
+      - '--port=8484'
+    networks:
+      - monitor
+
+  grafana:
+    image: grafana/grafana:latest
+    networks:
+      - traefik_proxy
+      - monitor
+    volumes:
+      - grafana-data-volumes:/var/lib/grafana
+    configs:
+      - source: grafana-datasource-conf
+        target: /etc/grafana/provisioning/datasources/datasource.yml
+    labels:
+      - "traefik.enable=true"
+      - "traefik.frontend.rule=Host:graf.example1.com"
+      - "traefik.port=3000"
+      - "traefik.docker.network=traefik_proxy"
+
+volumes:
+  prometheus-data-volumes:
+  grafana-data-volumes:
+
+configs:
+  prometheus-conf:
+    file: ./conf/prometheus-conf/prometheus.yml
+  grafana-datasource-conf:
+    file: ./conf/grafana-conf/datasource.yml
+
+networks:
+  traefik_proxy:
+    external: true
+  monitor:
+    driver: overlay