Monitoring with grafana

 Tool required:

1. Grafana
2. Prometheus
3. Alertmanager
4. Node exporter
5. service script

Grafana

Prometheus

cat /etc/prometheus/prometheus.yml  
# my global config
global:
 scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
 evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
 # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
 alertmanagers:
 - static_configs:
   - targets: ['localhost:9093']


# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
 # - "first_rules.yml"
 # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
 # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.

 - job_name: "prometheus"
   static_configs:
     - targets: ["localhost:9090"] # Prometheus self-monitoring metrics

 # Add a new job configuration for Node Exporter
 - job_name: "node_exporter"
   static_configs:
     - targets: ["localhost:9100"] # Node Exporter metrics endpoint

 - job_name: 'alertmanager'
   static_configs:
   - targets: ['localhost:9093']

Alertmanager .

cat /opt/alertmanager/alertmanager.yml
route:
 group_by: ['alertname']
 group_wait: 30s
 group_interval: 5m
 repeat_interval: 1m
 receiver: 'gmail'
receivers:
- name: 'web.hook'
 webhook_configs:
 - url: 'http://127.0.0.1:5001/'
- name: 'gmail'
 email_configs:
 - to: 'manjeetxxxxx19@gmail.com'
   from: 'manjeetyadavtemp3@gmail.com'
   smarthost: smtp.gmail.com:587
   auth_username: 'manjeetxxxxxtemp3@gmail.com'
   auth_identity: 'manjeetxxxxxtemp3@gmail.com'
   auth_password: 'abc'     

inhibit_rules:
 - source_match:
     severity: 'critical'
   target_match:
     severity: 'warning'
   equal: ['alertname', 'dev', 'instance']

systemctl restart alertmanager.service

Node exporter

cat /etc/systemd/system/node_exporter.service

[Unit]

Description=Prometheus Node Exporter

Wants=network-online.target

After=network-online.target

[Service]

User=nodeusr

ExecStart=/usr/local/bin/node_exporter --collector.textfile.directory=/var/lib/node_exporter/textfile_collector

[Install]

WantedBy=default.target

Create Textfile Collector Directory: Node Exporter has a textfile collector directory where custom scripts can drop metrics in a .prom file.

sudo mkdir -p /var/lib/node_exporter/textfile_collector

sudo adduser 

sudo chown -R nodeusr:nodeusr /var/lib/node_exporter

sudo systemctl daemon-reload

sudo systemctl enable node_exporter

sudo systemctl start node_exporter

service script

cat /usr/local/bin/check_docker_status.sh
#!/bin/bash
if systemctl is-active --quiet docker; then
 echo 'docker_status 1' > /var/lib/node_exporter/textfile_collector/docker_status.prom
else
 echo 'docker_status 0' > /var/lib/node_exporter/textfile_collector/docker_status.prom
fi

cat /usr/local/bin/check_elasticsearch_status.sh  
#!/bin/bash
if systemctl is-active --quiet elasticsearch; then
 echo 'elasticsearch_status 1' > /var/lib/node_exporter/textfile_collector/elasticsearch_status.prom
else
 echo 'elasticsearch_status 0' > /var/lib/node_exporter/textfile_collector/elasticsearch_status.prom
fi

cat /usr/local/bin/check_mysql_status.sh      
#!/bin/bash

# File to store the MySQL status
OUTPUT_FILE="/var/lib/node_exporter/textfile_collector/mysql_status.prom"

# Check MySQL status
if systemctl is-active --quiet mysql; then
   echo 'mysql_status 1' > $OUTPUT_FILE
else
   echo 'mysql_status 0' > $OUTPUT_FILE
fi

sudo chmod +x /usr/local/bin/check_docker_status.sh

sudo chmod +x /usr/local/bin/check_elasticsearch_status.sh

sudo chmod +x /usr/local/bin/check_mysql_status.sh

cron the script for monitoring service:

* * * * * /usr/local/bin/check_docker_status.sh

* * * * * /usr/local/bin/check_elasticsearch_status.sh