Here is an exemple i took from nomad website on monitoring section :
1- in this exemple i try to install prometheus with a docker image
prom/prometheus:latest and copy file webserver_alert.yml in docker image
2- copy paste the code below in visual studio code to see the ligne
3- At line 20 we created file webserver_alert.yml in the nomad
file system
4-from line 22 to 91 the content of yml file
5- we ask nomad to mount local/webserver_alert.yml in the docker
file system /etc/prometheus/webserver_alert.yml at startup of
docker
job "prometheus" {
datacenters = ["dc1"]
type = "service"
group "monitoring" {
count = 1
restart {
attempts = 2
interval = "30m"
delay = "15s"
mode = "fail"
}
ephemeral_disk {
size = 300
}
task "prometheus" {
template {
change_mode = "noop"
destination = "local/webserver_alert.yml"
data = <<EOH
---
groups:
- name: prometheus_alerts
rules:
- alert: Webserver down
expr: absent(up{job="webserver"})
for: 10s
labels:
severity: critical
annotations:
description: "Our webserver is down."
EOH
}
template {
change_mode = "noop"
destination = "local/prometheus.yml"
data = <<EOH
---
global:
scrape_interval: 5s
evaluation_interval: 5s
alerting:
alertmanagers:
- consul_sd_configs:
- server: '{{ env "NOMAD_IP_prometheus_ui" }}:8500'
services: ['alertmanager']
rule_files:
- "webserver_alert.yml"
scrape_configs:
- job_name: 'alertmanager'
consul_sd_configs:
- server: '{{ env "NOMAD_IP_prometheus_ui" }}:8500'
services: ['alertmanager']
- job_name: 'nomad_metrics'
consul_sd_configs:
- server: '{{ env "NOMAD_IP_prometheus_ui" }}:8500'
services: ['nomad-client', 'nomad']
relabel_configs:
- source_labels: ['__meta_consul_tags']
regex: '(.*)http(.*)'
action: keep
scrape_interval: 5s
metrics_path: /v1/metrics
params:
format: ['prometheus']
- job_name: 'webserver'
consul_sd_configs:
- server: '{{ env "NOMAD_IP_prometheus_ui" }}:8500'
services: ['webserver']
metrics_path: /metrics
- job_name: 'node_exporter'
static_configs:
- targets: ['167.86.106.9:9100','167.86.106.9:9107']
EOH
}
driver = "docker"
config {
image = "prom/prometheus:latest"
volumes = [
"local/webserver_alert.yml:/etc/prometheus/webserver_alert.yml",
"local/prometheus.yml:/etc/prometheus/prometheus.yml"
]
port_map {
prometheus_ui = 9090
}
}
resources {
network {
mbits = 10
port "prometheus_ui" {}
}
}
service {
name = "prometheus"
tags = ["urlprefix-/"]
port = "prometheus_ui"
check {
name = "prometheus_ui port alive"
type = "http"
path = "/-/healthy"
interval = "10s"
timeout = "2s"
}
}
}
}
}
if other people have #nomad - #consul - #vault or hashistack issue tag me stackoverflow