Prometheus

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

# yum search prome|grep exporter
apache_exporter.x86_64 : Prometheus exporter Apache webserver mertics.
artifactory_exporter.x86_64 : Prometheus exporter for JFrog Artifactory stats.
bareos_exporter.x86_64 : Prometheus exporter for BareOS data recovery system
bind_exporter.x86_64 : Prometheus exporter for Bind nameserver
collectd_exporter.x86_64 : Collectd stats exporter for Prometheus.
consul_exporter.x86_64 : Consul stats exporter for Prometheus.
couchbase_exporter.x86_64 : Prometheus exporter for Couchbase server metrics.
domain_exporter.x86_64 : Prometheus exporter for domain expiration time metrics
ebpf_exporter.x86_64 : Prometheus exporter for custom eBPF metrics
elasticsearch_exporter.x86_64 : Elasticsearch stats exporter for Prometheus.
exporter_exporter.x86_64 : Simple reverse proxy for Prometheus exporters
frr_exporter.x86_64 : Prometheus exporter for FRR metrics
golang-github-prometheus-node-exporter.x86_64 : Exporter for machine metrics
graphite_exporter.x86_64 : Server that accepts metrics via the Graphite protocol
haproxy_exporter.x86_64 : This is a simple server that scrapes HAProxy stats and
influxdb_exporter.x86_64 : InfluxDB stats exporter for Prometheus.
iperf3_exporter.x86_64 : Prometheus exporter for iPerf3 probing.
jmx_exporter.noarch : Prometheus exporter for mBeans scrape and expose.
jolokia_exporter.x86_64 : Prometheus exporter for jolokia metrics
json_exporter.x86_64 : A Prometheus exporter which scrapes remote JSON by
junos_exporter.x86_64 : Prometheus exporter for Junos device metrics.
kafka_exporter.x86_64 : Kafka exporter for Prometheus.
keepalived_exporter.x86_64 : Prometheus exporter for Keepalived metrics
memcached_exporter.x86_64 : Memcached stats exporter for Prometheus.
mongodb_exporter.x86_64 : A Prometheus exporter for MongoDB including sharding,
mysqld_exporter.x86_64 : Prometheus exporter for MySQL server metrics.
nginx_exporter.x86_64 : NGINX Prometheus Exporter for NGINX and NGINX Plus.
node_exporter.x86_64 : Prometheus exporter for machine metrics, written in Go
openstack_exporter.x86_64 : Prometheus exporter for OpenStack metrics.
pgbouncer_exporter.x86_64 : Prometheus exporter for PgBouncer.
phpfpm_exporter.x86_64 : A prometheus exporter for PHP-FPM. The exporter
postgres_exporter.x86_64 : Prometheus exporter for PostgreSQL server metrics
process_exporter.x86_64 : Process exporter for Prometheus.
rabbitmq_exporter.x86_64 : Prometheus exporter for RabbitMQ metrics
redis_exporter.x86_64 : Prometheus exporter for Redis server metrics.
snmp_exporter.x86_64 : Prometheus SNMP exporter.
ssl_exporter.x86_64 : Prometheus exporter for SSL certificates.
statsd_exporter.x86_64 : Export StatsD metrics in Prometheus format.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

yum -y install node_exporter
# 设置下开机自启动
cat <<EOF > /etc/systemd/system/node_exporter.service
[Unit]
Description = Prometheus node exporter with machine statistics

[Service]
ExecStart = /usr/bin/node_exporter

[Install]
WantedBy = multi-user.target
EOF
systemctl daemon-reload
systemctl enable node_exporter

1
2

yum -y install blackbox_exporter
systemctl start blackbox_exporter && systemctl enable blackbox_exporter

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# /etc/prometheus/prometheus.yml
- job_name: 'consul-prometheus'
  consul_sd_configs:
    - server: 'localhost:8500'
      services: []
  relabel_configs:
    - source_labels: [__meta__consul_tags]
      regex: ^xxx.*
      action: keep
    - regex: __meta_consul_service_metadata_(.+)
      action: labelmap

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ['localhost:9090']
  #- job_name: node
  #  static_configs:
  #    - targets: ['localhost:9100']

  #- job_name: windows
  #  static_configs:
  #    - targets: ['172.16.27.68:9182']
  - job_name: 'consul-prometheus'
    consul_sd_configs:
      - server: 'localhost:8500'
        services: ["node-exporter-172.16.27.71"]
    relabel_configs:
      - source_labels: [__meta__consul_tags]
        regex: .*
        action: keep
      - regex: __meta_consul_service_metadata_(.+)
        action: labelmap

1
2
3
4
5
6
7

yum -y install alertmanager
systemctl start alertmanager && systemctl enable alertmanager
#prometheus.yml增加如下配置并reload
alerting:
  alertmanagers:
    - static_configs:
        - targets: ['localhost:9093']

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

global:
  # The smarthost and SMTP sender used for mail notifications.
  smtp_smarthost: 'localhost:25'
  smtp_from: 'alertmanager@example.org'
  smtp_auth_username: 'alertmanager'
  smtp_auth_password: 'password'

# The directory from which notification templates are read.
templates:
- '/etc/alertmanager/template/*.tmpl'

# The root route on which each incoming alert enters.
route:
  # The labels by which incoming alerts are grouped together. For example,
  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
  # be batched into a single group.
  #
  # To aggregate by all possible labels use '...' as the sole label name.
  # This effectively disables aggregation entirely, passing through all
  # alerts as-is. This is unlikely to be what you want, unless you have
  # a very low alert volume or your upstream notification system performs
  # its own grouping. Example: group_by: [...]
  group_by: ['alertname', 'cluster', 'service']

  # When a new group of alerts is created by an incoming alert, wait at
  # least 'group_wait' to send the initial notification.
  # This way ensures that you get multiple alerts for the same group that start
  # firing shortly after another are batched together on the first
  # notification.
  group_wait: 30s

  # When the first notification was sent, wait 'group_interval' to send a batch
  # of new alerts that started firing for that group.
  group_interval: 30s

  # If an alert has successfully been sent, wait 'repeat_interval' to
  # resend them.
  repeat_interval: 30s

  # A default receiver
  receiver: team-X-mails

  # All the above attributes are inherited by all child routes and can
  # overwritten on each.

  # The child route trees.
  routes:
  # This routes performs a regular expression match on alert labels to
  # catch alerts that are related to a list of services.
  - matchers:
    - service=~"foo1|foo2|baz"
    receiver: team-X-mails
    # The service has a sub-route for critical alerts, any alerts
    # that do not match, i.e. severity != critical, fall-back to the
    # parent node and are sent to 'team-X-mails'
    routes:
    - matchers:
      - severity="critical"
      receiver: team-X-pager
  - matchers:
    - service="files"
    receiver: team-Y-mails

    routes:
    - matchers:
      - severity="critical"
      receiver: team-Y-pager

  # This route handles all alerts coming from a database service. If there's
  # no team to handle it, it defaults to the DB team.
  - matchers:
    - service="database"
    receiver: team-DB-pager
    # Also group alerts by affected database.
    group_by: [alertname, cluster, database]
    routes:
    - matchers:
      - owner="team-X"
      receiver: team-X-pager
      continue: true
    - matchers:
      - owner="team-Y"
      receiver: team-Y-pager


# Inhibition rules allow to mute a set of alerts given that another alert is
# firing.
# We use this to mute any warning-level notifications if the same alert is
# already critical.
inhibit_rules:
- source_matchers: [ severity="critical" ]
  target_matchers: [ severity="warning" ]
  # Apply inhibition if the alertname is the same.
  # CAUTION:
  #   If all label names listed in `equal` are missing
  #   from both the source and target alerts,
  #   the inhibition rule will apply!
  equal: [ alertname, cluster, service ]


receivers:
- name: 'team-X-mails'
  email_configs:
  - to: 'team-X+alerts@example.org'

- name: 'team-X-pager'
  email_configs:
  - to: 'team-X+alerts-critical@example.org'
  pagerduty_configs:
  - service_key: <team-X-key>

- name: 'team-Y-mails'
  email_configs:
  - to: 'team-Y+alerts@example.org'

- name: 'team-Y-pager'
  pagerduty_configs:
  - service_key: <team-Y-key>

- name: 'team-DB-pager'
  pagerduty_configs:
  - service_key: <team-DB-key>

1
2
3
4
5
6
7
8
9

yum -y install consul_exporter
systemctl start consul_exporter && systemctl enable consul_exporter
# prometheus.yml追加job
  - job_name: "consul"
    static_configs:
      - targets: ["localhost:9107"]
curl localhost:9107/metrics|grep -i consul
# 会发现很多consul相关的指标
# 这个时候继续设置后面的rules

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

global:
  smtp_smarthost: 'smtp.qiye.aliyun.com:465'
  smtp_from: 'lqx@liuliancao.com'
  smtp_require_tls: false
  smtp_auth_username: 'lqx@liuliancao.com'
  smtp_auth_password: 'blog.liuliancao.com'
# 中间的routes自己写哈，检查通过promtool检查好了
receivers:
- name: "liuliancao-dev-mails"
  email_configs:
  - to: "lqx@liuliancao.com"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

cd /tmp
wget https://ghproxy.com/https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v2.0.0/prometheus-webhook-dingtalk-2.0.0.linux-amd64.tar.gz
tar xf prometheus-webhook-dingtalk-2.0.0.linux-amd64.tar.gz
cd prometheus-webhook-dingtalk-2.0.0.linux-amd64

cp  prometheus-webhook-dingtalk /usr/local/bin/
# edit something about config.yml like
## Request timeout
# timeout: 5s

## Uncomment following line in order to write template from scratch (be careful!)
#no_builtin_template: true

## Customizable templates path
#templates:
#  - contrib/templates/legacy/template.tmpl

## You can also override default template using `default_message`
## The following example to use the 'legacy' template from v0.3.0
#default_message:
#  title: '{{ template "legacy.title" . }}'
#  text: '{{ template "legacy.content" . }}'

## Targets, previously was known as "profiles"
targets:
  cf-ops-test:
    url: https://oapi.dingtalk.com/robot/send?access_token=cf603722233780cc901b8935be2848dd6aa338d1c17ae856863bc361fbb6a99a
    # secret for signature
    secret: SEC86a6afe11c5414e411c2b1e4ab8ef51516126fd845c4e63df7ca0850a0d286c0
  webhook_legacy:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    # Customize template content
    message:
      # Use legacy template
      title: '{{ template "legacy.title" . }}'
      text: '{{ template "legacy.content" . }}'
  webhook_mention_all:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    mention:
      all: true
  webhook_mention_users:
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    mention:
      mobiles: ['156xxxx8827', '189xxxx8325']
cat <<EOF >/etc/systemd/system/alertmanager-dingtalk.service
[Unit]
Description = Prometheus node exporter with machine statistics

[Service]
ExecStart = /usr/local/bin/prometheus-webhook-dingtalk --config.file=config.yml

[Install]
WantedBy = multi-user.target

EOF
systemctl daemon-reload && systemctl enable alertmanager-dingtalk && systemctl start alertmanager-dingtalk

1
2
3

# cat /etc/default/prometheus
PROMETHEUS_OPTS='--config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus/data --web.console.libraries=/usr/share/prometheus/console_libraries\
 --web.console.templates=/usr/share/prometheus/consoles --web.external-url=http://xxxx:9090/'

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

cat /etc/prometheus/rules/alert-consul.rules
groups:
- name: consul
  rules:
  - alert: 服务器Consul状态监控
    expr: consul_serf_lan_member_status{job="consul"} != 1
    for: 15s
    labels:
      severity: critical
    annotations:
      summary: 服务器{{ $labels.instance }}服务异常
      description: this is description

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

#!/usr/bin/env bash 
# used for probe mtr in linux by lqx at 2022-07-04.
#
# PROBE_TYPE:
# default
# tcp
# udp
# PROBE_HOST:
# IP
# IP:PORT

while getopts h:p:t:s:k:r: OPTION
do
    case $OPTION in
        p)PROBE_TYPE=$OPTARG;;
        h)PROBE_HOST=$OPTARG;;
	k)PROBE_PORT=$OPTARG;;
        t)PROBE_TIMEOUT=$OPTARG;;
	s)ES_SERVER=$OPTARG;;
	r)REGION=$OPTARG;;
        ?)echo "use bash $0 -p PROBE_TYPE -h PROBE_HOST -t PROBE_TIMEOUT -k PROBE_PORT -s ES_SERVER -r REGION" && exit 1;;
    esac
done
[[ -z $PROBE_HOST ]] && echo use bash $0 -p PROBE_TYPE -h PROBE_HOST -t PROBE_TIMEOUT -k PROBE_PORT -s UPLOAD_SERVER -h PROBE_HOST cannot be null ! && exit 2
[[ -z $PROBE_TYPE ]] && PROBE_TYPE="default"
[[ -z $PROBE_PORT ]] && PROBE_PORT=0

if [[ $PROBE_TYPE == "default" ]];then
    traceroute_out=$(timeout $PROBE_TIMEOUT traceroute $PROBE_HOST)
elif [[ $PROBE_TYPE == "tcp" ]];then
    traceroute_out=$(timeout $PROBE_TIMEOUT traceroute -T $PROBE_HOST -p $PROBE_PORT)
elif [[ $PROBE_TYPE == "udp" ]];then
    traceroute_out=$(timeout $PROBE_TIMEOUT traceroute -U $PROBE_HOST -p $PROBE_PORT)
else
    echo unsupport probe type $PROBE_TYPE && exit 3
fi
traceroute_result=$(echo $traceroute_out|sed 's/\n/ /g')

cat <<EOF | curl -XPOST -H "Content-Type: application/json" --data-binary @- $ES_SERVER/traceroute-statistics/doc
{"region": "$REGION","traceroute": "$traceroute_result", "protocol": "$PROBE_TYPE", "host": "$PROBE_HOST", "port": "$PROBE_PORT"}
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

cd /usr/local/bin
wget https://ghproxy.com/https://github.com/alxrem/prometheus-logstash-exporter/releases/download/0.7.0/prometheus-logstash-exporter_0.7.0_linux_amd64
chmod u+x prometheus-logstash-exporter/releases/download/0.7.0/prometheus-logstash-exporter_0.7.0_linux_amd64
cat <<EOF >/etc/systemd/system/logstash-exporter.service 
[Unit]
Description=Prometheus logstash exporter not offical
Documentation=https://github.com/alxrem/prometheus-logstash-exporter
After=network-online.target
[Service]
Type=simple
ExecStart=/usr/local/bin/prometheus-logstash-exporter_0.7.0_linux_amd64 -logstash.host localhost -logstash.port 9600
Restart=always
RestartSecs=30s
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload && systemctl start logstash-exporter
systemctl status logstash-exporter

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

# logstash_exporter.pp
define logstash_exporter_init($port,) {
  notify { "port is $port.": }
  $exporter_port = $port - 9600 + 9304
  file { "/etc/systemd/system/logstash_exporter_${port}.service":
    content => template("prometheus/logstash_exporter/logstash_exporter.service.erb"),
    ensure => file,
    mode => '0644',
    notify => Class['systemd::daemon_reload'],
  }

  service { "logstash_exporter_${port}":
    ensure => running,
    hasrestart => true,
    hasstatus => true,
    subscribe => [Class['systemd::daemon_reload'], File['/usr/local/bin/logstash_exporter']],
  }
}

class prometheus::logstash_exporter($ports=[9600], ) { 
  include systemd::daemon_reload
  file { "/usr/local/bin/logstash_exporter":
    source => "puppet:///modules/prometheus/logstash_exporter/prometheus-logstash-exporter_0.7.0_linux_amd64",
    ensure => file,
    purge => true,
    mode => '0700',
  }
  each($ports) |$value| {  
    logstash_exporter_init{ "logstash-$value":
       port => $value 
    }
  }
}

# 引用
node 'logstash-server01' {
  class { 'prometheus::logstash_exporter':
    ports => [9600, 9602],
  }
}

1
2

# cat /etc/default/elasticsearch_exporter 
ELASTICSEARCH_EXPORTER_OPTS="--es.uri=http://xxx.xxx.xxx.xxx:9200 --es.all --es.cluster_settings --es.indices"