基于Docker部署Prometheus

1.部署Docker（使用docker仓库进行安装）

1.设置仓库 （安装所需的软件包）
yum install -y yum-utils device-mapper-persistent-data  lvm2
2.添加仓库
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
3.列出并排序您存储库中可用的版本。此示例按版本号（从高到低）对结果进行排序
yum list docker-ce --showduplicates | sort -r
4.安装Docker
yum install docker-ce-26.1.4-1.el7  docker-ce-cli-26.1.4-1.el7 containerd.io
5.设置开机自启动
systemctl enable docker
6.启动Docker
systemctl start docker
7.配置Docker镜像加速器
国内镜像加速器地址
网易：https://hub-mirror.c.163.com/
阿里云：https://<你的ID>.mirror.aliyuncs.com
七牛云加速器：https://reg-mirror.qiniu.com

[root@prometheus docker-compose]# cat /etc/docker/daemon.json 
{
    "registry-mirrors": [
        "https://docker.1ms.run",
        "https://docker.xuanyuan.me"
    ],
    "data-root": "/data/docker"
}

systemctl daemon-reload
systemctl restart docker
8.测试Docker
[root@prometheus docker-compose]# docker run hello-world
Unable to find image 'hello-world:latest' locally
latest: Pulling from library/hello-world

[root@prometheus docker-compose]# docker images
REPOSITORY    TAG       IMAGE ID       CREATED       SIZE
hello-world   latest    74cc54e27dc4   8 weeks ago   10.1kB

2.安装docker-compose

1.sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
2.sudo chmod +x /usr/local/bin/docker-compose
3.docker-compose --version

3.准备镜像

1.准备工作目录
[root@prometheus_servers_131 ~]# mkdir /data/docker-compose/
[root@prometheus_servers_131 ~]# cd /data/docker-compose/
[root@prometheus_servers_131 docker-compose]# mkdir {alertmanager,prometheus,data/prometheus/wal} -p
2.修改配置文件
[root@prometheus_servers_131 docker-compose]# cat alertmanager/alertmanager.yml
global:
  resolve_timeout: 30s
  smtp_smarthost: 'smtp.163.com:25' 
  smtp_from: 'your_email' 
  smtp_auth_username: 'your_email' 
  smtp_auth_password: 'VZffFwPPQAWzAPhw' # 邮箱授权码，需要在设置，开启SMTP服务，然后复制授权码到这里
  smtp_hello: '163.com'
  smtp_require_tls: false

templates:
  - '/etc/alertmanager/email.tmpl'

route:
  group_by: ['env','instance','type','group','job','alertname']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 40s
  receiver: 'email'

receivers:
  - name: 'email'
    email_configs:
      - to: 'your_email'
        html: '{{ template "email.to.html" . }}'
        send_resolved: true

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']
    
    
 [root@prometheus_servers_131 docker-compose]# cat alertmanager/email.tmpl
{{ define "email.to.html" }}
{{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }}
<h2>@告警通知</h2>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} <br>
告警类型: {{ .Labels.alertname }} <br>
故障主机: {{ .Labels.instance }} <br>
告警主题: {{ .Annotations.summary }} <br>
告警详情: {{ .Annotations.description }} <br>
触发时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} <br>
{{ end }}{{ end -}}
{{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }}
<h2>@告警恢复</h2>
告警程序: prometheus_alert <br>
告警级别: {{ .Labels.severity }} <br>
故障主机: {{ .Labels.instance }}<br>
故障主题: {{ .Annotations.summary }}<br>
告警详情: {{ .Annotations.description }}<br>
告警时间: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}<br>
恢复时间: {{ .EndsAt.Local.Format "2006-01-02 15:04:05" }}<br>
{{ end }}{{ end -}}
{{- end }}

[root@prometheus_servers_131 docker-compose]# cat prometheus/prometheus.yml 
global:
  scrape_interval:     1s
  evaluation_interval: 1s
rule_files:
- /etc/prometheus/rules.yml
alerting:
  alertmanagers:
    - static_configs:
      - targets: ['10.0.0.131:9093']
scrape_configs:

  - job_name: 'prometheus'
    static_configs:  
      - targets: ['10.0.0.131:9090']

  - job_name: 'node-export'
    static_configs:  
      - targets: ['10.0.0.131:9100']

[root@prometheus_servers_131 docker-compose]# cat prometheus/rules.yml
groups:
- name: server-alarm
  rules:
  - alert: "内存告警"
    expr: (1 - (node_memory_MemAvailable_bytes / (node_memory_MemTotal_bytes))) * 100 > 68
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高内存 使用率！"
      description: "{{$labels.instance}}: 内存使用率在 80% 以上 (当前使用值为:{{ printf \"%.0f\" $value }}% )"


  - alert: "CPU告警"
    expr: (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) * 100 > 80
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高CPU 使用率！"
      description: "{{$labels.instance}}: CPU使用率在 80% 以上 (当前使用值为:{{ printf \"%.0f\" $value }}% )"


  - alert: "磁盘告警"
    expr: 100 - (node_filesystem_free_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} / node_filesystem_size_bytes{fstype=~"tmpfs|ext4|xfs",mountpoint=~"/"} * 100) > 85
    for: 30s
    labels:
      severity: warning
    annotations:
      summary: "{{$labels.instance}}: 检测到 高磁盘 使用率！"
      description: "{{$labels.instance}}: / 磁盘使用率在 85% 以上 (当前使用值为:{{ printf \"%.0f\" $value }}% )"
     
  - alert: FlinkJobDown
    #expr: flink_jobmanager_job_downtime > 0
    expr: flink_jobmanager_numRunningJobs <= 0
    for: 1m
    labels:
      #severity: critical
      severity: 严重告警
    annotations:
      summary: "Flink 作业已停止"
      description: "作业{{ $labels.job }}已停止,当前运行job个数为:{{ $value }}。" 

  - alert: FlinkJobDown
    #expr: flink_jobmanager_job_downtime > 0
    expr: flink_jobmanager_numRunningJobs <= 0
    for: 1m
    labels:
      #severity: critical
      severity: 一般告警
    annotations:
      summary: "Flink 作业已停止"
      description: "作业{{ $labels.job }}已停止,当前运行job个数为:{{ $value }}。" 
   
[root@prometheus_servers_131 ~]# cat /data/docker-compose/docker-compose.yaml  
  version: '2'

networks:
  monitor:
    driver: bridge

services:
  prometheus:
    image: prom/prometheus
    container_name: prometheus
    hostname: prometheus
    restart: always
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules.yml:/etc/prometheus/rules.yml
      - /etc/localtime:/etc/localtime
      - /data/docker-compose/data/prometheus/wal:/prometheus/wal
    ports:
      - "9090:9090"
    networks:
      - monitor

  alertmanager:
    image: prom/alertmanager
    container_name: alertmanager
    hostname: alertmanager
    restart: always
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
      - ./alertmanager/email.tmpl:/etc/alertmanager/email.tmpl
      - /etc/localtime:/etc/localtime
    ports:
      - "9093:9093"
    networks:
      - monitor

   grafana:
     image: grafana/grafana
     container_name: grafana
     hostname: grafana
     restart: always
     volumes:
       - /etc/localtime:/etc/localtime
       - /data/docker-compose/grafana/data:/var/lib/grafana
     ports:
       - "3000:3000"
     networks:
       - monitor

  cadvisor:
    image: google/cadvisor:latest
    container_name: cadvisor
    hostname: cadvisor
    restart: always
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:rw
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /etc/localtime:/etc/localtime
    ports:
      - "8899:8080"
    networks:
      - monitor

3.拉取镜像
docker pull prom/alertmanager
docker pull prom/prometheus      
docker pull grafana/grafana
docker pull google/cadvisor

4.创建容器
docker-compose up -d

4.Prometheus监控nginx

1.安装nginx
rpm -ivh pcre2-10.23-2.el7.x86_64.rpm
rpm -ivh nginx-1.26.1-1.el7.ngx.x86_64.rpm 

2.设置开机自启
systemctl enable nginx
systemctl start nginx

3.检查nginx是否启动成功
ps -ef|grep nginx
ss -ntl |grep 80

4.启用Nginx的stub_status模块
nginx -V 2>&1 | grep -o with-http_stub_status_module

5.vim /etc/nginx/conf.d/status.conf 
[root@flink nginx]# cat /etc/nginx/conf.d/status.conf 
server {
    listen 80;
    server_name 10.0.0.100;
    
    location /nginx_status {
        stub_status on;
        access_log off;
        allow 10.0.0.1;      # 允许客户端访问，按需修改
        allow 10.0.0.100;
        allow 10.0.0.131;
        allow 127.0.0.1;     # 仅允许本地访问，按需修改
        deny all;
    }
}

6.重启nginx
systemctl restart nginx  或者 nginx -s reload

7.验证状态页
http://10.0.0.100/nginx_status

8.curl验证
[root@flink nginx]# curl http://10.0.0.100/nginx_status
Active connections: 4 
server accepts handled requests
 11 11 7455 
Reading: 0 Writing: 1 Waiting: 3 

9.部署Nginx Prometheus Exporter （使用第三方Exporter将stub_status数据转换为Prometheus格式。）
下载Exporter：wget https://github.com/nginxinc/nginx-prometheus-exporter/releases/download/v0.11.0/nginx-prometheus-exporter_0.11.0_linux_amd64.tar.gz
运行Exporter：./nginx-prometheus-exporter -nginx.scrape-uri http://localhost:80/nginx_status

配置nginx-exporer.service
[root@flink nginx]# cat /etc/systemd/system/nginx-exporter.service 
[Unit]
Description=Linux Node Exporter
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target

[Service]
ExecStart=/data/nginx/nginx-prometheus-exporter -nginx.scrape-uri http://10.0.0.100:80/nginx_status

[Install]
WantedBy=multi-user.target

重新加载：systemctl daemon-reload
开机自启：systemctl enable nginx-exporter
启动nginx-exporter：systemctl start  nginx-exporter

检查nginx-exporter
ps -ef|grep nginx
root       8750      1  0 10:56 ?        00:01:12 /data/nginx/nginx-prometheus-exporter -nginx.scrape-uri http://10.0.0.100:80/nginx_status
ss -ntl |grep 9113 #nginx-exporter 默认9113端口

10.修改Prometheus配置文件
 vim /data/docker-compose/prometheus/prometheus.yml
  - job_name: 'nginx'
    static_configs:
      - targets: ['10.0.0.100:9113']
      
11.重启Prometheus
docker restart 容器id

12.登录Prometheus页面查看是否采集成功
curl http://10.0.0.100:9113/metrics|grep nginx

13，监控nginx服务状态指标nginx up
nginx up 0 #代表服务可能停止运行，异常
nginx up 1 #服务正常

14.修改告警规则配置文件
vim /data/docker-compose/prometheus/rules.yml
  - alert: "nginx 服务异常"
    expr: nginx_up == 0
    for: 30s
    labels:
      severity: "严重告警"
    annotations:
      summary: "Nginx 服务不可用"
      description: "Prometheus 无法连接到 Nginx 服务实例 {{ $labels.instance }}，Nginx 服务可能已停止运行。" 

5.监控node-exporter

1.部署node-exporter （https://github.com/prometheus/node_exporter/releases/download）
准备软件包node_exporter-1.8.2.linux-amd64.tar.gz
2.解压软件包
tar xf node_exporter-1.8.2.linux-amd64.tar.gz
3.配置systemcd服务node-exporter.service
[root@prometheus_servers_131 ~]# cat /etc/systemd/system/node-exporter.service 
[Unit]
Description=Linux Node Exporter
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target

[Service]
ExecStart=/data/docker-compose/node_exporter-1.8.2.linux-amd64/node_exporter  # --web.listen-address=:9100 --collector.textfile.directory=/data/docker-compose/node_exporter-1.8.2.linux-amd64/key/ 

[Install]
WantedBy=multi-user.target

重新加载：systemctl daemon-reload
开机自启：systemctl enable node-exporter
启动：systemctl start node-exporter

4.Prometheus配置监控node-exporter
   cat /data/docker-compose/prometheus/prometheus.yml
  - job_name: 'node-export'
    static_configs:  
      - targets: ['10.0.0.131:9100']  # 集群可以['10.0.0.141:9100','10.0.0.142:9100','10.0.0.143:9100']
      
5.重启Prometheus
docker restart 容器id

6.导入摸id=1860

6.监控zookeeper集群

1.下载zookeeper_exporter （集群每一台执行）
mv zookeeper_exporter /usr/local/sbin/ && chmod +x /usr/local/sbin/zookeeper_exporter
2.启动  （集群每一台执行）
nohup /usr/local/sbin/zookeeper_exporter -zookeeper=10.0.0.108:2181 -zookeeper=10.0.0.109:2181 -zookeeper=10.0.0.110:2181 >> /data/exporter/logs/zookeeper.log 2>&1 &
3.访问
http://10.0.0.108:9141/metrics
http://10.0.0.109:9141/metrics
http://10.0.0.110:9141/metrics
4.修改zoo.cfg配置文件
4lw.commands.whitelist=srvr,ruok,mntr,stat
5.配置systemcd服务zookeeper-exporter.service
[root@kafka108 ~]# vim /etc/systemd/system/zookeeper-exporter.service 
[Unit]
Description=Linux Node Exporter
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target

[Service]
ExecStart=/usr/local/sbin/zookeeper_exporter -zookeeper=10.0.0.108:2181 -zookeeper=10.0.0.109:2181 -zookeeper=10.0.0.110:2181

[Install]
WantedBy=multi-user.target

重新加载：systemctl daemon-reload
开机自启：systemctl enable zookeeper-exporter
启动：systemctl start zookeeper-exporter
6.Prometheus配置监控zookeeper-exporter
vim /data/docker-compose/prometheus/prometheus.yml
  ...
  - job_name: zookeeper-exporter
    static_configs:
      - targets: ['10.0.0.108:9141','10.0.0.109:9141','10.0.0.110:9141']
      
7.重启Prometheus
docker restart 容器id

8.granfan导入模版 id=15026

7.监控kafka集群

1.下载kafka_exporter （集群每一台执行）
tar xf kafka_exporter-1.8.0.linux-amd64.tar.gz -C /data/
2.启动kafka-exporter
nohup /data/kafka_exporter-1.8.0.linux-amd64/kafka_exporter --kafka.server=10.0.0.108:9092 --kafka.server=10.0.0.109:9092 --kafka.server=10.0.0.110:9092 --sasl.enabled --sasl.mechanism=PLAIN --sasl.username="admin" --sasl.password="admin@2024" >> /logs/kafka-exporter/kafka-exporter.log 2>&1 &
3.验证端口（默认端口9308）
ss -ntl|grep 9308
http://10.0.0.108:9308/metrics
4.配置Prometheus
vim /data/docker-compose/prometheus/prometheus.yml
  - job_name: kafka-exporter
    static_configs:
      - targets: ['10.0.0.108:9308','10.0.0.109:9308','10.0.0.110:9308']
      
5.重启Prometheus
docker restart 容器id

8.监控es集群

1.下载elasticsearch_exporter
https://github.com/prometheus-community/elasticsearch_exporter/releases
2.准备工作目录
mkdir /data/export/logs -p
3.解压软件包
tar xf elasticsearch_exporter-1.6.0.linux-amd64.tar.gz -C /data/exporter/
4。需要配置环境变量
cat /etc/profile
...
export ES_USERNAME=your_name   #es用户
export ES_PASSWORD=your_password #es密码
5.启动es-exporter
nohup ./elasticsearch_exporter --es.uri=http://10.0.0.105:9200 --es.ca=/zhuyang/app/elasticsearch-7.3.1/config/elasticsearch-certificates.p12   >> /data/exporter/logs/es.log 2>&1 &

--es.uri=http://10.0.0.105:9200 #集群每台执行一次并且ip地址换成本机ip
6.配置Prometheus
vim /data/docker-compose/prometheus/prometheus.yml
  - job_name: 'es-exporter'
    static_configs:
      - targets: ['10.0.0.105:9114','10.0.0.106:9114','10.0.0.107:9114']
7.重启Prometheus
docker restart 容器id

8.导入granfan 14191

9.监控flink集群

1.修改配置文件
metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory
2.重启flink集群
stop-cluster.sh
start-cluster.sh
3.检查端口默认端口9249
ss -ntl|grep 9249
4.Prometheus配置flink
vim /data/docker-compose/prometheus/prometheus.yml
  - job_name: 'flink_job'
    static_configs:
      - targets: ['10.0.0.111:9249','10.0.0.112:9249','10.0.0.113:9249']
5.重启Prometheus
docker restart 容器id
6.导入granfan 11049 14911

10.监控redis

1.部署redis
下载redis离线包
redis-6.2.5-1.el7.remi.x86_64.rpm
2.安装redis
rpm -ivh redis-6.2.5-1.el7.remi.x86_64.rpm
3.修改配置文件
sed -i '/ *#/d; /^$/d' #把所有注释项全部为空
vim /etc/redis/redis.conf
...
daemonize yes
bind 10.0.0.105
masterauth "Admin@2024"
requirepass "Admin@2024"
4.启动redis
/usr/bin/redis-server /etc/redis/redis.conf
5.停止redis
/usr/bin/redis-cli -h 10.0.0.105 -p 7000 -a Admin@2024 shutdown
6.安装redis_exporter
https://github.com/oliver006/redis_exporter/releases/download
7.解压软件包
tar xf redis_exporter-v1.69.0.linux-386.tar.gz -C /data/exporter
8.启动redis_exporter
nohup /data/exporter/redis_exporter-v1.69.0.linux-386/redis_exporter -redis.addr 10.0.0.105:7000 -redis.password Admin@2024 >> /data/exporter/logs/redis.log 2>&1 &
9.验证端口默认9121
ss -ntl|grep 9121 
10.Prometheus配置flink
vim /data/docker-compose/prometheus/prometheus.yml
  - job_name: 'redis_exporter'
    static_configs:
      - targets: ['10.0.0.105:9121']
11.重启Prometheus
docker restart 容器id     
12.导入granfan 763模版id或者 11835 id

11.监控MySQL

1.下载mysql_exporter
https://github.com/prometheus/mysqld_exporter/releases/download/

mysqld_exporter-0.17.1.linux-amd64.tar.gz
2.创建mysql_exporter用户
-- 登录到 MySQL 服务器
mysql -u root -p

-- 创建新用户
CREATE USER 'mysql_exporter'@'localhost' IDENTIFIED BY 'your_password';

-- 授予用户必要的权限
GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'mysql_exporter'@'localhost';

-- 刷新权限
FLUSH PRIVILEGES;
3. 配置并启动 MySQL Exporter
创建一个配置文件 my.cnf 来存储 MySQL 的连接信息：
vim /data/exporter/mysqld_exporter-0.17.1.linux-amd64/my.cnf

[root@elk107 mysqld_exporter-0.17.1.linux-amd64]# cat my.cnf 
[client]
user = mysql_exporter
password = your_password

启动 MySQL Exporter，并指定配置文件：
后台启动：
nohup /data/exporter/mysqld_exporter-0.17.1.linux-amd64/mysqld_exporter --config.my-cnf="/data/exporter/mysqld_exporter-0.17.1.linux-amd64/my.cnf" >> /data/logs/mysql_exporter.log 2>&1 &
4.验证端口默认9104
ss -ntl|grep 9104
5.Prometheus配置mysql
vim /data/docker-compose/prometheus/prometheus.yml
  - job_name: 'mysql_exporter'
    static_configs:
      - targets: ['10.0.0.107:9104']
6.重启Prometheus
docker restart 容器id
7.导入granfan 14031或者18949

12.监控java -jar运行的微服务进程

1.使用 node_exporter 的 textfile 收集器
创建目录：
mkdir /data/exporter/node_exporter-1.8.2.linux-amd64/textfile_collector
2.自定义脚本
[root@dev-dmp-md-1 textfile_collector]# cat java_process.sh 
#!/bin/bash

# 定义指标输出路径（需与 node_exporter 配置一致）
OUTPUT_DIR="/data/exporter/node_exporter-1.8.2.linux-amd64/textfile_collector"
METRIC_FILE="$OUTPUT_DIR/java_process_metrics.prom"

# 清理旧指标文件
rm -f $METRIC_FILE

# 检查 java -jar 进程是否存在
PROCESS_COUNT=$(pgrep -f 'java -jar' | wc -l)

# 采集进程资源使用（示例：内存和CPU）
if [ $PROCESS_COUNT -gt 0 ]; then
  # 获取第一个匹配进程的 PID
  PID=$(pgrep -f 'java -jar' | head -n 1)
  
  # 使用 ps 获取进程的 CPU 和内存占用
  CPU_USAGE=$(ps -p $PID -o %cpu --no-headers)
  MEM_USAGE=$(ps -p $PID -o %mem --no-headers)
  RSS_MEM=$(ps -p $PID -o rss --no-headers)  # 实际内存（KB）
else
  CPU_USAGE=0
  MEM_USAGE=0
  RSS_MEM=0
fi

# 生成 Prometheus 格式的指标
cat << EOF > $METRIC_FILE
# HELP java_process_running Whether the Java process is running (1) or not (0)
# TYPE java_process_running gauge
java_process_running $PROCESS_COUNT

# HELP java_process_cpu_usage CPU usage percentage of the Java process
# TYPE java_process_cpu_usage gauge
java_process_cpu_usage $CPU_USAGE

# HELP java_process_mem_usage Memory usage percentage of the Java process
# TYPE java_process_mem_usage gauge
java_process_mem_usage $MEM_USAGE

# HELP java_process_rss_mem_kb Resident Set Size memory of the Java process in KB
# TYPE java_process_rss_mem_kb gauge
java_process_rss_mem_kb $RSS_MEM
EOF
创建文件 touch /data/exporter/node_exporter-1.8.2.linux-amd64/textfile_collector/java_process_metrics.prom
4.修改启动命令
[root@dev-dmp-md-1 textfile_collector]# cat /etc/systemd/system/node-exporter.service 
[Unit]
Description=Linux Node Exporter
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target

[Service]
ExecStart=/data/exporter/node_exporter-1.8.2.linux-amd64/node_exporter --collector.textfile.directory=/data/exporter/node_exporter-1.8.2.linux-amd64/textfile_collector --collector.textfile 
#指定node_exporter 的 textfile 收集器的目录路劲
[Install]
WantedBy=multi-user.target

5.重启node-exporter
systemctl restart node-exporter