标签归档：prometheus

Prometheus 部署 Black Exporter 黑盒监控 DNS-TCP-ICMP

发表评论

建议软件包安装，二进制安装请自行解决配置问题，docker安装需考虑网络问题。

linux 软件包直接部署

apt install prometheus-blackbox-exporter

二进制部署

# 下载安装
wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.23.0/blackbox_exporter-0.23.0.linux-amd64.tar.gz
tar -xvf  blackbox_exporter-0.23.0.linux-amd64.tar.gz
mv blackbox_exporter-0.23.0.linux-amd64/ /usr/local/blackbox_exporter

# 创建配置文件
cat > /data/black_exporter/black_exporter.yml << EOF
modules:
  http_2xx:
    prober: http
    timeout: 20s
    http:
      preferred_ip_protocol: "ip4"
  http_post_2xx_query:
    prober: http
    timeout: 20s
    http:
      preferred_ip_protocol: "ip4" ##使用ipv4
      method: POST
      headers:
        Content-Type: application/json ##header头
      body: '{"hmac":"","params":{"publicFundsKeyWords":"xxx"}}' ##传参
  tls_connect_tls:
    prober: tcp
    timeout: 5s
    tcp:
      tls: true
  tcp_connect:
    prober: tcp
    timeout: 5s
 #
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp
    timeout: 20s
EOF

# 配置systemd
cat > /etc/systemd/system/blackbox_exporter.service <<EOF
[Unit]
Description=blackbox_exporter
Documentation=https://prometheus.io/
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=/usr/local/bin/blackbox_exporter  --config.file=/data/blackbox-exporter/black-exporter.yml
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

# 启动服务
systemctl enable blackbox_exporter.service 
systemctl start blackbox_exporter.service 
systemctl status blackbox_exporter.service

Docker 部署

docker run --rm \
  -p 9115/tcp \
  --name blackbox_exporter \
  -v $(pwd):/config \
  quay.io/prometheus/blackbox-exporter:latest --config.file=/config/blackbox.yml

# 创建配置文件
cat > /data/docker/blackbox-exporter/black-exporter.yml << EOF
modules:
  http_2xx:
    prober: http
    timeout: 20s
    http:
      preferred_ip_protocol: "ip4"
  http_post_2xx_query:
    prober: http
    timeout: 20s
    http:
      preferred_ip_protocol: "ip4" ##使用ipv4
      method: POST
      headers:
        Content-Type: application/json ##header头
      body: '{"hmac":"","params":{"publicFundsKeyWords":"xxx"}}' ##传参
  tls_connect_tls:
    prober: tcp
    timeout: 5s
    tcp:
      tls: true
  tcp_connect:
    prober: tcp
    timeout: 5s
 #
  pop3s_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^+OK"
      tls: true
      tls_config:
        insecure_skip_verify: false
  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
  irc_banner:
    prober: tcp
    tcp:
      query_response:
      - send: "NICK prober"
      - send: "USER prober prober prober :prober"
      - expect: "PING :([^ ]+)"
        send: "PONG ${1}"
      - expect: "^:[^ ]+ 001"
  icmp:
    prober: icmp
    timeout: 20s
EOF

docker run -d \
  --net myDefault \
  --restart always \
  -p 9115:9115/tcp \
  --name blackbox-exporter \
  -v /data/docker/blackbox-exporter:/config \
  songtianlun/blackbox-exporter:v0.23.0 --config.file=/config/black-exporter.yml

测试使用

curl http://192.168.5.152:9115/probe?target=www.frytea.com&module=http_2xx

prometeus 使用

# https monitor
  - job_name: 'hci-https-monitor'
    metrics_path: /probe
    params:
      module: [tls_connect_tls]
    static_configs:
      - targets:
        - 'https://192.168.5.221:8006'
        - 'https://192.168.5.222:8006'
        - 'https://192.168.5.187:8006'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.19.0.1:9115  # The blackbox exporter's real hostname:port.

# http monitor
  - job_name: 'hci-http-monitor'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
        - 'http://192.168.5.221:3000'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.19.0.1:9115  # The blackbox exporter's real hostname:port.

  # icmp 监控
  - job_name: 'hci-icmp-monitor'
    scrape_interval: 1m
    metrics_path: /probe
    params:
      module: [ "icmp" ]
    static_configs:
      - targets:
        - 192.168.5.254
        - 192.168.5.221
        - 192.168.5.222
        - 192.168.5.187
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 10.19.0.1:9115  # The blackbox exporter's real hostname:port.

Grafana导入Dashboard-ID：7587

参考文献

部署 Prometheus 主机监控完全体

发表评论

监控完全体 = Prometheus + Node Exporter + cadvisor + grafana

Prometheus 作为轮转数据库，从各个服务器采集数据； node-exporter 作为数据采集器，接受 prometheus 采集请求，上报主机各项参数； vadvisor 作为容器数据采集器，接受 prometheus 采集请求，上报主机 docker 各项参数； grafana 作为看板，从 prometheus 查询数据做数据可视化。

部署 Promoetheus

准备配置文件 /data/docker/prometheus/prometheus.yml

global:
  scrape_interval:     15s # By default, scrape targets every 15 seconds.

  # Attach these labels to any time series or alerts when communicating with
  # external systems (federation, remote storage, Alertmanager).
  external_labels:
    monitor: 'codelab-monitor'

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=
<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # Override the global default and scrape targets from this job every 5 seconds.
    scrape_interval: 5s

    static_configs:
      - targets: ['localhost:9090']

  # 采集node exporter监控数据
  - job_name: 'vps2'
    static_configs:
      - targets: ['10.28.0.1:9100']

准备数据文件夹，并设置访问权限为 nobody

# 创建文件夹
mkdir -p /data/docker/prometheus/data

# 取得nobody的uid
docker run --rm quay.io/prometheus/busybox cat /etc/passwd
···
nobody:x:65534:65534:nobody:/home:/bin/false

# 配置文件夹权限
chown 65534:65534 -R /data/docker/prometheus/data

启动 prometheus

$ docker run -d \
    --name=prometheus \
    -p 9090:9090 \
    --network=myDefault \
    --restart always \
    --volume /data/docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
    --volume /data/docker/prometheus/data:/prometheus \
    prom/prometheus:v2.40.4 \
    --web.enable-lifecycle \
    --config.file=/etc/prometheus/prometheus.yml \
    --storage.tsdb.path=/prometheus \
    --storage.tsdb.retention=365d

$ docker run -d \
    --name=prometheus \
    -p 9090:9090 \
    --network=myDefault \
    --restart always \
    --volume /data/docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
    --volume /data/docker/prometheus/data:/prometheus \
    prom/prometheus:v2.40.4 \
    --web.enable-lifecycle \
    --config.file=/etc/prometheus/prometheus.yml \
    --storage.tsdb.path=/prometheus \
    --storage.tsdb.retention=365d
# --web.enable-lifecycle 允许热加载配置文件
# 注意放在镜像名后面的内容为映射到内部的配置标志

部署 Node Exporter 采集主机数据

可采用编译安装或包管理器直接安装。

编译安装

# x86
$ curl -OL https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-amd64.tar.gz
$ tar -xzf node_exporter-1.5.0.linux-amd64.tar.gz
$ cp node_exporter-1.5.0.linux-amd64/node_exporter /usr/local/bin/
# arm64
$ curl -OL https://github.com/prometheus/node_exporter/releases/download/v1.5.0/node_exporter-1.5.0.linux-arm64.tar.gz
$ tar -xzf node_exporter-1.5.0.linux-arm64.tar.gz
$ cp node_exporter-1.5.0.linux-arm64/node_exporter /usr/local/bin/

$ sudo useradd -rs /bin/false nodeusr

$ echo "[Unit]
Description=Node Exporter
After=network.target

[Service]
User=nodeusr
Group=nodeusr
Type=simple
ExecStart=/usr/local/bin/node_exporter

[Install]
WantedBy=multi-user.target" > /etc/systemd/system/node_exporter.service

$ sudo systemctl daemon-reload
$ sudo systemctl enable node_exporter && systemctl start node_exporter

Ubuntu 等直接安装：

$ apt search node_exporter
Sorting... Done
Full Text Search... Done
prometheus-node-exporter-collectors/focal 0+git20200110.fc91c86-1 all
  Supplemental textfile collector scripts for Prometheus node_exporter

$ apt install -y prometheus-node-exporter-collectors

# 启动服务
$ systemctl enable prometheus-node-exporter.service && systemctl start prometheus-node-exporter.service
Synchronizing state of prometheus-node-exporter.service with SysV service script with /lib/systemd/systemd-sysv-install.
Executing: /lib/systemd/systemd-sysv-install enable prometheus-node-exporter

# OpenWRT
opkg update
opkg install prometheus-node-exporter-lua \
prometheus-node-exporter-lua-nat_traffic \
prometheus-node-exporter-lua-netstat \
prometheus-node-exporter-lua-openwrt \
prometheus-node-exporter-lua-wifi \
prometheus-node-exporter-lua-wifi_stations

curl localhost:9100/metrics

cat /etc/config/prometheus-node-exporter-lua
config prometheus-node-exporter-lua 'main'
        option listen_ipv6 '0'
        option listen_port '9100'
        option listen_interface 'lan'

/etc/init.d/prometheus-node-exporter-lua restart

重载 prometheus 配置文件

# 方法一  使用 SIGHUP 信号
kill -HUP $(pidof prometheus)
# 方法二 使用 POST 请求，需 --web.enable-lifecycle
curl -X POST http://localhost:9090/-/reload
# 实测两种方法在 Docker 下均不生效，猜测外部修改必须重启才能映射入容器，原因待查

部署 cadvisor

# 这里使用了我在 docker hub 镜像的官方 gcr 源镜像
# 避免无法拉取问题
# 普通 docker hub 镜像即可拉取
$ sudo docker run \
  --restart always \
  --volume=/:/rootfs:ro \
  --volume=/var/run:/var/run:ro \
  --volume=/sys:/sys:ro \
  --volume=/var/lib/docker/:/var/lib/docker:ro \
  --volume=/dev/disk/:/dev/disk:ro \
  --publish=8080:8080 \
  --detach=true \
  --name=cadvisor \
  --privileged \
  --device=/dev/kmsg \
  --net=myDefault \
  songtianlun/cadvisor:v0.46.0
$ curl http://localhost:8080

部署 grafana

docker run -d \
    --name=grafana \
    -p 3000:3000 \
    --network=myDefault \
    --restart always \
    grafana/grafana

默认帐号： admin/admin

看板模版在这里找： https://grafana.com/grafana/dashboards/

参考文献