使用上述方案,黑盒監(jiān)控與自建cmdb 平臺很容易進(jìn)行集成,使其監(jiān)控自動化,不需要過多的人工干預(yù),可以省去大量的人工成本,grafana 的配置這里就不進(jìn)行過多介紹,自行通過谷歌完成。
前言:
是Prometheus 官方提供的 exporter 之一,主要提供http、dns、tcp、icmp 的監(jiān)控數(shù)據(jù)采集。
主要提供,服務(wù)發(fā)現(xiàn),健康檢查,等功能,本次集成主要使用到服務(wù)發(fā)現(xiàn)功能。
本文主要實(shí)現(xiàn),基于consul_sd_config & consul 的 prometheus 服務(wù)發(fā)現(xiàn),實(shí)現(xiàn)網(wǎng)路設(shè)備ping監(jiān)控,站點(diǎn)可用行監(jiān)控,以及證書相關(guān)信息監(jiān)控。
安裝環(huán)境:
- k8s
- consul
- Prometheus
- blackbox_exporter
1: Consul 安裝
1.1:使用helm 安裝 consul
Bash
# 添加 consul helm 源
helm repo add hashicorp https://helm.releases.hashicorp.com
# 安裝consul
helm -n consul install \
--set storageClass=alicloud-disk-efficiency \
consul hashicorp/consul \
--version=0.32.1
1.2:查看服務(wù)安裝狀態(tài)
Bash
[root@xxxxxxxx consul_install]# kubectl -n consul get pods
NAME READY STATUS RESTARTS AGE
consul-consul-9lxfc 1/1 Running 0 6d1h
consul-consul-ntqcf 1/1 Running 0 6d1h
consul-consul-q7c6f 1/1 Running 0 6d1h
consul-consul-server-0 1/1 Running 0 6d1h
consul-consul-server-1 1/1 Running 0 6d1h
consul-consul-server-2 1/1 Running 0 6d1h
1.3:nginx-ingress consul
Bash
# consul.xxxxxx.cn -----> 替換為正確域名
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: consul-ingress
namespace: consul
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/rewrite-target: /
spec:
rules:
- host: consul.xxxxxx.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: consul-consul-ui
port:
number: 80
Bash
kubectl apply -f consul_ingress.yml
1.4:訪問測試
2: Blackbox_export
2.1:blackbox 安裝
- blackbox-exporter-config.yaml
Bash
apiVersion: v1
kind: ConfigMap
metadata:
name: blackbox-exporter
labels:
app: blackbox-exporter
data:
blackbox.yml: |-
modules:
## ----------- DNS 檢測配置 -----------
dns_tcp:
prober: dns
dns:
transport_protocol: "tcp"
preferred_ip_protocol: "ip4"
query_name: "kubernetes.default.svc.cluster.local" # 用于檢測域名可用的網(wǎng)址
query_type: "A"
## ----------- TCP 檢測模塊配置 -----------
tcp_connect:
prober: tcp
timeout: 5s
## ----------- ICMP 檢測配置 -----------
ping:
prober: icmp
timeout: 5s
icmp:
preferred_ip_protocol: "ip4"
## ----------- HTTP GET 2xx 檢測模塊配置 -----------
http_get_2xx:
prober: http
timeout: 10s
http:
method: GET
preferred_ip_protocol: "ip4"
valid_http_versions: ["HTTP/1.1","HTTP/2"]
valid_status_codes: [200] # 驗(yàn)證的HTTP狀態(tài)碼,默認(rèn)為2xx
no_follow_redirects: false # 是否不跟隨重定向
## ----------- HTTP GET 3xx 檢測模塊配置 -----------
http_get_3xx:
prober: http
timeout: 10s
http:
method: GET
preferred_ip_protocol: "ip4"
valid_http_versions: ["HTTP/1.1","HTTP/2"]
valid_status_codes: [301,302,304,305,306,307] # 驗(yàn)證的HTTP狀態(tài)碼,默認(rèn)為2xx
no_follow_redirects: false # 是否不跟隨重定向
## ----------- HTTP POST 監(jiān)測模塊 -----------
http_post_2xx:
prober: http
timeout: 10s
http:
method: POST
preferred_ip_protocol: "ip4"
valid_http_versions: ["HTTP/1.1", "HTTP/2"]
#headers: # HTTP頭設(shè)置
# Content-Type: application/json
#body: '{}' # 請求體設(shè)置
- blackbox-exporter-deploy.yaml
Bash
apiVersion: v1
kind: Service
metadata:
name: blackbox-exporter
labels:
k8s-app: blackbox-exporter
spec:
type: ClusterIP
ports:
- name: http
port: 9115
targetPort: 9115
selector:
k8s-app: blackbox-exporter
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: blackbox-exporter
labels:
k8s-app: blackbox-exporter
spec:
replicas: 1
selector:
matchLabels:
k8s-app: blackbox-exporter
template:
metadata:
labels:
k8s-app: blackbox-exporter
spec:
containers:
- name: blackbox-exporter
image: prom/blackbox-exporter:v0.19.0
args:
- --config.file=/etc/blackbox_exporter/blackbox.yml
- --web.listen-address=:9115
- --log.level=info
ports:
- name: http
containerPort: 9115
resources:
limits:
cpu: 3
memory: 6000Mi
requests:
cpu: 100m
memory: 50Mi
livenessProbe:
tcpSocket:
port: 9115
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
readinessProbe:
tcpSocket:
port: 9115
initialDelaySeconds: 5
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
volumeMounts:
- name: config
mountPath: /etc/blackbox_exporter
volumes:
- name: config
configMap:
name: blackbox-exporter
defaultMode: 420
Bash
kubectl apply -f blackbox-exporter-deploy.yaml
kubectl apply -f blackbox-exporter-config.yaml
2.2:nginx ingress blackbox-exporter ? blackbox_ingress.yml
Bash
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: blackbox-ingress
namespace: monitoring
annotations:
kubernetes.io/ingress.class: nginx
nginx.ingress.kubernetes.io/rewrite-target: /
spec:
rules:
- host: blackbox-devops.lululemon.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: blackbox-exporter
port:
number: 9115
Bash
kubectl apply -f blackbox_ingress.yml
3: rometheus 添加 服務(wù)動態(tài)發(fā)現(xiàn)
Bash
##### http_get_2xx 數(shù)據(jù)獲取
- job_name: http_get_2xx
params:
module:
- http_get_2xx
scrape_interval: 2s
scrape_timeout: 2s
metrics_path: /probe
consul_sd_configs:
# consul 服務(wù)地址
- server: consul-consul-server.consul.svc.cluster.local:8500
tag_separator: ','
services:
- http_get_2xx
relabel_configs:
- source_labels: ['__meta_consul_service_address']
target_label: __param_target
- source_labels: ['__meta_consul_service_address']
target_label: instance
- target_label: __address__
## blackbox-export 地址
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
####### icmp 配置
- job_name: blackbox_icmp
params:
module:
- ping
scrape_interval: 2s
scrape_timeout: 2s
metrics_path: /probe
consul_sd_configs:
# consul 服務(wù)地址
- server: consul-consul-server.consul.svc.cluster.local:8500
tag_separator: ','
services:
- ping
relabel_configs:
- source_labels: ['__meta_consul_service_address']
target_label: __param_target
- source_labels: ['__meta_consul_service_address']
target_label: instance
- target_label: __address__
## blackbox-export 地址
replacement: blackbox-exporter.monitoring.svc.cluster.local:9115
4:添加 icmp 監(jiān)控
4.1:添加監(jiān)控地址到consul
Bash
192.168.1.1
192.168.1.2
- add_consul_service_icmp.sh
Bash
#!/usr/bin/env bash
ip_addr=$1
if test "$ip_addr";then
curl -X PUT -d '{
"id": "icmp_'${ip_addr}'",
"name": "ping",
"address": "'${ip_addr}'",
"port": 443,
"Meta": {
"env": "prod",
"team": "network",
"project": "network",
"owner": "Mike"
},
"tags": ["node"],
"checks": [{"http": "http://blackbox-exporter.monitoring.svc.cluster.local:9115/","interval": "15s"}]}' \
http://consul-consul-server:8500/v1/agent/service/register
else
echo "請輸入?yún)?shù)"
fi
Bash
for i in `cat icmp_list`;do bash add_consul_service_icmp.sh $i;done
4.2:查看consul 服務(wù)

4.3:刪除ping 監(jiān)控地址腳本
Bash
#!/usr/bin/env bash
ip_addr=$1
curl -X PUT http://consul-consul-server:8500/v1/agent/service/deregister/icmp_${ip_addr}
5: 添加http_get_2xx
5.1:添加監(jiān)控域名
Bash
wwww.baidu.com
wwww.1111.com
wwww.2222.com
- add_consul_service_http_get_2xx.sh
Bash
#!/usr/bin/env bash
service_name=$1
if test "$service_name";then
curl -X PUT -d '{
"id": "http_get_2xx_'${service_name}'",
"name": "http_get_2xx",
"address": "https://'${service_name}'",
"port": 443,
"Meta": {
"env": "prod",
"team": "web",
"project": "web",
"owner": "Devops"
},
"tags": ["node"],
"checks": [{"http": "http://blackbox-exporter.monitoring.svc.cluster.local:9115/","interval": "15s"}]}' \
http://consul-consul-server:8500/v1/agent/service/register
else
echo "請輸入?yún)?shù)"
fi
Bash
for i in `cat domain_name_list`;do bash add_consul_service_http_get_2xx.sh $i;done
5.2:查看consul 服務(wù)

5.3:刪除域名監(jiān)控腳本
- del_consul_service_http_get_2xx.sh
Bash
#!/usr/bin/env bash
ip_addr=$1
curl -X PUT http://consul-consul-server:8500/v1/agent/service/deregister/http_get_2xx_${ip_addr}
6:查看prometheus 監(jiān)控

總結(jié):
使用上述方案,黑盒監(jiān)控與自建cmdb 平臺很容易進(jìn)行集成,使其監(jiān)控自動化,不需要過多的人工干預(yù),可以省去大量的人工成本,grafana 的配置這里就不進(jìn)行過多介紹,自行通過谷歌完成。