采用prometheus方式進(jìn)行部署，監(jiān)控本地服務(wù)器、網(wǎng)絡(luò)線(xiàn)路、域名訪(fǎng)問(wèn)達(dá)到閾值觸發(fā)告警，從而快速定為問(wèn)題源加快響應(yīng)速度

1、創(chuàng)建所需系統(tǒng)結(jié)構(gòu)目錄并給目錄授權(quán)

2、prometheus部署

1）創(chuàng)建docker-compose文件

vim docker-compose.yml

-------------------------------------包含部署grafana、consul、alertmanager----------------------------------

version: '3.7'

services:

prometheus:
depends_on:
- alertmanager
image: prom/prometheus:latest
restart: always
container_name: prometheus
environment:
- TZ=CST-8
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
- ./rules:/etc/prometheus/rules/
- ./prometheus/data:/prometheus
- /etc/hosts:/etc/hosts
- /etc/localtime:/etc/localtime
ports:
- "9090:9090"
networks:
- prom

alertmanager:
image: prom/alertmanager:latest
restart: always
container_name: alertmanager
volumes:
- ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
- ./alertmanager/mail.tmpl:/etc/alertmanager/mail.tmpl
- /etc/localtime:/etc/localtime
environment:
TZ: Asia/Shanghai
ports:
- "9093:9093"
- "9094:9094"
networks:
- prom

grafana:
depends_on:
- prometheus
container_name: grafana
image: grafana/grafana:latest
restart: always
volumes:
- ./grafana:/var/lib/grafana
- /etc/localtime:/etc/localtime
ports:
- "3000:3000"

blackbox_exporter:
image: quay.io/prometheus/blackbox-exporter:latest
restart: always
container_name: blackbox_exporter
volumes:
- /etc/localtime:/etc/localtime
- ./blackbox_exporter/blackbox.yml:/config/blackbox.yml
ports:
- "9115:9115"
command: --config.file=/config/blackbox.yml
networks:
- prom

vmware_exporter:
image: ?pryorda/vmware_exporter
restart: always
container_name: vmware_exporter
env_file:
- ./vm_config/vm_config.env
ports:
- "9273:9272"
networks:
- prom

nginx:
image: nginx:latest
restart: always
container_name: nginx
volumes:
- /usr/share/zoneinfo/PRC:/etc/localtime
- ./nginx/html:/usr/share/nginx/html
- ./nginx/conf.d:/etc/nginx/conf.d
- ./nginx/image:/image
ports:
- "80:80"
networks:
- prom

vmware_exporter2:
image: ?pryorda/vmware_exporter
restart: always
container_name: vmware_exporter2
env_file:
- ./vm_config/vm2_config.env
ports:
- "9272:9272"
networks:
- prom

vmware_exporter-jk:
image: ?pryorda/vmware_exporter
restart: always
container_name: vmware_exporter-jk
env_file:
- ./vm_config/vm3_config.env
ports:
- "9274:9272"
networks:
- prom

snmp-AD:
image: prom/snmp-exporter
restart: always
container_name: snmp_exporter-AD
volumes:
- ./snmp_exporter/snmp_exporter-AD/snmp_exporter/generator/snmp.yml:/etc/snmp_exporter/snmp.yml
ports:
- 9116:9116
command: --config.file=/etc/snmp_exporter/snmp.yml
networks:
- prom

snmp-AD-2:
image: prom/snmp-exporter
restart: always
container_name: snmp_exporter-AD-2
ports:
- "9117:9116"
command: --config.file=/etc/snmp_exporter/snmp.yml
networks:
- prom

networks:
prom:
driver: bridge

-------------------------------------------------------------------------------------------------------------------

2）配置prometheus配置文件

vim prometheus.yml

global:

scrape_interval: 15s

evaluation_interval: 15s

rule_files:

- '/etc/prometheus/rules/ad-alert/*.yml'

- '/etc/prometheus/rules/https-alert/*.yml'

- '/etc/prometheus/rules/https-duration/*.yml'

- '/etc/prometheus/rules/node-alert/*.yml'

alerting:

alertmanagers:

- static_configs:

- targets:

- 'X.X.X.X:9093'

scrape_configs:

- job_name: 'prometheus'

static_configs:

- targets:

- 'X.X.X.X:9090'

- job_name: 'vm-exporter'

static_configs:

- targets:

- 'X.X.X.X:9272' #本地物理機(jī)監(jiān)控

- 'X.X.X.X:9273'

- 'X.X.X.X:9274'

- job_name: 'federate' #聯(lián)邦集群

metrics_path: '/federate'

honor_labels: true

params:

'match[]':

- '{job="prometheus"}'

- '{__name__=~".*"}'

static_configs:

- targets:

- 'X.X.X.X:9090'

- job_name: 'consul-node-exporter' #consul自動(dòng)注冊(cè)

metrics_path: /metrics

scheme: http

scrape_interval: 15s

scrape_timeout: 5s

consul_sd_configs:

- server: 'X.X.X.X:8500'

refresh_interval: 30s

services: ['node-exporter']

relabel_configs:

- source_labels: [__meta_consul_tags]

regex: .*,instance=([^,]*).*

target_label: instance

- source_labels: [__meta_consul_service_address]

target_label: 'ipaddress'

- source_labels: [__meta_consul_service_id]

target_label: 'hostname'

- source_labels: [__meta_consul_service_metadata_group]

target_label: 'localhost'

- source_labels: [__meta_consul_service_metadata_environment]

target_label: 'environment'

- source_labels: [__meta_consul_service_metadata_Project]

target_label: 'Project'

- source_labels: [__meta_consul_service]

target_label: 'service'

- job_name: 'snmp' #本地網(wǎng)絡(luò)設(shè)備監(jiān)控（自定義監(jiān)控項(xiàng)）

metrics_path: /snmp

params:

module: [sangfor]

relabel_configs:

- source_labels: [__address__]

target_label: __param_target

- source_labels: [__param_target]

target_label: instance

- target_label: __address__

replacement: X.X.X.X:9116

static_configs:

- targets:

- '1x.x.x.x'? ? ? ? ? ? ?? #被監(jiān)控網(wǎng)絡(luò)設(shè)備 地址

labels:

hostname: AD

group: snmp

scrape_interval: 30s

scrape_timeout: 30s

- job_name: 'snmp-1' #本地網(wǎng)絡(luò)設(shè)備監(jiān)控

metrics_path: /snmp

params:

module: [if_mib]

relabel_configs:

- source_labels: [__address__]

target_label: __param_target

- source_labels: [__param_target]

target_label: instance

- target_label: __address__

replacement: X.X.X.X:9117

static_configs:

- targets:

- 'x.x.x.x'? ? ? ? ? #被監(jiān)控網(wǎng)絡(luò)設(shè)備 地址

labels:

hostname: AD-2

group: snmp

scrape_interval: 30s

scrape_timeout: 30s

- job_name: 'blackbox' #黑盒監(jiān)控-域名質(zhì)量訪(fǎng)問(wèn)監(jiān)控

metrics_path: /probe

params:

module: [http_2xx] # Look for an HTTP 200 response.

relabel_configs:

- source_labels: [__address__]

target_label: __param_target

- source_labels: [__param_target]

target_label: instance

- target_label: __address__

replacement: X.X.X.X:9115 # The blackbox exporter's real hostname:port.

static_configs:

- targets:

- 'https://www.baidu.com'

- 'https://www.google.com'

- 'https://www.github.com'

- 'https://www.youtube.com'

- 'https://activity.huaweicloud.com'

- 'https://www.aliyun.com'

- 'https://cloud.tencent.com'

- 'https://www.tapd.cn'

- 'https://www.openai.com'

- 'https://www.pinterest.com'

- 'https://www.qq.com'

- 'https://www.bilibili.com'

3、添加告警rules

1）服務(wù)器告警：

groups:

• name: node-alert
  rules:

• • alert: NodeDown
    expr: up{job="consul-node-exporter"} == 0
    for: 5m
    labels:
    severity: critical
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} down"
    description: "主機(jī): {{ $labels.instance }} 已經(jīng)宕機(jī) 5分鐘"
    value: "{{ $value }}"
  • alert: NodeCpuHigh
    expr: 100 * (1 - avg by(instance) (irate(node_cpu_seconds_total{mode="idle",job="consul-node-exporter"}[5m]))) ?> 80
    for: 1m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} cpu使用率過(guò)高"
    description: "CPU 使用率超過(guò) 80%"
    value: "{{ $value }}"
  • alert: NodeCpuIowaitHigh
    expr: avg by (instance) (irate(node_cpu_seconds_total{job="consul-node-exporter",mode="iowait"}[5m])) * 100 > 50
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} cpu iowait 使用率過(guò)高"
    description: "CPU iowait 使用率超過(guò) 50%"
    value: "{{ $value }}"
  • alert: NodeLoad5High
    expr: node_load5 > (count by (instance) (node_cpu_seconds_total{job="consul-node-exporter",mode='system'})) * 1.2
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} load(5m) 過(guò)高"
    description: "Load(5m) 過(guò)高，超出cpu核數(shù) 1.2倍"
    value: "{{ $value }}"
  • alert: NodeMemoryHigh
    expr: (1 - node_memory_MemAvailable_bytes{job="consul-node-exporter"} / node_memory_MemTotal_bytes{job="consul-node-exporter"}) * 100 > 90
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} memory 使用率過(guò)高"
    description: "Memory 使用率超過(guò) 90%"
    value: "{{ $value }}"
  • alert: NodeDiskRootHigh
    expr: (1 - node_filesystem_avail_bytes{job="consul-node-exporter",fstype=~"ext.|xfs",mountpoint ="/"} / node_filesystem_size_bytes{job="consul-node-exporter",fstype=~"ext.|xfs",mountpoint ="/"}) * 100 > 90
    for: 10m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk(/ 分區(qū)) 使用率過(guò)高"
    description: "Disk(/ 分區(qū)) 使用率超過(guò) 90%"
    value: "{{ $value }}"
  • alert: NodeDiskBootHigh
    expr: (1 - node_filesystem_avail_bytes{job="consul-node-exporter",fstype=~"ext.|xfs",mountpoint ="/boot"} / node_filesystem_size_bytes{job="consul-node-exporter",fstype=~"ext.|xfs",mountpoint ="/boot"}) * 100 > 80
    for: 10m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk(/boot 分區(qū)) 使用率過(guò)高"
    description: "Disk(/boot 分區(qū)) 使用率超過(guò) 80%"
    value: "{{ $value }}"
  • alert: NodeDiskReadHigh
    expr: irate(node_disk_read_bytes_total{job="consul-node-exporter"}[5m]) > 20 * (1024 ^ 2)
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk 讀取字節(jié)數(shù) 速率過(guò)高"
    description: "Disk 讀取字節(jié)數(shù) 速率超過(guò) 20 MB/s"
    value: "{{ $value }}"
  • alert: NodeDiskWriteHigh
    expr: irate(node_disk_written_bytes_total{job="consul-node-exporter"}[5m]) > 20 * (1024 ^ 2)
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk 寫(xiě)入字節(jié)數(shù) 速率過(guò)高"
    description: "Disk 寫(xiě)入字節(jié)數(shù) 速率超過(guò) 20 MB/s"
    value: "{{ $value }}"
  • alert: NodeDiskReadRateCountHigh
    expr: irate(node_disk_reads_completed_total{job="consul-node-exporter"}[5m]) > 3000
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk iops 每秒讀取速率過(guò)高"
    description: "Disk iops 每秒讀取速率超過(guò) 3000 iops"
    value: "{{ $value }}"
  • alert: NodeDiskWriteRateCountHigh
    expr: irate(node_disk_writes_completed_total{job="consul-node-exporter"}[5m]) > 3000
    for: 5m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk iops 每秒寫(xiě)入速率過(guò)高"
    description: "Disk iops 每秒寫(xiě)入速率超過(guò) 3000 iops"
    value: "{{ $value }}"
  • alert: NodeInodeRootUsedPercentHigh
    expr: (1 - node_filesystem_files_free{job="consul-node-exporter",fstype=~"ext4|xfs",mountpoint="/"} / node_filesystem_files{job="consul-node-exporter",fstype=~"ext4|xfs",mountpoint="/"}) * 100 > 80
    for: 10m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk(/ 分區(qū)) inode 使用率過(guò)高"
    description: "Disk (/ 分區(qū)) inode 使用率超過(guò) 80%"
    value: "{{ $value }}"
  • alert: NodeInodeBootUsedPercentHigh
    expr: (1 - node_filesystem_files_free{job="consul-node-exporter",fstype=~"ext4|xfs",mountpoint="/boot"} / node_filesystem_files{job="consul-node-exporter",fstype=~"ext4|xfs",mountpoint="/boot"}) * 100 > 80
    for: 10m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} disk(/boot 分區(qū)) inode 使用率過(guò)高"
    description: "Disk (/boot 分區(qū)) inode 使用率超過(guò) 80%"
    value: "{{ $value }}"
  • alert: NodeFilefdAllocatedPercentHigh
    expr: node_filefd_allocated{job="consul-node-exporter"} / node_filefd_maximum{job="consul-node-exporter"} * 100 > 80
    for: 10m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} filefd 打開(kāi)百分比過(guò)高"
    description: "Filefd 打開(kāi)百分比 超過(guò) 80%"
    value: "{{ $value }}"
  • alert: NodeNetworkNetinBitRateHigh
    expr: avg by (instance) (irate(node_network_receive_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) 8) > 20 (1024 ^ 2) * 8
    for: 3m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} network 接收比特?cái)?shù) 速率過(guò)高"
    description: "Network 接收比特?cái)?shù) 速率超過(guò) 20MB/s"
    value: "{{ $value }}"
  • alert: NodeNetworkNetoutBitRateHigh
    expr: avg by (instance) (irate(node_network_transmit_bytes_total{device=~"eth0|eth1|ens33|ens37"}[1m]) 8) > 20 (1024 ^ 2) * 8
    for: 3m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} network 發(fā)送比特?cái)?shù) 速率過(guò)高"
    description: "Network 發(fā)送比特?cái)?shù) 速率超過(guò) 20MB/s"
    value: "{{ $value }}"
  • alert: NodeNetworkNetinPacketErrorRateHigh
    expr: avg by (instance) (irate(node_network_receive_errs_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
    for: 3m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} 接收錯(cuò)誤包 速率過(guò)高"
    description: "Network 接收錯(cuò)誤包 速率超過(guò) 15個(gè)/秒"
    value: "{{ $value }}"
  • alert: NodeNetworkNetoutPacketErrorRateHigh
    expr: avg by (instance) (irate(node_network_transmit_packets_total{device=~"eth0|eth1|ens33|ens37"}[1m])) > 15
    for: 3m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} 發(fā)送錯(cuò)誤包 速率過(guò)高"
    description: "Network 發(fā)送錯(cuò)誤包 速率超過(guò) 15個(gè)/秒"
    value: "{{ $value }}"
  • alert: NodeProcessBlockedHigh
    expr: node_procs_blocked{job="consul-node-exporter"} > 10
    for: 10m
    labels:
    severity: warning
    instance: "{{ $labels.instance }}"
    annotations:
    summary: "instance: {{ $labels.instance }} 當(dāng)前被阻塞的任務(wù)的數(shù)量過(guò)多"
    description: "Process 當(dāng)前被阻塞的任務(wù)的數(shù)量超過(guò) 10個(gè)"
    value: "{{ $value }}"
  • alert: NodeTimeOffsetHigh
    expr: abs(node_timex_offset_seconds{job="consul-node-exporter"}) > 3 * 60
    for: 2m
    labels:
    severity: info{{ $labels.instance }} 時(shí)間偏差過(guò)大"
    description: "Time 節(jié)點(diǎn)的時(shí)間偏差超過(guò) 3m"
    value: "{{ $value }}"

2）域名探測(cè)延遲告警

groups:

• name: httsp-duration
  rules:

• • alert: DomainAccessDelayExceeds5s
    annotations:
    description: 域名：{{ $labels.instance }} 探測(cè)延遲大于 5 秒，當(dāng)前延遲為：{{ $value }}
    summary: 域名探測(cè)，訪(fǎng)問(wèn)延遲超過(guò) 5 秒
    expr: sum(probe_http_duration_seconds{job=~"blackbox"}) by (instance) > 5
    for: 30m
    labels:
    severity: warning
    type: blackbox

3）域名連接告警

groups:

• name: https-alert
  rules:

• • alert: https-domain-access
    annotations:
    description: 域名：{{ $labels.instance }} 失去連接
    summary: 域名失去連接5分鐘
    expr: sum(probe_success{job=~"blackbox"}) by (instance) <= 0
    for: 30m
    labels:
    severity: warning
    type: blackbox

4）線(xiàn)路狀態(tài)告警、網(wǎng)絡(luò)流量線(xiàn)路告警

groups:

• name: 本地網(wǎng)絡(luò)接口線(xiàn)路掉線(xiàn)告警
  rules:

• • alert: "100M電信CN2海外"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth1", ifIndex="18", ifName="eth1", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "100M電信CN2海外線(xiàn)路異常"
    description: "AD-eth1接口-100M電信CN2海外線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "900M聯(lián)通"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth2", ifIndex="19", ifName="eth2", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "900M聯(lián)通線(xiàn)路異常"
    description: "AD-eth2接口-900M聯(lián)通線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "1000M騰訊云"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ?ifDescr="eth3", ifIndex="20", ifName="eth3", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "1000M騰訊云線(xiàn)路異常"
    description: "AD-eth3接口-1000M騰訊云線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "200M中信"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth4", ifIndex="21", ifName="eth4", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "200M中信線(xiàn)路異常"
    description: "AD-eth4接口-200M中信線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "100M專(zhuān)線(xiàn)(IOA)"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth5", ifIndex="22", ifName="eth5", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "100M專(zhuān)線(xiàn)(IOA)線(xiàn)路異常"
    description: "AD-eth5接口-100M專(zhuān)線(xiàn)(IOA)線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "AD撥號(hào)-1000M電信"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth6", ifIndex="23", ifName="eth6", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "AD撥號(hào)-1000M電信線(xiàn)路異常"
    description: "AD-eth6接口-AD撥號(hào)-1000M電信線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "LAN"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ?ifDescr="eth7", ifIndex="24", ifName="eth7", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "LAN線(xiàn)路異常"
    description: "AD-eth7接口-LAN線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "100M中信備線(xiàn)"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth9", ifIndex="26", ifName="eth9", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "100M中信備線(xiàn)"
    description: "AD-eth9接口-100M中信備線(xiàn)線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "AD撥號(hào)-1000M聯(lián)通"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth10", ifIndex="27", ifName="eth10", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "AD撥號(hào)-1000M聯(lián)通"
    description: "AD-eth10接口-AD撥號(hào)-1000M聯(lián)通線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "200M新加坡"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth11", ifIndex="28", ifName="eth11", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "200M新加坡"
    description: "AD-eth11接口-200M新加坡線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"
  • alert: "1000M電信-路由2"
    expr: ifOperStatus{group="snmp", hostname="AD-2", ifDescr="eth12", ifIndex="29", ifName="eth12", instance="X.X.X.X", job="snmp-1"} == 2
    for: 1s
    labels:
    severity: critical
    annotations:
    summary: "1000M電信-路由2"
    description: "AD-eth11接口-1000M電信-路由2線(xiàn)路連接斷開(kāi)，請(qǐng)及時(shí)處理！"

--------------------------------------------------------------------------------------------------------------------------------------------------------------------

groups:

• name: 本地網(wǎng)絡(luò)線(xiàn)路流量告警
  rules:

• • alert: 1000M電信-AD撥號(hào)上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth6"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "1000M電信-AD撥號(hào)線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M電信-AD撥號(hào)下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth6"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "1000M電信-AD撥號(hào)線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M聯(lián)通-AD撥號(hào)下行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth10"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "1000M聯(lián)通-AD撥號(hào)線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M聯(lián)通-AD撥號(hào)下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X" , hostname='AD-2',ifName="eth10"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "1000M聯(lián)通-AD撥號(hào)線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M騰訊云上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth3"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "1000M騰訊云線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M騰訊云下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth3"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "1000M騰訊云線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M電信-路由上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth12"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "1000M騰訊云線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 1000M電線(xiàn)-路由下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth12"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "1000M騰訊云線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 100M-IOA專(zhuān)線(xiàn)上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth5"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "100M-IOA線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: ?100M-IOA專(zhuān)線(xiàn)下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth5"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "100M-IOA線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 100M電信CN2上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth1"}[5m])) * 8 > 80e6
    for: 15s
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "100M電信CN2線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 100M電信CN2下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth1"}[5m])) * 8 > 80e6
    for: 15s
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "100M電信CN2線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 100M中信備線(xiàn)上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth9"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "100M中信備線(xiàn)線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 100M中信備線(xiàn)下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth9"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "100M中信備線(xiàn)線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 200M新加坡上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth11"}[5m])) * 8 > 160e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "200M新加坡線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 200M新加坡下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth11"}[5m])) * 8 > 160e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "200M新加坡上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 200M中信上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth4"}[5m])) * 8 > 160e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "200M中信線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 200M中信下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth4"}[5m])) * 8 > 160e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "200M中信線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 900M聯(lián)通上行線(xiàn)路
    expr: (irate(ifHCOutOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth2"}[5m])) * 8 > 80e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "900M聯(lián)通線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: 900M聯(lián)通下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth2"}[5m])) * 8 > 800e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "900M聯(lián)通線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: LAN上行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth7"}[5m])) * 8 > 500e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到上行流量過(guò)高"
    description: "LAN線(xiàn)路上行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"
  • alert: LAN下行線(xiàn)路
    expr: (irate(ifHCInOctets{job="snmp-1",instance="X.X.X.X", hostname='AD-2',ifName="eth7"}[5m])) * 8 > 4000e6
    for: 1m
    labels:
    severity: warning
    annotations:
    summary: "檢測(cè)到下行流量過(guò)高"
    description: "LAN線(xiàn)路下行流量超過(guò)線(xiàn)路總流量的80%,請(qǐng)及時(shí)處理！"

4、配置alertmanager

1）配置alertmanager配置文件 vim alertmanager.yml

global:
resolve_timeout: 5m
smtp_smarthost: smtp.163.com:25
smtp_from: X.X.X.X@163.com
smtp_auth_username: X.X.X.X@163.com
smtp_auth_password: X.X.X.X
smtp_require_tls: false

templates:

• /etc/alertmanager/mail.tmpl

route:
group_by: ['alertname']
group_wait: 30s
group_interval: 5m
repeat_interval: 6h
receiver: 'email'
routes:
- match:
severity: 'critical' ? ? ?#線(xiàn)路掉線(xiàn)告警、宕機(jī)告警
receiver: 'webhook-critical'
- match:
severity: 'critical'
receiver: 'email'
- match:
secerity: 'P1,P2' ? ? ? ? #服務(wù)告警
receiver: 'email'
- match:
secerity: 'warning' ? ? ? #線(xiàn)路告警
receiver: 'https-alert'
receivers:

• name: 'email'
  email_configs:

• • to: 'bingbingtang@chaocanshu.ai, davisdeng@chaocanshu.ai, ivanli@chaocanshu.ai'
    html: '{{ template "email.to.html" . }}'
    send_resolved: true

• name: 'webhook-critical'
  webhook_configs:

• • url: 'https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=463d19a3-c3df-4345-a1ff-ff6e7c2a869d'
    email_configs:
  • to: 'bingbingtang@chaocanshu.ai, davisdeng@chaocanshu.ai, ivanli@chaocanshu.ai'
    html: '{{ template "email.to.html" . }}'
    send_resolved: true

• name: 'https-alert'
  email_configs:

• • to: 'bingbingtang@chaocanshu.ai, davisdeng@chaocanshu.ai, ivanli@chaocanshu.ai'
    html: '{{ template "email03.to.html" . }}'
    send_resolved: true

inhibit_rules:

• source_match:
  severity: 'critical'
  target_match:
  severity: 'warning'
  equal: ['alertname', 'cluster', 'service']
• source_match:
  severity: 'warning'
  target_match:
  severity: 'info'

2）配置告警模板 vimmail.tmpl (網(wǎng)絡(luò)線(xiàn)路告警和機(jī)器告警不使用同一個(gè)告警模板)

{{ define "email.to.html" }}

{{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }}

<h2 style="color:red">==========異常告警==========</h2>

告警程序: prometheus_alert <br>

告警級(jí)別: {{ .Labels.severity }} 級(jí) <br>

告警類(lèi)型: {{ .Labels.alertname }} <br>

故障主機(jī): {{ .Labels.instance }} <br>

告警主題: {{ .Annotations.summary }} <br>

告警詳情: {{ .Annotations.description }} <br>

觸發(fā)時(shí)間: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} <br>

{{ end }}{{ end -}}

{{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }}

<h2 style="color:green">==========異?；謴?fù)==========</h2>

告警程序: prometheus_alert <br>

故障主機(jī): {{ .Labels.instance }}<br>

故障主題: {{ .Annotations.summary }}<br>

告警詳情: {{ .Annotations.description }}<br>

告警時(shí)間: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}<br>

恢復(fù)時(shí)間: {{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}<br>

{{ end }}{{ end -}}

{{- end }}

{{ define "email03.to.html" }}

{{- if gt (len .Alerts.Firing) 0 -}}{{ range .Alerts }}

<h2 style="color:red">==========異常告警==========</h2>

告警程序: prometheus_alert <br>

告警級(jí)別: {{ .Labels.severity }} 級(jí) <br>

告警主題: {{ .Annotations.summary }} <br>

告警詳情: {{ .Annotations.description }} <br>

觸發(fā)時(shí)間: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }} <br>

{{ end }}{{ end -}}

{{- if gt (len .Alerts.Resolved) 0 -}}{{ range .Alerts }}

<h2 style="color:green">==========異常恢復(fù)==========</h2>

告警程序: prometheus_alert <br>

告警級(jí)別: {{ .Labels.severity }} 級(jí) <br>

故障主題: {{ .Annotations.summary }}<br>

告警詳情: {{ .Annotations.description }}<br>

告警時(shí)間: {{ .StartsAt.Local.Format "2006-01-02 15:04:05" }}<br>

恢復(fù)時(shí)間: {{ (.EndsAt.Add 28800e9).Format "2006-01-02 15:04:05" }}<br>

{{ end }}{{ end -}}

{{- end }}

5、配置成consul

1)采用分離部署，consul單獨(dú)寫(xiě)一個(gè)docker-compose.yml,集群部署方式

2）vim docker-compose.yml

version: '3.7'
services:
consul1:
image: hashicorp/consul:latest
container_name: consul1
restart: always
command: agent -server -client=0.0.0.0 -bootstrap-expect=3 -node=consul1
volumes:
- ./consul1/data:/consul/data
- ./consul1/config:/consul/config
networks:
- prom
consul2:
image: hashicorp/consul:latest
container_name: consul2
restart: always
command: agent -server -client=0.0.0.0 -retry-join=consul1 -node=consul2
volumes:
- ./consul2/data:/consul/data
- ./consul2/config:/consul/config
networks:
- prom
consul3:
image: hashicorp/consul:latest
container_name: consul3
restart: always
command: agent -server -client=0.0.0.0 -retry-join=consul1 -node=consul3
volumes:
- ./consul3/data:/consul/data
- ./consul3/config:/consul/config
networks:
- prom
consul4:
image: hashicorp/consul:latest
container_name: consul4
restart: always
ports:
- 8500:8500
command: agent -client=0.0.0.0 -retry-join=consul1 -ui -node=client1
volumes:
- ./consul4/data:/consul/data
- ./consul4/config:/consul/config
networks:
- prom

networks:
prom:
driver: bridge

3) 配置自動(dòng)注冊(cè)腳本、hosts注冊(cè)主機(jī)信息(注意這兩個(gè)文件必須放在同一目錄下)

[root@prometheus-2 consul]# cat hosts

prometheus X.X.X.X

prometheus-test X.X.X.X

test-nginx X.X.X.X

snipeit X.X.X.X

prometheus-2 X.X.X.X

--------------------------------------------------------------------------------------------

[root@prometheus-2 consul]# cat linux-node.sh

#!/bin/bash

CONSUL_SERVER="X.X.X.X"

while read -r host_name host_addr

do

payload='{"id": "'"$host_addr"'","name": "node-exporter","address": "'"$host_addr"'","port":9100,"tags": ["linux-node", "instance='"$host_name"'"],"checks": [{"http": "http://'"$host_addr"':9100/","interval": "15s"}]}'

curl -X PUT -d "$payload" "http://$CONSUL_SERVER:8500/v1/agent/service/register"

done < hosts

-------------------------------------------------------------------------------------------

6、配置本地實(shí)體物理機(jī)

1）進(jìn)入vm_config目錄（注意，每臺(tái)實(shí)體機(jī)監(jiān)控都需要配置一個(gè)地址賬號(hào)密碼環(huán)境變量）

[root@prometheus-2 vm_config]# ls

vm2_config.env vm3_config.env vm_config.env

[root@prometheus-2 vm_config]# cat *

VSPHERE_USER=root

VSPHERE_PASSWORD=x.x.x.x

VSPHERE_HOST=x.x.x.x

VSPHERE_IGNORE_SSL=TRUE

VSPHERE_SPECS_SIZE=2000

----------------------------------------------------------------------------------------------------------

VSPHERE_USER=root

VSPHERE_PASSWORD=x.x.x.x

VSPHERE_HOST=x.x.x.x

VSPHERE_IGNORE_SSL=TRUE

VSPHERE_SPECS_SIZE=2000

----------------------------------------------------------------------------------------------------------

VSPHERE_USER=root

VSPHERE_PASSWORD=x.x.x.x

VSPHERE_HOST=x.x.x.x

VSPHERE_IGNORE_SSL=TRUE

VSPHERE_SPECS_SIZE=2000

----------------------------------------------------------------------------------------------------------

7、配置blackbox_exporter

1)拉取github項(xiàng)目 wget clone https://github.com/prometheus/blackbox_exporter.git

2)修改blackbox.yml配置文件 vim blackbox.yml

modules:
http_2xx:
prober: http
http:
preferred_ip_protocol: "ip4"
http_post_2xx:
prober: http
http:
method: POST
tcp_connect:
prober: tcp
pop3s_banner:
prober: tcp
tcp:
query_response:
- expect: "^+OK"
tls: true
tls_config:
insecure_skip_verify: false
grpc:
prober: grpc
grpc:
tls: true
preferred_ip_protocol: "ip4"
grpc_plain:
prober: grpc
grpc:
tls: false
service: "service1"
ssh_banner:
prober: tcp
tcp:
query_response:
- expect: "^SSH-2.0-"
- send: "SSH-2.0-blackbox-ssh-check"
irc_banner:
prober: tcp
tcp:
query_response:
- send: "NICK prober"
- send: "USER prober prober prober :prober"
- expect: "PING :([^ ]+)"
send: "PONG ${1}"
- expect: ":[ ]+ 001"
icmp:
prober: icmp
icmp_ttl5:
prober: icmp
timeout: 5s
icmp:
ttl: 5

8、配置snmp_exporter

1）拉取github項(xiàng)目wget clone https://github.com/prometheus/snmp_exporter.git

2) 確保系統(tǒng)有GO環(huán)境、使用生成器生成配置信息

cd snmp_exporter/generator

make generator mibs

make generate

3）在mibs文件夾中添加被監(jiān)控機(jī)器的mib文件信息

4）配置generator.yml文件

modules:
sangfor:
walk:
- sfSysDevName ? ? ? ? # 系統(tǒng)主機(jī)名稱(chēng)
- adStandByState ? ? ? # 雙機(jī)主備狀態(tài)
- sfCpuLoadLast1Min ? ?# cpu過(guò)去1分鐘的平均負(fù)載
- sfCpuLoadLast5Min ? ?# cpu過(guò)去5分鐘的平均負(fù)載
- sfCpuLoadLast15Min ? # cpu過(guò)去15分鐘的平均負(fù)載
- sfCpuTemp ? ? ? ? ? ?# cpu溫度
- sfSysTotalMemory ? ? # 內(nèi)存總大小(KB)
- sfSysFreeMemory ? ? ?# 內(nèi)存可用大小(KB)
- sfDiskSize ? ? ? ? ? # 磁盤(pán)大小(M)
- sfFilesystemName ? ? # 磁盤(pán)分區(qū)名稱(chēng)
- sfDiskUsed ? ? ? ? ? # 磁盤(pán)使用的空間(M)
- sfDiskAvail ? ? ? ? ?# 磁盤(pán)剩余空間(M)
- sfDiskUsedPercent ? ?# 磁盤(pán)使用率(%)
- sfDeviceStatus ? ? ? # 磁盤(pán)狀態(tài)
- sfFanName ? ? ? ? ? ?# 風(fēng)扇名稱(chēng)
- sfFanSpeed ? ? ? ? ? # 風(fēng)扇轉(zhuǎn)速
- sfFanState ? ? ? ? ? # 風(fēng)扇狀態(tài)
- sfPowerState ? ? ? ? # 電源狀態(tài)
- adConns ? ? ? ? ? ? ?# 系統(tǒng)并發(fā)連接數(shù)
- adNewConns ? ? ? ? ? # 系統(tǒng)新建連接數(shù)
- adVsConns ? ? ? ? ? ?# 所有虛擬服務(wù)并發(fā)連接數(shù)
- adVsNewConns ? ? ? ? # 所有虛擬服務(wù)新建連接數(shù)
- adUplinkThroughput ? # 所有鏈路上行流量（整型）
- adDownlinkThroughput # 所有鏈路下行流量 （整型）
- adMemCostRate ? ? ? ?# 內(nèi)存使用率
- adHttpRequest ? ? ? ?# 當(dāng)前設(shè)備http請(qǐng)求速率
- adVsNumber ? ? ? ? ? # 虛擬服務(wù)數(shù)量
- adPoolNumber ? ? ? ? # 節(jié)點(diǎn)池?cái)?shù)量
- adNodeNumber ? ? ? ? # 節(jié)點(diǎn)數(shù)量
- adLinkName ? ? ? ? ? # 鏈路名稱(chēng)
- adLinkType ? ? ? ? ? # 鏈路類(lèi)型
- adLinkIfName ? ? ? ? # 鏈路引用的網(wǎng)口
- adLinkStatus ? ? ? ? # 鏈路狀態(tài)，0為離線(xiàn)，1為正常，2為繁忙
- adLinkBitIn ? ? ? ? ?# 鏈路上行流量
- adLinkBitOut ? ? ? ? # 鏈路下行流量
- adLinkNumber ? ? ? ? # 設(shè)備鏈路個(gè)數(shù)
- adCpuCostRate ? ? ? ?# CPU使用率
- adUptime ? ? ? ? ? ? # 系統(tǒng)運(yùn)行時(shí)間
- adInterfaceName ? ? ?# 網(wǎng)口名稱(chēng)
- adInterfaceBitIn ? ? # 網(wǎng)口上行數(shù)據(jù)

max_repetitions: 25
retries: 3
timeout: 5s
version: 2 #snmpV2版本
auth:
community: public #設(shè)備團(tuán)體名

lookups:

• source_indexes: [LinkIndex]
  lookup: adLinkType
• source_indexes: [LinkIndex]
  lookup: adLinkIfName
• source_indexes: [LinkIndex]
  lookup: adLinkName

overrides:
sfSysDevName:
type: DisplayString

sfSysCpuCostRate:
type: DisplayString

sfCpuLoadLast1Min:
type: DisplayString

sfCpuTemp:
type: DisplayString

adStandByState:
type: DisplayString

adLinkName:
type: DisplayString

5）使用配置器生成snmp.yml配置文件 （用docker生成）

docker run -it -v "${PWD}:/opt/" prom/snmp-generator:master generate

9、配置nginx代理

1）進(jìn)入nginx文件夾修改配置信息

目錄結(jié)構(gòu)[root@prometheus-2 nginx]# tree

├── conf.d

│?? └── default.conf #nginx.config配置文件

├── html

│?? └── index.html #訪(fǎng)問(wèn)展示頁(yè)面

└── image #存放image圖像

├── alertmanager.png

├── consul.png

├── grafana.jpg

├── x.x.x.x.png

└── prometheus.png

vim nginx/conf.d/default.conf

server {
listen       80;
listen  [::]:80;
server_name  localhost;

#access_log  /var/log/nginx/host.access.log  main;

location / {
    root   /usr/share/nginx/html;
    index  index.html index.htm;
}

#error_page  404              /404.html;

# redirect server error pages to the static page /50x.html
#
error_page   500 502 503 504  /50x.html;
location = /50x.html {
    root   /usr/share/nginx/html;
}

# Add a new location for image storage                     #配置可以識(shí)別image圖片
location /image/ {
    alias /image/;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
#    proxy_pass   http://127.0.0.1;
#}

# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
#    root           html;
#    fastcgi_pass   127.0.0.1:9000;
#    fastcgi_index  index.php;
#    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
#    include        fastcgi_params;
#}

# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
#    deny  all;
#}

2）配置訪(fǎng)問(wèn)展示頁(yè)面

1. [root@prometheus-2 html]# cat index.html
2. <!DOCTYPE html>
3. <html>
4. <head>
5. <title>Monitoring</title>
6. <style>
7. .app-wrapper {
8. display: flex;
9. justify-content: center;
10. align-items: center;
11. }
13. .app-container {
14. width: 250px;
15. height: 300px;
16. margin: 10px;
17. padding: 10px;
18. border: 1px solid #ccc;
19. border-radius: 5px;
20. text-align: center;
21. }
23. .app-container img {
24. width: 150px;
25. height: 150px;
26. object-fit: contain;
27. margin-bottom: 10px;
28. }
30. .app-description {
31. margin-bottom: 10px;
32. }
34. .app-button-container {
35. display: flex;
36. justify-content: center;
37. }
39. .app-button {
40. display: inline-block;
41. padding: 8px 16px;
42. background-color: #4CAF50;
43. color: white;
44. text-decoration: none;
45. border-radius: 4px;
46. }
47. </style>
48. </head>
49. <body>
51. <div class="app-wrapper">
52. <div class="app-container">
53. <a target="_blank">
54. <img src="/image/prometheus.png" alt="prometheus">
55. </a>
56. <div class="app-description">Prometheus Monitoring</div>
57. <div class="app-button-container">
58. <a class="app-button" target="_blank">Click to Enter</a>
59. </div>
60. </div>
62. <div class="app-container">
63. <a target="_blank">
64. <img src="/image/grafana.jpg" alt="grafana">
65. </a>
66. <div class="app-description">Grafana Dashboard</div>
67. <div class="app-button-container">
68. <a class="app-button" target="_blank">Click to Enter</a>
69. </div>
70. </div>
72. <div class="app-container">
73. <a target="_blank">
74. <img src="/image/alertmanager.png" alt="alertmanager">
75. </a>
76. <div class="app-description">Alertmanager Notifications</div>
77. <div class="app-button-container">
78. <a class="app-button" target="_blank">Click to Enter</a>
79. </div>
80. </div>
82. <div class="app-container">
83. <a target="_blank">
84. <img src="/image/consul.png" alt="consul">
85. </a>
86. <div class="app-description">Consul Service Discovery</div>
87. <div class="app-button-container">
88. <a class="app-button" target="_blank">Click to Enter</a>
89. </div>
90. </div>
93. <div class="app-container">
94. <a target="_blank">
95. <img src="/image/IOA.png" alt="grafana">
96. </a>
97. <div class="app-description">IOA-Grafana</div>
98. <div class="app-button-container">
99. <a class="app-button" target="_blank">Click to Enter</a>
100. </div>
101. </div>
102. </div>
106. </body>
107. </html>

10、可用性驗(yàn)證

1）訪(fǎng)問(wèn)nginx代理域名

2）prometheus展示

3）grafana展示

4）alertmanager展示

5）consul展示

文章來(lái)源地址http://www.zghlxwxcb.cn/news/detail-722214.html