Grafana
簡(jiǎn)介
Grafana 是一款開(kāi)源的數(shù)據(jù)可視化工具，使用 Grafana 可以非常輕松的將數(shù)據(jù)轉(zhuǎn)成圖表(如下圖)的展現(xiàn)形式來(lái)做到數(shù)據(jù)監(jiān)控以及數(shù)據(jù)統(tǒng)計(jì)。
下載軟件包
wget https://dl.grafana.com/enterprise/release/grafana-enterprise-9.1.6.linux-amd64.tar.gz
安裝部署
解壓
tar -xvzf grafana-enterprise-9.1.6.linux-arm64.tar.gz
mv grafana-9.1.6 /data/apps/
cd /data/apps/grafana-9.1.6/conf
cp sample.ini grafana.ini
修改配置文件
vim grafana.ini
[paths]
# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
;data = /var/lib/grafana
data = /data/apps/grafana-9.1.6/data
logs = /data/apps/grafana-9.1.6/logs
plugins = /data/apps/grafana-9.1.6/plugins
[log]
mode = file
level = warn
創(chuàng)建用戶
groupadd -g 9100 monitor
useradd -g 9100 -u 9100 -s /sbin/nologin -M monitor
mkdir data && mkdir logs && mkdir plugins
chown -R monitor:monitor grafana-9.1.6
創(chuàng)建Systemd服務(wù)
vim /usr/lib/systemd/system/grafana.service
[Unit]
Description=grafana service
After=network.target
[Service]
User=monitor
Group=monitor
KillMode=control-group
Restart=on-failure
RestartSec=60
ExecStart=/data/apps/grafana-9.1.6/bin/grafana-server -config /data/apps/grafana-9.1.6/conf/grafana.ini -pidfile /data/apps/grafana-9.1.6/grafana.pid -homepath /data/apps/grafana-9.1.6
[Install]
WantedBy=multi-user.target
啟動(dòng) Grafana
systemctl daemon-reload
systemctl restart grafana.service
systemctl enable grafana.service
systemctl status grafana
Spark應(yīng)用監(jiān)控 Graphite_exporter
配置 mapping 文件
cd /usr/local/graphite_exporter
vim graphite_exporter_mapping
graphite_exportergraphite_exportengs:
- match: '*.*.executor.filesystem.*.*'
name: spark_app_filesystem_usage
labels:
application: $1
executor_id: $2
fs_type: $3
qty: $4
- match: '*.*.jvm.*.*'
name: spark_app_jvm_memory_usage
labels:
application: $1
executor_id: $2
mem_type: $3
qty: $4
- match: '*.*.executor.jvmGCTime.count'
name: spark_app_jvm_gcTime_count
labels:
application: $1
executor_id: $2
- match: '*.*.jvm.pools.*.*'
name: spark_app_jvm_memory_pools
labels:
application: $1
executor_id: $2
mem_type: $3
qty: $4
- match: '*.*.executor.threadpool.*'
name: spark_app_executor_tasks
labels:
application: $1
executor_id: $2
qty: $3
- match: '*.*.BlockManager.*.*'
name: spark_app_block_manager
labels:
application: $1
executor_id: $2
type: $3
qty: $4
- match: '*.*.DAGScheduler.*.*'
name: spark_app_dag_scheduler
labels:
application: $1
executor_id: $2
type: $3
qty: $4
- match: '*.*.CodeGenerator.*.*'
name: spark_app_code_generator
labels:
application: $1
executor_id: $2
type: $3
qty: $4
- match: '*.*.HiveExternalCatalog.*.*'
name: spark_app_hive_external_catalog
labels:
application: $1
executor_id: $2
type: $3
qty: $4
- match: '*.*.*.StreamingMetrics.*.*'
name: spark_app_streaming_metrics
labels:
application: $1
executor_id: $2
app_name: $3
type: $4
qty: $5
修改spark的metrics.properties配置文件，讓其推送metrics到Graphite_exporter
*.sink.graphite.class=org.apache.spark.metrics.sink.GraphiteSink
*.sink.graphite.host=10.253.128.31
*.sink.graphite.port=9108
*.sink.graphite.period=10
*.sink.graphite.unit=seconds
#*.sink.graphite.prefix=<optional_value>
HDFS 監(jiān)控
namenode.yaml
---
startDelaySeconds: 0
hostPort: localhost:1234 #master為本機(jī)IP（一般可設(shè)置為localhost）；1234為想設(shè)置的jmx端口
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
datanode.yaml
---
startDelaySeconds: 0
hostPort: localhost:1244 #master為本機(jī)IP（一般可設(shè)置為localhost）；1244為想設(shè)置的jmx端口（可設(shè)置為未被占用的端口）
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
配置 hadoop-env.sh
export HADOOP_NAMENODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1234 -javaagent:/jmx_prometheus_javaagent-0.8.jar=9211:/namenode.yaml"
export HADOOP_DATANODE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=1244 -javaagent:/jmx_prometheus_javaagent-0.8.jar=9212:/datanode.yaml"
YARN 監(jiān)控
yarn.yaml
---
startDelaySeconds: 0
hostPort: localhost:2111 #master為本機(jī)IP（一般可設(shè)置為localhost）；1234為想設(shè)置的jmx端口
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
配置 yarn-env.sh
export YARN_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=2111 -javaagent:/jmx_prometheus_javaagent-0.8.jar=9323:/yarn.yaml"
HBase 監(jiān)控
master.yaml
---
startDelaySeconds: 0
hostPort: IP:1254 #master為本機(jī)IP（一般可設(shè)置為localhost）；1234為想設(shè)置的jmx端口（可設(shè)
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
regionserver.yaml
---
startDelaySeconds: 0
hostPort: IP:1255 #master為本機(jī)IP（一般可設(shè)置為localhost）；1234為想設(shè)置的jmx端口
#jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:1234/jmxrmi
ssl: false
lowercaseOutputName: false
lowercaseOutputLabelNames: false
配置 hbase-env.sh文章來(lái)源：http://www.zghlxwxcb.cn/news/detail-420353.html
HBASE_M_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmx remote.port=1254 -javaagent:/jmx_prometheus_javaagent-0.8.jar=9523:/hbasem.yaml"
#======================================= prometheus jmx export start===================================
HBASE_JMX_OPTS="-Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxre mote.port=1255 -javaagent:/jmx_prometheus_javaagent-0.8.jar=9522:/hbase.yaml"
#======================================= prometheus jmx export end ===================================
希望對(duì)正在查看文章的您有所幫助，記得關(guān)注、評(píng)論、收藏，謝謝您文章來(lái)源地址http://www.zghlxwxcb.cn/news/detail-420353.html
到了這里，關(guān)于【大數(shù)據(jù)監(jiān)控】Grafana、Spark、HDFS、YARN、Hbase指標(biāo)性能監(jiān)控安裝部署詳細(xì)文檔的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！
