夜鶯初探四·mtail插件采集日志指標(biāo)

前言

上一篇介紹了Categraf的配置，這篇我們嘗試通過使用google開源的mtail工具來作為Categraf的插件，從應(yīng)用日志中提取指標(biāo)數(shù)據(jù)。

mtail項(xiàng)目介紹和配置文件說明

通過mtail -h可以很方便看到參數(shù)詳細(xì)，也推薦喬克-從日志中提取指標(biāo)的瑞士軍刀或者Dream運(yùn)維夢(mèng)工廠-categraf-mtail日志收集插件詳解來了解更多,我就不再班門弄斧了。
當(dāng)然也可以通過官方來了解詳情新手村介紹 和高手入門

Categraf采集插件

categraf-mtail插件地址

https://github.com/flashcatcloud/categraf/tree/main/inputs/mtail

源碼解讀

package mtail
...

//常量值
const inputName = `mtail`
const description = ` extract internal monitoring data from application logs`

//配置
// MTail holds the configuration for the plugin.
type MTail struct {
    config.PluginConfig
    Instances []*Instance `toml:"instances"`
}
//配置文件中instances對(duì)象需要參數(shù)結(jié)構(gòu)體
type Instance struct {
	config.InstanceConfig


	/**
		type InternalConfig struct {
			// append labels
			Labels map[string]string `toml:"labels"`

			// metrics drop and pass filter
			MetricsDrop       []string `toml:"metrics_drop"`
			MetricsPass       []string `toml:"metrics_pass"`
			MetricsDropFilter filter.Filter
			MetricsPassFilter filter.Filter

			// metric name prefix
			MetricsNamePrefix string `toml:"metrics_name_prefix"`

			// mapping value
			ProcessorEnum []*ProcessorEnum `toml:"processor_enum"`

			// whether instance initial success
			inited bool `toml:"-"`
		}
		type InstanceConfig struct {
			InternalConfig
			IntervalTimes int64 `toml:"interval_times"`
		}
	**/


	NamePrefix           string        `toml:"name_prefix"`
	Progs                string        `toml:"progs"` 							//規(guī)則文件（xxx.mtail）的目錄
	Logs                 []string      `toml:"logs"` 							//要監(jiān)控的日志文件
	IgnoreFileRegPattern string        `toml:"ignore_filename_regex_pattern"`
	OverrideTimeZone     string        `toml:"override_timezone"`				//指定時(shí)區(qū)
	EmitProgLabel        string        `toml:"emit_prog_label"`					//是否導(dǎo)出label標(biāo)簽 string類型的bool值 
	emitProgLabel        bool          `toml:"-"`
	EmitMetricTimestamp  string        `toml:"emit_metric_timestamp"`			//metrics是否帶時(shí)間戳 string類型的bool值
	emitMetricTimestamp  bool          `toml:"-"`
	PollInterval         time.Duration `toml:"poll_interval"`
	PollLogInterval      time.Duration `toml:"poll_log_interval"`
	MetricPushInterval   time.Duration `toml:"metric_push_interval"`
	MaxRegexpLen         int           `toml:"max_regexp_length"`
	MaxRecursionDepth    int           `toml:"max_recursion_depth"`

	SyslogUseCurrentYear string `toml:"syslog_use_current_year"` // true
	sysLogUseCurrentYear bool   `toml:"-"`
	LogRuntimeErrors     string `toml:"vm_logs_runtime_errors"` // true
	logRuntimeErrors     bool   `toml:"-"`
	//
	ctx    context.Context    `toml:"-"`
	cancel context.CancelFunc `toml:"-"`
	m      *mtail.Server
}
//配置文件中instances對(duì)象的Init函數(shù)，調(diào)用mtail
func (ins *Instance) Init() error {

	//初始化檢查，設(shè)置默認(rèn)值

	if len(ins.Progs) == 0 || len(ins.Logs) == 0 {
		return types.ErrInstancesEmpty
	}

	// set default value
	ins.sysLogUseCurrentYear = ins.SyslogUseCurrentYear == "true"
	ins.logRuntimeErrors = ins.LogRuntimeErrors == "true"
	ins.emitProgLabel = ins.EmitProgLabel == "true"
	ins.emitMetricTimestamp = ins.EmitMetricTimestamp == "true"

	if ins.PollLogInterval == 0 {
		ins.PollLogInterval = 250 * time.Millisecond
	}
	if ins.PollInterval == 0 {
		ins.PollInterval = 250 * time.Millisecond
	}
	if ins.MetricPushInterval == 0 {
		ins.MetricPushInterval = 1 * time.Minute
	}
	if ins.MaxRegexpLen == 0 {
		ins.MaxRegexpLen = 1024
	}
	if ins.MaxRecursionDepth == 0 {
		ins.MaxRecursionDepth = 100
	}
	buildInfo := mtail.BuildInfo{
		Version: config.Version,
	}
	//時(shí)區(qū)設(shè)置
	loc, err := time.LoadLocation(ins.OverrideTimeZone)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse timezone %q: %s", ins.OverrideTimeZone, err)
		return err
	}
	//mtail參數(shù)設(shè)置
	opts := []mtail.Option{
		mtail.ProgramPath(ins.Progs),
		mtail.LogPathPatterns(ins.Logs...),
		mtail.IgnoreRegexPattern(ins.IgnoreFileRegPattern),
		mtail.SetBuildInfo(buildInfo),
		mtail.OverrideLocation(loc),
		mtail.MetricPushInterval(ins.MetricPushInterval), // keep it here ?
		mtail.MaxRegexpLength(ins.MaxRegexpLen),
		mtail.MaxRecursionDepth(ins.MaxRecursionDepth),
		mtail.LogRuntimeErrors,
	}
	if ins.cancel != nil {
		ins.cancel()
	} else {
		ins.ctx, ins.cancel = context.WithCancel(context.Background()) //父級(jí)ctx
	}
	//mtail配置，每隔1h啟動(dòng) 清理過期日志
	staleLogGcWaker := waker.NewTimed(ins.ctx, time.Hour)
	opts = append(opts, mtail.StaleLogGcWaker(staleLogGcWaker))

	if ins.PollInterval > 0 {
		logStreamPollWaker := waker.NewTimed(ins.ctx, ins.PollInterval)
		logPatternPollWaker := waker.NewTimed(ins.ctx, ins.PollLogInterval)
		opts = append(opts, mtail.LogPatternPollWaker(logPatternPollWaker), mtail.LogstreamPollWaker(logStreamPollWaker))
	}
	if ins.sysLogUseCurrentYear {
		opts = append(opts, mtail.SyslogUseCurrentYear)
	}
	if !ins.emitProgLabel {
		opts = append(opts, mtail.OmitProgLabel)
	}
	if ins.emitMetricTimestamp {
		opts = append(opts, mtail.EmitMetricTimestamp)
	}
	//指標(biāo)結(jié)果存儲(chǔ)對(duì)象
	store := metrics.NewStore()
	//間隔1h清理歷史指標(biāo)
	store.StartGcLoop(ins.ctx, time.Hour)

	m, err := mtail.New(ins.ctx, store, opts...)
	if err != nil {
		log.Println(err)
		ins.cancel()
		return err
	}
	ins.m = m

	return nil
}
//銷毀取消所有任務(wù)
func (ins *Instance) Drop() {
	ins.cancel()
}
//對(duì)象初始化方法
func init() {
	inputs.Add(inputName, func() inputs.Input {
		return &MTail{}
	})
}
//對(duì)象復(fù)制返回新建對(duì)象
func (s *MTail) Clone() inputs.Input {
	return &MTail{}
}

func (s *MTail) Name() string {
	return inputName
}
//MTail獲取配置文件中所有instances
func (s *MTail) GetInstances() []inputs.Instance {
	ret := make([]inputs.Instance, len(s.Instances))
	for i := 0; i < len(s.Instances); i++ {
		ret[i] = s.Instances[i]
	}
	return ret
}

// Description returns a one-sentence description on the input.
func (s *MTail) Description() string {
	return description
}
//抓取數(shù)據(jù)方法？
// Gather retrieves all the configured fields and tables.
// Any error encountered does not halt the process. The errors are accumulated
// and returned at the end.
// func (s *Instance) Gather(acc telegraf.Accumulator) error {
func (ins *Instance) Gather(slist *types.SampleList) {
	//獲取到prometheus注冊(cè)器
	reg := ins.m.GetRegistry()
	mfs, done, err := prometheus.ToTransactionalGatherer(reg).Gather()
	if err != nil {
		log.Println(err)
		return
	}
	defer done()
	//遍歷所有指標(biāo)向量？
	for _, mf := range mfs {
		metricName := mf.GetName()
		//遍歷所有指標(biāo)
		for _, m := range mf.Metric {
			//加入配置的Lables
			tags := util.MakeLabels(m, ins.GetLabels())
			//處理不同指標(biāo)類型
			if mf.GetType() == dto.MetricType_SUMMARY {
				util.HandleSummary(inputName, m, tags, metricName, ins.GetLogMetricTime, slist)
			} else if mf.GetType() == dto.MetricType_HISTOGRAM {
				util.HandleHistogram(inputName, m, tags, metricName, ins.GetLogMetricTime, slist)
			} else {
				util.HandleGaugeCounter(inputName, m, tags, metricName, ins.GetLogMetricTime, slist)
			}
		}
	}
}
//返回時(shí)間戳
func (p *Instance) GetLogMetricTime(ts int64) time.Time {
	var tm time.Time
	if ts <= 0 || !p.emitMetricTimestamp {
		return tm
	}
	sec := ts / 1000
	ms := ts % 1000 * 1e6
	tm = time.Unix(sec, ms)
	return tm
}

整體理解下來，Categraf有效的通過統(tǒng)一的文件完成了多個(gè)目錄，多個(gè)規(guī)則的日志采集，簡(jiǎn)化許多操作。

最后感謝看完,由于作者水平有限,使用很多工具并不熟悉,如有錯(cuò)誤和遺漏歡迎指出,感謝諒解。
以上內(nèi)容來源于官方推出的夜鶯黃埔營(yíng)的免費(fèi)培訓(xùn)活動(dòng)，加入 QQ 群查看直播視頻，還可以在官方答疑站點(diǎn)獲得更多支持 https://answer.flashcat.cloud/文章來源地址http://www.zghlxwxcb.cn/news/detail-442171.html

到了這里，關(guān)于夜鶯初探四·mtail插件采集日志指標(biāo)的文章就介紹完了。如果您還想了解更多內(nèi)容，請(qǐng)?jiān)谟疑辖撬阉鱐OY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！