init repo

.gitignore

@@ -0,0 +1,4 @@
+# this is .gitignore file.
+out
+*.exe
+.idea

README.md

@@ -0,0 +1,62 @@
+# exporterpush
+Remote data writing tool.(remote write data to prometheus and pushgateway)
+
+# 用途
+推送exporter数据到Prometheus、Pushgateway中，除了上述的目标数据源外还对接了腾讯的barad监控系统的指标推送。
+
+# 使用方式
+### 编译
+```
+go build -o exporterpush
+```
+
+### 启动
+```
+./exporterpush -config config.yaml
+```
+
+# 配置文件
+```
+# my global config
+global:
+  scrape_interval: 15 # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  disk: /dev/vda
+  net_interface: eth0
+  scrape_target_types:
+    node_exporter: http://127.0.0.1:9100/metrics #--指定抓取的exporter路径(目前这里暂时支持node_exporter和ck的exporter)
+    clickhouse_exporter: http://127.0.0.1:9363/metrics
+  log:
+    log_save_path: /tmp/logs #--日志路径
+    log_file_name: exportpush #-日志文件名
+    log_file_ext: .log。     #--日志文件后缀
+
+# push plugin configuration
+barad:   #--------- 腾讯barad监控系统对接配置
+  is_use: false
+  app_id: 1
+  instance_id: 22
+  node_id: 33
+  project_id: 0
+  namespace: pce/upclickhouse
+  static_configs:
+    - destination:
+        - http://xxx.barad.tencentyun.com/upclickhouse.cgi
+
+prometheus: #---数据写入远程prometheus配置
+  is_use: true
+  static_configs:
+    - destination:
+        - http://127.0.0.1:9090/api/v1/write
+      labels:
+        cluster_name: test #--自定标签，推送指标的时候添加自定义的标签
+
+pushgateway: #---数据写入远程pushgateway配置
+  is_use: false
+  job_name: push_job
+  static_configs:
+    - destination:
+        - http://127.0.0.1:9091
+      labels:
+        cluster_name: test 
+
+```

config/config.go

@@ -0,0 +1,104 @@
+package config
+
+import (
+	"flag"
+	"fmt"
+	"github.com/exporterpush/global"
+	"github.com/exporterpush/pkg/logger"
+	setting2 "github.com/exporterpush/pkg/setting"
+	"github.com/natefinch/lumberjack"
+	"log"
+)
+
+var (
+	configPath string
+)
+
+func init() {
+	err := setupFlag()
+	if err != nil {
+		log.Fatalf("init.setupFlag err: %v", err)
+	}
+
+	// for test
+	//configPath = "config/config.yaml"
+
+	err = setupSetting()
+	if err != nil {
+		log.Fatalf("init.setupSetting err: %v", err)
+	}
+
+	err = setupLogger()
+	if err != nil {
+		log.Fatalf("init.setupLogger err: %v", err)
+	}
+}
+
+func setupFlag() error {
+	flag.StringVar(&configPath, "config", "config/config.yaml", "指定要使用的配置文件路径")
+	flag.Parse()
+
+	return nil
+}
+
+func setupSetting() error {
+	setting, err := setting2.NewSetting(configPath)
+	if err != nil {
+		return err
+	}
+
+	err = setting.ReadSection("global", &global.GlobalSetting)
+	if err != nil {
+		return err
+	}
+
+	if global.GlobalSetting.ScrapeInterval < 15 {
+		return fmt.Errorf("The value of global.scrape_interval must be greater than 15 ")
+	}
+
+	err = setting.ReadSection("barad", &global.BaradSetting)
+	if err != nil {
+		return err
+	}
+
+	err = setting.ReadSection("prometheus", &global.PrometheusSetting)
+	if err != nil {
+		return err
+	}
+
+	err = setting.ReadSection("pushgateway", &global.PushgatewaySetting)
+	if err != nil {
+		return err
+	}
+
+	if len(global.BaradSetting.StaticConfigs) <= 0 {
+		return fmt.Errorf("barad static_configs is nil")
+	}
+
+	if len(global.BaradSetting.StaticConfigs[0].Destination) <= 0 {
+		return fmt.Errorf("barad destination is nil")
+	}
+
+	if len(global.PushgatewaySetting.StaticConfigs) <= 0 {
+		return fmt.Errorf("pushgateway static_configs is nil")
+	}
+
+	if len(global.PushgatewaySetting.StaticConfigs[0].Destination) <= 0 {
+		return fmt.Errorf("pushgateway destination is nil")
+	}
+
+	return nil
+}
+
+func setupLogger() error {
+
+	global.LogObj = logger.NewLogger(&lumberjack.Logger{
+		Filename: global.GlobalSetting.LogSetting.LogSavePath + "/" + global.GlobalSetting.LogSetting.LogFileName +
+			global.GlobalSetting.LogSetting.LogFileExt,
+		MaxSize:   200,
+		MaxAge:    10,
+		LocalTime: true,
+	}, "", log.LstdFlags).WithCallers(2)
+
+	return nil
+}

config/config.yaml

@@ -0,0 +1,41 @@
+# my global config
+global:
+  scrape_interval: 15 # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  disk: /dev/vda
+  net_interface: eth0
+  scrape_target_types:
+    node_exporter: http://127.0.0.1:9100/metrics
+    clickhouse_exporter: http://127.0.0.1:9363/metrics
+  log:
+    log_save_path: /tmp/logs
+    log_file_name: exportpush
+    log_file_ext: .log
+
+# push plugin configuration
+barad:
+  is_use: false
+  app_id: 1
+  instance_id: 22
+  node_id: 33
+  project_id: 0
+  namespace: pce/upclickhouse
+  static_configs:
+    - destination:
+        - http://receiver.barad.tencentyun.com/upclickhouse.cgi
+
+prometheus:
+  is_use: true
+  static_configs:
+    - destination:
+        - http://127.0.0.1:9090/api/v1/write
+      labels:
+        cluster_name: test
+
+pushgateway:
+  is_use: false
+  job_name: push_job
+  static_configs:
+    - destination:
+        - http://127.0.0.1:9091
+      labels:
+        cluster_name: test

global/model.go

@@ -0,0 +1,9 @@
+package global
+
+// MetricPoint is struct convert to timeSeries
+type MetricPoint struct {
+	Metric   string            `json:"metric"` // 指标名称
+	LabelMap map[string]string `json:"label"`  // 数据标签
+	Time     int64             `json:"time"`   // 时间戳，单位是秒
+	Value    float64           `json:"value"`  // 内部字段，最终转换之后的float64数值
+}

global/setting.go

@@ -0,0 +1,59 @@
+package global
+
+import (
+	"github.com/exporterpush/pkg/logger"
+	"github.com/exporterpush/pkg/setting"
+)
+
+var (
+	GlobalSetting      *setting.GlobalS
+	BaradSetting       *setting.BaradS
+	PrometheusSetting  *setting.PrometheusS
+	PushgatewaySetting *setting.PushgatewayS
+	LogObj             *logger.Logger
+)
+
+const (
+	/*
+		node_exporter
+	*/
+
+	NodeDisKReadsCompeted    = "node_disk_reads_completed"
+	NodeDiskWritesComplete   = "node_disk_writes_completed"
+	NodeDiskBytesWritten     = "node_disk_bytes_written"
+	NodeDiskBytesRead        = "node_disk_bytes_read"
+	NodeFileSystemSize       = "node_filesystem_size"
+	NodeFileSystemFree       = "node_filesystem_free"
+	NodeFileSystemAvail      = "node_filesystem_avail"
+	NodeFileSystemFiles      = "node_filesystem_files"
+	NodeFileSystemFilesFree  = "node_filesystem_files_free"
+	NodeCpu                  = "node_cpu"
+	NodeMemoryMemFree        = "node_memory_MemFree"
+	NodeMemoryBuffers        = "node_memory_Buffers"
+	NodeMemoryCached         = "node_memory_Cached"
+	NodeMemoryMemTotal       = "node_memory_MemTotal"
+	NodeMemorySlab           = "node_memory_Slab"
+	NodeNetworkTransmitBytes = "node_network_transmit_bytes"
+	NodeNetworkReceiveBytes  = "node_network_receive_bytes"
+
+	/*
+		clickhouse_exporter
+	*/
+
+	ClickhouseTcpConnection                = "ClickHouseMetrics_TCPConnection"
+	ClickhouseHttpConnection               = "ClickHouseMetrics_HTTPConnection"
+	ClickhouseMysqlConnection              = "ClickHouseMetrics_MySQLConnection"
+	ClickhouseInterServerConnection        = "ClickHouseMetrics_InterserverConnection"
+	ClickhousePostgreSQLConnection         = "ClickHouseMetrics_PostgreSQLConnection"
+	ClickhouseFailedQueryCount             = "ClickHouseProfileEvents_FailedQuery"
+	ClickhouseQueryCount                   = "ClickHouseProfileEvents_Query"
+	ClickhouseDelayedInsertsCount          = "ClickHouseProfileEvents_DelayedInserts"
+	ClickhouseMergeCount                   = "ClickHouseProfileEvents_Merge"
+	ClickhouseReplicatedPartMutationsCount = "ClickHouseProfileEvents_ReplicatedPartMutations"
+	ClickhouseInsertedRowsCount            = "ClickHouseProfileEvents_InsertedRows"
+	ClickhouseInsertedBytesSize            = "ClickHouseProfileEvents_InsertedBytes"
+	ClickhouseTPS_InsertQueryCount         = "ClickHouseProfileEvents_InsertQuery"
+	ClickhouseQPS_QueryCount               = "ClickHouseProfileEvents_Query"
+	ClickhouseDataPartsGauge               = "ClickHouseMetrics_PartsCommitted"
+	//ClickhouseDataPartsGauge               = "ClickHouseMetrics_PartsActive"
+)

go.mod

@@ -0,0 +1,49 @@
+module github.com/exporterpush
+
+go 1.17
+
+require (
+	github.com/golang/protobuf v1.5.2
+	github.com/golang/snappy v0.0.4
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369
+	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/prometheus/client_golang v1.12.2
+	github.com/prometheus/client_model v0.2.0
+	github.com/prometheus/common v0.34.0
+	github.com/prometheus/prometheus v0.36.1
+	github.com/shirou/gopsutil/v3 v3.22.5
+	github.com/spf13/viper v1.12.0
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/fsnotify/fsnotify v1.5.4 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.6 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/pelletier/go-toml v1.9.5 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/spf13/afero v1.8.2 // indirect
+	github.com/spf13/cast v1.5.0 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.3.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
+)