夜莺Nightingale V5 通过Exporter实现告警升级

nightingale

由于夜莺监控没有告警升级这个功能，可以通过自定义exporter去夜莺数据库查询活跃告警暴露指标出来实现，代码如下

夜莺版本：V5.15.0

开发环境：Golang1.18

package main

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"log"
	"net/http"
	"strconv"
	"time"
)

var (
	alertEventMetric = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name: "alert_event_trigger_time",
			Help: "Trigger time of the alert events in UNIX timestamp",
		},
		[]string{"id", "cluster", "group_id", "severity", "description", "Group", "Region"}, #此处标签根据业务实际情况添加
	)
)

func init() {
	// Register custom metrics with Prometheus's default registry.
	prometheus.MustRegister(alertEventMetric)
}
func queryDatabaseAndUpdateMetrics() {
	dsn := "xxx:xxx@tcp(xxxx:3306)/n9e_v5"
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		log.Fatalf("Failed to connect to MySQL: %v", err)
	}
	defer db.Close()

	// Ensure the connection is valid.
	err = db.Ping()
	if err != nil {
		log.Fatalf("MySQL connection error: %v", err)
	}

	// Query the alert_cur_event table.
	query := `SELECT id, cluster, group_id, severity, rule_name, trigger_time FROM alert_cur_event where severity < 3;`

	rows, err := db.Query(query)
	if err != nil {
		log.Fatalf("Failed to execute query: %v", err)
	}
	defer rows.Close()

	// Clear existing metrics before updating.
	alertEventMetric.Reset()
	currentTime := time.Now().Unix()
	// Iterate over the result set and update Prometheus metrics.
	for rows.Next() {
		var id int
		var cluster string
		var groupID int
		var severity int
		var ruleName string
		var triggerTime int64
		var region string

		err := rows.Scan(&id, &cluster, &groupID, &severity, &ruleName, &triggerTime)
		if err != nil {
			log.Printf("Error scanning row: %v", err)
			continue
		}
		region = "告警升级"
		var metricValue float64
		if currentTime-triggerTime > 8*3600 { //8小时升级，可根据实际需求修改
			metricValue = 1 // Indicates that the event is older than 8 hours.
		} else {
			metricValue = 0 // Indicates that the event is within 8 hours.
		}
		// Update the metric with label values.
		alertEventMetric.WithLabelValues(
			strconv.Itoa(id),
			cluster,
			strconv.Itoa(groupID),
			strconv.Itoa(severity),
			ruleName,
			cluster,
			region,
		).Set(float64(metricValue))
	}

	if err = rows.Err(); err != nil {
		log.Fatalf("Error during rows iteration: %v", err)
	}
}

func main() {
	go func() {
		for {
			queryDatabaseAndUpdateMetrics()
			time.Sleep(30 * time.Second) // Query interval
		}
	}()

	http.Handle("/metrics", promhttp.Handler())
	log.Println("Starting HTTP server on :8080")
	log.Fatal(http.ListenAndServe(":8080", nil))
}