2020-02-16 19:59:02 +01:00
package main
import (
2020-03-13 11:19:31 +01:00
"context"
2020-02-16 19:59:02 +01:00
"flag"
2020-03-13 11:19:31 +01:00
"fmt"
2020-04-01 17:17:53 +02:00
"net/url"
"os"
2020-06-21 12:32:46 +02:00
"strconv"
2020-03-13 11:19:31 +01:00
"strings"
"time"
2020-02-16 19:59:02 +01:00
2020-10-20 09:15:21 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/config"
2020-02-16 19:59:02 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/datasource"
2020-04-06 13:44:03 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/notifier"
2020-06-28 13:26:22 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remoteread"
2020-04-27 23:18:02 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert/remotewrite"
2020-02-16 19:59:02 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/envflag"
2020-05-14 21:01:51 +02:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fasttime"
2020-03-29 00:48:30 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/flagutil"
2020-02-16 19:59:02 +01:00
"github.com/VictoriaMetrics/VictoriaMetrics/lib/httpserver"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/procutil"
2020-04-11 21:42:01 +02:00
"github.com/VictoriaMetrics/metrics"
2020-02-16 19:59:02 +01:00
)
var (
2020-04-12 13:47:26 +02:00
rulePath = flagutil . NewArray ( "rule" , ` Path to the file with alert rules .
Supports patterns . Flag can be specified multiple times .
2020-03-29 00:48:30 +01:00
Examples :
2020-08-20 23:36:38 +02:00
- rule = "/path/to/file" . Path to a single file with alerting rules
- rule = "dir/*.yaml" - rule = "/*.yaml" . Relative path to all . yaml files in "dir" folder ,
2020-08-13 15:43:55 +02:00
absolute path to all . yaml files in root .
Rule files may contain % { ENV_VAR } placeholders , which are substituted by the corresponding env vars . ` )
2020-06-06 22:27:09 +02:00
2020-06-28 13:26:22 +02:00
httpListenAddr = flag . String ( "httpListenAddr" , ":8880" , "Address to listen for http connections" )
evaluationInterval = flag . Duration ( "evaluationInterval" , time . Minute , "How often to evaluate the rules" )
2020-06-06 22:27:09 +02:00
validateTemplates = flag . Bool ( "rule.validateTemplates" , true , "Whether to validate annotation and label templates" )
validateExpressions = flag . Bool ( "rule.validateExpressions" , true , "Whether to validate rules expressions via MetricsQL engine" )
2020-06-21 12:32:46 +02:00
externalURL = flag . String ( "external.url" , "" , "External URL is used as alert's source for sent alerts to the notifier" )
externalAlertSource = flag . String ( "external.alert.source" , "" , ` External Alert Source allows to override the Source link for alerts sent to AlertManager for cases where you want to build a custom link to Grafana , Prometheus or any other service .
eg . ' explore ? orgId = 1 & left = [ \ "now-1h\",\"now\",\"VictoriaMetrics\",{\"expr\": \"{{$expr|quotesEscape|pathEscape}}\"},{\"mode\":\"Metrics\"},{\"ui\":[true,true,true,\"none\" ] } ] ' . If empty ' / api / v1 / : groupID / alertID / status ' is used ` )
2020-07-28 13:20:31 +02:00
externalLabels = flagutil . NewArray ( "external.label" , "Optional label in the form 'name=value' to add to all generated recording rules and alerts. " +
"Pass multiple -label flags in order to add multiple label sets." )
2020-06-28 13:26:22 +02:00
remoteReadLookBack = flag . Duration ( "remoteRead.lookback" , time . Hour , "Lookback defines how far to look into past for alerts timeseries." +
" For example, if lookback=1h then range from now() to now()-1h will be scanned." )
2020-10-20 09:15:21 +02:00
dryRun = flag . Bool ( "dryRun" , false , "Whether to check only config files without running vmalert. The rules file are validated. The `-rule` flag must be specified." )
2020-02-16 19:59:02 +01:00
)
func main ( ) {
2020-05-16 10:59:30 +02:00
// Write flags and help message to stdout, since it is easier to grep or pipe.
flag . CommandLine . SetOutput ( os . Stdout )
2020-06-05 09:42:56 +02:00
flag . Usage = usage
2020-02-16 19:59:02 +01:00
envflag . Parse ( )
buildinfo . Init ( )
logger . Init ( )
2020-06-23 21:45:45 +02:00
2020-10-20 09:15:21 +02:00
if * dryRun {
u , _ := url . Parse ( "https://victoriametrics.com/" )
notifier . InitTemplateFunc ( u )
groups , err := config . Parse ( * rulePath , true , true )
if err != nil {
logger . Fatalf ( err . Error ( ) )
}
if len ( groups ) == 0 {
logger . Fatalf ( "No rules for validation. Please specify path to file(s) with alerting and/or recording rules using `-rule` flag" )
}
return
}
2020-06-28 13:26:22 +02:00
ctx , cancel := context . WithCancel ( context . Background ( ) )
manager , err := newManager ( ctx )
2020-06-23 21:45:45 +02:00
if err != nil {
2020-06-28 13:26:22 +02:00
logger . Fatalf ( "failed to init: %s" , err )
2020-04-27 23:18:02 +02:00
}
2020-06-06 22:27:09 +02:00
if err := manager . start ( ctx , * rulePath , * validateTemplates , * validateExpressions ) ; err != nil {
2020-05-10 18:58:17 +02:00
logger . Fatalf ( "failed to start: %s" , err )
}
2020-05-09 11:32:12 +02:00
2020-05-10 18:58:17 +02:00
go func ( ) {
// init reload metrics with positive values to improve alerting conditions
configSuccess . Set ( 1 )
2020-05-14 21:01:51 +02:00
configTimestamp . Set ( fasttime . UnixTimestamp ( ) )
2020-05-10 18:58:17 +02:00
sigHup := procutil . NewSighupChan ( )
for {
<- sigHup
configReloads . Inc ( )
logger . Infof ( "SIGHUP received. Going to reload rules %q ..." , * rulePath )
2020-06-06 22:27:09 +02:00
if err := manager . update ( ctx , * rulePath , * validateTemplates , * validateExpressions , false ) ; err != nil {
2020-05-10 18:58:17 +02:00
configReloadErrors . Inc ( )
configSuccess . Set ( 0 )
logger . Errorf ( "error while reloading rules: %s" , err )
continue
}
configSuccess . Set ( 1 )
2020-05-14 21:01:51 +02:00
configTimestamp . Set ( fasttime . UnixTimestamp ( ) )
2020-05-10 18:58:17 +02:00
logger . Infof ( "Rules reloaded successfully from %q" , * rulePath )
}
} ( )
2020-05-09 11:32:12 +02:00
2020-05-10 18:58:17 +02:00
rh := & requestHandler { m : manager }
2020-05-18 10:55:16 +02:00
go httpserver . Serve ( * httpListenAddr , rh . handler )
2020-04-06 13:44:03 +02:00
2020-02-16 19:59:02 +01:00
sig := procutil . WaitForSigterm ( )
logger . Infof ( "service received signal %s" , sig )
2020-02-21 22:15:05 +01:00
if err := httpserver . Stop ( * httpListenAddr ) ; err != nil {
logger . Fatalf ( "cannot stop the webservice: %s" , err )
}
2020-03-13 11:19:31 +01:00
cancel ( )
2020-05-10 18:58:17 +02:00
manager . close ( )
2020-03-13 11:19:31 +01:00
}
2020-04-11 21:42:01 +02:00
var (
2020-05-10 18:58:17 +02:00
configReloads = metrics . NewCounter ( ` vmalert_config_last_reload_total ` )
configReloadErrors = metrics . NewCounter ( ` vmalert_config_last_reload_errors_total ` )
configSuccess = metrics . NewCounter ( ` vmalert_config_last_reload_successful ` )
configTimestamp = metrics . NewCounter ( ` vmalert_config_last_reload_success_timestamp_seconds ` )
2020-04-11 21:42:01 +02:00
)
2020-06-28 13:26:22 +02:00
func newManager ( ctx context . Context ) ( * manager , error ) {
q , err := datasource . Init ( )
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "failed to init datasource: %w" , err )
2020-06-28 13:26:22 +02:00
}
eu , err := getExternalURL ( * externalURL , * httpListenAddr , httpserver . IsTLS ( ) )
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "failed to init `external.url`: %w" , err )
2020-06-28 13:26:22 +02:00
}
notifier . InitTemplateFunc ( eu )
aug , err := getAlertURLGenerator ( eu , * externalAlertSource , * validateTemplates )
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "failed to init `external.alert.source`: %w" , err )
2020-06-28 13:26:22 +02:00
}
2020-06-29 21:21:03 +02:00
nts , err := notifier . Init ( aug )
2020-06-28 13:26:22 +02:00
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "failed to init notifier: %w" , err )
2020-06-28 13:26:22 +02:00
}
manager := & manager {
2020-06-29 21:21:03 +02:00
groups : make ( map [ uint64 ] * Group ) ,
querier : q ,
notifiers : nts ,
2020-07-28 13:20:31 +02:00
labels : map [ string ] string { } ,
2020-06-28 13:26:22 +02:00
}
rw , err := remotewrite . Init ( ctx )
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "failed to init remoteWrite: %w" , err )
2020-06-28 13:26:22 +02:00
}
manager . rw = rw
rr , err := remoteread . Init ( )
if err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "failed to init remoteRead: %w" , err )
2020-06-28 13:26:22 +02:00
}
manager . rr = rr
2020-07-28 13:20:31 +02:00
for _ , s := range * externalLabels {
2020-12-08 13:53:41 +01:00
if len ( s ) == 0 {
continue
}
2020-07-28 13:20:31 +02:00
n := strings . IndexByte ( s , '=' )
if n < 0 {
return nil , fmt . Errorf ( "missing '=' in `-label`. It must contain label in the form `name=value`; got %q" , s )
}
manager . labels [ s [ : n ] ] = s [ n + 1 : ]
}
2020-06-28 13:26:22 +02:00
return manager , nil
}
2020-04-01 17:17:53 +02:00
func getExternalURL ( externalURL , httpListenAddr string , isSecure bool ) ( * url . URL , error ) {
if externalURL != "" {
return url . Parse ( externalURL )
2020-03-13 11:19:31 +01:00
}
2020-04-01 17:17:53 +02:00
hname , err := os . Hostname ( )
2020-03-13 11:19:31 +01:00
if err != nil {
2020-04-01 17:17:53 +02:00
return nil , err
2020-03-13 11:19:31 +01:00
}
2020-04-01 17:17:53 +02:00
port := ""
if ipport := strings . Split ( httpListenAddr , ":" ) ; len ( ipport ) > 1 {
port = ":" + ipport [ 1 ]
}
schema := "http://"
if isSecure {
schema = "https://"
2020-03-13 11:19:31 +01:00
}
2020-04-01 17:17:53 +02:00
return url . Parse ( fmt . Sprintf ( "%s%s%s" , schema , hname , port ) )
2020-02-16 19:59:02 +01:00
}
2020-06-21 12:32:46 +02:00
func getAlertURLGenerator ( externalURL * url . URL , externalAlertSource string , validateTemplate bool ) ( notifier . AlertURLGenerator , error ) {
if externalAlertSource == "" {
return func ( alert notifier . Alert ) string {
return fmt . Sprintf ( "%s/api/v1/%s/%s/status" , externalURL , strconv . FormatUint ( alert . GroupID , 10 ) , strconv . FormatUint ( alert . ID , 10 ) )
} , nil
}
if validateTemplate {
if err := notifier . ValidateTemplates ( map [ string ] string {
"tpl" : externalAlertSource ,
} ) ; err != nil {
2020-06-30 21:58:18 +02:00
return nil , fmt . Errorf ( "error validating source template %s: %w" , externalAlertSource , err )
2020-06-21 12:32:46 +02:00
}
}
m := map [ string ] string {
"tpl" : externalAlertSource ,
}
return func ( alert notifier . Alert ) string {
templated , err := alert . ExecTemplate ( m )
if err != nil {
logger . Errorf ( "can not exec source template %s" , err )
}
return fmt . Sprintf ( "%s/%s" , externalURL , templated [ "tpl" ] )
} , nil
}
2020-06-05 09:42:56 +02:00
func usage ( ) {
const s = `
vmalert processes alerts and recording rules .
See the docs at https : //github.com/VictoriaMetrics/VictoriaMetrics/blob/master/app/vmalert/README.md .
`
2020-12-03 20:40:30 +01:00
flagutil . Usage ( s )
2020-06-05 09:42:56 +02:00
}