VictoriaMetrics/lib/promscrape/discovery/kubernetes/api.go

package kubernetes

import (
	"encoding/base64"
	"fmt"
	"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
	"gopkg.in/yaml.v2"
	"net"
	"os"
	"strings"

	"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)

// apiConfig contains config for API server
type apiConfig struct {
	aw *apiWatcher
}

type Config struct {
	Kind        string      `yaml:"kind,omitempty"`
	APIVersion  string      `yaml:"apiVersion,omitempty"`
	Preferences Preferences `yaml:"preferences"`
	Clusters    []struct {
		Name    string   `yaml:"name"`
		Cluster *Cluster `yaml:"cluster"`
	} `yaml:"clusters"`
	AuthInfos []struct {
		Name     string    `yaml:"name"`
		AuthInfo *AuthInfo `yaml:"user"`
	} `yaml:"users"`
	Contexts []struct {
		Name    string   `yaml:"name"`
		Context *Context `yaml:"context"`
	} `yaml:"contexts"`
	CurrentContext string `yaml:"current-context"`
}

type Preferences struct {
	Colors bool `yaml:"colors,omitempty"`
}

type Cluster struct {
	LocationOfOrigin         string
	Server                   string `yaml:"server"`
	TLSServerName            string `yaml:"tls-server-name,omitempty"`
	InsecureSkipTLSVerify    bool   `yaml:"insecure-skip-tls-verify,omitempty"`
	CertificateAuthority     string `yaml:"certificate-authority,omitempty"`
	CertificateAuthorityData string `yaml:"certificate-authority-data,omitempty"`
	ProxyURL                 string `yaml:"proxy-url,omitempty"`
}

// AuthInfo contains information that describes identity information.  This is use to tell the kubernetes cluster who you are.
type AuthInfo struct {
	LocationOfOrigin      string
	ClientCertificate     string   `yaml:"client-certificate,omitempty"`
	ClientCertificateData string   `yaml:"client-certificate-data,omitempty"`
	ClientKey             string   `yaml:"client-key,omitempty"`
	ClientKeyData         string   `yaml:"client-key-data,omitempty"`
	Token                 string   `yaml:"token,omitempty"`
	TokenFile             string   `yaml:"tokenFile,omitempty"`
	Impersonate           string   `yaml:"act-as,omitempty"`
	ImpersonateUID        string   `yaml:"act-as-uid,omitempty"`
	ImpersonateGroups     []string `yaml:"act-as-groups,omitempty"`
	ImpersonateUserExtra  []string `yaml:"act-as-user-extra,omitempty"`
	Username              string   `yaml:"username,omitempty"`
	Password              string   `yaml:"password,omitempty"`
}

// Context is a tuple of references to a cluster (how do I communicate with a kubernetes cluster), a user (how do I identify myself), and a namespace (what subset of resources do I want to work with)
type Context struct {
	LocationOfOrigin string
	Cluster          string `yaml:"cluster"`
	AuthInfo         string `yaml:"user"`
	Namespace        string `yaml:"namespace,omitempty"`
}

type ApiConfig struct {
	basicAuth *promauth.BasicAuthConfig
	server    string
	token     string
	tokenFile string
	tlsConfig *promauth.TLSConfig
}

func buildConfig(sdc *SDConfig) (*ApiConfig, error) {

	data, err := fs.ReadFileOrHTTP(sdc.KubeConfig)
	if err != nil {
		return nil, fmt.Errorf("cannot read kubeConfig from %q: %w", sdc.KubeConfig, err)
	}
	var config Config
	if err = yaml.Unmarshal(data, &config); err != nil {
		return nil, fmt.Errorf("cannot parse %q: %w", sdc.KubeConfig, err)
	}

	var authInfos = make(map[string]*AuthInfo)
	for _, obj := range config.AuthInfos {
		authInfos[obj.Name] = obj.AuthInfo
	}
	var clusterInfos = make(map[string]*Cluster)
	for _, obj := range config.Clusters {
		clusterInfos[obj.Name] = obj.Cluster
	}
	var contexts = make(map[string]*Context)
	for _, obj := range config.Contexts {
		contexts[obj.Name] = obj.Context
	}

	contextName := config.CurrentContext
	configContext, exists := contexts[contextName]
	if !exists {
		return nil, fmt.Errorf("context %q does not exist", contextName)
	}

	clusterInfoName := configContext.Cluster
	configClusterInfo, exists := clusterInfos[clusterInfoName]
	if !exists {
		return nil, fmt.Errorf("cluster %q does not exist", clusterInfoName)
	}

	authInfoName := configContext.AuthInfo
	configAuthInfo, exists := authInfos[authInfoName]
	if authInfoName != "" && !exists {
		return nil, fmt.Errorf("auth info %q does not exist", authInfoName)
	}

	var apiConfig ApiConfig

	apiConfig.tlsConfig = &promauth.TLSConfig{
		CAFile:             configClusterInfo.CertificateAuthority,
		ServerName:         configClusterInfo.TLSServerName,
		InsecureSkipVerify: configClusterInfo.InsecureSkipTLSVerify,
	}

	if len(configClusterInfo.CertificateAuthorityData) != 0 {
		apiConfig.tlsConfig.CA, err = base64.StdEncoding.DecodeString(configClusterInfo.CertificateAuthorityData)
		if err != nil {
			return nil, fmt.Errorf("cannot base64-decode configClusterInfo.CertificateAuthorityData %q: %w", configClusterInfo.CertificateAuthorityData, err)
		}
	}

	if configAuthInfo != nil {
		apiConfig.tlsConfig.CertFile = configAuthInfo.ClientCertificate
		apiConfig.tlsConfig.KeyFile = configAuthInfo.ClientKey
		apiConfig.token = configAuthInfo.Token
		apiConfig.tokenFile = configAuthInfo.TokenFile
		if len(configAuthInfo.ClientCertificateData) != 0 {
			apiConfig.tlsConfig.Cert, err = base64.StdEncoding.DecodeString(configAuthInfo.ClientCertificateData)
			if err != nil {
				return nil, fmt.Errorf("cannot base64-decode configAuthInfo.ClientCertificateData %q: %w", configClusterInfo.CertificateAuthorityData, err)
			}
		}
		if len(configAuthInfo.ClientKeyData) != 0 {
			apiConfig.tlsConfig.Key, err = base64.StdEncoding.DecodeString(configAuthInfo.ClientKeyData)
			if err != nil {
				return nil, fmt.Errorf("cannot base64-decode configAuthInfo.ClientKeyData %q: %w", configClusterInfo.CertificateAuthorityData, err)
			}
		}
		if len(configAuthInfo.Username) > 0 || len(configAuthInfo.Password) > 0 {
			apiConfig.basicAuth = &promauth.BasicAuthConfig{
				Username: configAuthInfo.Username,
				Password: promauth.NewSecret(configAuthInfo.Password),
			}
		}
	}

	apiConfig.server = configClusterInfo.Server

	return &apiConfig, nil
}

func newAPIConfig(sdc *SDConfig, baseDir string, swcFunc ScrapeWorkConstructorFunc) (*apiConfig, error) {
	role := sdc.role()
	switch role {
	case "node", "pod", "service", "endpoints", "endpointslice", "ingress":
	default:
		return nil, fmt.Errorf("unexpected `role`: %q; must be one of `node`, `pod`, `service`, `endpoints`, `endpointslice` or `ingress`", role)
	}
	ac, err := sdc.HTTPClientConfig.NewConfig(baseDir)
	if err != nil {
		return nil, fmt.Errorf("cannot parse auth config: %w", err)
	}
	apiServer := sdc.APIServer

	if len(sdc.KubeConfig) != 0 {
		config, err := buildConfig(sdc)
		if err != nil {
			return nil, fmt.Errorf("cannot parse kube config: %w", err)
		}
		acNew, err := promauth.NewConfig(".", nil, config.basicAuth, config.token, config.tokenFile, nil, config.tlsConfig)
		if err != nil {
			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
		}
		ac = acNew
		apiServer = config.server
	}

	if len(apiServer) == 0 {
		// Assume we run at k8s pod.
		// Discover apiServer and auth config according to k8s docs.
		// See https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller
		host := os.Getenv("KUBERNETES_SERVICE_HOST")
		port := os.Getenv("KUBERNETES_SERVICE_PORT")
		if len(host) == 0 {
			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_HOST env var; it must be defined when running in k8s; " +
				"probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?")
		}
		if len(port) == 0 {
			return nil, fmt.Errorf("cannot find KUBERNETES_SERVICE_PORT env var; it must be defined when running in k8s; "+
				"KUBERNETES_SERVICE_HOST=%q", host)
		}
		apiServer = "https://" + net.JoinHostPort(host, port)
		tlsConfig := promauth.TLSConfig{
			CAFile: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt",
		}
		acNew, err := promauth.NewConfig(".", nil, nil, "", "/var/run/secrets/kubernetes.io/serviceaccount/token", nil, &tlsConfig)
		if err != nil {
			return nil, fmt.Errorf("cannot initialize service account auth: %w; probably, `kubernetes_sd_config->api_server` is missing in Prometheus configs?", err)
		}
		ac = acNew
	}
	if !strings.Contains(apiServer, "://") {
		proto := "http"
		if sdc.HTTPClientConfig.TLSConfig != nil {
			proto = "https"
		}
		apiServer = proto + "://" + apiServer
	}
	for strings.HasSuffix(apiServer, "/") {
		apiServer = apiServer[:len(apiServer)-1]
	}
	aw := newAPIWatcher(apiServer, ac, sdc, swcFunc)
	cfg := &apiConfig{
		aw: aw,
	}
	return cfg, nil
}