diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ff4226f27c..a9eb933795 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,7 @@ sort: 15 * FEATURE: vmagent: add ability to read scrape configs from multiple files specified in `scrape_config_files` section. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1559). * FEATURE: vmagent: reduce memory usage and CPU usage when Prometheus staleness tracking is enabled for metrics exported from the deleted or disappeared scrape targets. +* FEATURE: vmagent: discover `role: ingress` and `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) via v1 API instead of v1beta1 API if Kubernetes supports it. This fixes service discovery in Kubernetes v1.22 and newer versions. See [these docs](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#ingress-v122). * FEATURE: take into account failed queries in `vm_request_duration_seconds` summary at `/metrics`. Previously only successful queries were taken into account. This could result in skewed summary. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1537). * FEATURE: vmalert: add `-disableAlertgroupLabel` command-line flag for disabling the label with alert group name. This may be needed for proper deduplication in Alertmanager. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1532). * FEATURE: update Go builder from v1.16.7 to v1.17.0. This improves data ingestion and query performance by up to 5% according to benchmarks. See [the release post for Go1.17](https://go.dev/blog/go1.17). diff --git a/lib/promscrape/discovery/kubernetes/api_watcher.go b/lib/promscrape/discovery/kubernetes/api_watcher.go index 9171ab8563..b12d2e62c5 100644 --- a/lib/promscrape/discovery/kubernetes/api_watcher.go +++ b/lib/promscrape/discovery/kubernetes/api_watcher.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/VictoriaMetrics/VictoriaMetrics/lib/logger" @@ -159,6 +160,14 @@ func (aw *apiWatcher) getScrapeWorkObjects() []interface{} { // groupWatcher watches for Kubernetes objects on the given apiServer with the given namespaces, // selectors using the given client. type groupWatcher struct { + // Old Kubernetes doesn't support /apis/networking.k8s.io/v1/, so /apis/networking.k8s.io/v1beta1/ must be used instead. + // This flag is used for automatic substitution of v1 API path with v1beta1 API path during requests to apiServer. + useNetworkingV1Beta1 uint32 + + // Old Kubernetes doesn't support /apis/discovery.k8s.io/v1/, so discovery.k8s.io/v1beta1/ must be used instead. + // This flag is used for automatic substitution of v1 API path with v1beta1 API path during requests to apiServer. + useDiscoveryV1Beta1 uint32 + apiServer string namespaces []string selectors []Selector @@ -294,6 +303,14 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) { // doRequest performs http request to the given requestURL. func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) { + if strings.Contains(requestURL, "/apis/networking.k8s.io/v1/") && atomic.LoadUint32(&gw.useNetworkingV1Beta1) == 1 { + // Update networking URL for old Kubernetes API, which supports only v1beta1 path. + requestURL = strings.Replace(requestURL, "/apis/networking.k8s.io/v1/", "/apis/networking.k8s.io/v1beta1/", 1) + } + if strings.Contains(requestURL, "/apis/discovery.k8s.io/v1/") && atomic.LoadUint32(&gw.useDiscoveryV1Beta1) == 1 { + // Update discovery URL for old Kuberentes API, which supports only v1beta1 path. + requestURL = strings.Replace(requestURL, "/apis/discovery.k8s.io/v1/", "/apis/discovery.k8s.io/v1beta1/", 1) + } req, err := http.NewRequest("GET", requestURL, nil) if err != nil { logger.Fatalf("cannot create a request for %q: %s", requestURL, err) @@ -301,7 +318,21 @@ func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) { if ah := gw.getAuthHeader(); ah != "" { req.Header.Set("Authorization", ah) } - return gw.client.Do(req) + resp, err := gw.client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode == http.StatusNotFound { + if strings.Contains(requestURL, "/apis/networking.k8s.io/v1/") && atomic.LoadUint32(&gw.useNetworkingV1Beta1) == 0 { + atomic.StoreUint32(&gw.useNetworkingV1Beta1, 1) + return gw.doRequest(requestURL) + } + if strings.Contains(requestURL, "/apis/discovery.k8s.io/v1/") && atomic.LoadUint32(&gw.useDiscoveryV1Beta1) == 0 { + atomic.StoreUint32(&gw.useDiscoveryV1Beta1, 1) + return gw.doRequest(requestURL) + } + } + return resp, nil } func (gw *groupWatcher) registerPendingAPIWatchers() { @@ -682,10 +713,10 @@ func getAPIPath(objectType, namespace, query string) string { suffix += "?" + query } if objectType == "ingresses" { - return "/apis/networking.k8s.io/v1beta1/" + suffix + return "/apis/networking.k8s.io/v1/" + suffix } if objectType == "endpointslices" { - return "/apis/discovery.k8s.io/v1beta1/" + suffix + return "/apis/discovery.k8s.io/v1/" + suffix } return "/api/v1/" + suffix } diff --git a/lib/promscrape/discovery/kubernetes/api_watcher_test.go b/lib/promscrape/discovery/kubernetes/api_watcher_test.go index 35b2b181cc..bc619d416b 100644 --- a/lib/promscrape/discovery/kubernetes/api_watcher_test.go +++ b/lib/promscrape/discovery/kubernetes/api_watcher_test.go @@ -129,7 +129,7 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) { }) // role=endpointslices - f("endpointslices", nil, nil, []string{"/apis/discovery.k8s.io/v1beta1/endpointslices"}) + f("endpointslices", nil, nil, []string{"/apis/discovery.k8s.io/v1/endpointslices"}) f("endpointslices", []string{"x", "y"}, []Selector{ { Role: "endpointslices", @@ -137,12 +137,12 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) { Label: "label", }, }, []string{ - "/apis/discovery.k8s.io/v1beta1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field", - "/apis/discovery.k8s.io/v1beta1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field", + "/apis/discovery.k8s.io/v1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field", + "/apis/discovery.k8s.io/v1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field", }) // role=ingress - f("ingress", nil, nil, []string{"/apis/networking.k8s.io/v1beta1/ingresses"}) + f("ingress", nil, nil, []string{"/apis/networking.k8s.io/v1/ingresses"}) f("ingress", []string{"x", "y"}, []Selector{ { Role: "node", @@ -161,8 +161,8 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) { Label: "baaa", }, }, []string{ - "/apis/networking.k8s.io/v1beta1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc", - "/apis/networking.k8s.io/v1beta1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc", + "/apis/networking.k8s.io/v1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc", + "/apis/networking.k8s.io/v1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc", }) } @@ -577,9 +577,9 @@ func TestGetScrapeWorkObjects(t *testing.T) { initAPIObjectsByRole: map[string][]byte{ "ingress": []byte(`{ "kind": "IngressList", - "apiVersion": "extensions/v1beta1", + "apiVersion": "extensions/v1", "metadata": { - "selfLink": "/apis/extensions/v1beta1/ingresses", + "selfLink": "/apis/extensions/v1/ingresses", "resourceVersion": "351452" }, "items": [ @@ -678,9 +678,9 @@ func TestGetScrapeWorkObjects(t *testing.T) { initAPIObjectsByRole: map[string][]byte{ "endpointslices": []byte(`{ "kind": "EndpointSliceList", - "apiVersion": "discovery.k8s.io/v1beta1", + "apiVersion": "discovery.k8s.io/v1", "metadata": { - "selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices", + "selfLink": "/apis/discovery.k8s.io/v1/endpointslices", "resourceVersion": "1177" }, "items": [ diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go index 68b5c3c3df..a4a9b21b9e 100644 --- a/lib/promscrape/discovery/kubernetes/endpoints.go +++ b/lib/promscrape/discovery/kubernetes/endpoints.go @@ -79,7 +79,7 @@ type ObjectReference struct { // EndpointPort implements k8s endpoint port. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointport-v1beta1-discovery-k8s-io +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointport-v1-discovery-k8s-io type EndpointPort struct { AppProtocol string Name string diff --git a/lib/promscrape/discovery/kubernetes/endpointslices.go b/lib/promscrape/discovery/kubernetes/endpointslices.go index 4ac24b4750..37053de79e 100644 --- a/lib/promscrape/discovery/kubernetes/endpointslices.go +++ b/lib/promscrape/discovery/kubernetes/endpointslices.go @@ -146,16 +146,17 @@ func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp En return m } -// EndpointSliceList - implements kubernetes endpoint slice list object, -// that groups service endpoints slices. -// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io +// EndpointSliceList - implements kubernetes endpoint slice list object, that groups service endpoints slices. +// +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslicelist-v1-discovery-k8s-io type EndpointSliceList struct { Metadata ListMeta Items []*EndpointSlice } // EndpointSlice - implements kubernetes endpoint slice. -// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io +// +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslice-v1-discovery-k8s-io type EndpointSlice struct { Metadata ObjectMeta Endpoints []Endpoint @@ -164,7 +165,8 @@ type EndpointSlice struct { } // Endpoint implements kubernetes object endpoint for endpoint slice. -// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoint-v1beta1-discovery-k8s-io +// +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpoint-v1-discovery-k8s-io type Endpoint struct { Addresses []string Conditions EndpointConditions @@ -174,7 +176,8 @@ type Endpoint struct { } // EndpointConditions implements kubernetes endpoint condition. -// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointconditions-v1beta1-discovery-k8s-io +// +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointconditions-v1-discovery-k8s-io type EndpointConditions struct { Ready bool } diff --git a/lib/promscrape/discovery/kubernetes/endpointslices_test.go b/lib/promscrape/discovery/kubernetes/endpointslices_test.go index 4114e0cff6..f647695f47 100644 --- a/lib/promscrape/discovery/kubernetes/endpointslices_test.go +++ b/lib/promscrape/discovery/kubernetes/endpointslices_test.go @@ -32,9 +32,9 @@ func TestParseEndpointSliceListFail(t *testing.T) { func TestParseEndpointSliceListSuccess(t *testing.T) { data := `{ "kind": "EndpointSliceList", - "apiVersion": "discovery.k8s.io/v1beta1", + "apiVersion": "discovery.k8s.io/v1", "metadata": { - "selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices", + "selfLink": "/apis/discovery.k8s.io/v1/endpointslices", "resourceVersion": "1177" }, "items": [ @@ -42,7 +42,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) { "metadata": { "name": "kubernetes", "namespace": "default", - "selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/default/endpointslices/kubernetes", + "selfLink": "/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes", "uid": "a60d9173-5fe4-4bc3-87a6-269daee71f8a", "resourceVersion": "159", "generation": 1, @@ -54,7 +54,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) { { "manager": "kube-apiserver", "operation": "Update", - "apiVersion": "discovery.k8s.io/v1beta1", + "apiVersion": "discovery.k8s.io/v1", "time": "2020-09-07T14:27:22Z", "fieldsType": "FieldsV1", "fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:labels":{".":{},"f:kubernetes.io/service-name":{}}},"f:ports":{}} @@ -85,7 +85,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) { "name": "kube-dns-22mvb", "generateName": "kube-dns-", "namespace": "kube-system", - "selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/kube-system/endpointslices/kube-dns-22mvb", + "selfLink": "/apis/discovery.k8s.io/v1/namespaces/kube-system/endpointslices/kube-dns-22mvb", "uid": "7c95c854-f34c-48e1-86f5-bb8269113c11", "resourceVersion": "604", "generation": 5, @@ -111,7 +111,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) { { "manager": "kube-controller-manager", "operation": "Update", - "apiVersion": "discovery.k8s.io/v1beta1", + "apiVersion": "discovery.k8s.io/v1", "time": "2020-09-07T14:28:35Z", "fieldsType": "FieldsV1", "fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:annotations":{".":{},"f:endpoints.kubernetes.io/last-change-trigger-time":{}},"f:generateName":{},"f:labels":{".":{},"f:endpointslice.kubernetes.io/managed-by":{},"f:kubernetes.io/service-name":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"509e80d8-6d05-487b-bfff-74f5768f1024\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:ports":{}} diff --git a/lib/promscrape/discovery/kubernetes/ingress.go b/lib/promscrape/discovery/kubernetes/ingress.go index cca6bdb240..deedd32f2f 100644 --- a/lib/promscrape/discovery/kubernetes/ingress.go +++ b/lib/promscrape/discovery/kubernetes/ingress.go @@ -33,7 +33,7 @@ func parseIngress(data []byte) (object, error) { // IngressList represents ingress list in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresslist-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingresslist-v1-networking-k8s-io type IngressList struct { Metadata ListMeta Items []*Ingress @@ -41,7 +41,7 @@ type IngressList struct { // Ingress represents ingress in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingress-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingress-v1-networking-k8s-io type Ingress struct { Metadata ObjectMeta Spec IngressSpec @@ -49,7 +49,7 @@ type Ingress struct { // IngressSpec represents ingress spec in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressspec-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressspec-v1-networking-k8s-io type IngressSpec struct { TLS []IngressTLS `json:"tls"` Rules []IngressRule @@ -57,14 +57,14 @@ type IngressSpec struct { // IngressTLS represents ingress TLS spec in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresstls-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingresstls-v1-networking-k8s-io type IngressTLS struct { Hosts []string } // IngressRule represents ingress rule in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressrule-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressrule-v1-networking-k8s-io type IngressRule struct { Host string HTTP HTTPIngressRuleValue `json:"http"` @@ -72,14 +72,14 @@ type IngressRule struct { // HTTPIngressRuleValue represents HTTP ingress rule value in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#httpingressrulevalue-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#httpingressrulevalue-v1-networking-k8s-io type HTTPIngressRuleValue struct { Paths []HTTPIngressPath } // HTTPIngressPath represents HTTP ingress path in k8s. // -// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#httpingresspath-v1beta1-extensions +// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#httpingresspath-v1-networking-k8s-io type HTTPIngressPath struct { Path string } diff --git a/lib/promscrape/discovery/kubernetes/ingress_test.go b/lib/promscrape/discovery/kubernetes/ingress_test.go index f3440ac23d..149660b80f 100644 --- a/lib/promscrape/discovery/kubernetes/ingress_test.go +++ b/lib/promscrape/discovery/kubernetes/ingress_test.go @@ -30,9 +30,9 @@ func TestParseIngressListSuccess(t *testing.T) { data := ` { "kind": "IngressList", - "apiVersion": "extensions/v1beta1", + "apiVersion": "extensions/v1", "metadata": { - "selfLink": "/apis/extensions/v1beta1/ingresses", + "selfLink": "/apis/extensions/v1/ingresses", "resourceVersion": "351452" }, "items": [ @@ -40,13 +40,13 @@ func TestParseIngressListSuccess(t *testing.T) { "metadata": { "name": "test-ingress", "namespace": "default", - "selfLink": "/apis/extensions/v1beta1/namespaces/default/ingresses/test-ingress", + "selfLink": "/apis/extensions/v1/namespaces/default/ingresses/test-ingress", "uid": "6d3f38f9-de89-4bc9-b273-c8faf74e8a27", "resourceVersion": "351445", "generation": 1, "creationTimestamp": "2020-04-13T16:43:52Z", "annotations": { - "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"networking.k8s.io/v1beta1\",\"kind\":\"Ingress\",\"metadata\":{\"annotations\":{},\"name\":\"test-ingress\",\"namespace\":\"default\"},\"spec\":{\"backend\":{\"serviceName\":\"testsvc\",\"servicePort\":80}}}\n" + "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"networking.k8s.io/v1\",\"kind\":\"Ingress\",\"metadata\":{\"annotations\":{},\"name\":\"test-ingress\",\"namespace\":\"default\"},\"spec\":{\"backend\":{\"serviceName\":\"testsvc\",\"servicePort\":80}}}\n" } }, "spec": { @@ -85,7 +85,7 @@ func TestParseIngressListSuccess(t *testing.T) { expectedLabelss := [][]prompbmarshal.Label{ discoveryutils.GetSortedLabels(map[string]string{ "__address__": "foobar", - "__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration": `{"apiVersion":"networking.k8s.io/v1beta1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n", + "__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration": `{"apiVersion":"networking.k8s.io/v1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n", "__meta_kubernetes_ingress_annotationpresent_kubectl_kubernetes_io_last_applied_configuration": "true", "__meta_kubernetes_ingress_host": "foobar", "__meta_kubernetes_ingress_name": "test-ingress",