diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index bce8f8c6d..baf134e92 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -38,6 +38,7 @@ The following tip changes can be tested by building VictoriaMetrics components f * BUGFIX: [vmui](https://docs.victoriametrics.com/#vmui): fix possible UI freeze after querying `node_uname_info` time series. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2115). * BUGFIX: show the original location of the warning or error message when logging throttled messages. Previously the location inside `lib/logger/throttler.go` was shown. This could increase the complexity of debugging. * BUGFIX: vmalert: fix links at web UI. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2167). +* BUGFIX: vmagent: properly discover pods without exposed ports for the given service for `role: endpoints` and `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config). See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134). ## [v1.72.0](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.72.0) diff --git a/lib/promscrape/discovery/kubernetes/endpoints.go b/lib/promscrape/discovery/kubernetes/endpoints.go index 8c1fe6b9d..a9ddc6ecc 100644 --- a/lib/promscrape/discovery/kubernetes/endpoints.go +++ b/lib/promscrape/discovery/kubernetes/endpoints.go @@ -169,11 +169,16 @@ func getEndpointLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, eps *Endpoi return m } p.appendCommonLabels(m) + // always add pod targetRef, even if epp port doesn't match container port + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134 + if _, ok := podPortsSeen[p]; !ok { + podPortsSeen[p] = []int{} + } for _, c := range p.Spec.Containers { for _, cp := range c.Ports { if cp.ContainerPort == epp.Port { - p.appendContainerLabels(m, c, &cp) podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort) + p.appendContainerLabels(m, c, &cp) break } } diff --git a/lib/promscrape/discovery/kubernetes/endpoints_test.go b/lib/promscrape/discovery/kubernetes/endpoints_test.go index 51b30d091..f26fe405c 100644 --- a/lib/promscrape/discovery/kubernetes/endpoints_test.go +++ b/lib/promscrape/discovery/kubernetes/endpoints_test.go @@ -113,3 +113,203 @@ func TestParseEndpointsListSuccess(t *testing.T) { t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, expectedLabelss) } } + +func TestGetEndpointLabels(t *testing.T) { + type testArgs struct { + containerPorts map[string][]ContainerPort + endpointPorts []EndpointPort + } + f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) { + t.Run(name, func(t *testing.T) { + eps := Endpoints{ + Metadata: ObjectMeta{ + Name: "test-eps", + Namespace: "default", + }, + Subsets: []EndpointSubset{ + { + Ports: args.endpointPorts, + Addresses: []EndpointAddress{ + { + IP: "10.13.15.15", + TargetRef: ObjectReference{ + Kind: "Pod", + Namespace: "default", + Name: "test-pod", + }, + }, + }, + }, + }, + } + svc := Service{ + Metadata: ObjectMeta{ + Name: "test-eps", + Namespace: "default", + }, + Spec: ServiceSpec{ + Ports: []ServicePort{ + { + Name: "test-port", + Port: 8081, + }, + }, + }, + } + pod := Pod{ + Metadata: ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Status: PodStatus{PodIP: "192.168.15.1"}, + } + for cn, ports := range args.containerPorts { + pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports}) + } + var gw groupWatcher + gw.m = map[string]*urlWatcher{ + "pod": &urlWatcher{ + role: "pod", + objectsByKey: map[string]object{ + "default/test-pod": &pod, + }, + }, + "service": &urlWatcher{ + role: "service", + objectsByKey: map[string]object{ + "default/test-eps": &svc, + }, + }, + } + var sortedLabelss [][]prompbmarshal.Label + gotLabels := eps.getTargetLabels(&gw) + for _, lbs := range gotLabels { + sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs)) + } + if !areEqualLabelss(sortedLabelss, wantLabels) { + t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels) + + } + }) + } + + f("1 port from endpoint", testArgs{ + endpointPorts: []EndpointPort{ + { + Name: "web", + Port: 8081, + }, + }, + }, [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.13.15.15:8081", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "test-pod", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_name": "test-eps", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_host_ip": "", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "", + "__meta_kubernetes_service_cluster_ip": "", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "", + }), + }) + + f("1 port from endpoint and 1 from pod", testArgs{ + containerPorts: map[string][]ContainerPort{"metrics": []ContainerPort{{ + Name: "http-metrics", + ContainerPort: 8428, + }}}, + endpointPorts: []EndpointPort{ + { + Name: "web", + Port: 8081, + }, + }, + }, [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.13.15.15:8081", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "test-pod", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_name": "test-eps", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_host_ip": "", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "", + "__meta_kubernetes_service_cluster_ip": "", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "", + }), + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "192.168.15.1:8428", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_container_name": "metrics", + "__meta_kubernetes_pod_container_port_name": "http-metrics", + "__meta_kubernetes_pod_container_port_number": "8428", + "__meta_kubernetes_pod_container_port_protocol": "", + "__meta_kubernetes_pod_host_ip": "", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "", + "__meta_kubernetes_service_cluster_ip": "", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "", + }), + }) + + f("1 port from endpoint", testArgs{ + containerPorts: map[string][]ContainerPort{"metrics": []ContainerPort{{ + Name: "web", + ContainerPort: 8428, + }}}, + endpointPorts: []EndpointPort{ + { + Name: "web", + Port: 8428, + }, + }, + }, [][]prompbmarshal.Label{ + discoveryutils.GetSortedLabels(map[string]string{ + "__address__": "10.13.15.15:8428", + "__meta_kubernetes_endpoint_address_target_kind": "Pod", + "__meta_kubernetes_endpoint_address_target_name": "test-pod", + "__meta_kubernetes_endpoint_port_name": "web", + "__meta_kubernetes_endpoint_port_protocol": "", + "__meta_kubernetes_endpoint_ready": "true", + "__meta_kubernetes_endpoints_name": "test-eps", + "__meta_kubernetes_namespace": "default", + "__meta_kubernetes_pod_container_name": "metrics", + "__meta_kubernetes_pod_container_port_name": "web", + "__meta_kubernetes_pod_container_port_number": "8428", + "__meta_kubernetes_pod_container_port_protocol": "", + "__meta_kubernetes_pod_host_ip": "", + "__meta_kubernetes_pod_ip": "192.168.15.1", + "__meta_kubernetes_pod_name": "test-pod", + "__meta_kubernetes_pod_node_name": "", + "__meta_kubernetes_pod_phase": "", + "__meta_kubernetes_pod_ready": "unknown", + "__meta_kubernetes_pod_uid": "", + "__meta_kubernetes_service_cluster_ip": "", + "__meta_kubernetes_service_name": "test-eps", + "__meta_kubernetes_service_type": "", + }), + }) +} diff --git a/lib/promscrape/discovery/kubernetes/endpointslice.go b/lib/promscrape/discovery/kubernetes/endpointslice.go index 37053de79..727edd2fc 100644 --- a/lib/promscrape/discovery/kubernetes/endpointslice.go +++ b/lib/promscrape/discovery/kubernetes/endpointslice.go @@ -101,12 +101,17 @@ func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr s if ea.TargetRef.Kind != "Pod" || p == nil { return m } + // always add pod targetRef, even if epp port doesn't match container port. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2134 + if _, ok := podPortsSeen[p]; !ok { + podPortsSeen[p] = []int{} + } p.appendCommonLabels(m) for _, c := range p.Spec.Containers { for _, cp := range c.Ports { if cp.ContainerPort == epp.Port { - p.appendContainerLabels(m, c, &cp) podPortsSeen[p] = append(podPortsSeen[p], cp.ContainerPort) + p.appendContainerLabels(m, c, &cp) break } } @@ -117,7 +122,6 @@ func getEndpointSliceLabelsForAddressAndPort(podPortsSeen map[*Pod][]int, addr s // //getEndpointSliceLabels builds labels for given EndpointSlice func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp EndpointPort) map[string]string { - addr = discoveryutils.JoinHostPort(addr, epp.Port) m := map[string]string{ "__address__": addr,