Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-11-23 20:37:12 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

lib/promscrape/discovery/kubernetes: allow attaching node-level labels to role: endpoints and role: endpointlice targets in the same way as Prometheus does

Browse Source 
See https://github.com/prometheus/prometheus/pull/10759
This commit is contained in:
Aliaksandr Valialkin 2022-07-06 23:18:55 +03:00
parent a6bd442ef9
commit b186b63e07
No known key found for this signature in database
GPG Key ID: A72BEC6CD3D0DED1
3 changed files with 226 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/CHANGELOG.md
View File

@ -23,6 +23,7 @@ If you use alerting rules or Grafana dashboards, which rely on this metric, then
* FEATURE: [vmui](https://docs.victoriametrics.com/#vmui): add an UI for [query tracing](https://docs.victoriametrics.com/#query-tracing). It can be enabled by clicking `enable query tracing` checkbox and re-running the query. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2703).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `-remoteWrite.headers` command-line option for specifying optional HTTP headers to send to the configured `-remoteWrite.url`. For example, `-remoteWrite.headers='Foo:Bar^^Baz:x'` would send `Foo: Bar` and `Baz: x` HTTP headers with every request to `-remoteWrite.url`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/2805).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): push per-target `scrape_samples_limit` metric to the cofigured `-remoteWrite.url` if `sample_limit` option is set for this target in [scrape_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). See [this feature request](https://github.com/VictoriaMetrics/operator/issues/497).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): attach node-level labels to [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) targets if `attach_metadata: {"node": true}` is set for `role: endpoints` and `role: endpointslice`. This is a feature backport from Prometheus 2.37 - see [this pull request](https://github.com/prometheus/prometheus/pull/10759).
* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add ability to specify additional HTTP headers to send to scrape targets via `headers` section in `scrape_configs`. This can be used when the scrape target requires custom authorization and authentication like in [this stackoverflow question](https://stackoverflow.com/questions/66032498/prometheus-scrape-metric-with-custom-header). For example, the following config instructs sending `My-Auth: top-secret` and `TenantID: FooBar` headers with each request to `http://host123:8080/metrics`:
```yaml

6
lib/promscrape/discovery/kubernetes/api_watcher.go
View File

@ -345,7 +345,7 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
gw.startWatchersForRole("pod", nil)
gw.startWatchersForRole("service", nil)
}
if gw.attachNodeMetadata && role == "pod" {
if gw.attachNodeMetadata && (role == "pod" || role == "endpoints" || role == "endpointslice") {
gw.startWatchersForRole("node", nil)
}
paths := getAPIPathsWithNamespaces(role, gw.namespaces, gw.selectors)
@ -803,8 +803,8 @@ func (uw *urlWatcher) maybeUpdateDependedScrapeWorksLocked() {
uwx.needRecreateScrapeWorks = true
continue
}
if attachNodeMetadata && role == "node" && uwx.role == "pod" {
// pod objects depend on node objects if attachNodeMetadata is set
if attachNodeMetadata && role == "node" && (uwx.role == "pod" || uwx.role == "endpoints" || uwx.role == "endpointslice") {
// pod, endpoints and enpointslices objects depend on node objects if attachNodeMetadata is set
uwx.needRecreateScrapeWorks = true
continue
}

396
lib/promscrape/discovery/kubernetes/endpoints_test.go
View File

@ -119,197 +119,245 @@ func TestGetEndpointLabels(t *testing.T) {
containerPorts map[string][]ContainerPort
endpointPorts []EndpointPort
}
f := func(name string, args testArgs, wantLabels [][]prompbmarshal.Label) {
t.Run(name, func(t *testing.T) {
eps := Endpoints{
Metadata: ObjectMeta{
Name: "test-eps",
Namespace: "default",
},
Subsets: []EndpointSubset{
{
Ports: args.endpointPorts,
Addresses: []EndpointAddress{
{
IP: "10.13.15.15",
TargetRef: ObjectReference{
Kind: "Pod",
Namespace: "default",
Name: "test-pod",
},
f := func(t *testing.T, args testArgs, wantLabels [][]prompbmarshal.Label) {
t.Helper()
eps := Endpoints{
Metadata: ObjectMeta{
Name: "test-eps",
Namespace: "default",
},
Subsets: []EndpointSubset{
{
Ports: args.endpointPorts,
Addresses: []EndpointAddress{
{
IP: "10.13.15.15",
TargetRef: ObjectReference{
Kind: "Pod",
Namespace: "default",
Name: "test-pod",
},
},
},
},
}
svc := Service{
Metadata: ObjectMeta{
Name: "test-eps",
Namespace: "default",
},
Spec: ServiceSpec{
Ports: []ServicePort{
{
Name: "test-port",
Port: 8081,
},
},
}
svc := Service{
Metadata: ObjectMeta{
Name: "test-eps",
Namespace: "default",
},
Spec: ServiceSpec{
ClusterIP: "1.2.3.4",
Type: "service-type",
Ports: []ServicePort{
{
Name: "test-port",
Port: 8081,
},
},
}
pod := Pod{
Metadata: ObjectMeta{
Name: "test-pod",
Namespace: "default",
},
Status: PodStatus{PodIP: "192.168.15.1"},
}
for cn, ports := range args.containerPorts {
pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
}
var gw groupWatcher
gw.m = map[string]*urlWatcher{
"pod": {
role: "pod",
objectsByKey: map[string]object{
"default/test-pod": &pod,
},
}
pod := Pod{
Metadata: ObjectMeta{
UID: "pod-uid",
Name: "test-pod",
Namespace: "default",
},
Spec: PodSpec{
NodeName: "test-node",
},
Status: PodStatus{
Phase: "abc",
PodIP: "192.168.15.1",
HostIP: "4.5.6.7",
},
}
node := Node{
Metadata: ObjectMeta{
Labels: []prompbmarshal.Label{
{
Name: "node-label",
Value: "xyz",
},
},
"service": {
role: "service",
objectsByKey: map[string]object{
"default/test-eps": &svc,
},
},
}
for cn, ports := range args.containerPorts {
pod.Spec.Containers = append(pod.Spec.Containers, Container{Name: cn, Ports: ports})
}
var gw groupWatcher
gw.m = map[string]*urlWatcher{
"pod": {
role: "pod",
objectsByKey: map[string]object{
"default/test-pod": &pod,
},
}
var sortedLabelss [][]prompbmarshal.Label
gotLabels := eps.getTargetLabels(&gw)
for _, lbs := range gotLabels {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
}
if !areEqualLabelss(sortedLabelss, wantLabels) {
t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
}
})
},
"service": {
role: "service",
objectsByKey: map[string]object{
"default/test-eps": &svc,
},
},
"node": {
role: "node",
objectsByKey: map[string]object{
"/test-node": &node,
},
},
}
gw.attachNodeMetadata = true
var sortedLabelss [][]prompbmarshal.Label
gotLabels := eps.getTargetLabels(&gw)
for _, lbs := range gotLabels {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(lbs))
}
if !areEqualLabelss(sortedLabelss, wantLabels) {
t.Fatalf("unexpected labels:\ngot\n%v\nwant\n%v", sortedLabelss, wantLabels)
}
}
f("1 port from endpoint", testArgs{
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8081,
t.Run("1 port from endpoint", func(t *testing.T) {
f(t, testArgs{
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8081,
Protocol: "foobar",
},
},
},
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8081",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8081",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "foobar",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_node_label_node_label": "xyz",
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "test-node",
"__meta_kubernetes_pod_phase": "abc",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "pod-uid",
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "service-type",
}),
})
})
f("1 port from endpoint and 1 from pod", testArgs{
containerPorts: map[string][]ContainerPort{"metrics": {{
Name: "http-metrics",
ContainerPort: 8428,
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8081,
t.Run("1 port from endpoint and 1 from pod", func(t *testing.T) {
f(t, testArgs{
containerPorts: map[string][]ContainerPort{"metrics": {{
Name: "http-metrics",
ContainerPort: 8428,
Protocol: "foobar",
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8081,
Protocol: "https",
},
},
},
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8081",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "192.168.15.1:8428",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "http-metrics",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8081",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "https",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_node_label_node_label": "xyz",
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "test-node",
"__meta_kubernetes_pod_phase": "abc",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "pod-uid",
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "service-type",
}),
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "192.168.15.1:8428",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_node_label_node_label": "xyz",
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "http-metrics",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "foobar",
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "test-node",
"__meta_kubernetes_pod_phase": "abc",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "pod-uid",
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "service-type",
}),
})
})
f("1 port from endpoint", testArgs{
containerPorts: map[string][]ContainerPort{"metrics": {{
Name: "web",
ContainerPort: 8428,
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8428,
t.Run("1 port from endpoint", func(t *testing.T) {
f(t, testArgs{
containerPorts: map[string][]ContainerPort{"metrics": {{
Name: "web",
ContainerPort: 8428,
Protocol: "sdc",
}}},
endpointPorts: []EndpointPort{
{
Name: "web",
Port: 8428,
Protocol: "xabc",
},
},
},
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8428",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "",
"__meta_kubernetes_pod_host_ip": "",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "",
"__meta_kubernetes_pod_phase": "",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "",
"__meta_kubernetes_service_cluster_ip": "",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "",
}),
}, [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.13.15.15:8428",
"__meta_kubernetes_endpoint_address_target_kind": "Pod",
"__meta_kubernetes_endpoint_address_target_name": "test-pod",
"__meta_kubernetes_endpoint_port_name": "web",
"__meta_kubernetes_endpoint_port_protocol": "xabc",
"__meta_kubernetes_endpoint_ready": "true",
"__meta_kubernetes_endpoints_name": "test-eps",
"__meta_kubernetes_namespace": "default",
"__meta_kubernetes_node_label_node_label": "xyz",
"__meta_kubernetes_node_labelpresent_node_label": "true",
"__meta_kubernetes_node_name": "test-node",
"__meta_kubernetes_pod_container_name": "metrics",
"__meta_kubernetes_pod_container_port_name": "web",
"__meta_kubernetes_pod_container_port_number": "8428",
"__meta_kubernetes_pod_container_port_protocol": "sdc",
"__meta_kubernetes_pod_host_ip": "4.5.6.7",
"__meta_kubernetes_pod_ip": "192.168.15.1",
"__meta_kubernetes_pod_name": "test-pod",
"__meta_kubernetes_pod_node_name": "test-node",
"__meta_kubernetes_pod_phase": "abc",
"__meta_kubernetes_pod_ready": "unknown",
"__meta_kubernetes_pod_uid": "pod-uid",
"__meta_kubernetes_service_cluster_ip": "1.2.3.4",
"__meta_kubernetes_service_name": "test-eps",
"__meta_kubernetes_service_type": "service-type",
}),
})
})
}