lib/promscrape/discovery/kubernetes: add support for __meta_kubernetes_pod_container_id

See https://github.com/prometheus/prometheus/issues/11843
and https://github.com/prometheus/prometheus/pull/11844

docs/CHANGELOG.md

@@ -23,6 +23,7 @@ The following tip changes can be tested by building VictoriaMetrics components f
 * FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): reduce memory usage when sending stale markers for targets, which expose big number of metrics. See [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3668) and [this](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3675) issues.
 * FEATURE: [vmauth](https://docs.victoriametrics.com/vmauth.html): allow limiting the number of concurrent requests sent to `vmauth` via `-maxConcurrentRequests` command-line flag. This allows controlling memory usage of `vmauth` and the resource usage of backends behind `vmauth`. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3346). Thanks to @dmitryk-dk for [the initial implementation](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/3486).
 * FEATURE: allow using VictoriaMetrics components behind proxies, which communicate with the backend via [proxy protocol](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3335). For example, [vmauth](https://docs.victoriametrics.com/vmauth.html) accepts proxy protocol connections when it starts with `-httpListenAddr.useProxyProtocol` command-line flag.
+* FEATURE: [vmagent](https://docs.victoriametrics.com/vmagent.html): add `__meta_kubernetes_pod_container_id` meta-label to the targets discovered via [kubernetes_sd_configs](https://docs.victoriametrics.com/sd_configs.html#kubernetes_sd_configs). This label has been added in Prometheus starting from `v2.42.0`. See [this feature request](https://github.com/prometheus/prometheus/issues/11843).
 * FEATURE: add `-internStringMaxLen` command-line flag, which can be used for fine-tuning RAM vs CPU usage in certain workloads. For example, if the stored time series contain long labels, then it may be useful reducing the `-internStringMaxLen` in order to reduce memory usage at the cost of increased CPU usage. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3692).
 
 * BUGFIX: fix a bug, which could prevent background merges for the previous partitions until restart if the storage didn't have enough disk space for final deduplication and down-sampling.

docs/sd_configs.md

@@ -788,7 +788,8 @@ One of the following `role` types can be configured to discover targets:
   * `__meta_kubernetes_pod_labelpresent_<labelname>`: "true" for each label from the pod object.
   * `__meta_kubernetes_pod_annotation_<annotationname>`: Each annotation from the pod object.
   * `__meta_kubernetes_pod_annotationpresent_<annotationname>`: "true" for each annotation from the pod object.
-  * `__meta_kubernetes_pod_container_init`: "true" if the container is an InitContainer
+  * `__meta_kubernetes_pod_container_id`: ID of the container in the form `<type>://<container_id>`.
+  * `__meta_kubernetes_pod_container_init`: "true" if the container is an InitContainer.
   * `__meta_kubernetes_pod_container_image`: Container image the target address points to.
   * `__meta_kubernetes_pod_container_name`: Name of the container the target address points to.
   * `__meta_kubernetes_pod_container_port_name`: Name of the container port.

lib/promscrape/discovery/kubernetes/pod.go

@@ -82,10 +82,12 @@ type ContainerPort struct {
 //
 // See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#podstatus-v1-core
 type PodStatus struct {
-	Phase      string
-	PodIP      string
-	HostIP     string
-	Conditions []PodCondition
+	Phase                 string
+	PodIP                 string
+	HostIP                string
+	Conditions            []PodCondition
+	ContainerStatuses     []ContainerStatus
+	InitContainerStatuses []ContainerStatus
 }
 
 // PodCondition implements k8s pod condition.
@@ -96,6 +98,27 @@ type PodCondition struct {
 	Status string
 }
 
+// ContainerStatus implements k8s container status.
+//
+// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core
+type ContainerStatus struct {
+	Name        string
+	ContainerID string
+}
+
+func getContainerID(p *Pod, containerName string, isInit bool) string {
+	css := p.Status.ContainerStatuses
+	if isInit {
+		css = p.Status.InitContainerStatuses
+	}
+	for _, cs := range css {
+		if cs.Name == containerName {
+			return cs.ContainerID
+		}
+	}
+	return ""
+}
+
 // getTargetLabels returns labels for each port of the given p.
 //
 // See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod
@@ -105,12 +128,12 @@ func (p *Pod) getTargetLabels(gw *groupWatcher) []*promutils.Labels {
 		return nil
 	}
 	var ms []*promutils.Labels
-	ms = appendPodLabels(ms, gw, p, p.Spec.Containers, "false")
-	ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, "true")
+	ms = appendPodLabels(ms, gw, p, p.Spec.Containers, false)
+	ms = appendPodLabels(ms, gw, p, p.Spec.InitContainers, true)
 	return ms
 }
 
-func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit string) []*promutils.Labels {
+func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Container, isInit bool) []*promutils.Labels {
 	for _, c := range cs {
 		for _, cp := range c.Ports {
 			ms = appendPodLabelsInternal(ms, gw, p, c, &cp, isInit)
@@ -122,14 +145,24 @@ func appendPodLabels(ms []*promutils.Labels, gw *groupWatcher, p *Pod, cs []Cont
 	return ms
 }
 
-func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit string) []*promutils.Labels {
+func appendPodLabelsInternal(ms []*promutils.Labels, gw *groupWatcher, p *Pod, c Container, cp *ContainerPort, isInit bool) []*promutils.Labels {
 	addr := p.Status.PodIP
 	if cp != nil {
 		addr = discoveryutils.JoinHostPort(addr, cp.ContainerPort)
 	}
 	m := promutils.GetLabels()
 	m.Add("__address__", addr)
-	m.Add("__meta_kubernetes_pod_container_init", isInit)
+	isInitStr := "false"
+	if isInit {
+		isInitStr = "true"
+	}
+	m.Add("__meta_kubernetes_pod_container_init", isInitStr)
+
+	containerID := getContainerID(p, c.Name, isInit)
+	if containerID != "" {
+		m.Add("__meta_kubernetes_pod_container_id", containerID)
+	}
+
 	p.appendCommonLabels(m, gw)
 	p.appendContainerLabels(m, c, cp)
 	return append(ms, m)

lib/promscrape/discovery/kubernetes/pod_test.go

@@ -262,6 +262,7 @@ func TestParsePodListSuccess(t *testing.T) {
 			"__meta_kubernetes_pod_controller_kind":          "Node",
 			"__meta_kubernetes_pod_controller_name":          "m01",
 			"__meta_kubernetes_pod_container_init":           "false",
+			"__meta_kubernetes_pod_container_id":             "docker://a28f0800855008485376c1eece1cf61de97cb7026b9188d138b0d55d92fc2f5c",
 
 			"__meta_kubernetes_pod_label_component": "etcd",
 			"__meta_kubernetes_pod_label_tier":      "control-plane",