Kiu/VictoriaMetrics
Kiu/VictoriaMetrics
1
0
Fork 0
mirror of https://github.com/VictoriaMetrics/VictoriaMetrics.git synced 2024-12-15 00:13:30 +01:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Adds dockerswarm sd (#818)

Browse Source 
* adds dockerswarm service discovery

https://github.com/VictoriaMetrics/VictoriaMetrics/issues/656

 Following roles supported: services, tasks and nodes.
 Basic, token and tls auth supported.
 Added tests for labels generation.

* added unix socket support to discovery utils

Co-authored-by: Aliaksandr Valialkin <valyala@gmail.com>
This commit is contained in:
Nikolay Khramchikhin 2020-10-12 13:38:21 +03:00 committed by Aliaksandr Valialkin
parent ac525462ce
commit 7f96712b38
14 changed files with 1585 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/vmagent/README.md
View File

@ -151,6 +151,8 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
* `openstack_sd_configs` - for scraping OpenStack targets. * `openstack_sd_configs` - for scraping OpenStack targets.
See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details. See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
[OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only. [OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
* `dockerswarm_sd_configs` - for scraping dockerswarm targets.
See [dockerswarm_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config) for details.
File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`. File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.

39
lib/promscrape/config.go
View File

@ -18,6 +18,7 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promrelabel"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/consul"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dns"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/dockerswarm"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes" "github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
@ -72,6 +73,7 @@ type ScrapeConfig struct {
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs"` KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs"`
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs"` OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs"`
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs"` ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs"`
DockerSwarmConfigs []dockerswarm.SDConfig `yaml:"dockerswarm_sd_configs"`
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs"` DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs"`
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs"` EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs"`
GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs"` GCESDConfigs []gce.SDConfig `yaml:"gce_sd_configs"`
@ -231,6 +233,34 @@ func (cfg *Config) getOpenStackSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
return dst return dst
} }
// getDockerSwarmSDScrapeWork returns `dockerswarm_sd_configs` ScrapeWork from cfg.
func (cfg *Config) getDockerSwarmSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
swsPrevByJob := getSWSByJob(prev)
var dst []ScrapeWork
for i := range cfg.ScrapeConfigs {
sc := &cfg.ScrapeConfigs[i]
dstLen := len(dst)
ok := true
for j := range sc.DockerSwarmConfigs {
sdc := &sc.DockerSwarmConfigs[j]
var okLocal bool
dst, okLocal = appendDockerSwarmScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
if ok {
ok = okLocal
}
}
if ok {
continue
}
swsPrev := swsPrevByJob[sc.swc.jobName]
if len(swsPrev) > 0 {
logger.Errorf("there were errors when discovering dockerswarm targets for job %q, so preserving the previous targets", sc.swc.jobName)
dst = append(dst[:dstLen], swsPrev...)
}
}
return dst
}
// getConsulSDScrapeWork returns `consul_sd_configs` ScrapeWork from cfg. // getConsulSDScrapeWork returns `consul_sd_configs` ScrapeWork from cfg.
func (cfg *Config) getConsulSDScrapeWork(prev []ScrapeWork) []ScrapeWork { func (cfg *Config) getConsulSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
swsPrevByJob := getSWSByJob(prev) swsPrevByJob := getSWSByJob(prev)
@ -483,6 +513,15 @@ func appendOpenstackScrapeWork(dst []ScrapeWork, sdc *openstack.SDConfig, baseDi
return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "openstack_sd_config"), true return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "openstack_sd_config"), true
} }
func appendDockerSwarmScrapeWork(dst []ScrapeWork, sdc *dockerswarm.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) {
targetLabels, err := dockerswarm.GetLabels(sdc, baseDir)
if err != nil {
logger.Errorf("error when discovering dockerswarm targets for `job_name` %q: %s; skipping it", swc.jobName, err)
return dst, false
}
return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "dockerswarm_sd_config"), true
}
func appendConsulScrapeWork(dst []ScrapeWork, sdc *consul.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) { func appendConsulScrapeWork(dst []ScrapeWork, sdc *consul.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) {
targetLabels, err := consul.GetLabels(sdc, baseDir) targetLabels, err := consul.GetLabels(sdc, baseDir)
if err != nil { if err != nil {

39
lib/promscrape/discovery/dockerswarm/api.go Normal file
View File

@ -0,0 +1,39 @@
package dockerswarm
import (
"fmt"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
var configMap = discoveryutils.NewConfigMap()
type apiConfig struct {
client *discoveryutils.Client
port int
}
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
if err != nil {
return nil, err
}
return v.(*apiConfig), nil
}
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
cfg := &apiConfig{
port: sdc.Port,
}
config, err := promauth.NewConfig(baseDir, sdc.BasicAuth, sdc.BearerToken, sdc.BearerTokenFile, sdc.TLSConfig)
if err != nil {
return nil, err
}
client, err := discoveryutils.NewClient(sdc.Host, config)
if err != nil {
return nil, fmt.Errorf("cannot create HTTP client for %q: %w", sdc.Host, err)
}
cfg.client = client
return cfg, nil
}

51
lib/promscrape/discovery/dockerswarm/dockerswarm.go Normal file
View File

@ -0,0 +1,51 @@
package dockerswarm
import (
"fmt"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
)
// SDConfig represents docker swarm service discovery configuration
//
// See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config
type SDConfig struct {
Host string `yaml:"host"`
Role string `yaml:"role"`
Port int `yaml:"port"`
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
BasicAuth *promauth.BasicAuthConfig `yaml:"basic_auth"`
BearerToken string `yaml:"bearer_token"`
BearerTokenFile string `yaml:"bearer_token_file"`
}
// joinLabels adds labels to destination from source with given key from destination matching given value.
func joinLabels(source []map[string]string, destination map[string]string, key, value string) map[string]string {
for _, sourceLabels := range source {
if sourceLabels[key] == value {
for k, v := range sourceLabels {
destination[k] = v
}
return destination
}
}
return destination
}
// GetLabels returns gce labels according to sdc.
func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
cfg, err := getAPIConfig(sdc, baseDir)
if err != nil {
return nil, fmt.Errorf("cannot get API config: %w", err)
}
switch sdc.Role {
case "tasks":
return getTasksLabels(cfg)
case "services":
return getServicesLabels(cfg)
case "nodes":
return getNodesLabels(cfg)
default:
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `tasks`, `services` or `nodes`; skipping it", sdc.Role)
}
}

61
lib/promscrape/discovery/dockerswarm/network.go Normal file
View File

@ -0,0 +1,61 @@
package dockerswarm
import (
"encoding/json"
"fmt"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// See https://docs.docker.com/engine/api/v1.40/#tag/Network
type network struct {
ID string
Name string
Scope string
Internal bool
Ingress bool
Labels map[string]string
}
func getNetworksLabels(cfg *apiConfig) ([]map[string]string, error) {
networks, err := getNetworks(cfg)
if err != nil {
return nil, err
}
return addNetworkLabels(networks), nil
}
func getNetworks(cfg *apiConfig) ([]network, error) {
resp, err := cfg.client.GetAPIResponse("/networks")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for networks: %w", err)
}
return parseNetworks(resp)
}
func parseNetworks(data []byte) ([]network, error) {
var networks []network
if err := json.Unmarshal(data, &networks); err != nil {
return nil, fmt.Errorf("cannot parse networks: %w", err)
}
return networks, nil
}
func addNetworkLabels(networks []network) []map[string]string {
var ms []map[string]string
for _, network := range networks {
m := map[string]string{
"__meta_dockerswarm_network_id": network.ID,
"__meta_dockerswarm_network_name": network.Name,
"__meta_dockerswarm_network_scope": network.Scope,
"__meta_dockerswarm_network_internal": strconv.FormatBool(network.Internal),
"__meta_dockerswarm_network_ingress": strconv.FormatBool(network.Ingress),
}
for k, v := range network.Labels {
m["__meta_dockerswarm_network_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms = append(ms, m)
}
return ms
}

166
lib/promscrape/discovery/dockerswarm/network_test.go Normal file
View File

@ -0,0 +1,166 @@
package dockerswarm
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
func Test_addNetworkLabels(t *testing.T) {
type args struct {
networks []network
}
tests := []struct {
name string
args args
want [][]prompbmarshal.Label
}{
{
name: "ingress network",
args: args{
networks: []network{
{
ID: "qs0hog6ldlei9ct11pr3c77v1",
Ingress: true,
Scope: "swarm",
Name: "ingress",
Labels: map[string]string{
"key1": "value1",
},
},
},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
})},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := addNetworkLabels(tt.args.networks)
var sortedLabelss [][]prompbmarshal.Label
for _, labels := range got {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {
t.Errorf("addNetworkLabels() \ngot %v, \nwant %v", sortedLabelss, tt.want)
}
})
}
}
func Test_parseNetworks(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []network
wantErr bool
}{
{
name: "parse two networks",
args: args{
data: []byte(`[
{
"Name": "ingress",
"Id": "qs0hog6ldlei9ct11pr3c77v1",
"Created": "2020-10-06T08:39:58.957083331Z",
"Scope": "swarm",
"Driver": "overlay",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": [
{
"Subnet": "10.0.0.0/24",
"Gateway": "10.0.0.1"
}
]
},
"Internal": false,
"Attachable": false,
"Ingress": true,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": null,
"Options": {
"com.docker.network.driver.overlay.vxlanid_list": "4096"
},
"Labels": {
"key1": "value1"
}
},
{
"Name": "host",
"Id": "317f0384d7e5f5c26304a0b04599f9f54bc08def4d0535059ece89955e9c4b7b",
"Created": "2020-10-06T08:39:52.843373136Z",
"Scope": "local",
"Driver": "host",
"EnableIPv6": false,
"IPAM": {
"Driver": "default",
"Options": null,
"Config": []
},
"Internal": false,
"Attachable": false,
"Ingress": false,
"ConfigFrom": {
"Network": ""
},
"ConfigOnly": false,
"Containers": {},
"Options": {},
"Labels": {
"key": "value"
}
}
]`),
},
want: []network{
{
ID: "qs0hog6ldlei9ct11pr3c77v1",
Ingress: true,
Scope: "swarm",
Name: "ingress",
Labels: map[string]string{
"key1": "value1",
},
},
{
ID: "317f0384d7e5f5c26304a0b04599f9f54bc08def4d0535059ece89955e9c4b7b",
Scope: "local",
Name: "host",
Labels: map[string]string{
"key": "value",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseNetworks(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseNetworks() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseNetworks() \ngot %v, \nwant %v", got, tt.want)
}
})
}
}

90
lib/promscrape/discovery/dockerswarm/nodes.go Normal file
View File

@ -0,0 +1,90 @@
package dockerswarm
import (
"encoding/json"
"fmt"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// See https://docs.docker.com/engine/api/v1.40/#tag/Node
type node struct {
ID string
Spec struct {
Labels map[string]string
Role string
Availability string
}
Description struct {
Hostname string
Platform struct {
Architecture string
OS string
}
Engine struct {
EngineVersion string
}
}
Status struct {
State string
Message string
Addr string
}
ManagerStatus *struct {
Leader bool
Reachability string
Addr string
}
}
func getNodesLabels(cfg *apiConfig) ([]map[string]string, error) {
nodes, err := getNodes(cfg)
if err != nil {
return nil, err
}
return addNodeLabels(nodes, cfg.port), nil
}
func getNodes(cfg *apiConfig) ([]node, error) {
resp, err := cfg.client.GetAPIResponse("/nodes")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for nodes: %w", err)
}
return parseNodes(resp)
}
func parseNodes(data []byte) ([]node, error) {
var nodes []node
if err := json.Unmarshal(data, &nodes); err != nil {
return nil, fmt.Errorf("cannot parse nodes: %w", err)
}
return nodes, nil
}
func addNodeLabels(nodes []node, port int) []map[string]string {
var ms []map[string]string
for _, node := range nodes {
m := map[string]string{
"__address__": discoveryutils.JoinHostPort(node.Status.Addr, port),
"__meta_dockerswarm_node_id": node.ID,
"__meta_dockerswarm_node_address": node.Status.Addr,
"__meta_dockerswarm_node_availability": node.Spec.Availability,
"__meta_dockerswarm_node_engine_version": node.Description.Engine.EngineVersion,
"__meta_dockerswarm_node_hostname": node.Description.Hostname,
"__meta_dockerswarm_node_platform_architecture": node.Description.Platform.Architecture,
"__meta_dockerswarm_node_platform_os": node.Description.Platform.OS,
"__meta_dockerswarm_node_role": node.Spec.Role,
"__meta_dockerswarm_node_status": node.Status.State,
}
if node.ManagerStatus != nil {
m["__meta_dockerswarm_node_manager_address"] = node.ManagerStatus.Addr
m["__meta_dockerswarm_node_manager_manager_reachability"] = node.ManagerStatus.Reachability
m["__meta_dockerswarm_node_manager_leader"] = fmt.Sprintf("%t", node.ManagerStatus.Leader)
}
for k, v := range node.Spec.Labels {
m["__meta_dockerswarm_node_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
ms = append(ms, m)
}
return ms
}

185
lib/promscrape/discovery/dockerswarm/nodes_test.go Normal file
View File

@ -0,0 +1,185 @@
package dockerswarm
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
func Test_parseNodes(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []node
wantErr bool
}{
{
name: "parse ok",
args: args{
data: []byte(`[
{
"ID": "qauwmifceyvqs0sipvzu8oslu",
"Version": {
"Index": 16
},
"Spec": {
"Role": "manager",
"Availability": "active"
},
"Description": {
"Hostname": "ip-172-31-40-97",
"Platform": {
"Architecture": "x86_64",
"OS": "linux"
},
"Resources": {
"NanoCPUs": 1000000000,
"MemoryBytes": 1026158592
},
"Engine": {
"EngineVersion": "19.03.11"
}
},
"Status": {
"State": "ready",
"Addr": "172.31.40.97"
}
}
]
`),
},
want: []node{
{
ID: "qauwmifceyvqs0sipvzu8oslu",
Spec: struct {
Labels map[string]string
Role string
Availability string
}{Role: "manager", Availability: "active"},
Status: struct {
State string
Message string
Addr string
}{State: "ready", Addr: "172.31.40.97"},
Description: struct {
Hostname string
Platform struct {
Architecture string
OS string
}
Engine struct{ EngineVersion string }
}{
Hostname: "ip-172-31-40-97",
Platform: struct {
Architecture string
OS string
}{
Architecture: "x86_64",
OS: "linux",
},
Engine: struct{ EngineVersion string }{
EngineVersion: "19.03.11",
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseNodes(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseNodes() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseNodes() \ngot %v, \nwant %v", got, tt.want)
}
})
}
}
func Test_addNodeLabels(t *testing.T) {
type args struct {
nodes []node
port int
}
tests := []struct {
name string
args args
want [][]prompbmarshal.Label
}{
{
name: "add labels to one node",
args: args{
nodes: []node{
{
ID: "qauwmifceyvqs0sipvzu8oslu",
Spec: struct {
Labels map[string]string
Role string
Availability string
}{Role: "manager", Availability: "active"},
Status: struct {
State string
Message string
Addr string
}{State: "ready", Addr: "172.31.40.97"},
Description: struct {
Hostname string
Platform struct {
Architecture string
OS string
}
Engine struct{ EngineVersion string }
}{
Hostname: "ip-172-31-40-97",
Platform: struct {
Architecture string
OS string
}{
Architecture: "x86_64",
OS: "linux",
},
Engine: struct{ EngineVersion string }{
EngineVersion: "19.03.11",
},
},
},
},
port: 9100,
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "172.31.40.97:9100",
"__meta_dockerswarm_node_address": "172.31.40.97",
"__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11",
"__meta_dockerswarm_node_hostname": "ip-172-31-40-97",
"__meta_dockerswarm_node_id": "qauwmifceyvqs0sipvzu8oslu",
"__meta_dockerswarm_node_platform_architecture": "x86_64",
"__meta_dockerswarm_node_platform_os": "linux",
"__meta_dockerswarm_node_role": "manager",
"__meta_dockerswarm_node_status": "ready",
})},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := addNodeLabels(tt.args.nodes, tt.args.port)
var sortedLabelss [][]prompbmarshal.Label
for _, labels := range got {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {
t.Errorf("addNodeLabels() \ngot %v, \nwant %v", sortedLabelss, tt.want)
}
})
}
}

139
lib/promscrape/discovery/dockerswarm/services.go Normal file
View File

@ -0,0 +1,139 @@
package dockerswarm
import (
"encoding/json"
"fmt"
"net"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// https://docs.docker.com/engine/api/v1.40/#tag/Service
type service struct {
ID string
Spec struct {
Labels map[string]string
Name string
TaskTemplate struct {
ContainerSpec struct {
Hostname string
Image string
}
}
Mode struct {
Global interface{}
Replicated interface{}
}
}
UpdateStatus *struct {
State string
}
Endpoint struct {
Ports []portConfig
VirtualIPs []struct {
NetworkID string
Addr string
}
}
}
type portConfig struct {
Protocol string
Name string
PublishMode string
PublishedPort int
}
func getServicesLabels(cfg *apiConfig) ([]map[string]string, error) {
services, err := getServices(cfg)
if err != nil {
return nil, err
}
networksLabels, err := getNetworksLabels(cfg)
if err != nil {
return nil, err
}
return addServicesLabels(services, networksLabels, cfg.port), nil
}
func getServices(cfg *apiConfig) ([]service, error) {
data, err := cfg.client.GetAPIResponse("/services")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for services: %w", err)
}
return parseServicesResponse(data)
}
func parseServicesResponse(data []byte) ([]service, error) {
var services []service
if err := json.Unmarshal(data, &services); err != nil {
return nil, fmt.Errorf("cannot parse services: %w", err)
}
return services, nil
}
func getServiceMode(svc service) string {
if svc.Spec.Mode.Global != nil {
return "global"
}
if svc.Spec.Mode.Replicated != nil {
return "replicated"
}
return ""
}
func addServicesLabels(services []service, networksLabels []map[string]string, port int) []map[string]string {
var ms []map[string]string
for _, service := range services {
m := map[string]string{
"__meta_dockerswarm_service_id": service.ID,
"__meta_dockerswarm_service_name": service.Spec.Name,
"__meta_dockerswarm_service_task_container_hostname": service.Spec.TaskTemplate.ContainerSpec.Hostname,
"__meta_dockerswarm_service_task_container_image": service.Spec.TaskTemplate.ContainerSpec.Image,
"__meta_dockerswarm_service_mode": getServiceMode(service),
}
if service.UpdateStatus != nil {
m["__meta_dockerswarm_service_updating_status"] = service.UpdateStatus.State
}
for k, v := range service.Spec.Labels {
m["__meta_dockerswarm_service_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
for _, vip := range service.Endpoint.VirtualIPs {
var added bool
ip, _, err := net.ParseCIDR(vip.Addr)
if err != nil {
logger.Errorf("cannot parse: %q as cidr for service label add, err: %v", vip.Addr, err)
continue
}
for _, ep := range service.Endpoint.Ports {
if ep.Protocol != "tcp" {
continue
}
lbls := map[string]string{
"__meta_dockerswarm_service_endpoint_port_name": ep.Name,
"__meta_dockerswarm_service_endpoint_port_publish_mode": ep.PublishMode,
"__address__": discoveryutils.JoinHostPort(ip.String(), ep.PublishedPort),
}
for k, v := range m {
lbls[k] = v
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", vip.NetworkID)
added = true
ms = append(ms, lbls)
}
if !added {
lbls := make(map[string]string, len(m))
for k, v := range m {
lbls[k] = v
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", vip.NetworkID)
lbls["__address__"] = discoveryutils.JoinHostPort(ip.String(), port)
ms = append(ms, lbls)
}
}
}
return ms
}

293
lib/promscrape/discovery/dockerswarm/services_test.go Normal file
View File

@ -0,0 +1,293 @@
package dockerswarm
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
func Test_parseServicesResponse(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []service
wantErr bool
}{
{
name: "parse ok",
args: args{
data: []byte(`[
{
"ID": "tgsci5gd31aai3jyudv98pqxf",
"Version": {
"Index": 25
},
"CreatedAt": "2020-10-06T11:17:31.948808444Z",
"UpdatedAt": "2020-10-06T11:17:31.950195138Z",
"Spec": {
"Name": "redis2",
"Labels": {},
"TaskTemplate": {
"ContainerSpec": {
"Image": "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
"Init": false,
"DNSConfig": {},
"Isolation": "default"
},
"Resources": {
"Limits": {},
"Reservations": {}
}
},
"Mode": {
"Replicated": {}
},
"EndpointSpec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 6379,
"PublishedPort": 8081,
"PublishMode": "ingress"
}
]
}
},
"Endpoint": {
"Spec": {
"Mode": "vip",
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 6379,
"PublishedPort": 8081,
"PublishMode": "ingress"
}
]
},
"Ports": [
{
"Protocol": "tcp",
"TargetPort": 6379,
"PublishedPort": 8081,
"PublishMode": "ingress"
}
],
"VirtualIPs": [
{
"NetworkID": "qs0hog6ldlei9ct11pr3c77v1",
"Addr": "10.0.0.3/24"
}
]
}
}
]`),
},
want: []service{
{
ID: "tgsci5gd31aai3jyudv98pqxf",
Spec: struct {
Labels map[string]string
Name string
TaskTemplate struct {
ContainerSpec struct {
Hostname string
Image string
}
}
Mode struct {
Global interface{}
Replicated interface{}
}
}{
Labels: map[string]string{},
Name: "redis2",
TaskTemplate: struct {
ContainerSpec struct {
Hostname string
Image string
}
}{
ContainerSpec: struct {
Hostname string
Image string
}{
Hostname: "",
Image: "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
},
},
Mode: struct {
Global interface{}
Replicated interface{}
}{
Replicated: map[string]interface{}{},
},
},
Endpoint: struct {
Ports []portConfig
VirtualIPs []struct {
NetworkID string
Addr string
}
}{Ports: []portConfig{
{
Protocol: "tcp",
PublishMode: "ingress",
PublishedPort: 8081,
},
}, VirtualIPs: []struct {
NetworkID string
Addr string
}{
{
NetworkID: "qs0hog6ldlei9ct11pr3c77v1",
Addr: "10.0.0.3/24",
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseServicesResponse(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseServicesResponse() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseServicesResponse() \ngot %v, \nwant %v", got, tt.want)
}
})
}
}
func Test_addServicesLabels(t *testing.T) {
type args struct {
services []service
networksLabels []map[string]string
port int
}
tests := []struct {
name string
args args
want [][]prompbmarshal.Label
}{
{
name: "add 2 services with network labels join",
args: args{
port: 9100,
networksLabels: []map[string]string{
{
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
},
},
services: []service{
{
ID: "tgsci5gd31aai3jyudv98pqxf",
Spec: struct {
Labels map[string]string
Name string
TaskTemplate struct {
ContainerSpec struct {
Hostname string
Image string
}
}
Mode struct {
Global interface{}
Replicated interface{}
}
}{
Labels: map[string]string{},
Name: "redis2",
TaskTemplate: struct {
ContainerSpec struct {
Hostname string
Image string
}
}{
ContainerSpec: struct {
Hostname string
Image string
}{
Hostname: "node1",
Image: "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
},
},
Mode: struct {
Global interface{}
Replicated interface{}
}{
Replicated: map[string]interface{}{},
},
},
Endpoint: struct {
Ports []portConfig
VirtualIPs []struct {
NetworkID string
Addr string
}
}{Ports: []portConfig{
{
Protocol: "tcp",
Name: "redis",
PublishMode: "ingress",
},
}, VirtualIPs: []struct {
NetworkID string
Addr string
}{
{
NetworkID: "qs0hog6ldlei9ct11pr3c77v1",
Addr: "10.0.0.3/24",
},
},
},
},
},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "10.0.0.3:0",
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
"__meta_dockerswarm_service_endpoint_port_name": "redis",
"__meta_dockerswarm_service_endpoint_port_publish_mode": "ingress",
"__meta_dockerswarm_service_id": "tgsci5gd31aai3jyudv98pqxf",
"__meta_dockerswarm_service_mode": "replicated",
"__meta_dockerswarm_service_name": "redis2",
"__meta_dockerswarm_service_task_container_hostname": "node1",
"__meta_dockerswarm_service_task_container_image": "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
})},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := addServicesLabels(tt.args.services, tt.args.networksLabels, tt.args.port)
var sortedLabelss [][]prompbmarshal.Label
for _, labels := range got {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {
t.Errorf("addServicesLabels() \ngot %v, \nwant %v", sortedLabelss, tt.want)
}
})
}
}

149
lib/promscrape/discovery/dockerswarm/tasks.go Normal file
View File

@ -0,0 +1,149 @@
package dockerswarm
import (
"encoding/json"
"fmt"
"net"
"strconv"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
// See https://docs.docker.com/engine/api/v1.40/#tag/Task
type task struct {
ID string
ServiceID string
NodeID string
Labels map[string]string
DesiredState string
NetworksAttachments []struct {
Addresses []string
Network struct {
ID string
}
}
Status struct {
State string
ContainerStatus *struct {
ContainerID string
}
PortStatus struct {
Ports []portConfig
}
}
Slot int
}
func getTasksLabels(cfg *apiConfig) ([]map[string]string, error) {
tasks, err := getTasks(cfg)
if err != nil {
return nil, err
}
services, err := getServices(cfg)
if err != nil {
return nil, err
}
networkLabels, err := getNetworksLabels(cfg)
if err != nil {
return nil, err
}
svcLabels := addServicesLabels(services, networkLabels, cfg.port)
nodeLabels, err := getNodesLabels(cfg)
if err != nil {
return nil, err
}
return addTasksLabels(tasks, nodeLabels, svcLabels, networkLabels, services, cfg.port), nil
}
func getTasks(cfg *apiConfig) ([]task, error) {
resp, err := cfg.client.GetAPIResponse("/tasks")
if err != nil {
return nil, fmt.Errorf("cannot query dockerswarm api for tasks: %w", err)
}
return parseTasks(resp)
}
func parseTasks(data []byte) ([]task, error) {
var tasks []task
if err := json.Unmarshal(data, &tasks); err != nil {
return nil, fmt.Errorf("cannot parse tasks: %w", err)
}
return tasks, nil
}
func addTasksLabels(tasks []task, nodesLabels, servicesLabels, networksLabels []map[string]string, services []service, port int) []map[string]string {
var ms []map[string]string
for _, task := range tasks {
m := map[string]string{
"__meta_dockerswarm_task_id": task.ID,
"__meta_dockerswarm_task_desired_state": task.DesiredState,
"__meta_dockerswarm_task_state": task.Status.State,
"__meta_dockerswarm_task_slot": strconv.Itoa(task.Slot),
}
if task.Status.ContainerStatus != nil {
m["__meta_dockerswarm_task_container_id"] = task.Status.ContainerStatus.ContainerID
}
for k, v := range task.Labels {
m["__meta_dockerswarm_task_label_"+discoveryutils.SanitizeLabelName(k)] = v
}
var svcPorts []portConfig
for i, v := range services {
if v.ID == task.ServiceID {
svcPorts = services[i].Endpoint.Ports
break
}
}
m = joinLabels(servicesLabels, m, "__meta_dockerswarm_service_id", task.ServiceID)
m = joinLabels(nodesLabels, m, "__meta_dockerswarm_node_id", task.NodeID)
for _, port := range task.Status.PortStatus.Ports {
if port.Protocol != "tcp" {
continue
}
lbls := make(map[string]string, len(m))
lbls["__meta_dockerswarm_task_port_publish_mode"] = port.PublishMode
lbls["__address__"] = discoveryutils.JoinHostPort(m["__meta_dockerswarm_node_address"], port.PublishedPort)
for k, v := range m {
lbls[k] = v
}
ms = append(ms, lbls)
}
for _, na := range task.NetworksAttachments {
for _, address := range na.Addresses {
ip, _, err := net.ParseCIDR(address)
if err != nil {
logger.Errorf("cannot parse task network attachments address: %s as net CIDR: %v", address, err)
continue
}
var added bool
for _, v := range svcPorts {
if v.Protocol != "tcp" {
continue
}
lbls := make(map[string]string, len(m))
for k, v := range m {
lbls[k] = v
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", na.Network.ID)
lbls["__address"] = discoveryutils.JoinHostPort(ip.String(), v.PublishedPort)
lbls["__meta_dockerswarm_task_port_publish_mode"] = v.PublishMode
ms = append(ms, lbls)
added = true
}
if !added {
lbls := make(map[string]string, len(m))
for k, v := range m {
lbls[k] = v
}
lbls = joinLabels(networksLabels, lbls, "__meta_dockerswarm_network_id", na.Network.ID)
lbls["__address__"] = discoveryutils.JoinHostPort(ip.String(), port)
ms = append(ms, lbls)
}
}
}
}
return ms
}

352
lib/promscrape/discovery/dockerswarm/tasks_test.go Normal file
View File

@ -0,0 +1,352 @@
package dockerswarm
import (
"reflect"
"testing"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
)
func Test_parseTasks(t *testing.T) {
type args struct {
data []byte
}
tests := []struct {
name string
args args
want []task
wantErr bool
}{
{
name: "parse ok",
args: args{
data: []byte(`[
{
"ID": "t4rdm7j2y9yctbrksiwvsgpu5",
"Version": {
"Index": 23
},
"Labels": {},
"Spec": {
"ContainerSpec": {
"Image": "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
"Init": false
},
"Resources": {
"Limits": {},
"Reservations": {}
},
"Placement": {
"Platforms": [
{
"Architecture": "amd64",
"OS": "linux"
}
]
},
"ForceUpdate": 0
},
"ServiceID": "t91nf284wzle1ya09lqvyjgnq",
"Slot": 1,
"NodeID": "qauwmifceyvqs0sipvzu8oslu",
"Status": {
"State": "running",
"ContainerStatus": {
"ContainerID": "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
"ExitCode": 0
},
"PortStatus": {}
},
"DesiredState": "running"
}
]
`),
},
want: []task{
{
ID: "t4rdm7j2y9yctbrksiwvsgpu5",
ServiceID: "t91nf284wzle1ya09lqvyjgnq",
NodeID: "qauwmifceyvqs0sipvzu8oslu",
Labels: map[string]string{},
DesiredState: "running",
Slot: 1,
Status: struct {
State string
ContainerStatus *struct{ ContainerID string }
PortStatus struct{ Ports []portConfig }
}{
State: "running",
ContainerStatus: &struct{ ContainerID string }{
ContainerID: "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
},
PortStatus: struct{ Ports []portConfig }{}},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseTasks(tt.args.data)
if (err != nil) != tt.wantErr {
t.Errorf("parseTasks() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("parseTasks() got = %v, want %v", got, tt.want)
}
})
}
}
func Test_addTasksLabels(t *testing.T) {
type args struct {
tasks []task
nodesLabels []map[string]string
servicesLabels []map[string]string
networksLabels []map[string]string
services []service
port int
}
tests := []struct {
name string
args args
want [][]prompbmarshal.Label
}{
{
name: "adds 1 task with nodes labels",
args: args{
port: 9100,
tasks: []task{
{
ID: "t4rdm7j2y9yctbrksiwvsgpu5",
ServiceID: "t91nf284wzle1ya09lqvyjgnq",
NodeID: "qauwmifceyvqs0sipvzu8oslu",
Labels: map[string]string{},
DesiredState: "running",
Slot: 1,
Status: struct {
State string
ContainerStatus *struct{ ContainerID string }
PortStatus struct{ Ports []portConfig }
}{
State: "running",
ContainerStatus: &struct{ ContainerID string }{
ContainerID: "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
},
PortStatus: struct{ Ports []portConfig }{
Ports: []portConfig{
{
PublishMode: "ingress",
Name: "redis",
Protocol: "tcp",
PublishedPort: 6379,
},
},
}},
},
},
nodesLabels: []map[string]string{
{
"__address__": "172.31.40.97:9100",
"__meta_dockerswarm_node_address": "172.31.40.97",
"__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11",
"__meta_dockerswarm_node_hostname": "ip-172-31-40-97",
"__meta_dockerswarm_node_id": "qauwmifceyvqs0sipvzu8oslu",
"__meta_dockerswarm_node_platform_architecture": "x86_64",
"__meta_dockerswarm_node_platform_os": "linux",
"__meta_dockerswarm_node_role": "manager",
"__meta_dockerswarm_node_status": "ready",
},
},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address__": "172.31.40.97:9100",
"__meta_dockerswarm_node_address": "172.31.40.97",
"__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11",
"__meta_dockerswarm_node_hostname": "ip-172-31-40-97",
"__meta_dockerswarm_node_id": "qauwmifceyvqs0sipvzu8oslu",
"__meta_dockerswarm_node_platform_architecture": "x86_64",
"__meta_dockerswarm_node_platform_os": "linux",
"__meta_dockerswarm_node_role": "manager",
"__meta_dockerswarm_node_status": "ready",
"__meta_dockerswarm_task_container_id": "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
"__meta_dockerswarm_task_desired_state": "running",
"__meta_dockerswarm_task_id": "t4rdm7j2y9yctbrksiwvsgpu5",
"__meta_dockerswarm_task_port_publish_mode": "ingress",
"__meta_dockerswarm_task_slot": "1",
"__meta_dockerswarm_task_state": "running",
})},
},
{
name: "adds 1 task with nodes, network and services labels",
args: args{
port: 9100,
tasks: []task{
{
ID: "t4rdm7j2y9yctbrksiwvsgpu5",
ServiceID: "tgsci5gd31aai3jyudv98pqxf",
NodeID: "qauwmifceyvqs0sipvzu8oslu",
Labels: map[string]string{},
DesiredState: "running",
Slot: 1,
NetworksAttachments: []struct {
Addresses []string
Network struct{ ID string }
}{
{
Network: struct {
ID string
}{
ID: "qs0hog6ldlei9ct11pr3c77v1",
},
Addresses: []string{"10.10.15.15/24"},
},
},
Status: struct {
State string
ContainerStatus *struct{ ContainerID string }
PortStatus struct{ Ports []portConfig }
}{
State: "running",
ContainerStatus: &struct{ ContainerID string }{
ContainerID: "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
},
PortStatus: struct{ Ports []portConfig }{}},
},
},
networksLabels: []map[string]string{
{
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
},
},
nodesLabels: []map[string]string{
{
"__address__": "172.31.40.97:9100",
"__meta_dockerswarm_node_address": "172.31.40.97",
"__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11",
"__meta_dockerswarm_node_hostname": "ip-172-31-40-97",
"__meta_dockerswarm_node_id": "qauwmifceyvqs0sipvzu8oslu",
"__meta_dockerswarm_node_platform_architecture": "x86_64",
"__meta_dockerswarm_node_platform_os": "linux",
"__meta_dockerswarm_node_role": "manager",
"__meta_dockerswarm_node_status": "ready",
},
},
services: []service{
{
ID: "tgsci5gd31aai3jyudv98pqxf",
Spec: struct {
Labels map[string]string
Name string
TaskTemplate struct {
ContainerSpec struct {
Hostname string
Image string
}
}
Mode struct {
Global interface{}
Replicated interface{}
}
}{
Labels: map[string]string{},
Name: "redis2",
TaskTemplate: struct {
ContainerSpec struct {
Hostname string
Image string
}
}{
ContainerSpec: struct {
Hostname string
Image string
}{
Hostname: "node1",
Image: "redis:3.0.6@sha256:6a692a76c2081888b589e26e6ec835743119fe453d67ecf03df7de5b73d69842",
},
},
Mode: struct {
Global interface{}
Replicated interface{}
}{
Replicated: map[string]interface{}{},
},
},
Endpoint: struct {
Ports []portConfig
VirtualIPs []struct {
NetworkID string
Addr string
}
}{Ports: []portConfig{
{
Protocol: "tcp",
Name: "redis",
PublishMode: "ingress",
},
}, VirtualIPs: []struct {
NetworkID string
Addr string
}{
{
NetworkID: "qs0hog6ldlei9ct11pr3c77v1",
Addr: "10.0.0.3/24",
},
},
},
},
},
servicesLabels: []map[string]string{},
},
want: [][]prompbmarshal.Label{
discoveryutils.GetSortedLabels(map[string]string{
"__address": "10.10.15.15:0",
"__address__": "172.31.40.97:9100",
"__meta_dockerswarm_network_id": "qs0hog6ldlei9ct11pr3c77v1",
"__meta_dockerswarm_network_ingress": "true",
"__meta_dockerswarm_network_internal": "false",
"__meta_dockerswarm_network_label_key1": "value1",
"__meta_dockerswarm_network_name": "ingress",
"__meta_dockerswarm_network_scope": "swarm",
"__meta_dockerswarm_node_address": "172.31.40.97",
"__meta_dockerswarm_node_availability": "active",
"__meta_dockerswarm_node_engine_version": "19.03.11",
"__meta_dockerswarm_node_hostname": "ip-172-31-40-97",
"__meta_dockerswarm_node_id": "qauwmifceyvqs0sipvzu8oslu",
"__meta_dockerswarm_node_platform_architecture": "x86_64",
"__meta_dockerswarm_node_platform_os": "linux",
"__meta_dockerswarm_node_role": "manager",
"__meta_dockerswarm_node_status": "ready",
"__meta_dockerswarm_task_container_id": "33034b69f6fa5f808098208752fd1fe4e0e1ca86311988cea6a73b998cdc62e8",
"__meta_dockerswarm_task_desired_state": "running",
"__meta_dockerswarm_task_id": "t4rdm7j2y9yctbrksiwvsgpu5",
"__meta_dockerswarm_task_port_publish_mode": "ingress",
"__meta_dockerswarm_task_slot": "1",
"__meta_dockerswarm_task_state": "running",
}),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := addTasksLabels(tt.args.tasks, tt.args.nodesLabels, tt.args.servicesLabels, tt.args.networksLabels, tt.args.services, tt.args.port)
var sortedLabelss [][]prompbmarshal.Label
for _, labels := range got {
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
}
if !reflect.DeepEqual(sortedLabelss, tt.want) {
t.Errorf("addTasksLabels() \ngot %v, \nwant %v", sortedLabelss, tt.want)
}
})
}
}

17
lib/promscrape/discoveryutils/client.go
View File

@ -41,11 +41,23 @@ type Client struct {
// NewClient returns new Client for the given apiServer and the given ac. // NewClient returns new Client for the given apiServer and the given ac.
func NewClient(apiServer string, ac *promauth.Config) (*Client, error) { func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
var u fasthttp.URI var (
dialFunc fasthttp.DialFunc
tlsCfg *tls.Config
u fasthttp.URI
)
u.Update(apiServer) u.Update(apiServer)
// special case for unix socket connection
if string(u.Scheme()) == "unix" {
dialAddr := string(u.Path())
apiServer = "http://"
dialFunc = func(_ string) (net.Conn, error) {
return net.Dial("unix", dialAddr)
}
}
hostPort := string(u.Host()) hostPort := string(u.Host())
isTLS := string(u.Scheme()) == "https" isTLS := string(u.Scheme()) == "https"
var tlsCfg *tls.Config
if isTLS && ac != nil { if isTLS && ac != nil {
tlsCfg = ac.NewTLSConfig() tlsCfg = ac.NewTLSConfig()
} }
@ -66,6 +78,7 @@ func NewClient(apiServer string, ac *promauth.Config) (*Client, error) {
WriteTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second,
MaxResponseBodySize: 300 * 1024 * 1024, MaxResponseBodySize: 300 * 1024 * 1024,
MaxConns: 2 * *maxConcurrency, MaxConns: 2 * *maxConcurrency,
Dial: dialFunc,
} }
return &Client{ return &Client{
hc: hc, hc: hc,

5
lib/promscrape/scraper.go
View File

@ -36,9 +36,11 @@ var (
gceSDCheckInterval = flag.Duration("promscrape.gceSDCheckInterval", time.Minute, "Interval for checking for changes in gce. "+ gceSDCheckInterval = flag.Duration("promscrape.gceSDCheckInterval", time.Minute, "Interval for checking for changes in gce. "+
"This works only if `gce_sd_configs` is configured in '-promscrape.config' file. "+ "This works only if `gce_sd_configs` is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details") "See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#gce_sd_config for details")
dockerswarmSDCheckInterval = flag.Duration("promscrape.dockerswarmSDCheckInterval", 30*time.Second, "Interval for checking for changes in dockerswarm. "+
"This works only if `dockerswarm_sd_configs` is configured in '-promscrape.config' file. "+
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dockerswarm_sd_config for details")
promscrapeConfigFile = flag.String("promscrape.config", "", "Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. "+ promscrapeConfigFile = flag.String("promscrape.config", "", "Optional path to Prometheus config file with 'scrape_configs' section containing targets to scrape. "+
"See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details") "See https://victoriametrics.github.io/#how-to-scrape-prometheus-exporters-such-as-node-exporter for details")
suppressDuplicateScrapeTargetErrors = flag.Bool("promscrape.suppressDuplicateScrapeTargetErrors", false, "Whether to suppress `duplicate scrape target` errors; "+ suppressDuplicateScrapeTargetErrors = flag.Bool("promscrape.suppressDuplicateScrapeTargetErrors", false, "Whether to suppress `duplicate scrape target` errors; "+
"see https://victoriametrics.github.io/vmagent.html#troubleshooting for details") "see https://victoriametrics.github.io/vmagent.html#troubleshooting for details")
) )
@ -96,6 +98,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) }) scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) })
scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) }) scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) })
scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) }) scs.add("gce_sd_configs", *gceSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getGCESDScrapeWork(swsPrev) })
scs.add("dockerswarm_sd_configs", *dockerswarmSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getDockerSwarmSDScrapeWork(swsPrev) })
sighupCh := procutil.NewSighupChan() sighupCh := procutil.NewSighupChan()