mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-11-23 12:31:07 +01:00
Adds openstack sd (#811)
* adds openstack service discovery https://github.com/VictoriaMetrics/VictoriaMetrics/issues/728 implemented hypervisors and instance discovery with openstack v3 api. Added tests for labeling and data parsing. Added token refresh. * Apply suggestions from code review * Apply suggestions from code review Co-authored-by: Aliaksandr Valialkin <valyala@gmail.com>
This commit is contained in:
Nikolay Khramchikhin
GitHub
parent
f42194d817
commit
cbe3cf683b
@ -148,6 +148,9 @@ The following scrape types in [scrape_config](https://prometheus.io/docs/prometh
|
||||
See [consul_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) for details.
|
||||
* `dns_sd_configs` - for scraping targets discovered from DNS records (SRV, A and AAAA).
|
||||
See [dns_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#dns_sd_config) for details.
|
||||
* `openstack_sd_configs` - for scraping OpenStack targets.
|
||||
See [openstack_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config) for details.
|
||||
[OpenStack identity API v3](https://docs.openstack.org/api-ref/identity/v3/) is supported only.
|
||||
|
||||
File feature requests at [our issue tracker](https://github.com/VictoriaMetrics/VictoriaMetrics/issues) if you need other service discovery mechanisms to be supported by `vmagent`.
|
||||
|
||||
|
||||
@ -21,6 +21,7 @@ import (
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/ec2"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/gce"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/kubernetes"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discovery/openstack"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
@ -69,6 +70,7 @@ type ScrapeConfig struct {
|
||||
StaticConfigs []StaticConfig `yaml:"static_configs"`
|
||||
FileSDConfigs []FileSDConfig `yaml:"file_sd_configs"`
|
||||
KubernetesSDConfigs []kubernetes.SDConfig `yaml:"kubernetes_sd_configs"`
|
||||
OpenStackSDConfigs []openstack.SDConfig `yaml:"openstack_sd_configs"`
|
||||
ConsulSDConfigs []consul.SDConfig `yaml:"consul_sd_configs"`
|
||||
DNSSDConfigs []dns.SDConfig `yaml:"dns_sd_configs"`
|
||||
EC2SDConfigs []ec2.SDConfig `yaml:"ec2_sd_configs"`
|
||||
@ -201,6 +203,34 @@ func (cfg *Config) getKubernetesSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
return dst
|
||||
}
|
||||
|
||||
// getOpenStackSDScrapeWork returns `openstack_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getOpenStackSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
var dst []ScrapeWork
|
||||
for i := range cfg.ScrapeConfigs {
|
||||
sc := &cfg.ScrapeConfigs[i]
|
||||
dstLen := len(dst)
|
||||
ok := true
|
||||
for j := range sc.OpenStackSDConfigs {
|
||||
sdc := &sc.OpenStackSDConfigs[j]
|
||||
var okLocal bool
|
||||
dst, okLocal = appendOpenstackScrapeWork(dst, sdc, cfg.baseDir, sc.swc)
|
||||
if ok {
|
||||
ok = okLocal
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
continue
|
||||
}
|
||||
swsPrev := swsPrevByJob[sc.swc.jobName]
|
||||
if len(swsPrev) > 0 {
|
||||
logger.Errorf("there were errors when discovering openstack targets for job %q, so preserving the previous targets", sc.swc.jobName)
|
||||
dst = append(dst[:dstLen], swsPrev...)
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// getConsulSDScrapeWork returns `consul_sd_configs` ScrapeWork from cfg.
|
||||
func (cfg *Config) getConsulSDScrapeWork(prev []ScrapeWork) []ScrapeWork {
|
||||
swsPrevByJob := getSWSByJob(prev)
|
||||
@ -444,6 +474,15 @@ func appendKubernetesScrapeWork(dst []ScrapeWork, sdc *kubernetes.SDConfig, base
|
||||
return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "kubernetes_sd_config"), true
|
||||
}
|
||||
|
||||
func appendOpenstackScrapeWork(dst []ScrapeWork, sdc *openstack.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) {
|
||||
targetLabels, err := openstack.GetLabels(sdc, baseDir)
|
||||
if err != nil {
|
||||
logger.Errorf("error when discovering openstack targets for `job_name` %q: %s; skipping it", swc.jobName, err)
|
||||
return dst, false
|
||||
}
|
||||
return appendScrapeWorkForTargetLabels(dst, swc, targetLabels, "openstack_sd_config"), true
|
||||
}
|
||||
|
||||
func appendConsulScrapeWork(dst []ScrapeWork, sdc *consul.SDConfig, baseDir string, swc *scrapeWorkConfig) ([]ScrapeWork, bool) {
|
||||
targetLabels, err := consul.GetLabels(sdc, baseDir)
|
||||
if err != nil {
|
||||
|
||||
203
lib/promscrape/discovery/openstack/api.go
Normal file
203
lib/promscrape/discovery/openstack/api.go
Normal file
@ -0,0 +1,203 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
const authHeaderName = "X-Auth-Token" // for making requests to openstack api
|
||||
|
||||
var configMap = discoveryutils.NewConfigMap()
|
||||
|
||||
// apiCredentials can be refreshed
|
||||
type apiCredentials struct {
|
||||
// computeURL obtained from auth response and maybe changed
|
||||
computeURL *url.URL
|
||||
// value of authHeaderName
|
||||
token string
|
||||
expiration time.Time
|
||||
}
|
||||
|
||||
type apiConfig struct {
|
||||
// client may use tls
|
||||
client *http.Client
|
||||
port int
|
||||
// tokenLock - guards creds refresh
|
||||
tokenLock sync.Mutex
|
||||
creds *apiCredentials
|
||||
// authTokenReq - request for apiCredentials
|
||||
authTokenReq []byte
|
||||
// keystone endpoint
|
||||
endpoint *url.URL
|
||||
allTenants bool
|
||||
region string
|
||||
// availability public, internal, admin for filtering compute endpoint
|
||||
availability string
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getFreshAPICredentials() (*apiCredentials, error) {
|
||||
cfg.tokenLock.Lock()
|
||||
defer cfg.tokenLock.Unlock()
|
||||
|
||||
if time.Until(cfg.creds.expiration) > 10*time.Second {
|
||||
|
||||
// Credentials aren't expired yet.
|
||||
return cfg.creds, nil
|
||||
}
|
||||
newCreds, err := getCreds(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed token refresh: %w", err)
|
||||
}
|
||||
logger.Infof("refreshed, next : %v", cfg.creds.expiration.String())
|
||||
cfg.creds = newCreds
|
||||
|
||||
return cfg.creds, nil
|
||||
}
|
||||
|
||||
func getAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
v, err := configMap.Get(sdc, func() (interface{}, error) { return newAPIConfig(sdc, baseDir) })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return v.(*apiConfig), nil
|
||||
}
|
||||
|
||||
func newAPIConfig(sdc *SDConfig, baseDir string) (*apiConfig, error) {
|
||||
cfg := &apiConfig{
|
||||
client: discoveryutils.GetHTTPClient(),
|
||||
availability: sdc.Availability,
|
||||
region: sdc.Region,
|
||||
allTenants: sdc.AllTenants,
|
||||
port: sdc.Port,
|
||||
}
|
||||
|
||||
if sdc.TLSConfig != nil {
|
||||
config, err := promauth.NewConfig(baseDir, nil, "", "", sdc.TLSConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: config.NewTLSConfig(),
|
||||
}
|
||||
cfg.client.Transport = tr
|
||||
}
|
||||
// use public compute endpoint by default
|
||||
if len(cfg.availability) == 0 {
|
||||
cfg.availability = "public"
|
||||
}
|
||||
|
||||
// create new variable to prevent side effects
|
||||
sdcAuth := *sdc
|
||||
// special case if identity_endpoint is not defined
|
||||
if len(sdc.IdentityEndpoint) == 0 {
|
||||
// override sdc
|
||||
sdcAuth = readCredentialsFromEnv()
|
||||
}
|
||||
|
||||
parsedURL, err := url.Parse(sdcAuth.IdentityEndpoint)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot parse identity_endpoint: %s as url, err: %w", sdcAuth.IdentityEndpoint, err)
|
||||
}
|
||||
cfg.endpoint = parsedURL
|
||||
tokenReq, err := buildAuthRequestBody(&sdcAuth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.authTokenReq = tokenReq
|
||||
token, err := getCreds(cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.creds = token
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// getCreds - make call to openstack keystone api and retrieves token and computeURL
|
||||
// https://docs.openstack.org/api-ref/identity/v3/
|
||||
func getCreds(cfg *apiConfig) (*apiCredentials, error) {
|
||||
|
||||
apiURL := *cfg.endpoint
|
||||
apiURL.Path = path.Join(apiURL.Path, "auth", "tokens")
|
||||
|
||||
resp, err := cfg.client.Post(apiURL.String(), "application/json", bytes.NewBuffer(cfg.authTokenReq))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed query openstack identity api, url: %s, err: %w", apiURL.String(), err)
|
||||
}
|
||||
r, err := ioutil.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read response from %q: %w", apiURL.String(), err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return nil, fmt.Errorf("auth failed, bad status code: %d, want: 201", resp.StatusCode)
|
||||
}
|
||||
|
||||
at := resp.Header.Get("X-Subject-Token")
|
||||
if len(at) == 0 {
|
||||
return nil, fmt.Errorf("auth failed, response without X-Subject-Token")
|
||||
}
|
||||
|
||||
var ar authResponse
|
||||
if err := json.Unmarshal(r, &ar); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse auth credentials response: %w", err)
|
||||
}
|
||||
|
||||
computeURL, err := getComputeEndpointURL(ar.Token.Catalog, cfg.availability, cfg.region)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot get computeEndpoint, account doesn't have enough permissions,"+
|
||||
" availability: %s, region: %s", cfg.availability, cfg.region)
|
||||
}
|
||||
|
||||
return &apiCredentials{
|
||||
token: at,
|
||||
expiration: ar.Token.ExpiresAt,
|
||||
computeURL: computeURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readResponseBody - reads body from http.Response.
|
||||
func readResponseBody(resp *http.Response, apiURL string) ([]byte, error) {
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot read response from %q: %w", apiURL, err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status code for %q; got %d; want %d; response body: %q",
|
||||
apiURL, resp.StatusCode, http.StatusOK, data)
|
||||
}
|
||||
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// getAPIResponse - makes api call to openstack and returns response body
|
||||
func getAPIResponse(href string, cfg *apiConfig) ([]byte, error) {
|
||||
token, err := cfg.getFreshAPICredentials()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed refresh api credentials: %w", err)
|
||||
}
|
||||
req, err := http.NewRequest("GET", href, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot create new request for openstack api href: %s, err: %w", href, err)
|
||||
}
|
||||
req.Header.Set(authHeaderName, token.token)
|
||||
resp, err := cfg.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed query openstack api, href: %s, err : %w", href, err)
|
||||
}
|
||||
|
||||
return readResponseBody(resp, href)
|
||||
|
||||
}
|
||||
373
lib/promscrape/discovery/openstack/auth.go
Normal file
373
lib/promscrape/discovery/openstack/auth.go
Normal file
@ -0,0 +1,373 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// authResponse - identity api response
|
||||
// https://docs.openstack.org/api-ref/identity/v3/?expanded=create-credential-detail,password-authentication-with-unscoped-authorization-detail#authentication-and-token-management
|
||||
type authResponse struct {
|
||||
Token struct {
|
||||
ExpiresAt time.Time `json:"expires_at,omitempty"`
|
||||
Catalog []catalogItem `json:"catalog,omitempty"`
|
||||
}
|
||||
}
|
||||
|
||||
type catalogItem struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Endpoints []endpoint `json:"endpoints"`
|
||||
}
|
||||
|
||||
// openstack api endpoint
|
||||
// https://docs.openstack.org/api-ref/identity/v3/?expanded=create-credential-detail,password-authentication-with-unscoped-authorization-detail,token-authentication-with-scoped-authorization-detail#list-endpoints
|
||||
type endpoint struct {
|
||||
RegionID string `json:"region_id"`
|
||||
RegionName string `json:"region_name"`
|
||||
URL string `json:"url"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Interface string `json:"interface"`
|
||||
}
|
||||
|
||||
// getComputeEndpointURL extracts compute url endpoint with given filters from keystone catalog
|
||||
func getComputeEndpointURL(catalog []catalogItem, availability, region string) (*url.URL, error) {
|
||||
for _, eps := range catalog {
|
||||
if eps.Type == "compute" {
|
||||
for _, ep := range eps.Endpoints {
|
||||
if ep.Interface == availability && (len(region) == 0 || region == ep.RegionID || region == ep.RegionName) {
|
||||
return url.Parse(ep.URL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("cannot excract compute url from catalog, availability: %s, region: %s ", availability, region)
|
||||
}
|
||||
|
||||
// buildAuthRequestBody - builds request for authentication.
|
||||
func buildAuthRequestBody(sdc *SDConfig) ([]byte, error) {
|
||||
|
||||
// fast path
|
||||
if len(sdc.Password) == 0 && len(sdc.ApplicationCredentialID) == 0 && len(sdc.ApplicationCredentialName) == 0 {
|
||||
return nil, fmt.Errorf("password and application credentials is missing")
|
||||
}
|
||||
|
||||
type domainReq struct {
|
||||
ID *string `json:"id,omitempty"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
type userReq struct {
|
||||
ID *string `json:"id,omitempty"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
Password *string `json:"password,omitempty"`
|
||||
Passcode *string `json:"passcode,omitempty"`
|
||||
Domain *domainReq `json:"domain,omitempty"`
|
||||
}
|
||||
|
||||
type passwordReq struct {
|
||||
User userReq `json:"user"`
|
||||
}
|
||||
|
||||
type tokenReq struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type applicationCredentialReq struct {
|
||||
ID *string `json:"id,omitempty"`
|
||||
Name *string `json:"name,omitempty"`
|
||||
User *userReq `json:"user,omitempty"`
|
||||
Secret *string `json:"secret,omitempty"`
|
||||
}
|
||||
|
||||
type identityReq struct {
|
||||
Methods []string `json:"methods"`
|
||||
Password *passwordReq `json:"password,omitempty"`
|
||||
Token *tokenReq `json:"token,omitempty"`
|
||||
ApplicationCredential *applicationCredentialReq `json:"application_credential,omitempty"`
|
||||
}
|
||||
|
||||
type authReq struct {
|
||||
Identity identityReq `json:"identity"`
|
||||
Scope map[string]interface{} `json:"scope,omitempty"`
|
||||
}
|
||||
|
||||
type request struct {
|
||||
Auth authReq `json:"auth"`
|
||||
}
|
||||
|
||||
// Populate the request structure based on the provided arguments. Create and return an error
|
||||
// if insufficient or incompatible information is present.
|
||||
var req request
|
||||
|
||||
if len(sdc.Password) == 0 {
|
||||
// There are three kinds of possible application_credential requests
|
||||
// 1. application_credential id + secret
|
||||
// 2. application_credential name + secret + user_id
|
||||
// 3. application_credential name + secret + username + domain_id / domain_name
|
||||
if len(sdc.ApplicationCredentialID) > 0 {
|
||||
if len(sdc.ApplicationCredentialSecret) == 0 {
|
||||
return nil, fmt.Errorf("ApplicationCredentialSecret is empty")
|
||||
}
|
||||
req.Auth.Identity.Methods = []string{"application_credential"}
|
||||
req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{
|
||||
ID: &sdc.ApplicationCredentialID,
|
||||
Secret: &sdc.ApplicationCredentialSecret,
|
||||
}
|
||||
|
||||
// fast path unscoped
|
||||
return json.Marshal(req)
|
||||
}
|
||||
|
||||
// application_credential_name auth
|
||||
if len(sdc.ApplicationCredentialSecret) == 0 {
|
||||
return nil, fmt.Errorf("application_credential_name is not empty and application_credential_secret is empty")
|
||||
}
|
||||
|
||||
var userRequest *userReq
|
||||
|
||||
if len(sdc.UserID) > 0 {
|
||||
// UserID could be used without the domain information
|
||||
userRequest = &userReq{
|
||||
ID: &sdc.UserID,
|
||||
}
|
||||
}
|
||||
|
||||
if userRequest == nil && len(sdc.Username) == 0 {
|
||||
// Make sure that Username or UserID are provided
|
||||
return nil, fmt.Errorf("username and userid is empty")
|
||||
}
|
||||
|
||||
if userRequest == nil && len(sdc.DomainID) > 0 {
|
||||
userRequest = &userReq{
|
||||
Name: &sdc.Username,
|
||||
Domain: &domainReq{ID: &sdc.DomainID},
|
||||
}
|
||||
}
|
||||
|
||||
if userRequest == nil && len(sdc.DomainName) > 0 {
|
||||
userRequest = &userReq{
|
||||
Name: &sdc.Username,
|
||||
Domain: &domainReq{Name: &sdc.DomainName},
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure that domain_id or domain_name are provided among username
|
||||
if userRequest == nil {
|
||||
return nil, fmt.Errorf("domain_id and domain_name is empty for application_credential_name auth")
|
||||
}
|
||||
|
||||
req.Auth.Identity.Methods = []string{"application_credential"}
|
||||
req.Auth.Identity.ApplicationCredential = &applicationCredentialReq{
|
||||
Name: &sdc.ApplicationCredentialName,
|
||||
User: userRequest,
|
||||
Secret: &sdc.ApplicationCredentialSecret,
|
||||
}
|
||||
|
||||
// fast path unscoped
|
||||
return json.Marshal(req)
|
||||
}
|
||||
|
||||
// Password authentication.
|
||||
req.Auth.Identity.Methods = append(req.Auth.Identity.Methods, "password")
|
||||
|
||||
// At least one of Username and UserID must be specified.
|
||||
if len(sdc.Username) == 0 && len(sdc.UserID) == 0 {
|
||||
return nil, fmt.Errorf("username and userid is empty for username/password auth")
|
||||
}
|
||||
|
||||
if len(sdc.Username) > 0 {
|
||||
// If Username is provided, UserID may not be provided.
|
||||
if len(sdc.UserID) > 0 {
|
||||
return nil, fmt.Errorf("both username and userid is present")
|
||||
}
|
||||
|
||||
// Either DomainID or DomainName must also be specified.
|
||||
if len(sdc.DomainID) == 0 && len(sdc.DomainName) == 0 {
|
||||
return nil, fmt.Errorf(" domain_id or domain_name is missing for username/password auth: %s", sdc.Username)
|
||||
}
|
||||
|
||||
if len(sdc.DomainID) > 0 {
|
||||
if sdc.DomainName != "" {
|
||||
return nil, fmt.Errorf("both domain_id and domain_name is present")
|
||||
}
|
||||
|
||||
// Configure the request for Username and Password authentication with a DomainID.
|
||||
if len(sdc.Password) > 0 {
|
||||
req.Auth.Identity.Password = &passwordReq{
|
||||
User: userReq{
|
||||
Name: &sdc.Username,
|
||||
Password: &sdc.Password,
|
||||
Domain: &domainReq{ID: &sdc.DomainID},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(sdc.DomainName) > 0 {
|
||||
// Configure the request for Username and Password authentication with a DomainName.
|
||||
if len(sdc.Password) > 0 {
|
||||
req.Auth.Identity.Password = &passwordReq{
|
||||
User: userReq{
|
||||
Name: &sdc.Username,
|
||||
Password: &sdc.Password,
|
||||
Domain: &domainReq{Name: &sdc.DomainName},
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(sdc.UserID) > 0 {
|
||||
// If UserID is specified, neither DomainID nor DomainName may be.
|
||||
if len(sdc.DomainID) > 0 {
|
||||
return nil, fmt.Errorf("both user_id and domain_id is present")
|
||||
}
|
||||
if len(sdc.DomainName) > 0 {
|
||||
return nil, fmt.Errorf("both user_id and domain_name is present")
|
||||
}
|
||||
|
||||
// Configure the request for UserID and Password authentication.
|
||||
if len(sdc.Password) > 0 {
|
||||
req.Auth.Identity.Password = &passwordReq{
|
||||
User: userReq{
|
||||
ID: &sdc.UserID,
|
||||
Password: &sdc.Password,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// build scope for password auth
|
||||
scope, err := buildScope(sdc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(scope) > 0 {
|
||||
req.Auth.Scope = scope
|
||||
}
|
||||
|
||||
return json.Marshal(req)
|
||||
}
|
||||
|
||||
// buildScope - adds scope information into auth request
|
||||
// https://docs.openstack.org/api-ref/identity/v3/?expanded=password-authentication-with-scoped-authorization-detail#password-authentication-with-unscoped-authorization
|
||||
func buildScope(sdc *SDConfig) (map[string]interface{}, error) {
|
||||
|
||||
// fast path
|
||||
if len(sdc.ProjectName) == 0 && len(sdc.ProjectID) == 0 && len(sdc.DomainID) == 0 && len(sdc.DomainName) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if len(sdc.ProjectName) > 0 {
|
||||
// ProjectName provided: either DomainID or DomainName must also be supplied.
|
||||
// ProjectID may not be supplied.
|
||||
if len(sdc.DomainID) == 0 && len(sdc.DomainName) == 0 {
|
||||
return nil, fmt.Errorf("both domain_id and domain_name present")
|
||||
}
|
||||
if len(sdc.ProjectID) > 0 {
|
||||
return nil, fmt.Errorf("both domain_id and domain_name present")
|
||||
}
|
||||
|
||||
if len(sdc.DomainID) > 0 {
|
||||
|
||||
// ProjectName + DomainID
|
||||
return map[string]interface{}{
|
||||
"project": map[string]interface{}{
|
||||
"name": &sdc.ProjectName,
|
||||
"domain": map[string]interface{}{"id": &sdc.DomainID},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
if len(sdc.DomainName) > 0 {
|
||||
|
||||
// ProjectName + DomainName
|
||||
return map[string]interface{}{
|
||||
"project": map[string]interface{}{
|
||||
"name": &sdc.ProjectName,
|
||||
"domain": map[string]interface{}{"name": &sdc.DomainName},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
} else if len(sdc.ProjectID) > 0 {
|
||||
// ProjectID provided. ProjectName, DomainID, and DomainName may not be provided.
|
||||
if len(sdc.DomainID) > 0 {
|
||||
return nil, fmt.Errorf("both project_id and domain_id present")
|
||||
}
|
||||
if len(sdc.DomainName) > 0 {
|
||||
return nil, fmt.Errorf("both project_id and domain_name present")
|
||||
}
|
||||
|
||||
// ProjectID
|
||||
return map[string]interface{}{
|
||||
"project": map[string]interface{}{
|
||||
"id": &sdc.ProjectID,
|
||||
},
|
||||
}, nil
|
||||
} else if len(sdc.DomainID) > 0 {
|
||||
// DomainID provided. ProjectID, ProjectName, and DomainName may not be provided.
|
||||
if len(sdc.DomainName) > 0 {
|
||||
return nil, fmt.Errorf("both domain_id and domain_name present")
|
||||
}
|
||||
|
||||
// DomainID
|
||||
return map[string]interface{}{
|
||||
"domain": map[string]interface{}{
|
||||
"id": &sdc.DomainID,
|
||||
},
|
||||
}, nil
|
||||
} else if len(sdc.DomainName) > 0 {
|
||||
|
||||
// DomainName
|
||||
return map[string]interface{}{
|
||||
"domain": map[string]interface{}{
|
||||
"name": &sdc.DomainName,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// readCredentialsFromEnv - obtains serviceDiscoveryConfig from env variables for openstack
|
||||
func readCredentialsFromEnv() SDConfig {
|
||||
authURL := os.Getenv("OS_AUTH_URL")
|
||||
username := os.Getenv("OS_USERNAME")
|
||||
userID := os.Getenv("OS_USERID")
|
||||
password := os.Getenv("OS_PASSWORD")
|
||||
tenantID := os.Getenv("OS_TENANT_ID")
|
||||
tenantName := os.Getenv("OS_TENANT_NAME")
|
||||
domainID := os.Getenv("OS_DOMAIN_ID")
|
||||
domainName := os.Getenv("OS_DOMAIN_NAME")
|
||||
applicationCredentialID := os.Getenv("OS_APPLICATION_CREDENTIAL_ID")
|
||||
applicationCredentialName := os.Getenv("OS_APPLICATION_CREDENTIAL_NAME")
|
||||
applicationCredentialSecret := os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET")
|
||||
// If OS_PROJECT_ID is set, overwrite tenantID with the value.
|
||||
if v := os.Getenv("OS_PROJECT_ID"); v != "" {
|
||||
tenantID = v
|
||||
}
|
||||
|
||||
// If OS_PROJECT_NAME is set, overwrite tenantName with the value.
|
||||
if v := os.Getenv("OS_PROJECT_NAME"); v != "" {
|
||||
tenantName = v
|
||||
}
|
||||
return SDConfig{
|
||||
IdentityEndpoint: authURL,
|
||||
Username: username,
|
||||
UserID: userID,
|
||||
Password: password,
|
||||
ProjectName: tenantName,
|
||||
ProjectID: tenantID,
|
||||
DomainName: domainName,
|
||||
DomainID: domainID,
|
||||
ApplicationCredentialName: applicationCredentialName,
|
||||
ApplicationCredentialID: applicationCredentialID,
|
||||
ApplicationCredentialSecret: applicationCredentialSecret,
|
||||
}
|
||||
}
|
||||
122
lib/promscrape/discovery/openstack/auth_test.go
Normal file
122
lib/promscrape/discovery/openstack/auth_test.go
Normal file
@ -0,0 +1,122 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func Test_buildAuthRequestBody1(t *testing.T) {
|
||||
type args struct {
|
||||
sdc *SDConfig
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want []byte
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "empty config",
|
||||
args: args{
|
||||
sdc: &SDConfig{},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "username password auth with domain",
|
||||
args: args{
|
||||
sdc: &SDConfig{
|
||||
Username: "some-user",
|
||||
Password: "some-password",
|
||||
DomainName: "some-domain",
|
||||
},
|
||||
},
|
||||
want: []byte(`{"auth":{"identity":{"methods":["password"],"password":{"user":{"name":"some-user","password":"some-password","domain":{"name":"some-domain"}}}},"scope":{"domain":{"name":"some-domain"}}}}`),
|
||||
},
|
||||
{
|
||||
name: "application credentials auth",
|
||||
args: args{
|
||||
sdc: &SDConfig{
|
||||
ApplicationCredentialID: "some-id",
|
||||
ApplicationCredentialSecret: "some-secret",
|
||||
},
|
||||
},
|
||||
want: []byte(`{"auth":{"identity":{"methods":["application_credential"],"application_credential":{"id":"some-id","secret":"some-secret"}}}}`),
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := buildAuthRequestBody(tt.args.sdc)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("buildAuthRequestBody() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("buildAuthRequestBody() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_getComputeEndpointURL1(t *testing.T) {
|
||||
type args struct {
|
||||
catalog []catalogItem
|
||||
availability string
|
||||
region string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "bad catalog data",
|
||||
args: args{
|
||||
catalog: []catalogItem{
|
||||
{
|
||||
Type: "keystone",
|
||||
Endpoints: []endpoint{},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "good private url",
|
||||
args: args{
|
||||
availability: "private",
|
||||
catalog: []catalogItem{
|
||||
{
|
||||
Type: "compute",
|
||||
Endpoints: []endpoint{
|
||||
{
|
||||
Interface: "private",
|
||||
Type: "compute",
|
||||
URL: "https://compute.test.local:8083/v2.1",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Type: "keystone",
|
||||
Endpoints: []endpoint{},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: "https://compute.test.local:8083/v2.1",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := getComputeEndpointURL(tt.args.catalog, tt.args.availability, tt.args.region)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("getComputeEndpointURL() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if !tt.wantErr && !reflect.DeepEqual(got.String(), tt.want) {
|
||||
t.Errorf("getComputeEndpointURL() got = %v, want %v", got.String(), tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
91
lib/promscrape/discovery/openstack/hypervisor.go
Normal file
91
lib/promscrape/discovery/openstack/hypervisor.go
Normal file
@ -0,0 +1,91 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
// https://docs.openstack.org/api-ref/compute/?expanded=list-servers-detailed-detail#list-hypervisors-details
|
||||
type hypervisorDetail struct {
|
||||
Hypervisors []hypervisor `json:"hypervisors"`
|
||||
Links []struct {
|
||||
HREF string `json:"href"`
|
||||
Rel string `json:"rel,omitempty"`
|
||||
} `json:"hypervisors_links,omitempty"`
|
||||
}
|
||||
|
||||
type hypervisor struct {
|
||||
HostIP string `json:"host_ip"`
|
||||
ID int `json:"id"`
|
||||
Hostname string `json:"hypervisor_hostname"`
|
||||
Status string `json:"status"`
|
||||
State string `json:"state"`
|
||||
Type string `json:"hypervisor_type"`
|
||||
}
|
||||
|
||||
func parseHypervisorDetail(data []byte) (*hypervisorDetail, error) {
|
||||
var hvsd hypervisorDetail
|
||||
if err := json.Unmarshal(data, &hvsd); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse hypervisorDetail: %w", err)
|
||||
}
|
||||
|
||||
return &hvsd, nil
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getHypervisors() ([]hypervisor, error) {
|
||||
computeURL := *cfg.creds.computeURL
|
||||
computeURL.Path = path.Join(computeURL.Path, "os-hypervisors", "detail")
|
||||
nextLink := computeURL.String()
|
||||
var hvs []hypervisor
|
||||
for {
|
||||
resp, err := getAPIResponse(nextLink, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
detail, err := parseHypervisorDetail(resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hvs = append(hvs, detail.Hypervisors...)
|
||||
|
||||
if len(detail.Links) > 0 {
|
||||
nextLink = detail.Links[0].HREF
|
||||
continue
|
||||
}
|
||||
|
||||
return hvs, nil
|
||||
}
|
||||
}
|
||||
|
||||
func addHypervisorLabels(hvs []hypervisor, port int) []map[string]string {
|
||||
var ms []map[string]string
|
||||
for _, hv := range hvs {
|
||||
addr := discoveryutils.JoinHostPort(hv.HostIP, port)
|
||||
m := map[string]string{
|
||||
"__address__": addr,
|
||||
"__meta_openstack_hypervisor_type": hv.Type,
|
||||
"__meta_openstack_hypervisor_status": hv.Status,
|
||||
"__meta_openstack_hypervisor_hostname": hv.Hostname,
|
||||
"__meta_openstack_hypervisor_state": hv.State,
|
||||
"__meta_openstack_hypervisor_host_ip": hv.HostIP,
|
||||
"__meta_openstack_hypervisor_id": strconv.Itoa(hv.ID),
|
||||
}
|
||||
ms = append(ms, m)
|
||||
}
|
||||
|
||||
return ms
|
||||
}
|
||||
|
||||
func getHypervisorLabels(cfg *apiConfig) ([]map[string]string, error) {
|
||||
hvs, err := cfg.getHypervisors()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot get hypervisors: %w", err)
|
||||
}
|
||||
|
||||
return addHypervisorLabels(hvs, cfg.port), nil
|
||||
}
|
||||
152
lib/promscrape/discovery/openstack/hypervisor_test.go
Normal file
152
lib/promscrape/discovery/openstack/hypervisor_test.go
Normal file
@ -0,0 +1,152 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
func Test_parseHypervisorDetail(t *testing.T) {
|
||||
type args struct {
|
||||
data []byte
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want hypervisorDetail
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "bad data",
|
||||
args: args{
|
||||
data: []byte(`{ff}`),
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "1 hypervisor",
|
||||
args: args{
|
||||
data: []byte(`{
|
||||
"hypervisors": [
|
||||
{
|
||||
"cpu_info": {
|
||||
"arch": "x86_64",
|
||||
"model": "Nehalem",
|
||||
"vendor": "Intel",
|
||||
"features": [
|
||||
"pge",
|
||||
"clflush"
|
||||
],
|
||||
"topology": {
|
||||
"cores": 1,
|
||||
"threads": 1,
|
||||
"sockets": 4
|
||||
}
|
||||
},
|
||||
"current_workload": 0,
|
||||
"status": "enabled",
|
||||
"state": "up",
|
||||
"disk_available_least": 0,
|
||||
"host_ip": "1.1.1.1",
|
||||
"free_disk_gb": 1028,
|
||||
"free_ram_mb": 7680,
|
||||
"hypervisor_hostname": "host1",
|
||||
"hypervisor_type": "fake",
|
||||
"hypervisor_version": 1000,
|
||||
"id": 2,
|
||||
"local_gb": 1028,
|
||||
"local_gb_used": 0,
|
||||
"memory_mb": 8192,
|
||||
"memory_mb_used": 512,
|
||||
"running_vms": 0,
|
||||
"service": {
|
||||
"host": "host1",
|
||||
"id": 6,
|
||||
"disabled_reason": null
|
||||
},
|
||||
"vcpus": 2,
|
||||
"vcpus_used": 0
|
||||
}
|
||||
]}`),
|
||||
},
|
||||
want: hypervisorDetail{
|
||||
Hypervisors: []hypervisor{
|
||||
{
|
||||
HostIP: "1.1.1.1",
|
||||
ID: 2,
|
||||
Hostname: "host1",
|
||||
Status: "enabled",
|
||||
State: "up",
|
||||
Type: "fake",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseHypervisorDetail(tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseHypervisorDetail() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !tt.wantErr && !reflect.DeepEqual(*got, tt.want) {
|
||||
t.Errorf("parseHypervisorDetail() got = %v, want %v", *got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_addHypervisorLabels(t *testing.T) {
|
||||
type args struct {
|
||||
hvs []hypervisor
|
||||
port int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want [][]prompbmarshal.Label
|
||||
}{
|
||||
{
|
||||
name: "",
|
||||
args: args{
|
||||
port: 9100,
|
||||
hvs: []hypervisor{
|
||||
{
|
||||
Type: "fake",
|
||||
ID: 5,
|
||||
State: "enabled",
|
||||
Status: "up",
|
||||
Hostname: "fakehost",
|
||||
HostIP: "1.2.2.2",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "1.2.2.2:9100",
|
||||
"__meta_openstack_hypervisor_host_ip": "1.2.2.2",
|
||||
"__meta_openstack_hypervisor_hostname": "fakehost",
|
||||
"__meta_openstack_hypervisor_id": "5",
|
||||
"__meta_openstack_hypervisor_state": "enabled",
|
||||
"__meta_openstack_hypervisor_status": "up",
|
||||
"__meta_openstack_hypervisor_type": "fake",
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := addHypervisorLabels(tt.args.hvs, tt.args.port)
|
||||
var sortedLabelss [][]prompbmarshal.Label
|
||||
for _, labels := range got {
|
||||
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
|
||||
}
|
||||
if !reflect.DeepEqual(sortedLabelss, tt.want) {
|
||||
t.Errorf("addHypervisorLabels() = %v, want %v", sortedLabelss, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
141
lib/promscrape/discovery/openstack/instance.go
Normal file
141
lib/promscrape/discovery/openstack/instance.go
Normal file
@ -0,0 +1,141 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
// https://docs.openstack.org/api-ref/compute/?expanded=list-servers-detailed-detail#list-servers
|
||||
type serversDetail struct {
|
||||
Servers []server `json:"servers"`
|
||||
Links []struct {
|
||||
HREF string `json:"href"`
|
||||
Rel string `json:"rel"`
|
||||
} `json:"servers_links,omitempty"`
|
||||
}
|
||||
|
||||
type server struct {
|
||||
ID string `json:"id"`
|
||||
TenantID string `json:"tenant_id"`
|
||||
UserID string `json:"user_id"`
|
||||
Name string `json:"name"`
|
||||
HostID string `json:"hostid"`
|
||||
Status string `json:"status"`
|
||||
Addresses map[string][]struct {
|
||||
Address string `json:"addr"`
|
||||
Version int `json:"version"`
|
||||
Type string `json:"OS-EXT-IPS:type"`
|
||||
} `json:"addresses"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Flavor struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"flavor"`
|
||||
}
|
||||
|
||||
func parseServersDetail(data []byte) (*serversDetail, error) {
|
||||
var srvd serversDetail
|
||||
if err := json.Unmarshal(data, &srvd); err != nil {
|
||||
return nil, fmt.Errorf("cannot parse serversDetail: %w", err)
|
||||
}
|
||||
|
||||
return &srvd, nil
|
||||
}
|
||||
|
||||
func addInstanceLabels(servers []server, port int) []map[string]string {
|
||||
var ms []map[string]string
|
||||
for _, server := range servers {
|
||||
m := map[string]string{
|
||||
"__meta_openstack_instance_id": server.ID,
|
||||
"__meta_openstack_instance_status": server.Status,
|
||||
"__meta_openstack_instance_name": server.Name,
|
||||
"__meta_openstack_project_id": server.TenantID,
|
||||
"__meta_openstack_user_id": server.UserID,
|
||||
"__meta_openstack_instance_flavor": server.Flavor.ID,
|
||||
}
|
||||
|
||||
for k, v := range server.Metadata {
|
||||
m["__meta_openstack_tag_"+discoveryutils.SanitizeLabelName(k)] = v
|
||||
}
|
||||
for pool, addresses := range server.Addresses {
|
||||
if len(addresses) == 0 {
|
||||
// pool with zero addresses skip it
|
||||
continue
|
||||
}
|
||||
var publicIP string
|
||||
// its possible to have only one floating ip per pool
|
||||
for _, ip := range addresses {
|
||||
if ip.Type != "floating" {
|
||||
continue
|
||||
}
|
||||
publicIP = ip.Address
|
||||
break
|
||||
}
|
||||
for _, ip := range addresses {
|
||||
// fast return
|
||||
if len(ip.Address) == 0 || ip.Type == "floating" {
|
||||
continue
|
||||
}
|
||||
// copy labels
|
||||
lbls := make(map[string]string, len(m))
|
||||
for k, v := range m {
|
||||
lbls[k] = v
|
||||
}
|
||||
lbls["__meta_openstack_address_pool"] = pool
|
||||
lbls["__meta_openstack_private_ip"] = ip.Address
|
||||
if len(publicIP) > 0 {
|
||||
lbls["__meta_openstack_public_ip"] = publicIP
|
||||
}
|
||||
lbls["__address__"] = discoveryutils.JoinHostPort(ip.Address, port)
|
||||
ms = append(ms, lbls)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return ms
|
||||
}
|
||||
|
||||
func (cfg *apiConfig) getServers() ([]server, error) {
|
||||
computeURL := *cfg.creds.computeURL
|
||||
computeURL.Path = path.Join(computeURL.Path, "servers", "detail")
|
||||
// by default, query fetches data from all tenants
|
||||
if !cfg.allTenants {
|
||||
q := computeURL.Query()
|
||||
q.Set("all_tenants", "false")
|
||||
computeURL.RawQuery = q.Encode()
|
||||
}
|
||||
|
||||
nextLink := computeURL.String()
|
||||
|
||||
var servers []server
|
||||
for {
|
||||
resp, err := getAPIResponse(nextLink, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serversDetail, err := parseServersDetail(resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
servers = append(servers, serversDetail.Servers...)
|
||||
|
||||
if len(serversDetail.Links) > 0 {
|
||||
nextLink = serversDetail.Links[0].HREF
|
||||
continue
|
||||
}
|
||||
|
||||
return servers, nil
|
||||
}
|
||||
}
|
||||
|
||||
func getInstancesLabels(cfg *apiConfig) ([]map[string]string, error) {
|
||||
srv, err := cfg.getServers()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return addInstanceLabels(srv, cfg.port), nil
|
||||
|
||||
}
|
||||
270
lib/promscrape/discovery/openstack/instance_test.go
Normal file
270
lib/promscrape/discovery/openstack/instance_test.go
Normal file
@ -0,0 +1,270 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/prompbmarshal"
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promscrape/discoveryutils"
|
||||
)
|
||||
|
||||
func Test_addInstanceLabels(t *testing.T) {
|
||||
type args struct {
|
||||
servers []server
|
||||
port int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want [][]prompbmarshal.Label
|
||||
}{
|
||||
{
|
||||
name: "empty response",
|
||||
args: args{
|
||||
port: 9100,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "1 server",
|
||||
args: args{
|
||||
port: 9100,
|
||||
servers: []server{
|
||||
{
|
||||
ID: "10",
|
||||
Status: "enabled",
|
||||
Name: "server-1",
|
||||
HostID: "some-host-id",
|
||||
TenantID: "some-tenant-id",
|
||||
UserID: "some-user-id",
|
||||
Flavor: struct {
|
||||
ID string `json:"id"`
|
||||
}{ID: "5"},
|
||||
Addresses: map[string][]struct {
|
||||
Address string `json:"addr"`
|
||||
Version int `json:"version"`
|
||||
Type string `json:"OS-EXT-IPS:type"`
|
||||
}{
|
||||
"test": {
|
||||
{
|
||||
Address: "192.168.0.1",
|
||||
Version: 4,
|
||||
Type: "fixed",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "192.168.0.1:9100",
|
||||
"__meta_openstack_address_pool": "test",
|
||||
"__meta_openstack_instance_flavor": "5",
|
||||
"__meta_openstack_instance_id": "10",
|
||||
"__meta_openstack_instance_name": "server-1",
|
||||
"__meta_openstack_instance_status": "enabled",
|
||||
"__meta_openstack_private_ip": "192.168.0.1",
|
||||
"__meta_openstack_project_id": "some-tenant-id",
|
||||
"__meta_openstack_user_id": "some-user-id",
|
||||
}),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with public ip",
|
||||
args: args{
|
||||
port: 9100,
|
||||
servers: []server{
|
||||
{
|
||||
ID: "10",
|
||||
Status: "enabled",
|
||||
Name: "server-2",
|
||||
HostID: "some-host-id",
|
||||
TenantID: "some-tenant-id",
|
||||
UserID: "some-user-id",
|
||||
Flavor: struct {
|
||||
ID string `json:"id"`
|
||||
}{ID: "5"},
|
||||
Addresses: map[string][]struct {
|
||||
Address string `json:"addr"`
|
||||
Version int `json:"version"`
|
||||
Type string `json:"OS-EXT-IPS:type"`
|
||||
}{
|
||||
"test": {
|
||||
{
|
||||
Address: "192.168.0.1",
|
||||
Version: 4,
|
||||
Type: "fixed",
|
||||
},
|
||||
{
|
||||
Address: "1.5.5.5",
|
||||
Version: 4,
|
||||
Type: "floating",
|
||||
},
|
||||
},
|
||||
"internal": {
|
||||
{
|
||||
Address: "10.10.0.1",
|
||||
Version: 4,
|
||||
Type: "fixed",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
want: [][]prompbmarshal.Label{
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "192.168.0.1:9100",
|
||||
"__meta_openstack_address_pool": "test",
|
||||
"__meta_openstack_instance_flavor": "5",
|
||||
"__meta_openstack_instance_id": "10",
|
||||
"__meta_openstack_instance_name": "server-2",
|
||||
"__meta_openstack_instance_status": "enabled",
|
||||
"__meta_openstack_private_ip": "192.168.0.1",
|
||||
"__meta_openstack_public_ip": "1.5.5.5",
|
||||
"__meta_openstack_project_id": "some-tenant-id",
|
||||
"__meta_openstack_user_id": "some-user-id",
|
||||
}),
|
||||
discoveryutils.GetSortedLabels(map[string]string{
|
||||
"__address__": "10.10.0.1:9100",
|
||||
"__meta_openstack_address_pool": "internal",
|
||||
"__meta_openstack_instance_flavor": "5",
|
||||
"__meta_openstack_instance_id": "10",
|
||||
"__meta_openstack_instance_name": "server-2",
|
||||
"__meta_openstack_instance_status": "enabled",
|
||||
"__meta_openstack_private_ip": "10.10.0.1",
|
||||
"__meta_openstack_project_id": "some-tenant-id",
|
||||
"__meta_openstack_user_id": "some-user-id",
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := addInstanceLabels(tt.args.servers, tt.args.port)
|
||||
|
||||
var sortedLabelss [][]prompbmarshal.Label
|
||||
for _, labels := range got {
|
||||
sortedLabelss = append(sortedLabelss, discoveryutils.GetSortedLabels(labels))
|
||||
}
|
||||
if !reflect.DeepEqual(sortedLabelss, tt.want) {
|
||||
t.Errorf("addInstanceLabels() = \n got: %v,\nwant: %v", sortedLabelss, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_parseServersDetail(t *testing.T) {
|
||||
type args struct {
|
||||
data []byte
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want serversDetail
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "parse ok",
|
||||
args: args{
|
||||
data: []byte(`{
|
||||
"servers":[
|
||||
{
|
||||
"id":"c9f68076-01a3-489a-aebe-8b773c71e7f3",
|
||||
"name":"test10",
|
||||
"status":"ACTIVE",
|
||||
"tenant_id":"d34be4e44f9c444eab9a5ec7b953951f",
|
||||
"user_id":"e55737f142ac42f18093037760656bd7",
|
||||
"metadata":{
|
||||
|
||||
},
|
||||
"hostId":"e26db8db23736877aa92ebbbe11743b2a2a3b107aada00a8a0cf474b",
|
||||
"image":{
|
||||
"id":"253f7a69-dc79-4fb2-86f8-9ec92c94107a",
|
||||
"links":[
|
||||
{
|
||||
"rel":"bookmark",
|
||||
"href":"http://10.20.20.1:8774/images/253f7a69-dc79-4fb2-86f8-9ec92c94107a"
|
||||
}
|
||||
]
|
||||
},
|
||||
"flavor":{
|
||||
"id":"1"
|
||||
},
|
||||
"addresses":{
|
||||
"test":[
|
||||
{
|
||||
"version":4,
|
||||
"addr":"192.168.222.15",
|
||||
"OS-EXT-IPS:type":"fixed",
|
||||
"OS-EXT-IPS-MAC:mac_addr":"fa:16:3e:b0:40:af"
|
||||
},
|
||||
{
|
||||
"version":4,
|
||||
"addr":"10.20.20.69",
|
||||
"OS-EXT-IPS:type":"floating",
|
||||
"OS-EXT-IPS-MAC:mac_addr":"fa:16:3e:b0:40:af"
|
||||
}
|
||||
]
|
||||
},
|
||||
"accessIPv4":"",
|
||||
"accessIPv6":"",
|
||||
"key_name":"microstack",
|
||||
"security_groups":[
|
||||
{
|
||||
"name":"default"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}`),
|
||||
},
|
||||
want: serversDetail{
|
||||
Servers: []server{
|
||||
{
|
||||
Flavor: struct {
|
||||
ID string `json:"id"`
|
||||
}{ID: "1"},
|
||||
ID: "c9f68076-01a3-489a-aebe-8b773c71e7f3",
|
||||
TenantID: "d34be4e44f9c444eab9a5ec7b953951f",
|
||||
UserID: "e55737f142ac42f18093037760656bd7",
|
||||
Name: "test10",
|
||||
HostID: "e26db8db23736877aa92ebbbe11743b2a2a3b107aada00a8a0cf474b",
|
||||
Status: "ACTIVE",
|
||||
Metadata: map[string]string{},
|
||||
Addresses: map[string][]struct {
|
||||
Address string `json:"addr"`
|
||||
Version int `json:"version"`
|
||||
Type string `json:"OS-EXT-IPS:type"`
|
||||
}{
|
||||
"test": {
|
||||
{
|
||||
Address: "192.168.222.15",
|
||||
Version: 4,
|
||||
Type: "fixed",
|
||||
},
|
||||
{
|
||||
Address: "10.20.20.69",
|
||||
Version: 4,
|
||||
Type: "floating",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := parseServersDetail(tt.args.data)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseServersDetail() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !tt.wantErr && !reflect.DeepEqual(*got, tt.want) {
|
||||
t.Errorf("parseServersDetail() \ngot = %v,\nwant= %v", *got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
44
lib/promscrape/discovery/openstack/openstack.go
Normal file
44
lib/promscrape/discovery/openstack/openstack.go
Normal file
@ -0,0 +1,44 @@
|
||||
package openstack
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promauth"
|
||||
)
|
||||
|
||||
// SDConfig is the configuration for OpenStack based service discovery.
|
||||
type SDConfig struct {
|
||||
IdentityEndpoint string `yaml:"identity_endpoint"`
|
||||
Username string `yaml:"username"`
|
||||
UserID string `yaml:"userid"`
|
||||
Password string `yaml:"password"`
|
||||
ProjectName string `yaml:"project_name"`
|
||||
ProjectID string `yaml:"project_id"`
|
||||
DomainName string `yaml:"domain_name"`
|
||||
DomainID string `yaml:"domain_id"`
|
||||
ApplicationCredentialName string `yaml:"application_credential_name"`
|
||||
ApplicationCredentialID string `yaml:"application_credential_id"`
|
||||
ApplicationCredentialSecret string `yaml:"application_credential_secret"`
|
||||
Role string `yaml:"role"`
|
||||
Region string `yaml:"region"`
|
||||
Port int `yaml:"port"`
|
||||
AllTenants bool `yaml:"all_tenants"`
|
||||
TLSConfig *promauth.TLSConfig `yaml:"tls_config"`
|
||||
Availability string `yaml:"availability"`
|
||||
}
|
||||
|
||||
// GetLabels returns gce labels according to sdc.
|
||||
func GetLabels(sdc *SDConfig, baseDir string) ([]map[string]string, error) {
|
||||
cfg, err := getAPIConfig(sdc, baseDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot get API config: %w", err)
|
||||
}
|
||||
switch sdc.Role {
|
||||
case "hypervisor":
|
||||
return getHypervisorLabels(cfg)
|
||||
case "instance":
|
||||
return getInstancesLabels(cfg)
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected `role`: %q; must be one of `instance` or `hypervisor`; skipping it", sdc.Role)
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,9 @@ var (
|
||||
kubernetesSDCheckInterval = flag.Duration("promscrape.kubernetesSDCheckInterval", 30*time.Second, "Interval for checking for changes in Kubernetes API server. "+
|
||||
"This works only if `kubernetes_sd_configs` is configured in '-promscrape.config' file. "+
|
||||
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config for details")
|
||||
openstackSDCheckInterval = flag.Duration("promscrape.openstackSDCheckInterval", 30*time.Second, "Interval for checking for changes in openstack API server. "+
|
||||
"This works only if `openstack_sd_configs` is configured in '-promscrape.config' file. "+
|
||||
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#openstack_sd_config for details")
|
||||
consulSDCheckInterval = flag.Duration("promscrape.consulSDCheckInterval", 30*time.Second, "Interval for checking for changes in consul. "+
|
||||
"This works only if `consul_sd_configs` is configured in '-promscrape.config' file. "+
|
||||
"See https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config for details")
|
||||
@ -85,6 +88,7 @@ func runScraper(configFile string, pushData func(wr *prompbmarshal.WriteRequest)
|
||||
scs.add("static_configs", 0, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getStaticScrapeWork() })
|
||||
scs.add("file_sd_configs", *fileSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getFileSDScrapeWork(swsPrev) })
|
||||
scs.add("kubernetes_sd_configs", *kubernetesSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getKubernetesSDScrapeWork(swsPrev) })
|
||||
scs.add("openstack_sd_configs", *openstackSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getOpenStackSDScrapeWork(swsPrev) })
|
||||
scs.add("consul_sd_configs", *consulSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getConsulSDScrapeWork(swsPrev) })
|
||||
scs.add("dns_sd_configs", *dnsSDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getDNSSDScrapeWork(swsPrev) })
|
||||
scs.add("ec2_sd_configs", *ec2SDCheckInterval, func(cfg *Config, swsPrev []ScrapeWork) []ScrapeWork { return cfg.getEC2SDScrapeWork(swsPrev) })
|
||||
|
||||
Loading…
Reference in New Issue
Block a user