mirror of
https://github.com/VictoriaMetrics/VictoriaMetrics.git
synced 2024-12-05 01:01:09 +01:00
a8b8e23d68
Target-level debugging is performed by clicking the 'debug' link at the corresponding target on either http://vmagent:8429/targets page or on http://vmagent:8428/service-discovery page. Metric-level debugging is perfromed at http://vmagent:8429/metric-relabel-debug page. Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/3407 See https://docs.victoriametrics.com/vmagent.html#relabel-debug
409 lines
18 KiB
Plaintext
409 lines
18 KiB
Plaintext
{% import (
|
|
"net/url"
|
|
"time"
|
|
|
|
"github.com/VictoriaMetrics/VictoriaMetrics/lib/promutils"
|
|
) %}
|
|
|
|
{% stripspace %}
|
|
|
|
{% func TargetsResponsePlain(tsr *targetsStatusResult, filter *requestFilter) %}
|
|
|
|
{% if tsr.err != nil %}
|
|
{%s= tsr.err.Error() %}
|
|
{% return %}
|
|
{% endif %}
|
|
|
|
{% for _, jts := range tsr.jobTargetsStatuses %}
|
|
job={%s= jts.jobName %}{% space %}({%d jts.upCount %}/{%d jts.targetsTotal %}{% space %}up)
|
|
{% newline %}
|
|
{% for _, ts := range jts.targetsStatus %}
|
|
{%s= "\t" %}
|
|
state={% if ts.up %}up{% else %}down{% endif %},{% space %}
|
|
endpoint={%s= ts.sw.Config.ScrapeURL %},{% space %}
|
|
labels={%s= ts.sw.Config.Labels.String() %},{% space %}
|
|
{% if filter.showOriginalLabels %}originalLabels={%s= ts.sw.Config.OriginalLabels.String() %},{% space %}{% endif %}
|
|
scrapes_total={%d ts.scrapesTotal %},{% space %}
|
|
scrapes_failed={%d ts.scrapesFailed %},{% space %}
|
|
last_scrape={%d int(ts.getDurationFromLastScrape().Milliseconds()) %}ms ago,{% space %}
|
|
scrape_duration={%d int(ts.scrapeDuration) %}ms,{% space %}
|
|
samples_scraped={%d ts.samplesScraped %},{% space %}
|
|
error={% if ts.err != nil %}{%s= ts.err.Error() %}{% endif %}
|
|
{% newline %}
|
|
{% endfor %}
|
|
{% endfor %}
|
|
|
|
{% for _, jobName := range tsr.emptyJobs %}
|
|
job={%s= jobName %}{% space %}(0/0 up)
|
|
{% newline %}
|
|
{% endfor %}
|
|
|
|
{% endfunc %}
|
|
|
|
{% func TargetsResponseHTML(tsr *targetsStatusResult, filter *requestFilter) %}
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
{%= commonHeader() %}
|
|
<title>Active Targets</title>
|
|
</head>
|
|
<body>
|
|
{%= navbar() %}
|
|
<div class="container-fluid">
|
|
{% if tsr.err != nil %}
|
|
{%= errorNotification(tsr.err) %}
|
|
{% endif %}
|
|
<div class="row">
|
|
<main class="col-12">
|
|
<h1>Active Targets</h1>
|
|
<hr />
|
|
{%= filtersForm(filter) %}
|
|
<hr />
|
|
{%= targetsTabs(tsr, filter, "scrapeTargets") %}
|
|
</main>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
{% endfunc %}
|
|
|
|
{% func ServiceDiscoveryResponse(tsr *targetsStatusResult, filter *requestFilter) %}
|
|
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
{%= commonHeader() %}
|
|
<title>Discovered Targets</title>
|
|
</head>
|
|
<body>
|
|
{%= navbar() %}
|
|
<div class="container-fluid">
|
|
{% if tsr.err != nil %}
|
|
{%= errorNotification(tsr.err) %}
|
|
{% endif %}
|
|
<div class="row">
|
|
<main class="col-12">
|
|
<h1>Discovered Targets</h1>
|
|
<hr />
|
|
{%= filtersForm(filter) %}
|
|
<hr />
|
|
{%= targetsTabs(tsr, filter, "discoveredTargets") %}
|
|
</main>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
{% endfunc %}
|
|
|
|
{% func commonHeader() %}
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
<link href="static/css/bootstrap.min.css" rel="stylesheet" />
|
|
{% endfunc %}
|
|
|
|
{% func navbar() %}
|
|
<div class="navbar navbar-dark bg-dark box-shadow">
|
|
<div class="d-flex justify-content-between">
|
|
<a href="/" class="navbar-brand d-flex align-items-center ms-3" title="The High Performance Open Source Time Series Database & Monitoring Solution ">
|
|
<svg xmlns="http://www.w3.org/2000/svg" id="VM_logo" viewBox="0 0 464.61 533.89" width="20" height="20" class="me-1"><defs><style>.cls-1{fill:#fff;}</style></defs><path class="cls-1" d="M459.86,467.77c9,7.67,24.12,13.49,39.3,13.69v0h1.68v0c15.18-.2,30.31-6,39.3-13.69,47.43-40.45,184.65-166.24,184.65-166.24,36.84-34.27-65.64-68.28-223.95-68.47h-1.68c-158.31.19-260.79,34.2-224,68.47C275.21,301.53,412.43,427.32,459.86,467.77Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,535.88c-9,7.67-24.12,13.5-39.3,13.7h-1.6c-15.18-.2-30.31-6-39.3-13.7-32.81-28-148.56-132.93-192.16-172.7v60.74c0,6.67,2.55,15.52,7.09,19.68,29.64,27.18,143.94,131.8,185.07,166.88,9,7.67,24.12,13.49,39.3,13.69v0h1.6v0c15.18-.2,30.31-6,39.3-13.69,41.13-35.08,155.43-139.7,185.07-166.88,4.54-4.16,7.09-13,7.09-19.68V363.18C688.66,403,572.91,507.9,540.1,535.88Z" transform="translate(-267.7 -233.05)"/><path class="cls-1" d="M540.1,678.64c-9,7.67-24.12,13.49-39.3,13.69v0h-1.6v0c-15.18-.2-30.31-6-39.3-13.69-32.81-28-148.56-132.94-192.16-172.7v60.73c0,6.67,2.55,15.53,7.09,19.69,29.64,27.17,143.94,131.8,185.07,166.87,9,7.67,24.12,13.5,39.3,13.7h1.6c15.18-.2,30.31-6,39.3-13.7,41.13-35.07,155.43-139.7,185.07-166.87,4.54-4.16,7.09-13,7.09-19.69V505.94C688.66,545.7,572.91,650.66,540.1,678.64Z" transform="translate(-267.7 -233.05)"/></svg>
|
|
<strong>VictoriaMetrics</strong>
|
|
</a>
|
|
</div>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func filtersForm(filter *requestFilter) %}
|
|
<div class="row g-3 align-items-center mb-3">
|
|
<div class="col-auto">
|
|
<button id="all-btn" type="button" class="btn{% space %}{% if !filter.showOnlyUnhealthy %}btn-secondary{% else %}btn-success{% endif %}"
|
|
onclick="location.href='?{%= queryArgs(filter, map[string]string{"show_only_unhealthy": "false"}) %}'">
|
|
All
|
|
</button>
|
|
</div>
|
|
<div class="col-auto">
|
|
<button id="unhealthy-btn" type="button" class="btn{% space %}{% if filter.showOnlyUnhealthy %}btn-secondary{% else %}btn-danger{% endif %}"
|
|
onclick="location.href='?{%= queryArgs(filter, map[string]string{"show_only_unhealthy": "true"}) %}'">
|
|
Unhealthy
|
|
</button>
|
|
</div>
|
|
<div class="col-auto">
|
|
<button type="button" class="btn btn-primary" onclick="document.querySelectorAll('.scrape-job').forEach((el) => { el.style.display = 'none'; })">
|
|
Collapse all
|
|
</button>
|
|
</div>
|
|
<div class="col-auto">
|
|
<button type="button" class="btn btn-secondary" onclick="document.querySelectorAll('.scrape-job').forEach((el) => { el.style.display = 'block'; })">
|
|
Expand all
|
|
</button>
|
|
</div>
|
|
<div class="col-auto">
|
|
<button type="button" class="btn btn-success" onclick="document.getElementById('filters').style.display='block'">
|
|
Filter targets
|
|
</button>
|
|
</div>
|
|
</div>
|
|
<div id="filters" {% if filter.endpointSearch == "" && filter.labelSearch == "" %}style="display:none"{% endif %}>
|
|
<form class="form-horizontal">
|
|
<div class="form-group mb-3">
|
|
<label for="endpoint_search" class="col-sm-10 control-label">Endpoint filter (<a target="_blank" href="https://github.com/google/re2/wiki/Syntax">Regexp</a> is accepted)</label>
|
|
<div class="col-sm-10">
|
|
<input type="text" id="endpoint_search" name="endpoint_search"
|
|
placeholder="For example, 127.0.0.1" class="form-control" value="{%s filter.endpointSearch %}"/>
|
|
</div>
|
|
</div>
|
|
<div class="form-group mb-3">
|
|
<label for="label_search" class="col-sm-10 control-label">Labels filter (<a target="_blank" href="https://prometheus.io/docs/prometheus/latest/querying/basics/#time-series-selectors">Arbitrary time series selectors</a> are accepted)</label>
|
|
<div class="col-sm-10">
|
|
<input type="text" id="label_search" name="label_search"
|
|
placeholder="For example, {instance=~'.+:9100'}" class="form-control" value="{%s filter.labelSearch %}"/>
|
|
</div>
|
|
</div>
|
|
<input type="hidden" name="show_only_unhealthy" value="{%v filter.showOnlyUnhealthy %}"/>
|
|
<input type="hidden" name="show_original_labels" value="{%v filter.showOriginalLabels %}"/>
|
|
<button type="submit" class="btn btn-success mb-3">Submit</button>
|
|
<button type="button" class="btn btn-danger mb-3" onclick="location.href='?'">Clear target filters</button>
|
|
</form>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func targetsTabs(tsr *targetsStatusResult, filter *requestFilter, activeTab string) %}
|
|
<ul class="nav nav-tabs" id="myTab" role="tablist">
|
|
<li class="nav-item" role="presentation">
|
|
<button class="nav-link{%if activeTab=="scrapeTargets"%}{% space %}active{%endif%}" type="button" role="tab"
|
|
onclick="location.href='targets?{%= queryArgs(filter, nil) %}'">
|
|
Active targets
|
|
</button>
|
|
</li>
|
|
<li class="nav-item" role="presentation">
|
|
<button class="nav-link{%if activeTab=="discoveredTargets"%}{% space %}active{%endif%}" type="button" role="tab"
|
|
onclick="location.href='service-discovery?{%= queryArgs(filter, nil) %}'">
|
|
Discovered targets
|
|
</button>
|
|
</li>
|
|
</ul>
|
|
<div class="tab-content">
|
|
<div class="tab-pane active" role="tabpanel">
|
|
{% switch activeTab %}
|
|
{% case "scrapeTargets" %}
|
|
{%= scrapeTargets(tsr) %}
|
|
{% case "discoveredTargets" %}
|
|
{%= discoveredTargets(tsr) %}
|
|
{% endswitch %}
|
|
</div>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func scrapeTargets(tsr *targetsStatusResult) %}
|
|
<div class="row mt-4">
|
|
<div class="col-12">
|
|
{% for i, jts := range tsr.jobTargetsStatuses %}
|
|
{%= scrapeJobTargets(i, jts) %}
|
|
{% endfor %}
|
|
{% for i, jobName := range tsr.emptyJobs %}
|
|
{% code
|
|
num := i + len(tsr.jobTargetsStatuses)
|
|
jts := &jobTargetsStatuses{
|
|
jobName: jobName,
|
|
}
|
|
%}
|
|
{%= scrapeJobTargets(num, jts) %}
|
|
{% endfor %}
|
|
</div>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func scrapeJobTargets(num int, jts *jobTargetsStatuses) %}
|
|
<div class="row mb-4">
|
|
<div class="col-12">
|
|
<h4>
|
|
<span class="me-2">{%s jts.jobName %}{% space %}({%d jts.upCount %}/{%d jts.targetsTotal %}{% space %}up)</span>
|
|
{%= showHideScrapeJobButtons(num) %}
|
|
</h4>
|
|
<div id="scrape-job-{%d num %}" class="scrape-job table-responsive">
|
|
<table class="table table-striped table-hover table-bordered table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col">Endpoint</th>
|
|
<th scope="col">State</th>
|
|
<th scope="col" title="target labels">Labels</th>
|
|
<th scope="col" title="debug relabeling">Debug relabeling</th>
|
|
<th scope="col" title="total scrapes">Scrapes</th>
|
|
<th scope="col" title="total scrape errors">Errors</th>
|
|
<th scope="col" title="the time of the last scrape">Last Scrape</th>
|
|
<th scope="col" title="the duration of the last scrape">Duration</th>
|
|
<th scope="col" title="the number of metrics scraped during the last scrape">Samples</th>
|
|
<th scope="col" title="error from the last scrape (if any)">Last error</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for _, ts := range jts.targetsStatus %}
|
|
{% code
|
|
endpoint := ts.sw.Config.ScrapeURL
|
|
// The target is uniquely identified by a pointer to its original labels.
|
|
targetID := getLabelsID(ts.sw.Config.OriginalLabels)
|
|
lastScrapeDuration := ts.getDurationFromLastScrape()
|
|
%}
|
|
<tr {% if !ts.up %}{%space%}class="alert alert-danger" role="alert" {% endif %}>
|
|
<td class="endpoint">
|
|
<a href="{%s endpoint %}" target="_blank">{%s endpoint %}</a> (
|
|
<a href="target_response?id={%s targetID %}" target="_blank"
|
|
title="click to fetch target response on behalf of the scraper">response</a>
|
|
)
|
|
</td>
|
|
<td>
|
|
{% if ts.up %}
|
|
<span class="badge bg-success">UP</span>
|
|
{% else %}
|
|
<span class="badge bg-danger">DOWN</span>
|
|
{% endif %}
|
|
</td>
|
|
<td class="labels">
|
|
<div title="click to show original labels"
|
|
onclick="document.getElementById('original-labels-{%s targetID %}').style.display='block'">
|
|
{%= formatLabels(ts.sw.Config.Labels) %}
|
|
</div>
|
|
<div style="display:none" id="original-labels-{%s targetID %}">
|
|
{%= formatLabels(ts.sw.Config.OriginalLabels) %}
|
|
</div>
|
|
</td>
|
|
<td>
|
|
<a href="target-relabel-debug?id={%s targetID %}" target="_blank">debug</a>
|
|
</td>
|
|
<td>{%d ts.scrapesTotal %}</td>
|
|
<td>{%d ts.scrapesFailed %}</td>
|
|
<td>
|
|
{% if lastScrapeDuration < 365*24*time.Hour %}
|
|
{%d int(lastScrapeDuration.Milliseconds()) %}ms ago
|
|
{% else %}
|
|
none
|
|
{% endif %}
|
|
<td>{%d int(ts.scrapeDuration) %}ms</td>
|
|
<td>{%d ts.samplesScraped %}</td>
|
|
<td>{% if ts.err != nil %}{%s ts.err.Error() %}{% endif %}</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func discoveredTargets(tsr *targetsStatusResult) %}
|
|
{% code tljs := tsr.getTargetLabelsByJob() %}
|
|
<div class="row mt-4">
|
|
<div class="col-12">
|
|
{% for i, tlj := range tljs %}
|
|
{%= discoveredJobTargets(i, tlj) %}
|
|
{% endfor %}
|
|
</div>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func discoveredJobTargets(num int, tlj *targetLabelsByJob) %}
|
|
<h4>
|
|
<span class="me-2">{%s tlj.jobName %}{% space %}({%d tlj.activeTargets %}/{%d tlj.activeTargets+tlj.droppedTargets %}{% space %}active)</span>
|
|
{%= showHideScrapeJobButtons(num) %}
|
|
</h4>
|
|
<div id="scrape-job-{%d num %}" class="scrape-job table-responsive">
|
|
<table class="table table-striped table-hover table-bordered table-sm">
|
|
<thead>
|
|
<tr>
|
|
<th scope="col" style="width: 5%">Status</th>
|
|
<th scope="col" style="width: 60%">Discovered Labels</th>
|
|
<th scope="col" style="width: 30%">Target Labels</th>
|
|
<th scope="col" stile="width: 5%">Debug relabeling</a>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{% for _, t := range tlj.targets %}
|
|
<tr
|
|
{% if !t.up %}
|
|
{% space %}role="alert"{% space %}
|
|
{% if t.labels.Len() > 0 %}
|
|
class="alert alert-danger"
|
|
{% else %}
|
|
class="alert alert-warning"
|
|
{% endif %}
|
|
{% endif %}
|
|
>
|
|
<td>
|
|
{% if t.up %}
|
|
<span class="badge bg-success">UP</span>
|
|
{% elseif t.labels.Len() > 0 %}
|
|
<span class="badge bg-danger">DOWN</span>
|
|
{% else %}
|
|
<span class="badge bg-warning">DROPPED</span>
|
|
{% endif %}
|
|
</td>
|
|
<td class="labels">
|
|
{%= formatLabels(t.originalLabels) %}
|
|
</td>
|
|
<td class="labels">
|
|
{%= formatLabels(t.labels) %}
|
|
</td>
|
|
<td>
|
|
{% code targetID := getLabelsID(t.originalLabels) %}
|
|
<a href="target-relabel-debug?id={%s targetID %}" target="_blank">debug</a>
|
|
</td>
|
|
</tr>
|
|
{% endfor %}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% func showHideScrapeJobButtons(num int) %}
|
|
<button type="button" class="btn btn-primary btn-sm me-1"
|
|
onclick="document.getElementById('scrape-job-{%d num %}').style.display='none'">
|
|
collapse
|
|
</button>
|
|
<button type="button" class="btn btn-secondary btn-sm me-1"
|
|
onclick="document.getElementById('scrape-job-{%d num %}').style.display='block'">
|
|
expand
|
|
</button>
|
|
{% endfunc %}
|
|
|
|
{% func queryArgs(filter *requestFilter, override map[string]string) %}
|
|
{% code
|
|
showOnlyUnhealthy := "false"
|
|
if filter.showOnlyUnhealthy {
|
|
showOnlyUnhealthy = "true"
|
|
}
|
|
m := map[string]string{
|
|
"show_only_unhealthy": showOnlyUnhealthy,
|
|
"endpoint_search": filter.endpointSearch,
|
|
"label_search": filter.labelSearch,
|
|
}
|
|
for k, v := range override {
|
|
m[k] = v
|
|
}
|
|
qa := make(url.Values, len(m))
|
|
for k, v := range m {
|
|
qa[k] = []string{v}
|
|
}
|
|
%}
|
|
{%s qa.Encode() %}
|
|
{% endfunc %}
|
|
|
|
{% func formatLabels(labels *promutils.Labels) %}
|
|
{% code labelsList := labels.GetLabels() %}
|
|
{
|
|
{% for i, label := range labelsList %}
|
|
{%s label.Name %}={%q label.Value %}
|
|
{% if i+1 < len(labelsList) %},{% space %}{% endif %}
|
|
{% endfor %}
|
|
}
|
|
{% endfunc %}
|
|
|
|
{% func errorNotification(err error) %}
|
|
<div class="alert alert-danger d-flex align-items-center" role="alert">
|
|
<svg class="bi flex-shrink-0 me-2" width="24" height="24" role="img" aria-label="Danger:">
|
|
<use xlink:href="#exclamation-triangle-fill"/></svg>
|
|
<div>
|
|
{%s err.Error() %}
|
|
</div>
|
|
</div>
|
|
{% endfunc %}
|
|
|
|
{% endstripspace %}
|