From 5f91a701fa11d1a089331d2babe60a1d9406b429 Mon Sep 17 00:00:00 2001 From: Aliaksandr Valialkin Date: Mon, 14 Jun 2021 14:01:13 +0300 Subject: [PATCH] lib/promscrape: show the number of samples collected during the last scrape at /targets and /api/v1/targets pages Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1377 --- docs/CHANGELOG.md | 1 + lib/promscrape/scrapework.go | 4 +- lib/promscrape/targets_response.qtpl | 7 +- lib/promscrape/targets_response.qtpl.go | 280 ++++++++++++------------ lib/promscrape/targetstatus.go | 11 +- 5 files changed, 160 insertions(+), 143 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0abaae99f6..2128402039 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,7 @@ sort: 15 ## tip * FEATURE: vmagent: add service discovery for DigitalOcean (aka [digitalocean_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#digitalocean_sd_config)). See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1367). +* FEATURE: vmagent: show the number of samples the target returned during the last scrape on `/targets` and `/api/v1/targets` pages. This should simplify debugging targets, which may return too big or too low number of samples. See [this feature request](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1377). ## [v1.61.1](https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/v1.61.1) diff --git a/lib/promscrape/scrapework.go b/lib/promscrape/scrapework.go index 7922344296..281a0e57e0 100644 --- a/lib/promscrape/scrapework.go +++ b/lib/promscrape/scrapework.go @@ -324,7 +324,7 @@ func (sw *scrapeWork) scrapeInternal(scrapeTimestamp, realTimestamp int64) error // body must be released only after wc is released, since wc refers to body. sw.prevBodyLen = len(body.B) leveledbytebufferpool.Put(body) - tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), err) + tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err) return err } @@ -391,7 +391,7 @@ func (sw *scrapeWork) scrapeStream(scrapeTimestamp, realTimestamp int64) error { sw.prevLabelsLen = len(wc.labels) wc.reset() writeRequestCtxPool.Put(wc) - tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), err) + tsmGlobal.Update(sw.Config, sw.ScrapeGroup, up == 1, realTimestamp, int64(duration*1000), samplesScraped, err) return err } diff --git a/lib/promscrape/targets_response.qtpl b/lib/promscrape/targets_response.qtpl index fabd7b1037..6120e3678d 100644 --- a/lib/promscrape/targets_response.qtpl +++ b/lib/promscrape/targets_response.qtpl @@ -19,7 +19,8 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) {% if showOriginLabels %}, originalLabels={%s= ol %}{% endif %}, last_scrape={%f.3 ts.lastScrapeTime.Seconds() %}s ago, scrape_duration={%f.3 ts.scrapeDuration.Seconds() %}s, - error={%q= ts.error %} + samples_scraped={%d ts.samplesScraped %}, + error={%q= ts.errMsg %} {% newline %} {% endfor %} {% endfor %} @@ -62,6 +63,7 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) Labels Last Scrape Scrape Duration + Samples Scraped Error @@ -76,7 +78,8 @@ job={%q= js.job %} ({%d js.upCount %}/{%d js.targetsTotal %} up) {%f.3 ts.lastScrapeTime.Seconds() %}s ago {%f.3 ts.scrapeDuration.Seconds() %}s - {%s ts.error %} + {%d ts.samplesScraped %} + {%s ts.errMsg %} {% endfor %} diff --git a/lib/promscrape/targets_response.qtpl.go b/lib/promscrape/targets_response.qtpl.go index 0c9d9699a9..bce59cdcde 100644 --- a/lib/promscrape/targets_response.qtpl.go +++ b/lib/promscrape/targets_response.qtpl.go @@ -94,280 +94,288 @@ func StreamTargetsResponsePlain(qw422016 *qt422016.Writer, jts []jobTargetsStatu //line lib/promscrape/targets_response.qtpl:21 qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3) //line lib/promscrape/targets_response.qtpl:21 - qw422016.N().S(`s, error=`) + qw422016.N().S(`s, samples_scraped=`) //line lib/promscrape/targets_response.qtpl:22 - qw422016.N().Q(ts.error) + qw422016.N().D(ts.samplesScraped) //line lib/promscrape/targets_response.qtpl:22 - qw422016.N().S(` `) + qw422016.N().S(`, error=`) //line lib/promscrape/targets_response.qtpl:23 + qw422016.N().Q(ts.errMsg) +//line lib/promscrape/targets_response.qtpl:23 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:24 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:23 +//line lib/promscrape/targets_response.qtpl:24 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:24 +//line lib/promscrape/targets_response.qtpl:25 } -//line lib/promscrape/targets_response.qtpl:24 +//line lib/promscrape/targets_response.qtpl:25 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:25 - } -//line lib/promscrape/targets_response.qtpl:25 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:26 + } +//line lib/promscrape/targets_response.qtpl:26 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:27 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:26 +//line lib/promscrape/targets_response.qtpl:27 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 } -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 func WriteTargetsResponsePlain(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, showOriginLabels bool) { -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 StreamTargetsResponsePlain(qw422016, jts, showOriginLabels) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 } -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 func TargetsResponsePlain(jts []jobTargetsStatuses, showOriginLabels bool) string { -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 WriteTargetsResponsePlain(qb422016, jts, showOriginLabels) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 qs422016 := string(qb422016.B) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 return qs422016 -//line lib/promscrape/targets_response.qtpl:28 +//line lib/promscrape/targets_response.qtpl:29 } -//line lib/promscrape/targets_response.qtpl:30 +//line lib/promscrape/targets_response.qtpl:31 func StreamTargetsResponseHTML(qw422016 *qt422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) { -//line lib/promscrape/targets_response.qtpl:30 +//line lib/promscrape/targets_response.qtpl:31 qw422016.N().S(` Scrape targets

Scrape targets

`) -//line lib/promscrape/targets_response.qtpl:51 +//line lib/promscrape/targets_response.qtpl:52 for _, js := range jts { -//line lib/promscrape/targets_response.qtpl:51 - qw422016.N().S(` `) //line lib/promscrape/targets_response.qtpl:52 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:53 if onlyUnhealthy && js.upCount == js.targetsTotal { -//line lib/promscrape/targets_response.qtpl:52 +//line lib/promscrape/targets_response.qtpl:53 continue -//line lib/promscrape/targets_response.qtpl:52 +//line lib/promscrape/targets_response.qtpl:53 } -//line lib/promscrape/targets_response.qtpl:52 +//line lib/promscrape/targets_response.qtpl:53 qw422016.N().S(`

`) -//line lib/promscrape/targets_response.qtpl:55 +//line lib/promscrape/targets_response.qtpl:56 qw422016.E().S(js.job) -//line lib/promscrape/targets_response.qtpl:55 +//line lib/promscrape/targets_response.qtpl:56 qw422016.N().S(` (`) -//line lib/promscrape/targets_response.qtpl:55 +//line lib/promscrape/targets_response.qtpl:56 qw422016.N().D(js.upCount) -//line lib/promscrape/targets_response.qtpl:55 +//line lib/promscrape/targets_response.qtpl:56 qw422016.N().S(`/`) -//line lib/promscrape/targets_response.qtpl:55 +//line lib/promscrape/targets_response.qtpl:56 qw422016.N().D(js.targetsTotal) -//line lib/promscrape/targets_response.qtpl:55 - qw422016.N().S(` up)

`) -//line lib/promscrape/targets_response.qtpl:69 +//line lib/promscrape/targets_response.qtpl:56 + qw422016.N().S(` up)
Endpoint State Labels Last Scrape Scrape Duration Error
`) +//line lib/promscrape/targets_response.qtpl:71 for _, ts := range js.targetsStatus { -//line lib/promscrape/targets_response.qtpl:69 +//line lib/promscrape/targets_response.qtpl:71 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:70 +//line lib/promscrape/targets_response.qtpl:72 if onlyUnhealthy && ts.up { -//line lib/promscrape/targets_response.qtpl:70 +//line lib/promscrape/targets_response.qtpl:72 continue -//line lib/promscrape/targets_response.qtpl:70 +//line lib/promscrape/targets_response.qtpl:72 } -//line lib/promscrape/targets_response.qtpl:70 +//line lib/promscrape/targets_response.qtpl:72 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:81 +//line lib/promscrape/targets_response.qtpl:84 } -//line lib/promscrape/targets_response.qtpl:81 +//line lib/promscrape/targets_response.qtpl:84 qw422016.N().S(`
Endpoint State Labels Last Scrape Scrape Duration Samples Scraped Error
`) -//line lib/promscrape/targets_response.qtpl:72 +//line lib/promscrape/targets_response.qtpl:74 qw422016.E().S(ts.endpoint) -//line lib/promscrape/targets_response.qtpl:72 +//line lib/promscrape/targets_response.qtpl:74 qw422016.N().S(`
 `) -//line lib/promscrape/targets_response.qtpl:73 +//line lib/promscrape/targets_response.qtpl:75 if ts.up { -//line lib/promscrape/targets_response.qtpl:73 +//line lib/promscrape/targets_response.qtpl:75 qw422016.N().S(`UP`) -//line lib/promscrape/targets_response.qtpl:73 +//line lib/promscrape/targets_response.qtpl:75 } else { -//line lib/promscrape/targets_response.qtpl:73 +//line lib/promscrape/targets_response.qtpl:75 qw422016.N().S(`DOWN`) -//line lib/promscrape/targets_response.qtpl:73 +//line lib/promscrape/targets_response.qtpl:75 } -//line lib/promscrape/targets_response.qtpl:73 +//line lib/promscrape/targets_response.qtpl:75 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:77 streamformatLabel(qw422016, ts.labels) -//line lib/promscrape/targets_response.qtpl:75 +//line lib/promscrape/targets_response.qtpl:77 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:77 +//line lib/promscrape/targets_response.qtpl:79 qw422016.N().FPrec(ts.lastScrapeTime.Seconds(), 3) -//line lib/promscrape/targets_response.qtpl:77 +//line lib/promscrape/targets_response.qtpl:79 qw422016.N().S(`s ago `) -//line lib/promscrape/targets_response.qtpl:78 +//line lib/promscrape/targets_response.qtpl:80 qw422016.N().FPrec(ts.scrapeDuration.Seconds(), 3) -//line lib/promscrape/targets_response.qtpl:78 +//line lib/promscrape/targets_response.qtpl:80 qw422016.N().S(`s `) -//line lib/promscrape/targets_response.qtpl:79 - qw422016.E().S(ts.error) -//line lib/promscrape/targets_response.qtpl:79 +//line lib/promscrape/targets_response.qtpl:81 + qw422016.N().D(ts.samplesScraped) +//line lib/promscrape/targets_response.qtpl:81 + qw422016.N().S(` `) +//line lib/promscrape/targets_response.qtpl:82 + qw422016.E().S(ts.errMsg) +//line lib/promscrape/targets_response.qtpl:82 qw422016.N().S(`
`) -//line lib/promscrape/targets_response.qtpl:85 +//line lib/promscrape/targets_response.qtpl:88 } -//line lib/promscrape/targets_response.qtpl:85 +//line lib/promscrape/targets_response.qtpl:88 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 } -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 func WriteTargetsResponseHTML(qq422016 qtio422016.Writer, jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) { -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 StreamTargetsResponseHTML(qw422016, jts, redirectPath, onlyUnhealthy) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 } -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 func TargetsResponseHTML(jts []jobTargetsStatuses, redirectPath string, onlyUnhealthy bool) string { -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 WriteTargetsResponseHTML(qb422016, jts, redirectPath, onlyUnhealthy) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 qs422016 := string(qb422016.B) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 return qs422016 -//line lib/promscrape/targets_response.qtpl:88 +//line lib/promscrape/targets_response.qtpl:91 } -//line lib/promscrape/targets_response.qtpl:90 +//line lib/promscrape/targets_response.qtpl:93 func streamformatLabel(qw422016 *qt422016.Writer, labels []prompbmarshal.Label) { -//line lib/promscrape/targets_response.qtpl:90 +//line lib/promscrape/targets_response.qtpl:93 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:94 for _, label := range labels { -//line lib/promscrape/targets_response.qtpl:91 +//line lib/promscrape/targets_response.qtpl:94 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:92 +//line lib/promscrape/targets_response.qtpl:95 qw422016.E().S(label.Name) -//line lib/promscrape/targets_response.qtpl:92 +//line lib/promscrape/targets_response.qtpl:95 qw422016.N().S(`=`) -//line lib/promscrape/targets_response.qtpl:92 +//line lib/promscrape/targets_response.qtpl:95 qw422016.E().Q(label.Value) -//line lib/promscrape/targets_response.qtpl:92 +//line lib/promscrape/targets_response.qtpl:95 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:92 +//line lib/promscrape/targets_response.qtpl:95 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:92 +//line lib/promscrape/targets_response.qtpl:95 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:93 +//line lib/promscrape/targets_response.qtpl:96 } -//line lib/promscrape/targets_response.qtpl:93 +//line lib/promscrape/targets_response.qtpl:96 qw422016.N().S(` `) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 } -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 func writeformatLabel(qq422016 qtio422016.Writer, labels []prompbmarshal.Label) { -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 qw422016 := qt422016.AcquireWriter(qq422016) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 streamformatLabel(qw422016, labels) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 qt422016.ReleaseWriter(qw422016) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 } -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 func formatLabel(labels []prompbmarshal.Label) string { -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 qb422016 := qt422016.AcquireByteBuffer() -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 writeformatLabel(qb422016, labels) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 qs422016 := string(qb422016.B) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 qt422016.ReleaseByteBuffer(qb422016) -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 return qs422016 -//line lib/promscrape/targets_response.qtpl:94 +//line lib/promscrape/targets_response.qtpl:97 } diff --git a/lib/promscrape/targetstatus.go b/lib/promscrape/targetstatus.go index 21998f47bc..eee6a548a5 100644 --- a/lib/promscrape/targetstatus.go +++ b/lib/promscrape/targetstatus.go @@ -88,7 +88,7 @@ func (tsm *targetStatusMap) Unregister(sw *ScrapeWork) { tsm.mu.Unlock() } -func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, err error) { +func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrapeTime, scrapeDuration int64, samplesScraped int, err error) { tsm.mu.Lock() ts := tsm.m[sw] if ts == nil { @@ -101,6 +101,7 @@ func (tsm *targetStatusMap) Update(sw *ScrapeWork, group string, up bool, scrape ts.scrapeGroup = group ts.scrapeTime = scrapeTime ts.scrapeDuration = scrapeDuration + ts.samplesScraped = samplesScraped ts.err = err tsm.mu.Unlock() } @@ -156,6 +157,7 @@ func (tsm *targetStatusMap) WriteActiveTargetsJSON(w io.Writer) { fmt.Fprintf(w, `,"lastError":%q`, errMsg) fmt.Fprintf(w, `,"lastScrape":%q`, time.Unix(st.scrapeTime/1000, (st.scrapeTime%1000)*1e6).Format(time.RFC3339Nano)) fmt.Fprintf(w, `,"lastScrapeDuration":%g`, (time.Millisecond * time.Duration(st.scrapeDuration)).Seconds()) + fmt.Fprintf(w, `,"lastSamplesScraped":%d`, st.samplesScraped) state := "up" if !st.up { state = "down" @@ -185,6 +187,7 @@ type targetStatus struct { scrapeGroup string scrapeTime int64 scrapeDuration int64 + samplesScraped int err error } @@ -270,7 +273,8 @@ type jobTargetStatus struct { originalLabels []prompbmarshal.Label lastScrapeTime time.Duration scrapeDuration time.Duration - error string + samplesScraped int + errMsg string } type jobTargetsStatuses struct { @@ -313,7 +317,8 @@ func (tsm *targetStatusMap) getTargetsStatusByJob() []jobTargetsStatuses { originalLabels: st.sw.OriginalLabels, lastScrapeTime: st.getDurationFromLastScrape(), scrapeDuration: time.Duration(st.scrapeDuration) * time.Millisecond, - error: errMsg, + samplesScraped: st.samplesScraped, + errMsg: errMsg, }) } jts = append(jts, jobTargetsStatuses{