2015-09-26 17:36:40 +02:00
|
|
|
// Copyright 2015 The Prometheus Authors
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2021-10-03 13:35:24 +02:00
|
|
|
//go:build !nofilesystem
|
2015-05-12 13:06:41 +02:00
|
|
|
// +build !nofilesystem
|
2014-06-05 21:44:44 +02:00
|
|
|
|
|
|
|
package collector
|
|
|
|
|
|
|
|
import (
|
|
|
|
"bufio"
|
2020-06-15 22:27:14 +02:00
|
|
|
"errors"
|
2018-10-30 22:12:42 +01:00
|
|
|
"fmt"
|
|
|
|
"io"
|
2014-06-05 21:44:44 +02:00
|
|
|
"os"
|
|
|
|
"strings"
|
2018-07-14 11:10:28 +02:00
|
|
|
"sync"
|
|
|
|
"time"
|
2014-06-05 21:44:44 +02:00
|
|
|
|
2023-03-07 09:25:05 +01:00
|
|
|
"github.com/alecthomas/kingpin/v2"
|
2020-11-14 11:53:51 +01:00
|
|
|
"github.com/go-kit/log"
|
|
|
|
"github.com/go-kit/log/level"
|
2019-05-10 20:04:06 +02:00
|
|
|
"golang.org/x/sys/unix"
|
2014-06-05 21:44:44 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2022-01-01 14:09:20 +01:00
|
|
|
defMountPointsExcluded = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/)"
|
2021-03-23 12:00:06 +01:00
|
|
|
defFSTypesExcluded = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
|
2014-06-05 21:44:44 +02:00
|
|
|
)
|
|
|
|
|
2019-09-13 21:13:59 +02:00
|
|
|
var mountTimeout = kingpin.Flag("collector.filesystem.mount-timeout",
|
|
|
|
"how long to wait for a mount to respond before marking it as stale").
|
|
|
|
Hidden().Default("5s").Duration()
|
2023-06-02 21:53:06 +02:00
|
|
|
var statWorkerCount = kingpin.Flag("collector.filesystem.stat-workers",
|
|
|
|
"how many stat calls to process simultaneously").
|
|
|
|
Hidden().Default("4").Int()
|
2018-07-14 11:10:28 +02:00
|
|
|
var stuckMounts = make(map[string]struct{})
|
|
|
|
var stuckMountsMtx = &sync.Mutex{}
|
|
|
|
|
2017-02-28 17:44:53 +01:00
|
|
|
// GetStats returns filesystem stats.
|
2017-02-28 19:47:20 +01:00
|
|
|
func (c *filesystemCollector) GetStats() ([]filesystemStats, error) {
|
2019-12-31 17:19:37 +01:00
|
|
|
mps, err := mountPointDetails(c.logger)
|
2014-06-05 21:44:44 +02:00
|
|
|
if err != nil {
|
2015-09-16 15:34:34 +02:00
|
|
|
return nil, err
|
2014-06-05 21:44:44 +02:00
|
|
|
}
|
2017-02-28 19:47:20 +01:00
|
|
|
stats := []filesystemStats{}
|
2023-06-02 21:53:06 +02:00
|
|
|
labelChan := make(chan filesystemLabels)
|
|
|
|
statChan := make(chan filesystemStats)
|
|
|
|
wg := sync.WaitGroup{}
|
2018-07-14 11:10:28 +02:00
|
|
|
|
2023-06-02 21:53:06 +02:00
|
|
|
workerCount := *statWorkerCount
|
|
|
|
if workerCount < 1 {
|
|
|
|
workerCount = 1
|
|
|
|
}
|
2019-12-31 17:19:37 +01:00
|
|
|
|
2023-06-02 21:53:06 +02:00
|
|
|
for i := 0; i < workerCount; i++ {
|
|
|
|
wg.Add(1)
|
|
|
|
go func() {
|
|
|
|
defer wg.Done()
|
|
|
|
for labels := range labelChan {
|
|
|
|
statChan <- c.processStat(labels)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
}
|
2015-07-03 01:20:49 +02:00
|
|
|
|
2023-06-02 21:53:06 +02:00
|
|
|
go func() {
|
|
|
|
for _, labels := range mps {
|
|
|
|
if c.excludedMountPointsPattern.MatchString(labels.mountPoint) {
|
|
|
|
level.Debug(c.logger).Log("msg", "Ignoring mount point", "mountpoint", labels.mountPoint)
|
|
|
|
continue
|
2018-07-16 15:56:27 +02:00
|
|
|
}
|
2023-06-02 21:53:06 +02:00
|
|
|
if c.excludedFSTypesPattern.MatchString(labels.fsType) {
|
|
|
|
level.Debug(c.logger).Log("msg", "Ignoring fs", "type", labels.fsType)
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
stuckMountsMtx.Lock()
|
|
|
|
if _, ok := stuckMounts[labels.mountPoint]; ok {
|
|
|
|
stats = append(stats, filesystemStats{
|
|
|
|
labels: labels,
|
|
|
|
deviceError: 1,
|
|
|
|
})
|
|
|
|
level.Debug(c.logger).Log("msg", "Mount point is in an unresponsive state", "mountpoint", labels.mountPoint)
|
|
|
|
stuckMountsMtx.Unlock()
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
stuckMountsMtx.Unlock()
|
|
|
|
labelChan <- labels
|
2015-11-10 09:25:04 +01:00
|
|
|
}
|
2023-06-02 21:53:06 +02:00
|
|
|
close(labelChan)
|
|
|
|
wg.Wait()
|
|
|
|
close(statChan)
|
|
|
|
}()
|
2015-11-10 09:25:04 +01:00
|
|
|
|
2023-06-02 21:53:06 +02:00
|
|
|
for stat := range statChan {
|
|
|
|
stats = append(stats, stat)
|
2014-06-05 21:44:44 +02:00
|
|
|
}
|
2015-09-16 15:34:34 +02:00
|
|
|
return stats, nil
|
2014-06-05 21:44:44 +02:00
|
|
|
}
|
|
|
|
|
2023-06-02 21:53:06 +02:00
|
|
|
func (c *filesystemCollector) processStat(labels filesystemLabels) filesystemStats {
|
|
|
|
success := make(chan struct{})
|
|
|
|
go stuckMountWatcher(labels.mountPoint, success, c.logger)
|
|
|
|
|
|
|
|
buf := new(unix.Statfs_t)
|
|
|
|
err := unix.Statfs(rootfsFilePath(labels.mountPoint), buf)
|
|
|
|
stuckMountsMtx.Lock()
|
|
|
|
close(success)
|
|
|
|
|
|
|
|
// If the mount has been marked as stuck, unmark it and log it's recovery.
|
|
|
|
if _, ok := stuckMounts[labels.mountPoint]; ok {
|
|
|
|
level.Debug(c.logger).Log("msg", "Mount point has recovered, monitoring will resume", "mountpoint", labels.mountPoint)
|
|
|
|
delete(stuckMounts, labels.mountPoint)
|
|
|
|
}
|
|
|
|
stuckMountsMtx.Unlock()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
level.Debug(c.logger).Log("msg", "Error on statfs() system call", "rootfs", rootfsFilePath(labels.mountPoint), "err", err)
|
|
|
|
return filesystemStats{
|
|
|
|
labels: labels,
|
|
|
|
deviceError: 1,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
var ro float64
|
|
|
|
for _, option := range strings.Split(labels.options, ",") {
|
|
|
|
if option == "ro" {
|
|
|
|
ro = 1
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return filesystemStats{
|
|
|
|
labels: labels,
|
|
|
|
size: float64(buf.Blocks) * float64(buf.Bsize),
|
|
|
|
free: float64(buf.Bfree) * float64(buf.Bsize),
|
|
|
|
avail: float64(buf.Bavail) * float64(buf.Bsize),
|
|
|
|
files: float64(buf.Files),
|
|
|
|
filesFree: float64(buf.Ffree),
|
|
|
|
ro: ro,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-14 11:10:28 +02:00
|
|
|
// stuckMountWatcher listens on the given success channel and if the channel closes
|
|
|
|
// then the watcher does nothing. If instead the timeout is reached, the
|
|
|
|
// mount point that is being watched is marked as stuck.
|
2019-12-31 17:19:37 +01:00
|
|
|
func stuckMountWatcher(mountPoint string, success chan struct{}, logger log.Logger) {
|
2021-10-24 12:48:57 +02:00
|
|
|
mountCheckTimer := time.NewTimer(*mountTimeout)
|
|
|
|
defer mountCheckTimer.Stop()
|
2018-07-14 11:10:28 +02:00
|
|
|
select {
|
|
|
|
case <-success:
|
|
|
|
// Success
|
2021-10-24 12:48:57 +02:00
|
|
|
case <-mountCheckTimer.C:
|
2018-07-14 11:10:28 +02:00
|
|
|
// Timed out, mark mount as stuck
|
|
|
|
stuckMountsMtx.Lock()
|
|
|
|
select {
|
|
|
|
case <-success:
|
|
|
|
// Success came in just after the timeout was reached, don't label the mount as stuck
|
|
|
|
default:
|
2019-12-31 17:19:37 +01:00
|
|
|
level.Debug(logger).Log("msg", "Mount point timed out, it is being labeled as stuck and will not be monitored", "mountpoint", mountPoint)
|
2018-07-14 11:10:28 +02:00
|
|
|
stuckMounts[mountPoint] = struct{}{}
|
|
|
|
}
|
|
|
|
stuckMountsMtx.Unlock()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-12-31 17:19:37 +01:00
|
|
|
func mountPointDetails(logger log.Logger) ([]filesystemLabels, error) {
|
2018-10-04 14:11:21 +02:00
|
|
|
file, err := os.Open(procFilePath("1/mounts"))
|
2020-06-15 22:27:14 +02:00
|
|
|
if errors.Is(err, os.ErrNotExist) {
|
2018-11-30 14:01:55 +01:00
|
|
|
// Fallback to `/proc/mounts` if `/proc/1/mounts` is missing due hidepid.
|
2019-12-31 17:19:37 +01:00
|
|
|
level.Debug(logger).Log("msg", "Reading root mounts failed, falling back to system mounts", "err", err)
|
2018-11-30 14:01:55 +01:00
|
|
|
file, err = os.Open(procFilePath("mounts"))
|
|
|
|
}
|
2014-06-05 21:44:44 +02:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
|
2018-10-30 22:12:42 +01:00
|
|
|
return parseFilesystemLabels(file)
|
|
|
|
}
|
|
|
|
|
|
|
|
func parseFilesystemLabels(r io.Reader) ([]filesystemLabels, error) {
|
|
|
|
var filesystems []filesystemLabels
|
|
|
|
|
|
|
|
scanner := bufio.NewScanner(r)
|
2014-06-05 21:44:44 +02:00
|
|
|
for scanner.Scan() {
|
|
|
|
parts := strings.Fields(scanner.Text())
|
2018-06-06 16:49:19 +02:00
|
|
|
|
2018-10-30 22:12:42 +01:00
|
|
|
if len(parts) < 4 {
|
|
|
|
return nil, fmt.Errorf("malformed mount point information: %q", scanner.Text())
|
|
|
|
}
|
|
|
|
|
2018-06-06 16:49:19 +02:00
|
|
|
// Ensure we handle the translation of \040 and \011
|
|
|
|
// as per fstab(5).
|
|
|
|
parts[1] = strings.Replace(parts[1], "\\040", " ", -1)
|
|
|
|
parts[1] = strings.Replace(parts[1], "\\011", "\t", -1)
|
|
|
|
|
2017-03-23 01:48:18 +01:00
|
|
|
filesystems = append(filesystems, filesystemLabels{
|
|
|
|
device: parts[0],
|
2019-07-19 16:51:17 +02:00
|
|
|
mountPoint: rootfsStripPrefix(parts[1]),
|
2017-03-23 01:48:18 +01:00
|
|
|
fsType: parts[2],
|
2018-07-16 15:56:27 +02:00
|
|
|
options: parts[3],
|
2017-03-23 01:48:18 +01:00
|
|
|
})
|
2014-06-05 21:44:44 +02:00
|
|
|
}
|
2018-10-30 22:12:42 +01:00
|
|
|
|
2017-01-03 15:55:40 +01:00
|
|
|
return filesystems, scanner.Err()
|
2014-06-05 21:44:44 +02:00
|
|
|
}
|