From bb8b3fca88538fde29266965a9936db10ab93405 Mon Sep 17 00:00:00 2001 From: Joe Handzik Date: Wed, 1 Feb 2017 17:32:26 -0600 Subject: [PATCH] ZFS Collector: Add zpool IO statistics Signed-Off-By: Joe Handzik --- collector/fixtures/e2e-output.txt | 48 +++++++++++++ .../fixtures/proc/spl/kstat/zfs/pool1/io | 3 + .../fixtures/proc/spl/kstat/zfs/poolz1/io | 3 + collector/zfs.go | 18 +++++ collector/zfs_linux.go | 71 ++++++++++++++++++- collector/zfs_linux_test.go | 41 +++++++++++ 6 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 collector/fixtures/proc/spl/kstat/zfs/pool1/io create mode 100644 collector/fixtures/proc/spl/kstat/zfs/poolz1/io diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 9a08a561..91485b18 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -2494,6 +2494,54 @@ node_zfs_zil_zil_itx_needcopy_bytes 0 # HELP node_zfs_zil_zil_itx_needcopy_count kstat.zfs.misc.zil.zil_itx_needcopy_count # TYPE node_zfs_zil_zil_itx_needcopy_count untyped node_zfs_zil_zil_itx_needcopy_count 0 +# HELP node_zfs_zpool_nread kstat.zfs.misc.io.nread +# TYPE node_zfs_zpool_nread untyped +node_zfs_zpool_nread{zpool="pool1"} 1.88416e+06 +node_zfs_zpool_nread{zpool="poolz1"} 2.82624e+06 +# HELP node_zfs_zpool_nwritten kstat.zfs.misc.io.nwritten +# TYPE node_zfs_zpool_nwritten untyped +node_zfs_zpool_nwritten{zpool="pool1"} 3.206144e+06 +node_zfs_zpool_nwritten{zpool="poolz1"} 2.680501248e+09 +# HELP node_zfs_zpool_rcnt kstat.zfs.misc.io.rcnt +# TYPE node_zfs_zpool_rcnt untyped +node_zfs_zpool_rcnt{zpool="pool1"} 0 +node_zfs_zpool_rcnt{zpool="poolz1"} 0 +# HELP node_zfs_zpool_reads kstat.zfs.misc.io.reads +# TYPE node_zfs_zpool_reads untyped +node_zfs_zpool_reads{zpool="pool1"} 22 +node_zfs_zpool_reads{zpool="poolz1"} 33 +# HELP node_zfs_zpool_rlentime kstat.zfs.misc.io.rlentime +# TYPE node_zfs_zpool_rlentime untyped +node_zfs_zpool_rlentime{zpool="pool1"} 1.04112268e+08 +node_zfs_zpool_rlentime{zpool="poolz1"} 6.472105124093e+12 +# HELP node_zfs_zpool_rtime kstat.zfs.misc.io.rtime +# TYPE node_zfs_zpool_rtime untyped +node_zfs_zpool_rtime{zpool="pool1"} 2.4168078e+07 +node_zfs_zpool_rtime{zpool="poolz1"} 9.82909164e+09 +# HELP node_zfs_zpool_rupdate kstat.zfs.misc.io.rupdate +# TYPE node_zfs_zpool_rupdate untyped +node_zfs_zpool_rupdate{zpool="pool1"} 7.921048984922e+13 +node_zfs_zpool_rupdate{zpool="poolz1"} 1.10734831944501e+14 +# HELP node_zfs_zpool_wcnt kstat.zfs.misc.io.wcnt +# TYPE node_zfs_zpool_wcnt untyped +node_zfs_zpool_wcnt{zpool="pool1"} 0 +node_zfs_zpool_wcnt{zpool="poolz1"} 0 +# HELP node_zfs_zpool_wlentime kstat.zfs.misc.io.wlentime +# TYPE node_zfs_zpool_wlentime untyped +node_zfs_zpool_wlentime{zpool="pool1"} 1.04112268e+08 +node_zfs_zpool_wlentime{zpool="poolz1"} 6.472105124093e+12 +# HELP node_zfs_zpool_writes kstat.zfs.misc.io.writes +# TYPE node_zfs_zpool_writes untyped +node_zfs_zpool_writes{zpool="pool1"} 132 +node_zfs_zpool_writes{zpool="poolz1"} 25294 +# HELP node_zfs_zpool_wtime kstat.zfs.misc.io.wtime +# TYPE node_zfs_zpool_wtime untyped +node_zfs_zpool_wtime{zpool="pool1"} 7.155162e+06 +node_zfs_zpool_wtime{zpool="poolz1"} 9.673715628e+09 +# HELP node_zfs_zpool_wupdate kstat.zfs.misc.io.wupdate +# TYPE node_zfs_zpool_wupdate untyped +node_zfs_zpool_wupdate{zpool="pool1"} 7.9210489694949e+13 +node_zfs_zpool_wupdate{zpool="poolz1"} 1.10734831833266e+14 # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. # TYPE process_cpu_seconds_total counter # HELP process_max_fds Maximum number of open file descriptors. diff --git a/collector/fixtures/proc/spl/kstat/zfs/pool1/io b/collector/fixtures/proc/spl/kstat/zfs/pool1/io new file mode 100644 index 00000000..ef2a58fe --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/pool1/io @@ -0,0 +1,3 @@ +12 3 0x00 1 80 79205351707403 395818011156865 +nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt +1884160 3206144 22 132 7155162 104112268 79210489694949 24168078 104112268 79210489849220 0 0 diff --git a/collector/fixtures/proc/spl/kstat/zfs/poolz1/io b/collector/fixtures/proc/spl/kstat/zfs/poolz1/io new file mode 100644 index 00000000..2e6677c5 --- /dev/null +++ b/collector/fixtures/proc/spl/kstat/zfs/poolz1/io @@ -0,0 +1,3 @@ +16 3 0x00 1 80 79568650431241 395832279341621 +nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt +2826240 2680501248 33 25294 9673715628 6472105124093 110734831833266 9829091640 6472105124093 110734831944501 0 0 diff --git a/collector/zfs.go b/collector/zfs.go index f7877bea..4d4c2370 100644 --- a/collector/zfs.go +++ b/collector/zfs.go @@ -45,12 +45,14 @@ func init() { type zfsCollector struct { zfsMetrics []zfsMetric linuxProcpathBase string + linuxZpoolIoPath string linuxPathMap map[string]string } func NewZFSCollector() (Collector, error) { var z zfsCollector z.linuxProcpathBase = "spl/kstat/zfs" + z.linuxZpoolIoPath = "/*/io" z.linuxPathMap = map[string]string{ "zfs_arc": "arcstats", "zfs_dmu_tx": "dmu_tx", @@ -98,3 +100,19 @@ func (c *zfsCollector) constSysctlMetric(subsystem string, sysctl zfsSysctl, val float64(value), ) } + +func (c *zfsCollector) constPoolMetric(poolName string, sysctl zfsSysctl, value int) prometheus.Metric { + metricName := sysctl.metricName() + + return prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(Namespace, "zfs_zpool", metricName), + string(sysctl), + []string{"zpool"}, + nil, + ), + prometheus.UntypedValue, + float64(value), + poolName, + ) +} diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go index 6ac5e5a4..2d228373 100644 --- a/collector/zfs_linux.go +++ b/collector/zfs_linux.go @@ -47,6 +47,35 @@ func (c *zfsCollector) updateZfsStats(subsystem string, ch chan<- prometheus.Met }) } +func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) (err error) { + zpoolPaths, err := filepath.Glob(procFilePath(filepath.Join(c.linuxProcpathBase, c.linuxZpoolIoPath))) + if err != nil { + return err + } + + if zpoolPaths == nil { + return nil + } + + for _, zpoolPath := range zpoolPaths { + file, err := os.Open(zpoolPath) + if err != nil { + log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", zpoolPath) + return zfsNotAvailableError + } + + err = c.parsePoolProcfsFile(file, zpoolPath, func(poolName string, s zfsSysctl, v int) { + ch <- c.constPoolMetric(poolName, s, v) + }) + file.Close() + if err != nil { + return err + } + } + + return nil +} + func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmtExt string, handler func(zfsSysctl, int)) (err error) { scanner := bufio.NewScanner(reader) @@ -81,6 +110,44 @@ func (c *zfsCollector) parseProcfsFile(reader io.Reader, fmtExt string, handler return scanner.Err() } -func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) (err error) { - return nil +func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, handler func(string, zfsSysctl, int)) (err error) { + scanner := bufio.NewScanner(reader) + + parseLine := false + var fields []string + for scanner.Scan() { + + line := strings.Fields(scanner.Text()) + + if !parseLine && len(line) >= 12 && line[0] == "nread" { + //Start parsing from here. + parseLine = true + fields = make([]string, len(line)) + copy(fields, line) + continue + } + if !parseLine { + continue + } + + zpoolPathElements := strings.Split(zpoolPath, "/") + pathLen := len(zpoolPathElements) + if pathLen < 2 { + return fmt.Errorf("zpool path did not return at least two elements") + } + zpoolName := zpoolPathElements[pathLen-2] + zpoolFile := zpoolPathElements[pathLen-1] + + for i, field := range fields { + key := fmt.Sprintf("kstat.zfs.misc.%s.%s", zpoolFile, field) + + value, err := strconv.Atoi(line[i]) + if err != nil { + return fmt.Errorf("could not parse expected integer value for %q: %v", key, err) + } + handler(zpoolName, zfsSysctl(key), value) + } + } + + return scanner.Err() } diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go index 8a102310..faad3bc5 100644 --- a/collector/zfs_linux_test.go +++ b/collector/zfs_linux_test.go @@ -15,6 +15,7 @@ package collector import ( "os" + "path/filepath" "testing" ) @@ -269,3 +270,43 @@ func TestDmuTxParsing(t *testing.T) { t.Fatal("DmuTx parsing handler was not called for some expected sysctls") } } + +func TestZpoolParsing(t *testing.T) { + zpoolPaths, err := filepath.Glob("fixtures/proc/spl/kstat/zfs/*/io") + if err != nil { + t.Fatal(err) + } + + c := zfsCollector{} + if err != nil { + t.Fatal(err) + } + + handlerCalled := false + for _, zpoolPath := range zpoolPaths { + file, err := os.Open(zpoolPath) + if err != nil { + t.Fatal(err) + } + + err = c.parsePoolProcfsFile(file, zpoolPath, func(poolName string, s zfsSysctl, v int) { + if s != zfsSysctl("kstat.zfs.misc.io.nread") { + return + } + + handlerCalled = true + + if v != int(1884160) && v != int(2826240) { + t.Fatalf("Incorrect value parsed from procfs data %v", v) + } + + }) + file.Close() + if err != nil { + t.Fatal(err) + } + } + if !handlerCalled { + t.Fatal("Zpool parsing handler was not called for some expected sysctls") + } +}