Merge pull request #2251 from DavidVentura/add-isolated-label-cpu-collector

Implement #2250: Add "isolated" metric on cpu collector on linux

collector/cpu_linux.go

@@ -18,6 +18,7 @@ package collector
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -27,6 +28,7 @@ import (
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/procfs"
+	"github.com/prometheus/procfs/sysfs"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
 
@@ -39,9 +41,11 @@ type cpuCollector struct {
 	cpuGuest           *prometheus.Desc
 	cpuCoreThrottle    *prometheus.Desc
 	cpuPackageThrottle *prometheus.Desc
+	cpuIsolated        *prometheus.Desc
 	logger             log.Logger
 	cpuStats           []procfs.CPUStat
 	cpuStatsMutex      sync.Mutex
+	isolatedCpus       []uint16
 
 	cpuFlagsIncludeRegexp *regexp.Regexp
 	cpuBugsIncludeRegexp  *regexp.Regexp
@@ -68,6 +72,20 @@ func NewCPUCollector(logger log.Logger) (Collector, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to open procfs: %w", err)
 	}
+
+	sysfs, err := sysfs.NewFS(*sysPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open sysfs: %w", err)
+	}
+
+	isolcpus, err := sysfs.IsolatedCPUs()
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("Unable to get isolated cpus: %w", err)
+		}
+		level.Debug(logger).Log("msg", "Could not open isolated file", "error", err)
+	}
+
 	c := &cpuCollector{
 		fs:  fs,
 		cpu: nodeCPUSecondsDesc,
@@ -101,7 +119,13 @@ func NewCPUCollector(logger log.Logger) (Collector, error) {
 			"Number of times this CPU package has been throttled.",
 			[]string{"package"}, nil,
 		),
-		logger: logger,
+		cpuIsolated: prometheus.NewDesc(
+			prometheus.BuildFQName(namespace, cpuCollectorSubsystem, "isolated"),
+			"Whether each core is isolated, information from /sys/devices/system/cpu/isolated.",
+			[]string{"cpu"}, nil,
+		),
+		logger:       logger,
+		isolatedCpus: isolcpus,
 	}
 	err = c.compileIncludeFlags(flagsInclude, bugsInclude)
 	if err != nil {
@@ -142,6 +166,9 @@ func (c *cpuCollector) Update(ch chan<- prometheus.Metric) error {
 	if err := c.updateStat(ch); err != nil {
 		return err
 	}
+	if c.isolatedCpus != nil {
+		c.updateIsolated(ch)
+	}
 	return c.updateThermalThrottle(ch)
 }
 
@@ -276,6 +303,14 @@ func (c *cpuCollector) updateThermalThrottle(ch chan<- prometheus.Metric) error
 	return nil
 }
 
+// updateIsolated reads /sys/devices/system/cpu/isolated through sysfs and exports isolation level metrics.
+func (c *cpuCollector) updateIsolated(ch chan<- prometheus.Metric) {
+	for _, cpu := range c.isolatedCpus {
+		cpuNum := strconv.Itoa(int(cpu))
+		ch <- prometheus.MustNewConstMetric(c.cpuIsolated, prometheus.GaugeValue, 1.0, cpuNum)
+	}
+}
+
 // updateStat reads /proc/stat through procfs and exports CPU-related metrics.
 func (c *cpuCollector) updateStat(ch chan<- prometheus.Metric) error {
 	stats, err := c.fs.Stat()

collector/fixtures/e2e-64k-page-output.txt

@@ -297,6 +297,13 @@ node_cpu_guest_seconds_total{cpu="6",mode="nice"} 0.07
 node_cpu_guest_seconds_total{cpu="6",mode="user"} 0.08
 node_cpu_guest_seconds_total{cpu="7",mode="nice"} 0.08
 node_cpu_guest_seconds_total{cpu="7",mode="user"} 0.09
+# HELP node_cpu_isolated Whether each core is isolated, information from /sys/devices/system/cpu/isolated.
+# TYPE node_cpu_isolated gauge
+node_cpu_isolated{cpu="1"} 1
+node_cpu_isolated{cpu="3"} 1
+node_cpu_isolated{cpu="4"} 1
+node_cpu_isolated{cpu="5"} 1
+node_cpu_isolated{cpu="9"} 1
 # HELP node_cpu_package_throttles_total Number of times this CPU package has been throttled.
 # TYPE node_cpu_package_throttles_total counter
 node_cpu_package_throttles_total{package="0"} 30

collector/fixtures/e2e-output.txt

@@ -319,6 +319,13 @@ node_cpu_info{cachesize="8192 KB",core="2",cpu="2",family="6",microcode="0xb4",m
 node_cpu_info{cachesize="8192 KB",core="2",cpu="6",family="6",microcode="0xb4",model="142",model_name="Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz",package="0",stepping="10",vendor="GenuineIntel"} 1
 node_cpu_info{cachesize="8192 KB",core="3",cpu="3",family="6",microcode="0xb4",model="142",model_name="Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz",package="0",stepping="10",vendor="GenuineIntel"} 1
 node_cpu_info{cachesize="8192 KB",core="3",cpu="7",family="6",microcode="0xb4",model="142",model_name="Intel(R) Core(TM) i7-8650U CPU @ 1.90GHz",package="0",stepping="10",vendor="GenuineIntel"} 1
+# HELP node_cpu_isolated Whether each core is isolated, information from /sys/devices/system/cpu/isolated.
+# TYPE node_cpu_isolated gauge
+node_cpu_isolated{cpu="1"} 1
+node_cpu_isolated{cpu="3"} 1
+node_cpu_isolated{cpu="4"} 1
+node_cpu_isolated{cpu="5"} 1
+node_cpu_isolated{cpu="9"} 1
 # HELP node_cpu_package_throttles_total Number of times this CPU package has been throttled.
 # TYPE node_cpu_package_throttles_total counter
 node_cpu_package_throttles_total{package="0"} 30

collector/fixtures/sys.ttar

@@ -3540,6 +3540,11 @@ Lines: 1
 1
 Mode: 644
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+Path: sys/devices/system/cpu/isolated
+Lines: 1
+1,3-5,9
+Mode: 664
+# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 Directory: sys/devices/system/edac
 Mode: 755
 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -