Skip to content

Commit

Permalink
fix: cpu stat in cgroup v2 (#3857)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevwan
kevwan authored Jan 17, 2024
1 parent 06d2c07 commit c08e741
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 74 deletions.
101 changes: 70 additions & 31 deletions core/stat/internal/cgroup_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package internal

import (
"bufio"
"errors"
"fmt"
"math"
"os"
Expand All @@ -18,6 +19,7 @@ import (

const (
cgroupDir = "/sys/fs/cgroup"
cpuMaxFile = cgroupDir + "/cpu.max"
cpuStatFile = cgroupDir + "/cpu.stat"
cpusetFile = cgroupDir + "/cpuset.cpus.effective"
)
Expand All @@ -30,10 +32,9 @@ var (
)

type cgroup interface {
cpuQuotaUs() (int64, error)
cpuPeriodUs() (uint64, error)
cpus() ([]uint64, error)
usageAllCpus() (uint64, error)
cpuQuota() (float64, error)
cpuUsage() (uint64, error)
effectiveCpus() (int, error)
}

func currentCgroup() (cgroup, error) {
Expand All @@ -48,13 +49,22 @@ type cgroupV1 struct {
cgroups map[string]string
}

func (c *cgroupV1) cpuQuotaUs() (int64, error) {
data, err := iox.ReadText(path.Join(c.cgroups["cpu"], "cpu.cfs_quota_us"))
func (c *cgroupV1) cpuQuota() (float64, error) {
quotaUs, err := c.cpuQuotaUs()
if err != nil {
return 0, err
}

return strconv.ParseInt(data, 10, 64)
if quotaUs == -1 {
return -1, nil
}

periodUs, err := c.cpuPeriodUs()
if err != nil {
return 0, err
}

return float64(quotaUs) / float64(periodUs), nil
}

func (c *cgroupV1) cpuPeriodUs() (uint64, error) {
Expand All @@ -66,16 +76,16 @@ func (c *cgroupV1) cpuPeriodUs() (uint64, error) {
return parseUint(data)
}

func (c *cgroupV1) cpus() ([]uint64, error) {
data, err := iox.ReadText(path.Join(c.cgroups["cpuset"], "cpuset.cpus"))
func (c *cgroupV1) cpuQuotaUs() (int64, error) {
data, err := iox.ReadText(path.Join(c.cgroups["cpu"], "cpu.cfs_quota_us"))
if err != nil {
return nil, err
return 0, err
}

return parseUints(data)
return strconv.ParseInt(data, 10, 64)
}

func (c *cgroupV1) usageAllCpus() (uint64, error) {
func (c *cgroupV1) cpuUsage() (uint64, error) {
data, err := iox.ReadText(path.Join(c.cgroups["cpuacct"], "cpuacct.usage"))
if err != nil {
return 0, err
Expand All @@ -84,38 +94,53 @@ func (c *cgroupV1) usageAllCpus() (uint64, error) {
return parseUint(data)
}

func (c *cgroupV1) effectiveCpus() (int, error) {
data, err := iox.ReadText(path.Join(c.cgroups["cpuset"], "cpuset.cpus"))
if err != nil {
return 0, err
}

cpus, err := parseUints(data)
if err != nil {
return 0, err
}

return len(cpus), nil
}

type cgroupV2 struct {
cgroups map[string]string
}

func (c *cgroupV2) cpuQuotaUs() (int64, error) {
data, err := iox.ReadText(path.Join(cgroupDir, "cpu.cfs_quota_us"))
func (c *cgroupV2) cpuQuota() (float64, error) {
data, err := iox.ReadText(cpuMaxFile)
if err != nil {
return 0, err
}

return strconv.ParseInt(data, 10, 64)
}
fields := strings.Fields(data)
if len(fields) != 2 {
return 0, fmt.Errorf("cgroup: bad /sys/fs/cgroup/cpu.max file: %s", data)
}

if fields[0] == "max" {
return -1, nil
}

func (c *cgroupV2) cpuPeriodUs() (uint64, error) {
data, err := iox.ReadText(path.Join(cgroupDir, "cpu.cfs_period_us"))
quotaUs, err := strconv.ParseInt(fields[0], 10, 64)
if err != nil {
return 0, err
}

return parseUint(data)
}

func (c *cgroupV2) cpus() ([]uint64, error) {
data, err := iox.ReadText(cpusetFile)
periodUs, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return nil, err
return 0, err
}

return parseUints(data)
return float64(quotaUs) / float64(periodUs), nil
}

func (c *cgroupV2) usageAllCpus() (uint64, error) {
func (c *cgroupV2) cpuUsage() (uint64, error) {
usec, err := parseUint(c.cgroups["usage_usec"])
if err != nil {
return 0, err
Expand All @@ -124,6 +149,20 @@ func (c *cgroupV2) usageAllCpus() (uint64, error) {
return usec * uint64(time.Microsecond), nil
}

func (c *cgroupV2) effectiveCpus() (int, error) {
data, err := iox.ReadText(cpusetFile)
if err != nil {
return 0, err
}

cpus, err := parseUints(data)
if err != nil {
return 0, err
}

return len(cpus), nil
}

func currentCgroupV1() (cgroup, error) {
cgroupFile := fmt.Sprintf("/proc/%d/cgroup", os.Getpid())
lines, err := iox.ReadTextLines(cgroupFile, iox.WithoutBlank())
Expand Down Expand Up @@ -200,7 +239,7 @@ func isCgroup2UnifiedMode() bool {
func parseUint(s string) (uint64, error) {
v, err := strconv.ParseInt(s, 10, 64)
if err != nil {
if err.(*strconv.NumError).Err == strconv.ErrRange {
if errors.Is(err, strconv.ErrRange) {
return 0, nil
}

Expand All @@ -225,21 +264,21 @@ func parseUints(val string) ([]uint64, error) {
for _, r := range cols {
if strings.Contains(r, "-") {
fields := strings.SplitN(r, "-", 2)
min, err := parseUint(fields[0])
minimum, err := parseUint(fields[0])
if err != nil {
return nil, fmt.Errorf("cgroup: bad int list format: %s", val)
}

max, err := parseUint(fields[1])
maximum, err := parseUint(fields[1])
if err != nil {
return nil, fmt.Errorf("cgroup: bad int list format: %s", val)
}

if max < min {
if maximum < minimum {
return nil, fmt.Errorf("cgroup: bad int list format: %s", val)
}

for i := min; i <= max; i++ {
for i := minimum; i <= maximum; i++ {
if _, ok := ints[i]; !ok {
ints[i] = lang.Placeholder
sets = append(sets, i)
Expand Down
18 changes: 13 additions & 5 deletions core/stat/internal/cgroup_linux_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,23 @@ func TestCgroupV1(t *testing.T) {
if isCgroup2UnifiedMode() {
cg, err := currentCgroupV1()
assert.NoError(t, err)
_, err = cg.cpus()
_, err = cg.effectiveCpus()
assert.Error(t, err)
_, err = cg.cpuPeriodUs()
_, err = cg.cpuQuota()
assert.Error(t, err)
_, err = cg.cpuQuotaUs()
assert.Error(t, err)
_, err = cg.usageAllCpus()
_, err = cg.cpuUsage()
assert.Error(t, err)
}

// test cgroup v2
cg, err := currentCgroupV2()
assert.NoError(t, err)
_, err = cg.effectiveCpus()
assert.NoError(t, err)
_, err = cg.cpuQuota()
assert.Error(t, err)
_, err = cg.cpuUsage()
assert.NoError(t, err)
}

func TestParseUint(t *testing.T) {
Expand Down
58 changes: 20 additions & 38 deletions core/stat/internal/cpu_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,40 +15,31 @@ const (
cpuTicks = 100
cpuFields = 8
cpuMax = 1000
statDir = "/proc/stat"
statFile = "/proc/stat"
)

var (
preSystem uint64
preTotal uint64
quota float64
limit float64
cores uint64
initOnce sync.Once
)

// if /proc not present, ignore the cpu calculation, like wsl linux
func initialize() {
cpus, err := cpuSets()
cpus, err := effectiveCpus()
if err != nil {
logx.Error(err)
return
}

cores = uint64(len(cpus))
quota = float64(len(cpus))
cq, err := cpuQuota()
if err == nil {
if cq != -1 {
period, err := cpuPeriod()
if err != nil {
logx.Error(err)
return
}

limit := float64(cq) / float64(period)
if limit < quota {
quota = limit
}
cores = uint64(cpus)
limit = float64(cpus)
quota, err := cpuQuota()
if err == nil && quota > 0 {
if quota < limit {
limit = quota
}
}

Expand All @@ -58,7 +49,7 @@ func initialize() {
return
}

preTotal, err = totalCpuUsage()
preTotal, err = cpuUsage()
if err != nil {
logx.Error(err)
return
Expand All @@ -69,7 +60,7 @@ func initialize() {
func RefreshCpu() uint64 {
initOnce.Do(initialize)

total, err := totalCpuUsage()
total, err := cpuUsage()
if err != nil {
return 0
}
Expand All @@ -83,7 +74,7 @@ func RefreshCpu() uint64 {
cpuDelta := total - preTotal
systemDelta := system - preSystem
if cpuDelta > 0 && systemDelta > 0 {
usage = uint64(float64(cpuDelta*cores*cpuMax) / (float64(systemDelta) * quota))
usage = uint64(float64(cpuDelta*cores*cpuMax) / (float64(systemDelta) * limit))
if usage > cpuMax {
usage = cpuMax
}
Expand All @@ -94,35 +85,35 @@ func RefreshCpu() uint64 {
return usage
}

func cpuQuota() (int64, error) {
func cpuQuota() (float64, error) {
cg, err := currentCgroup()
if err != nil {
return 0, err
}

return cg.cpuQuotaUs()
return cg.cpuQuota()
}

func cpuPeriod() (uint64, error) {
func cpuUsage() (uint64, error) {
cg, err := currentCgroup()
if err != nil {
return 0, err
}

return cg.cpuPeriodUs()
return cg.cpuUsage()
}

func cpuSets() ([]uint64, error) {
func effectiveCpus() (int, error) {
cg, err := currentCgroup()
if err != nil {
return nil, err
return 0, err
}

return cg.cpus()
return cg.effectiveCpus()
}

func systemCpuUsage() (uint64, error) {
lines, err := iox.ReadTextLines(statDir, iox.WithoutBlank())
lines, err := iox.ReadTextLines(statFile, iox.WithoutBlank())
if err != nil {
return 0, err
}
Expand Down Expand Up @@ -150,12 +141,3 @@ func systemCpuUsage() (uint64, error) {

return 0, errors.New("bad stats format")
}

func totalCpuUsage() (usage uint64, err error) {
var cg cgroup
if cg, err = currentCgroup(); err != nil {
return
}

return cg.usageAllCpus()
}

0 comments on commit c08e741

Please sign in to comment.