From b762d0bf965e4b9a08883cd430d8371cb5cf57df Mon Sep 17 00:00:00 2001 From: lantz Date: Sat, 22 Apr 2023 15:31:40 -0700 Subject: [PATCH] Add cgroup2 support (#1166) util.py: replace mountCgroups with checkCgroups, which returns the cgroup version (cgroup, cgroup2) node.py: handle cpu.max='quota period' for cgroup2 vs. cpu.{cfs_quota_us,cfs_period_us} for cgroup net.py: make _iperfVals more robust in the case of a missing UDP ACK examples/cpu.py: user _iperfVals to handle new iperf behavior of no output from telnet --- examples/cpu.py | 13 +++++++++---- mininet/net.py | 10 ++++++---- mininet/node.py | 33 ++++++++++++++++++++------------- mininet/util.py | 41 ++++++++++++++++++++++++++--------------- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/examples/cpu.py b/examples/cpu.py index a983517..d90e75a 100755 --- a/examples/cpu.py +++ b/examples/cpu.py @@ -68,11 +68,16 @@ def bwtest( cpuLimits, period_us=100000, seconds=10 ): # the client's buffer fill rate popen = server.popen( 'iperf -yc -s -p 5001' ) waitListening( client, server, 5001 ) - # ignore empty result from waitListening/telnet - popen.stdout.readline() client.cmd( 'iperf -yc -t %s -c %s' % ( seconds, server.IP() ) ) - result = decode( popen.stdout.readline() ).split( ',' ) - bps = float( result[ -1 ] ) + # ignore empty result from waitListening/telnet for old iperf + svals = {} + while not svals or int( svals[ 'rate' ] ) == 0: + line = decode( popen.stdout.readline() ) + # Probably shouldn't depend on an internal method, but + # this is the easiest way + svals = Mininet._iperfVals( # pylint: disable=protected-access + line, server.IP() ) + bps = float( svals[ 'rate' ] ) popen.terminate() net.stop() updated = results.get( sched, [] ) diff --git a/mininet/net.py b/mininet/net.py index ca76fad..3376d4d 100755 --- a/mininet/net.py +++ b/mininet/net.py @@ -789,8 +789,11 @@ class Mininet( object ): """ fields = 'date cip cport sip sport ipver interval sent rate' lines = iperfcsv.strip().split('\n') - if lines: - line = lines[ -1 ].split( ',' ) + svals = {} + for line in lines: + if ',' not in line: + continue + line = line.split( ',' ) svals = dict( zip( fields.split(), line ) ) # Return client in cip:cport, server in sip:sport if svals[ 'cip' ] == serverip: @@ -798,8 +801,7 @@ class Mininet( object ): svals[ 'sip' ], svals[ 'cip' ] ) svals[ 'cport' ], svals[ 'sport' ] = ( svals[ 'sport' ], svals[ 'cport' ] ) - return svals - return {} + return svals # XXX This should be cleaned up diff --git a/mininet/node.py b/mininet/node.py index 2c37b5b..0d219c5 100644 --- a/mininet/node.py +++ b/mininet/node.py @@ -708,20 +708,20 @@ class CPULimitedHost( Host ): def cgroupSet( self, param, value, resource='cpu' ): "Set a cgroup parameter and return its value" - cmd = 'cgset -r %s.%s=%s /%s' % ( - resource, param, value, self.name ) - quietRun( cmd ) - nvalue = int( self.cgroupGet( param, resource ) ) - if nvalue != value: + cmd = [ 'cgset', '-r', "%s.%s=%s" % ( + resource, param, value), '/' + self.name ] + errFail( cmd ) + nvalue = self.cgroupGet( param, resource ) + if nvalue != str( value ): error( '*** error: cgroupSet: %s set to %s instead of %s\n' % ( param, nvalue, value ) ) return nvalue def cgroupGet( self, param, resource='cpu' ): "Return value of cgroup parameter" - cmd = 'cgget -r %s.%s /%s' % ( - resource, param, self.name ) - return int( quietRun( cmd ).split()[ -1 ] ) + pname = '%s.%s' % ( resource, param ) + cmd = 'cgget -n -r %s /%s' % ( pname, self.name ) + return quietRun( cmd )[len(pname)+1:].strip() def cgroupDel( self ): "Clean up our cgroup" @@ -788,6 +788,8 @@ class CPULimitedHost( Host ): def cfsInfo( self, f ): "Internal method: return parameters for CFS bandwidth" pstr, qstr = 'cfs_period_us', 'cfs_quota_us' + if self.cgversion == 'cgroup2': + pstr, qstr = 'max', '' # CFS uses wall clock time for period and CPU time for quota. quota = int( self.period_us * f * numCores() ) period = self.period_us @@ -797,7 +799,7 @@ class CPULimitedHost( Host ): period = int( quota / f / numCores() ) # Reset to unlimited on negative quota if quota < 0: - quota = -1 + quota = 'max' if self.cgversion == 'cgroup2' else -1 return pstr, qstr, period, quota # BL comment: @@ -827,12 +829,16 @@ class CPULimitedHost( Host ): else: return # Set cgroup's period and quota - setPeriod = self.cgroupSet( pstr, period ) - setQuota = self.cgroupSet( qstr, quota ) + if self.cgversion == 'cgroup': + setPeriod = self.cgroupSet( pstr, period ) + setQuota = self.cgroupSet( qstr, quota ) + else: + setQuota, setPeriod = self.cgroupSet( + pstr, '%s %s' % (quota, period) ).split() if sched == 'rt': # Set RT priority if necessary sched = self.chrt() - info( '(%s %d/%dus) ' % ( sched, setQuota, setPeriod ) ) + info( '(%s %s/%dus) ' % ( sched, setQuota, int( setPeriod ) ) ) def setCPUs( self, cores, mems=0 ): "Specify (real) cores that our cgroup can run on" @@ -864,11 +870,12 @@ class CPULimitedHost( Host ): return r inited = False + cgversion = 'cgroup2' @classmethod def init( cls ): "Initialization for CPULimitedHost class" - mountCgroups() + cls.cgversion = mountCgroups() cls.inited = True diff --git a/mininet/util.py b/mininet/util.py index 8ccc567..e776e9f 100644 --- a/mininet/util.py +++ b/mininet/util.py @@ -5,6 +5,7 @@ import os import re import sys +from collections import namedtuple from fcntl import fcntl, F_GETFL, F_SETFL from functools import partial from os import O_NONBLOCK @@ -126,6 +127,8 @@ def oldQuietRun( *cmd ): # This is a bit complicated, but it enables us to # monitor command output as it is happening +CmdResult = namedtuple( 'CmdResult', 'out err ret' ) + # pylint: disable=too-many-branches,too-many-statements def errRun( *cmd, **kwargs ): """Run a command and return stdout, stderr and return code @@ -194,7 +197,8 @@ def errRun( *cmd, **kwargs ): if stderr == PIPE: popen.stderr.close() debug( out, err, returncode ) - return out, err, returncode + return CmdResult( out, err, returncode ) + # pylint: enable=too-many-branches def errFail( *cmd, **kwargs ): @@ -203,11 +207,11 @@ def errFail( *cmd, **kwargs ): if ret: raise Exception( "errFail: %s failed with return code %s: %s" % ( cmd, ret, err ) ) - return out, err, ret + return CmdResult( out, err, ret ) def quietRun( cmd, **kwargs ): "Run a command and return merged stdout and stderr" - return errRun( cmd, stderr=STDOUT, **kwargs )[ 0 ] + return errRun( cmd, stderr=STDOUT, **kwargs ).out def which(cmd, **kwargs ): "Run a command and return merged stdout and stderr" @@ -545,18 +549,25 @@ def fixLimits(): "Mininet's performance may be affected.\n" ) # pylint: enable=broad-except - -def mountCgroups(): - "Make sure cgroups file system is mounted" - mounts = quietRun( 'grep cgroup /proc/mounts' ) - cgdir = '/sys/fs/cgroup' - csdir = cgdir + '/cpuset' - if ('cgroup %s' % cgdir not in mounts and - 'cgroups %s' % cgdir not in mounts): - raise Exception( "cgroups not mounted on " + cgdir ) - if 'cpuset %s' % csdir not in mounts: - errRun( 'mkdir -p ' + csdir ) - errRun( 'mount -t cgroup -ocpuset cpuset ' + csdir ) +def mountCgroups( cgcontrol='cpu cpuacct cpuset' ): + """Mount cgroupfs if needed and return cgroup version + cgcontrol: cgroup controllers to check ('cpu cpuacct cpuset') + Returns: 'cgroup' | 'cgroup2' """ + # Try to read the cgroup controllers in cgcontrol + cglist = cgcontrol.split() + paths = ' '.join( '-g ' + c for c in cglist ) + cmd = 'cgget -n %s /' % paths + result = errRun( cmd ) + # If it failed, mount cgroupfs and retry + if result.ret or result.err or any( + c not in result.out for c in cglist ): + errFail( 'cgroupfs-mount' ) + result = errRun( cmd ) + errFail( cmd ) + # cpu.cfs_period_us is used for cgroup but not cgroup2 + if 'cpu.cfs_period_us' in result.out: + return 'cgroup' + return 'cgroup2' def natural( text ): "To sort sanely/alphabetically: sorted( l, key=natural )"